WEBDIENSTE/volatility3
1
0
Fork 0
mirror of https://github.com/volatilityfoundation/volatility3.git synced 2026-08-30 03:39:51 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'develop' into feature/cli-config-hierarchy

Browse Source
This commit is contained in:
ikelos
2024-02-08 11:10:28 +00:00
committed by GitHub
parent 2e4c257e15 795477e24b
commit da06d2e55a
240 changed files with 25494 additions and 9376 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/ISSUE_TEMPLATE/bug_report.md
+4 -2
View File
@@ -23,8 +23,10 @@ Steps to reproduce the behavior:
**Expected behavior**
A clear and concise description of what you expected to happen.
**Screenshots**
If applicable, add screenshots to help explain your problem.
**Example output**
Please copy and paste the text demonstrating the issue, ideally with verbose output turned on (`vol.py -vvv ...`).
Text is preferred to screenshots for searching and to talk about specific parts of the output.
**Additional information**
Add any other information about the problem here.
.github/workflows/black.yml
+13
View File
@@ -0,0 +1,13 @@
name: Black python linter
on: [push, pull_request]
jobs:
lint:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
- uses: psf/black@stable
with:
options: "--check --diff --verbose"
src: "./volatility3"
.github/workflows/build-pypi.yml
+8 -6
View File
@@ -15,14 +15,16 @@ on:
jobs:
build:
runs-on: ubuntu-latest
runs-on: ubuntu-20.04
strategy:
matrix:
python-version: ["3.7"]
steps:
- uses: actions/checkout@v2
- name: Set up Python 3.x
uses: actions/setup-python@v2
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: '3.x'
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
.github/workflows/codeql.yml
+74
View File
@@ -0,0 +1,74 @@
# For most projects, this workflow file will not need changing; you simply need
# to commit it to your repository.
#
# You may wish to alter this file to override the set of languages analyzed,
# or to provide custom queries or build logic.
#
# ******** NOTE ********
# We have attempted to detect the languages in your repository. Please check
# the `language` matrix defined below to confirm you have the correct set of
# supported CodeQL languages.
#
name: "CodeQL"
on:
push:
branches: [ "develop" ]
pull_request:
# The branches below must be a subset of the branches above
branches: [ "develop" ]
# schedule:
# - cron: '16 8 * * 0'
jobs:
analyze:
name: Analyze
runs-on: ubuntu-20.04
permissions:
actions: read
contents: read
security-events: write
strategy:
fail-fast: false
matrix:
language: [ 'python' ]
# CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
# Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
steps:
- name: Checkout repository
uses: actions/checkout@v3
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v2
with:
languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file.
# By default, queries listed here will override any specified in a config file.
# Prefix the list here with "+" to use these queries and those in the config file.
# Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
queries: security-and-quality # ,security-extended
# Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java).
# If this step fails, then you should remove it and run the build manually (see below)
- name: Autobuild
uses: github/codeql-action/autobuild@v2
# ℹ️ Command-line programs to run using the OS shell.
# 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
# If the Autobuild fails above, remove it and uncomment the following three lines.
# modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
# - run: |
# echo "Run, Build Application using script"
# ./location_of_script_within_repo/buildscript.sh
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v2
with:
category: "/language:${{matrix.language}}"
.github/workflows/install.yml
+31
View File
@@ -0,0 +1,31 @@
name: Install Volatility3 test
on: [push, pull_request]
jobs:
install_test:
runs-on: ${{ matrix.host }}
strategy:
fail-fast: false
matrix:
host: [ ubuntu-latest, windows-latest ]
python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ]
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Setup python-pip
run: python -m pip install --upgrade pip
- name: Install dependencies
run: |
pip install -r requirements.txt
- name: Install volatility3
run: pip install .
- name: Run volatility3
run: vol --help
.github/workflows/stale.yml
+23
View File
@@ -0,0 +1,23 @@
name: Close inactive issues
on:
schedule:
- cron: "30 1 * * *"
jobs:
close-issues:
runs-on: ubuntu-latest
permissions:
issues: write
pull-requests: write
steps:
- uses: actions/stale@v5
with:
days-before-issue-stale: 200
days-before-issue-close: 60
stale-issue-label: "stale"
stale-issue-message: "This issue is stale because it has been open for 200 days with no activity."
close-issue-message: "This issue was closed because it has been inactive for 60 days since being marked as stale."
days-before-pr-stale: -1
days-before-pr-close: -1
repo-token: ${{ secrets.GITHUB_TOKEN }}
exempt-issue-labels: "enhancement,plugin-request,question"
.github/workflows/test.yaml
+55
View File
@@ -0,0 +1,55 @@
name: Test Volatility3
on: [push, pull_request]
jobs:
build:
runs-on: ubuntu-20.04
strategy:
matrix:
python-version: ["3.7"]
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install Cmake
pip install setuptools wheel
pip install -r ./test/requirements-testing.txt
- name: Build PyPi packages
run: |
python setup.py sdist --formats=gztar,zip
python setup.py bdist_wheel
- name: Download images
run: |
curl -sLO "https://downloads.volatilityfoundation.org/volatility3/images/linux-sample-1.bin.gz"
gunzip linux-sample-1.bin.gz
curl -sLO "https://downloads.volatilityfoundation.org/volatility3/images/win-xp-laptop-2005-06-25.img.gz"
gunzip win-xp-laptop-2005-06-25.img.gz
- name: Download and Extract symbols
run: |
cd ./volatility3/symbols
curl -sLO https://downloads.volatilityfoundation.org/volatility3/symbols/linux.zip
unzip linux.zip
cd -
- name: Testing...
run: |
py.test ./test/test_volatility.py --volatility=vol.py --image win-xp-laptop-2005-06-25.img -k test_windows -v
py.test ./test/test_volatility.py --volatility=vol.py --image linux-sample-1.bin -k test_linux -v
- name: Clean up post-test
run: |
rm -rf *.lime
rm -rf *.img
cd volatility3/symbols
rm -rf linux
rm -rf linux.zip
cd -
.gitignore
+4
View File
@@ -38,3 +38,7 @@ ENV/
# Memory dump files
*.dmp
*.vmem
*.img
# PyTest cache files
.pytest_cache/
.readthedocs.yml
+5 -1
View File
@@ -12,8 +12,12 @@ sphinx:
# Optionally build your docs in additional formats such as PDF and ePub
formats: all
build:
os: ubuntu-22.04
tools:
python: "3.11"
# Optionally set the version of Python and requirements required to build your docs
python:
version: 3.7
install:
- requirements: doc/requirements.txt
.style.yapf
+1 -1
View File
@@ -107,7 +107,7 @@ each_dict_entry_on_separate_line=True
i18n_comment=
# The i18n function call names. The presence of this function stops
# reformattting on that line, because the string it has cannot be moved
# reformatting on that line, because the string it has cannot be moved
# away from the i18n comment.
i18n_function_call=
API_CHANGES.md
+20
View File
@@ -4,6 +4,26 @@ API Changes
When an addition to the existing API is made, the minor version is bumped.
When an API feature or function is removed or changed, the major version is bumped.
2.5.0
=====
Add in support for specifying a type override for object_from_symbol
2.4.0
=====
Add a `get_size()` method to Windows VAD structures and fix several off-by-one issues when calculating VAD sizes.
2.3.1
=====
Update in the windows `_EPROCESS.owning_process` method to support Windows Vista and later versions.
2.3.0
=====
Add in `child_template` to template class
2.2.0
=====
Changes to linux core calls
2.1.0
=====
Add in the linux `task.get_threads` method to the API.
CITATION.cff
+37
View File
@@ -0,0 +1,37 @@
# This CITATION.cff file was generated with cffinit.
# Visit https://bit.ly/cffinit to generate yours today!
cff-version: 1.2.0
title: Volatility 3
message: >-
If you reference this software, please feel free to cite
it using the information below.
type: software
authors:
- name: Volatility Foundation
country: US
website: 'https://www.volatilityfoundation.org/'
identifiers:
- type: url
value: 'https://github.com/volatilityfoundation/volatility3'
description: Volatility 3 source code respository
repository-code: 'https://github.com/volatilityfoundation/volatility3'
url: 'https://github.com/volatilityfoundation/volatility3'
abstract: >-
Volatility is the world's most widely used framework for
extracting digital artifacts from volatile memory (RAM)
samples. The extraction techniques are performed
completely independent of the system being investigated
but offer visibility into the runtime state of the system.
The framework is intended to introduce people to the
techniques and complexities associated with extracting
digital artifacts from volatile memory samples and provide
a platform for further work into this exciting area of
research.
keywords:
- malware
- forensics
- memory
- python
- ram
- volatility
LICENSE.txt
+1 -1
View File
@@ -31,7 +31,7 @@ If you make any Additions available to others, such as by providing copies of th
- You are responsible to ensure you have rights in Additions necessary to comply with this section.
Contributing
If you contribute (or offer to contribute) any materials to Volatility Foundation for the software, such as by submitting a pull request to the repository for the software or related content run by Volatility Foundation, you agree to contribute them under the under the BSD 2-Clause Plus Patent License (in the case of software) or the Creative Commons Zero Public Domain Dedication (in the case of content), unless you clearly mark them "Not a Contribution."
If you contribute (or offer to contribute) any materials to Volatility Foundation for the software, such as by submitting a pull request to the repository for the software or related content run by Volatility Foundation, you agree to contribute them under the BSD 2-Clause Plus Patent License (in the case of software) or the Creative Commons Zero Public Domain Dedication (in the case of content), unless you clearly mark them "Not a Contribution."
Trademarks
This license grants you no rights to any trademarks or service marks.
README.md
+5 -2
View File
@@ -20,7 +20,7 @@ more details.
## Requirements
Volatility 3 requires Python 3.6.0 or later. To install the most minimal set of dependencies (some plugins will not work) use a command such as:
Volatility 3 requires Python 3.7.0 or later. To install the most minimal set of dependencies (some plugins will not work) use a command such as:
```shell
pip3 install -r requirements-minimal.txt
@@ -94,6 +94,9 @@ Symbol tables zip files must be placed, as named, into the `volatility3/symbols`
Windows symbols that cannot be found will be queried, downloaded, generated and cached. Mac and Linux symbol tables must be manually produced by a tool such as [dwarf2json](https://github.com/volatilityfoundation/dwarf2json).
Important: The first run of volatility with new symbol files will require the cache to be updated. The symbol packs contain a large number of symbol files and so may take some time to update!
However, this process only needs to be run once on each new symbol file, so assuming the pack stays in the same location will not need to be done again. Please also note it can be interrupted and next run will restart itself.
Please note: These are representative and are complete up to the point of creation for Windows and Mac. Due to the ease of compiling Linux kernels and the inability to uniquely distinguish them, an exhaustive set of Linux symbol tables cannot easily be supplied.
## Documentation
@@ -104,7 +107,7 @@ The latest generated copy of the documentation can be found at: <https://volatil
## Licensing and Copyright
Copyright (C) 2007-2022 Volatility Foundation
Copyright (C) 2007-2024 Volatility Foundation
All Rights Reserved
development/mac-kdk/parse_pbzx2.py
+41 -48
View File
@@ -17,61 +17,54 @@ def seekread(f, offset = None, length = 0, relative = True):
f.seek(offset, [0, 1, 2][relative])
if length:
return f.read(length)
return None
def parse_pbzx(pbzx_path):
section = 0
xar_out_path = '%s.part%02d.cpio.xz' % (pbzx_path, section)
f = open(pbzx_path, 'rb')
# pbzx = f.read()
# f.close()
magic = seekread(f, length = 4)
if magic != 'pbzx':
raise RuntimeError("Error: Not a pbzx file")
# Read 8 bytes for initial flags
flags = seekread(f, length = 8)
# Interpret the flags as a 64-bit big-endian unsigned int
flags = struct.unpack('>Q', flags)[0]
xar_f = open(xar_out_path, 'wb')
while flags & (1 << 24):
# Read in more flags
with open(pbzx_path, 'rb') as f:
# pbzx = f.read()
# f.close()
magic = seekread(f, length = 4)
if magic != 'pbzx':
raise RuntimeError("Error: Not a pbzx file")
# Read 8 bytes for initial flags
flags = seekread(f, length = 8)
# Interpret the flags as a 64-bit big-endian unsigned int
flags = struct.unpack('>Q', flags)[0]
# Read in length
f_length = seekread(f, length = 8)
f_length = struct.unpack('>Q', f_length)[0]
xzmagic = seekread(f, length = 6)
if xzmagic != '\xfd7zXZ\x00':
# This isn't xz content, this is actually _raw decompressed cpio_ chunk of 16MB in size...
# Let's back up ...
seekread(f, offset = -6, length = 0)
# ... and split it out ...
f_content = seekread(f, length = f_length)
section += 1
decomp_out = '%s.part%02d.cpio' % (pbzx_path, section)
g = open(decomp_out, 'wb')
g.write(f_content)
g.close()
# Now to start the next section, which should hopefully be .xz (we'll just assume it is ...)
xar_f.close()
section += 1
new_out = '%s.part%02d.cpio.xz' % (pbzx_path, section)
xar_f = open(new_out, 'wb')
else:
f_length -= 6
# This part needs buffering
f_content = seekread(f, length = f_length)
tail = seekread(f, offset = -2, length = 2)
xar_f.write(xzmagic)
xar_f.write(f_content)
if tail != 'YZ':
xar_f.close()
raise RuntimeError("Error: Footer is not xar file footer")
try:
f.close()
xar_f.close()
except IOError:
pass
while flags & (1 << 24):
with open(xar_out_path, 'wb') as xar_f:
xar_f.seek(0, os.SEEK_END)
# Read in more flags
flags = seekread(f, length = 8)
flags = struct.unpack('>Q', flags)[0]
# Read in length
f_length = seekread(f, length = 8)
f_length = struct.unpack('>Q', f_length)[0]
xzmagic = seekread(f, length = 6)
if xzmagic != '\xfd7zXZ\x00':
# This isn't xz content, this is actually _raw decompressed cpio_ chunk of 16MB in size...
# Let's back up ...
seekread(f, offset = -6, length = 0)
# ... and split it out ...
f_content = seekread(f, length = f_length)
section += 1
decomp_out = '%s.part%02d.cpio' % (pbzx_path, section)
with open(decomp_out, 'wb') as g:
g.write(f_content)
# Now to start the next section, which should hopefully be .xz (we'll just assume it is ...)
section += 1
xar_out_path = '%s.part%02d.cpio.xz' % (pbzx_path, section)
else:
f_length -= 6
# This part needs buffering
f_content = seekread(f, length = f_length)
tail = seekread(f, offset = -2, length = 2)
xar_f.write(xzmagic)
xar_f.write(f_content)
if tail != 'YZ':
raise RuntimeError("Error: Footer is not xar file footer")
def main():
doc/requirements.txt
+5 -1
View File
@@ -1,4 +1,8 @@
# These packages are required for building the documentation.
sphinx>=4.0.0
sphinx>=4.0.0,<7
sphinx_autodoc_typehints>=1.4.0
sphinx-rtd-theme>=0.4.3
yara-python
pycryptodome
pefile
doc/source/basics.rst
+58 -15
View File
@@ -1,7 +1,7 @@
Volatility 3 Basics
===================
Volatility splits memory analysis down to several components:
Volatility splits memory analysis down to several components. The main ones are:
* Memory layers
* Templates and Objects
@@ -13,22 +13,65 @@ which acts as a container for all the various layers and tables necessary to con
Memory layers
-------------
A memory layer is a body of data that can be accessed by requesting data at a specific address. Memory is seen as
sequential when accessed through sequential addresses, however, there is no obligation for the data to be stored
sequentially, and modern processors tend to store the memory in a paged format. Moreover, there is no need for the data
to be stored in an easily accessible format, it could be encoded or encrypted or more, it could be the combination of
two other sources. These are typically handled by programs that process file formats, or the memory manager of the
processor, but these are all translations (either in the geometric or linguistic sense) of the original data.
A memory layer is a body of data that can be accessed by requesting data at a specific address. At its lowest level
this data is stored on a phyiscal medium (RAM) and very early computers addresses locations in memory directly. However,
as the size of memory increased and it became more difficult to manage memory most architectures moved to a "paged" model
of memory, where the available memory is cut into specific fixed-sized pages. To help further, programs can ask for any address
and the processor will look up their (virtual) address in a map, to find out where the (physical) address that it lives at is,
in the actual memory of the system.
In Volatility 3 this is represented by a directed graph, whose end nodes are
:py:class:`DataLayers <volatility3.framework.interfaces.layers.DataLayerInterface>` and whose internal nodes are
specifically called a :py:class:`TranslationLayer <volatility3.framework.interfaces.layers.TranslationLayerInterface>`.
In this way, a raw memory image in the LiME file format and a page file can be
combined to form a single Intel virtual memory layer. When requesting addresses from the Intel layer, it will use the
Intel memory mapping algorithm, along with the address of the directory table base or page table map, to translate that
Volatility can work with these layers as long as it knows the map (so, for example that virtual address `1` looks up at physical
address `9`). The automagic that runs at the start of every volatility session often locates the kernel's memory map, and creates
a kernel virtual layer, which allows for kernel addresses to be looked up and the correct data returned. There can, however, be
several maps, and in general there is a different map for each process (although a portion of the operating system's memory is
usually mapped to the same location across all processes). The maps may take the same address but point to a different part of
physical memory. It also means that two processes could theoretically share memory, but having an virtual address mapped to the
same physical address as another process. See the worked example below for more information.
To translate an address on a layer, call :py:meth:`layer.mapping(offset, length, ignore_errors) <volatility3.framework.interfaces.layers.TranslationLayerInterface.mapping>` and it will return a list of chunks without overlap, in order,
for the requested range. If a portion cannot be mapped, an exception will be thrown unless `ignore_errors` is true. Each
chunk will contain the original offset of the chunk, the translated offset, the original size and the translated size of
the chunk, as well as the lower layer the chunk lives within.
Worked example
^^^^^^^^^^^^^^
The operating system and two programs may all appear to have access to all of physical memory, but actually the maps they each have
mean they each see something different:
.. code-block::
:caption: Memory mapping example
Operating system map Physical Memory
1 -> 9 1 - Free
2 -> 3 2 - OS.4, Process 1.4, Process 2.4
3 -> 7 3 - OS.2
4 -> 2 4 - Free
5 - Free
Process 1 map 6 - Process 1.2, Process 2.3
1 -> 12 7 - OS.3
2 -> 6 8 - Process1.3
3 -> 8 9 - OS.1
4 -> 2 10 - Process2.1
11 - Free
Process 2 map 12 - Process1.1
1 -> 10 13 - Free
2 -> 15 14 - Free
3 -> 6 15 - Process2.2
4 -> 2 16 - Free
In this example, part of the operating system is visible across all processes (although not all processes can write to the memory, there
is a permissions model for intel addressing which is not discussed further here).)
In Volatility 3 mappings are represented by a directed graph of layers, whose end nodes are
:py:class:`DataLayers <volatility3.framework.interfaces.layers.DataLayerInterface>` and whose internal nodes are :py:class:`TranslationLayers <volatility3.framework.interfaces.layers.TranslationLayerInterface>`.
In this way, a raw memory image in the LiME file format and a page file can be combined to form a single Intel virtual
memory layer. When requesting addresses from the Intel layer, it will use the Intel memory mapping algorithm, along
with the address of the directory table base or page table map, to translate that
address into a physical address, which will then either be directed towards the swap layer or the LiME layer. Should it
be directed towards the LiME layer, the LiME file format algorithm will be translated to determine where within the file
the data is stored and that will be returned.
be directed towards the LiME layer, the LiME file format algorithm will be translate the new address to determine where
within the file the data is stored. When the :py:meth:`layer.read() <volatility3.framework.interfaces.layers.TranslationLayerInterface.read>`
method is called, the translation is done automatically and the correct data gathered and combined.
.. note:: Volatility 2 had a similar concept, called address spaces, but these could only stack linearly one on top of another.
doc/source/conf.py
+102 -45
View File
@@ -21,57 +21,72 @@ import sphinx.ext.apidoc
def setup(app):
volatility_directory = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', 'volatility3'))
volatility_directory = os.path.abspath(
os.path.join(os.path.dirname(__file__), "..", "..", "volatility3")
)
source_dir = os.path.abspath(os.path.dirname(__file__))
sphinx.ext.apidoc.main(argv = ['-e', '-M', '-f', '-T', '-o', source_dir, volatility_directory])
sphinx.ext.apidoc.main(
["-e", "-M", "-f", "-T", "-o", source_dir, volatility_directory]
)
# Go through the volatility3.framework.plugins files and change them to volatility3.plugins
for dir, _, files in os.walk(os.path.dirname(__file__)):
for filename in files:
if filename.startswith('volatility3.framework.plugins') and filename != 'volatility3.framework.plugins.rst':
if (
filename.startswith("volatility3.framework.plugins")
and filename != "volatility3.framework.plugins.rst"
):
# Change all volatility3.framework.plugins to volatility3.plugins in the file
# Rename the file
new_filename = filename.replace('volatility3.framework.plugins', 'volatility3.plugins')
new_filename = filename.replace(
"volatility3.framework.plugins", "volatility3.plugins"
)
replace_string = b"Submodules\n----------\n\n.. toctree::\n\n"
submodules = replace_string
# If file already exists, read out the subpackages entries from it add them to the new list
if os.path.exists(os.path.join(dir, new_filename)):
with open(os.path.join(dir, new_filename), 'rb') as newfile:
with open(os.path.join(dir, new_filename), "rb") as newfile:
data = newfile.read()
index = data.find(replace_string)
if index > -1:
submodules = data[index:]
with open(os.path.join(dir, new_filename), 'wb') as newfile:
with open(os.path.join(dir, new_filename), "wb") as newfile:
with open(os.path.join(dir, filename), "rb") as oldfile:
line = oldfile.read()
correct_plugins = line.replace(b'volatility3.framework.plugins', b'volatility3.plugins')
correct_submodules = correct_plugins.replace(replace_string, submodules)
correct_plugins = line.replace(
b"volatility3.framework.plugins", b"volatility3.plugins"
)
correct_submodules = correct_plugins.replace(
replace_string, submodules
)
newfile.write(correct_submodules)
os.remove(os.path.join(dir, filename))
elif filename == 'volatility3.framework.rst':
elif filename == "volatility3.framework.rst":
with open(os.path.join(dir, filename), "rb") as contents:
lines = contents.readlines()
plugins_seen = False
with open(os.path.join(dir, filename), "wb") as contents:
for line in lines:
if b'volatility3.framework.plugins' in line:
if b"volatility3.framework.plugins" in line:
plugins_seen = True
if plugins_seen and line == b'':
contents.write(b' volatility3.plugins')
if plugins_seen and line == b"":
contents.write(b" volatility3.plugins")
contents.write(line)
elif filename == 'volatility3.plugins.rst':
elif filename == "volatility3.plugins.rst":
with open(os.path.join(dir, filename), "rb") as contents:
lines = contents.readlines()
with open(os.path.join(dir, 'volatility3.framework.plugins.rst'), "rb") as contents:
with open(
os.path.join(dir, "volatility3.framework.plugins.rst"), "rb"
) as contents:
real_lines = contents.readlines()
# Process real_lines
for line_index in range(len(real_lines)):
if b'Submodules' in real_lines[line_index]:
if b"Submodules" in real_lines[line_index]:
break
else:
line_index = len(real_lines)
@@ -82,60 +97,79 @@ def setup(app):
for line in lines:
contents.write(line)
for line in submodule_lines:
contents.write(line.replace(b'volatility3.framework.plugins', b'volatility3.plugins'))
contents.write(
line.replace(
b"volatility3.framework.plugins", b"volatility3.plugins"
)
)
# Clear up the framework.plugins page
with open(os.path.join(os.path.dirname(__file__), 'volatility3.framework.plugins.rst'), "rb") as contents:
with open(
os.path.join(os.path.dirname(__file__), "volatility3.framework.plugins.rst"),
"rb",
) as contents:
real_lines = contents.readlines()
with open(os.path.join(os.path.dirname(__file__), 'volatility3.framework.plugins.rst'), "wb") as contents:
with open(
os.path.join(os.path.dirname(__file__), "volatility3.framework.plugins.rst"),
"wb",
) as contents:
for line in real_lines:
if b'volatility3.framework.plugins.' not in line:
if b"volatility3.framework.plugins." not in line:
contents.write(line)
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
sys.path.insert(0, os.path.abspath('../..'))
sys.path.insert(0, os.path.abspath("../.."))
from volatility3.framework import constants
# -- General configuration ------------------------------------------------
# If your documentation needs a minimal Sphinx version, state it here.
needs_sphinx = '2.0'
needs_sphinx = "2.0"
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
'sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.napoleon', 'sphinx.ext.intersphinx', 'sphinx.ext.todo',
'sphinx.ext.coverage', 'sphinx.ext.viewcode'
"sphinx.ext.autodoc",
"sphinx.ext.doctest",
"sphinx.ext.napoleon",
"sphinx.ext.intersphinx",
"sphinx.ext.todo",
"sphinx.ext.coverage",
"sphinx.ext.viewcode",
"sphinx.ext.autosectionlabel",
]
autosectionlabel_prefix_document = True
try:
import sphinx_autodoc_typehints
extensions.append('sphinx_autodoc_typehints')
extensions.append("sphinx_autodoc_typehints")
except ImportError:
# If the autodoc typehints extension isn't available, carry on regardless
pass
# Add any paths that contain templates here, relative to this directory.
# templates_path = ['tools/templates']
# The suffix of source filenames.
source_suffix = '.rst'
source_suffix = ".rst"
# The encoding of source files.
# source_encoding = 'utf-8-sig'
# The master toctree document.
master_doc = 'index'
master_doc = "index"
# General information about the project.
project = 'Volatility 3'
copyright = '2012-2022, Volatility Foundation'
project = "Volatility 3"
copyright = "2012-2024, Volatility Foundation"
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
@@ -144,7 +178,7 @@ copyright = '2012-2022, Volatility Foundation'
# The full version, including alpha/beta/rc tags.
release = constants.PACKAGE_VERSION
# The short X.Y version.
version = ".".join(release.split('.')[0:2])
version = ".".join(release.split(".")[0:2])
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
@@ -177,7 +211,7 @@ add_module_names = False
# show_authors = False
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'
pygments_style = "sphinx"
# A list of ignored prefixes for module index sorting.
# modindex_common_prefix = []
@@ -193,8 +227,8 @@ pygments_style = 'sphinx'
# html_theme = 'pydoctheme'
# html_theme_options = {'collapsiblesidebar': True}
# html_theme_path = ['tools']
html_theme = 'sphinx_rtd_theme'
html_theme_options = {'logo_only': True}
html_theme = "sphinx_rtd_theme"
html_theme_options = {"logo_only": True}
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
@@ -213,17 +247,17 @@ html_theme_options = {'logo_only': True}
# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
html_logo = '_static/vol.png'
html_logo = "_static/vol.png"
# The name of an image file (within the static path) to use as favicon of the
# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
html_favicon = '_static/favicon.ico'
html_favicon = "_static/favicon.ico"
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']
html_static_path = ["_static"]
# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
@@ -272,17 +306,15 @@ html_static_path = ['_static']
# html_file_suffix = None
# Output file base name for HTML help builder.
htmlhelp_basename = 'Volatilitydoc'
htmlhelp_basename = "Volatilitydoc"
# -- Options for LaTeX output ---------------------------------------------
latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
# 'papersize': 'letterpaper',
# The font size ('10pt', '11pt' or '12pt').
# 'pointsize': '10pt',
# Additional stuff for the LaTeX preamble.
# 'preamble': '',
}
@@ -291,7 +323,13 @@ latex_elements = {
# (source start file, target name, title,
# author, documentclass [howto, manual, or own class]).
latex_documents = [
('index', 'Volatility.tex', 'Volatility 3 Documentation', 'Volatility Foundation', 'manual'),
(
"index",
"Volatility.tex",
"Volatility 3 Documentation",
"Volatility Foundation",
"manual",
),
]
# The name of an image file (relative to this directory) to place at the top of
@@ -318,7 +356,15 @@ latex_documents = [
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [('vol-cli', 'volatility', 'Volatility 3 Documentation', ['Volatility Foundation'], 1)]
man_pages = [
(
"vol-cli",
"volatility",
"Volatility 3 Documentation",
["Volatility Foundation"],
1,
)
]
# If true, show URL addresses after external links.
# man_show_urls = False
@@ -329,8 +375,15 @@ man_pages = [('vol-cli', 'volatility', 'Volatility 3 Documentation', ['Volatilit
# (source start file, target name, title, author,
# dir menu entry, description, category)
texinfo_documents = [
('index', 'Volatility', 'Volatility 3 Documentation', 'Volatility Foundation', 'Volatility',
'Memory forensics framework.', 'Miscellaneous'),
(
"index",
"Volatility",
"Volatility 3 Documentation",
"Volatility Foundation",
"Volatility",
"Memory forensics framework.",
"Miscellaneous",
),
]
# Documents to append as an appendix to all manuals.
@@ -346,10 +399,14 @@ texinfo_documents = [
# texinfo_no_detailmenu = False
# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {'http://docs.python.org/': None}
intersphinx_mapping = {"python": ("http://docs.python.org/", None)}
# -- Autodoc options -------------------------------------------------------
# autodoc_member_order = 'groupwise'
autodoc_default_options = {'members': True, 'inherited-members': True, 'show-inheritance': True}
autoclass_content = 'both'
autodoc_default_options = {
"members": True,
"inherited-members": True,
"show-inheritance": True,
}
autoclass_content = "both"
doc/source/getting-started-linux-tutorial.rst
+192
View File
@@ -0,0 +1,192 @@
Linux Tutorial
==============
This guide will give you a brief overview of how volatility3 works as well as a demonstration of several of the plugins available in the suite.
Acquiring memory
----------------
Volatility3 does not provide the ability to acquire memory. Below are some examples of tools that can be used to acquire memory, but more are available:
* `AVML - Acquire Volatile Memory for Linux <https://github.com/microsoft/avml>`_
* `LiME - Linux Memory Extract <https://github.com/504ensicsLabs/LiME>`_
Procedure to create symbol tables for linux
--------------------------------------------
To create a symbol table please refer to :ref:`symbol-tables:Mac or Linux symbol tables`.
.. tip:: It may be possible to locate pre-made ISF files from the `Linux ISF Server <https://isf-server.techanarchy.net/>`_ ,
which is built and maintained by `kevthehermit <https://twitter.com/kevthehermit>`_.
After creating the file or downloading it from the ISF server, place the file under the directory ``volatility3/symbols/linux``.
If necessary create a linux directory under the symbols directory (this will become unnecessary in future versions).
Listing plugins
---------------
The following is a sample of the linux plugins available for volatility3, it is not complete and more more plugins may
be added. For a complete reference, please see the volatility 3 :doc:`list of plugins <volatility3.plugins>`.
For plugin requests, please create an issue with a description of the requested plugin.
.. code-block:: shell-session
$ python3 vol.py --help | grep -i linux. | head -n 5
banners.Banners Attempts to identify potential linux banners in an
linux.bash.Bash Recovers bash command history from memory.
linux.check_afinfo.Check_afinfo
linux.check_creds.Check_creds
linux.check_idt.Check_idt
.. note:: Here the the command is piped to grep and head in-order to provide the start of the list of linux plugins.
Using plugins
-------------
The following is the syntax to run the volatility CLI.
.. code-block:: shell-session
$ python3 vol.py -f <path to memory image> <plugin_name> <plugin_option>
Example
-------
banners
~~~~~~~
In this example we will be using a memory dump from the Insomni'hack teaser 2020 CTF Challenge called Getdents. We will limit the discussion to memory forensics with volatility 3 and not extend it to other parts of the challenge.
Thanks go to `stuxnet <https://github.com/stuxnet999/>`_ for providing this memory dump and `writeup <https://stuxnet999.github.io/insomnihack/2020/09/17/Insomihack-getdents.html>`_.
.. code-block:: shell-session
$ python3 vol.py -f memory.vmem banners
Volatility 3 Framework 2.0.1
Progress: 100.00 PDB scanning finished
Offset Banner
0x141c1390 Linux version 4.15.0-42-generic (buildd@lgw01-amd64-023) (gcc version 7.3.0 (Ubuntu 7.3.0-16ubuntu3)) #45-Ubuntu SMP Thu Nov 15 19:32:57 UTC 2018 (Ubuntu 4.15.0-42.45-generic 4.15.18)
0x63a00160 Linux version 4.15.0-72-generic (buildd@lcy01-amd64-026) (gcc version 7.4.0 (Ubuntu 7.4.0-1ubuntu1~18.04.1)) #81-Ubuntu SMP Tue Nov 26 12:20:02 UTC 2019 (Ubuntu 4.15.0-72.81-generic 4.15.18)
0x6455c4d4 Linux version 4.15.0-72-generic (buildd@lcy01-amd64-026) (gcc version 7.4.0 (Ubuntu 7.4.0-1ubuntu1~18.04.1)) #81-Ubuntu SMP Tue Nov 26 12:20:02 UTC 2019 (Ubuntu 4.15.0-72.81-generic 4.15.18)
0x6e1e055f Linux version 4.15.0-72-generic (buildd@lcy01-amd64-026) (gcc version 7.4.0 (Ubuntu 7.4.0-1ubuntu1~18.04.1)) #81-Ubuntu SMP Tue Nov 26 12:20:02 UTC 2019 (Ubuntu 4.15.0-72.81-generic 4.15.18)
0x7fde0010 Linux version 4.15.0-72-generic (buildd@lcy01-amd64-026) (gcc version 7.4.0 (Ubuntu 7.4.0-1ubuntu1~18.04.1)) #81-Ubuntu SMP Tue Nov 26 12:20:02 UTC 2019 (Ubuntu 4.15.0-72.81-generic 4.15.18)
The above command helps us to find the memory dump's kernel version and the distribution version. Now using the above banner we can search for the needed ISF file from the ISF server.
If ISF file cannot be found then, follow the instructions on :ref:`getting-started-linux-tutorial:Procedure to create symbol tables for linux`. After that, place the ISF file under the ``volatility3/symbols/linux`` directory.
.. tip:: Use the banner text which is most repeated to search from ISF Server.
linux.pslist
~~~~~~~~~~~~
.. code-block:: shell-session
$ python3 vol.py -f memory.vmem linux.pslist
Volatility 3 Framework 2.0.1 Stacking attempts finished
PID PPID COMM
1 0 systemd
2 0 kthreadd
3 2 kworker/0:0
4 2 kworker/0:0H
5 2 kworker/u256:0
6 2 mm_percpu_wq
7 2 ksoftirqd/0
8 2 rcu_sched
9 2 rcu_bh
10 2 migration/0
11 2 watchdog/0
12 2 cpuhp/0
13 2 kdevtmpfs
14 2 netns
15 2 rcu_tasks_kthre
16 2 kauditd
.....
``linux.pslist`` helps us to list the processes which are running, their PIDs and PPIDs.
linux.pstree
~~~~~~~~~~~~
.. code-block:: shell-session
$ python3 vol.py -f memory.vmem linux.pstree
Volatility 3 Framework 2.0.1
Progress: 100.00 Stacking attempts finished
PID PPID COMM
1 0 systemd
* 636 1 polkitd
* 514 1 acpid
* 1411 1 pulseaudio
* 517 1 rsyslogd
* 637 1 cups-browsed
* 903 1 whoopsie
* 522 1 ModemManager
* 525 1 cron
* 526 1 avahi-daemon
** 542 526 avahi-daemon
* 657 1 unattended-upgr
* 914 1 kerneloops
* 532 1 dbus-daemon
* 1429 1 ibus-x11
* 929 1 kerneloops
* 1572 1 gsd-printer
* 933 1 upowerd
* 1071 1 rtkit-daemon
* 692 1 gdm3
** 1234 692 gdm-session-wor
*** 1255 1234 gdm-x-session
**** 1257 1255 Xorg
**** 1266 1255 gnome-session-b
***** 1537 1266 gsd-clipboard
***** 1539 1266 gsd-color
***** 1542 1266 gsd-datetime
***** 2950 1266 deja-dup-monito
***** 1546 1266 gsd-housekeepin
***** 1548 1266 gsd-keyboard
***** 1550 1266 gsd-media-keys
``linux.pstree`` helps us to display the parent child relationships between processes.
linux.bash
~~~~~~~~~~
Now to find the commands that were run in the bash shell by using ``linux.bash``.
.. code-block:: shell-session
$ python3 vol.py -f memory.vmem linux.bash
Volatility 3 Framework 2.0.1
Progress: 100.00 Stacking attempts finished
PID Process CommandTime Command
1733 bash 2020-01-16 14:00:36.000000 sudo reboot
1733 bash 2020-01-16 14:00:36.000000 AWAVH��
1733 bash 2020-01-16 14:00:36.000000 sudo apt upgrade
1733 bash 2020-01-16 14:00:36.000000 sudo apt upgrade
1733 bash 2020-01-16 14:00:36.000000 sudo reboot
1733 bash 2020-01-16 14:00:36.000000 sudo apt update
1733 bash 2020-01-16 14:00:36.000000 sudo apt update
1733 bash 2020-01-16 14:00:36.000000 sudo reboot
1733 bash 2020-01-16 14:00:36.000000 sudo apt upgrade
1733 bash 2020-01-16 14:00:36.000000 sudo apt update
1733 bash 2020-01-16 14:00:36.000000 rub
1733 bash 2020-01-16 14:00:36.000000 sudo apt upgrade
1733 bash 2020-01-16 14:00:36.000000 uname -a
1733 bash 2020-01-16 14:00:36.000000 uname -a
1733 bash 2020-01-16 14:00:36.000000 sudo apt autoclean
1733 bash 2020-01-16 14:00:36.000000 sudo reboot
1733 bash 2020-01-16 14:00:36.000000 sudo apt upgrade
1733 bash 2020-01-16 14:00:41.000000 chmod +x meterpreter
1733 bash 2020-01-16 14:00:42.000000 sudo ./meterpreter
doc/source/getting-started-mac-tutorial.rst
+153
View File
@@ -0,0 +1,153 @@
macOS Tutorial
==============
This guide will give you a brief overview of how volatility3 works as well as a demonstration of several of the plugins available in the suite.
Acquiring memory
----------------
Volatility3 does not provide the ability to acquire memory. The example below is an open source tool. Other commercial tools are also available.
* `osxpmem <https://github.com/Velocidex/c-aff4/releases/download/3.2/osxpmem_3.2.zip>`_
Procedure to create symbol tables for macOS
--------------------------------------------
To create a symbol table please refer to :ref:`symbol-tables:Mac or Linux symbol tables`.
.. tip:: It may be possible to locate pre-made ISF files from the `download link <https://downloads.volatilityfoundation.org/volatility3/symbols/mac.zip>`_ ,
which is built and maintained by `volatilityfoundation <https://www.volatilityfoundation.org/>`_.
After creating the file or downloading it from the link, place the file under the directory ``volatility3/symbols/``.
Listing plugins
---------------
The following is a sample of the macOS plugins available for volatility3, it is not complete and more plugins may
be added. For a complete reference, please see the volatility 3 :doc:`list of plugins <volatility3.plugins>`.
For plugin requests, please create an issue with a description of the requested plugin.
.. code-block:: shell-session
$ python3 vol.py --help | grep -i mac. | head -n 4
mac.bash.Bash Recovers bash command history from memory.
mac.check_syscall.Check_syscall
mac.check_sysctl.Check_sysctl
mac.check_trap_table.Check_trap_table
.. note:: Here the the command is piped to grep and head in-order to provide the start of the list of macOS plugins.
Using plugins
-------------
The following is the syntax to run the volatility CLI.
.. code-block:: shell-session
$ python3 vol.py -f <path to memory image> <plugin_name> <plugin_option>
Example
-------
banners
~~~~~~~
In this example we will be using a memory dump from the Securinets CTF Quals 2019 Challenge called Contact_me. We will limit the discussion to memory forensics with volatility 3 and not extend it to other parts of the challenge.
Thanks go to `stuxnet <https://github.com/stuxnet999/>`_ for providing this memory dump and `writeup <https://stuxnet999.github.io/securinets-ctf/2019/08/24/SecurinetsQuals2019-Contact-Me.html>`_.
.. code-block:: shell-session
$ python3 vol.py -f contact_me banners.Banners
Volatility 3 Framework 2.4.2
Progress: 100.00 PDB scanning finished
Offset Banner
0x4d2c7d0 Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64
0xb42b180 Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64
0xcda9100 Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64
0x1275e7d0 Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64
0x1284fba4 Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64
0x34ad0180 Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64
The above command helps us to find the memory dump's Darwin kernel version. Now using the above banner we can search for the needed ISF file.
If ISF file cannot be found then, follow the instructions on :ref:`getting-started-mac-tutorial:Procedure to create symbol tables for macOS`. After that, place the ISF file under the ``volatility3/symbols`` directory.
mac.pslist
~~~~~~~~~~
.. code-block:: shell-session
$ python3 vol.py -f contact_me mac.pslist.PsList
Volatility 3 Framework 2.4.2
Progress: 100.00 Stacking attempts finished
PID PPID COMM
0 0 kernel_task
1 0 launchd
35 1 UserEventAgent
38 1 kextd
39 1 fseventsd
37 1 uninstalld
45 1 configd
46 1 powerd
52 1 logd
58 1 warmd
.....
``mac.pslist`` helps us to list the processes which are running, their PIDs and PPIDs.
mac.pstree
~~~~~~~~~~
.. code-block:: shell-session
$ python3 vol.py -f contact_me mac.pstree.PsTree
Volatility 3 Framework 2.4.2
Progress: 100.00 Stacking attempts finished
PID PPID COMM
35 1 UserEventAgent
38 1 kextd
39 1 fseventsd
37 1 uninstalld
204 1 softwareupdated
* 449 204 SoftwareUpdateCo
337 1 system_installd
* 455 337 update_dyld_shar
``mac.pstree`` helps us to display the parent child relationships between processes.
mac.ifconfig
~~~~~~~~~~~~
.. code-block:: shell-session
$ python3 vol.py -f contact_me mac.ifconfig.Ifconfig
Volatility 3 Framework 2.4.2
Progress: 100.00 Stacking attempts finished
Interface IP Address Mac Address Promiscuous
lo0 False
lo0 127.0.0.1 False
lo0 ::1 False
lo0 fe80:1::1 False
gif0 False
stf0 False
en0 00:0C:29:89:8B:F0 00:0C:29:89:8B:F0 False
en0 fe80:4::10fb:c89d:217f:52ae 00:0C:29:89:8B:F0 False
en0 192.168.140.128 00:0C:29:89:8B:F0 False
utun0 False
utun0 fe80:5::2a95:bb15:87e3:977c False
we can use the ``mac.ifconfig`` plugin to get information about the configuration of the network interfaces of the host under investigation.
doc/source/getting-started-windows-tutorial.rst
+124
View File
@@ -0,0 +1,124 @@
Windows Tutorial
================
This guide provides a brief introduction to how volatility3 works as a demonstration of several of the plugins available in the suite.
Acquiring memory
----------------
Volatility does not provide the ability to acquire memory.
Memory can be acquired using a number of tools, below are some examples but others exist:
* `WinPmem <https://github.com/Velocidex/WinPmem/releases/latest>`_
* `FTK Imager <https://accessdata.com/product-download/ftk-imager-version-4-5>`_
Listing Plugins
---------------
The following is a sample of the windows plugins available for volatility3, it is not complete and more more plugins may
be added. For a complete reference, please see the volatility 3 :doc:`list of plugins <volatility3.plugins>`.
For plugin requests, please create an issue with a description of the requested plugin.
.. code-block:: shell-session
$ python3 vol.py --help | grep windows | head -n 5
windows.bigpools.BigPools
windows.cmdline.CmdLine
windows.crashinfo.Crashinfo
windows.dlllist.DllList
.. note:: Here the the command is piped to grep and head in-order to provide the start of a list of the available windows plugins.
Using plugins
-------------
The following is the syntax to run the volatility CLI.
.. code-block:: shell-session
$ python3 vol.py -f <path to memory image> plugin_name plugin_option
Example
-------
windows.pslist
~~~~~~~~~~~~~~
In this example we will be using a memory dump from the PragyanCTF'22.
We will limit the discussion to memory forensics with volatility 3 and not extend it to other parts of the challenges.
When using windows plugins in volatility 3, the required ISF file can often be generated from PDB files automatically
downloaded from Microsoft servers, and therefore does not require locating or adding specific ISF files to the volatility 3 symbols directory.
.. code-block:: shell-session
$ python3 vol.py -f MemDump.DMP windows.pslist | head -n 10
Volatility 3 Framework 2.0.1 PDB scanning finished
PID PPID ImageFileName Offset(V) Threads Handles SessionId Wow64 CreateTime ExitTime File output
4 0 System 0xfa8000cbc040 85 492 N/A False 2022-02-07 16:30:12.000000 N/A Disabled
276 4 smss.exe 0xfa8001e04040 2 29 N/A False 2022-02-07 16:30:12.000000 N/A Disabled
352 336 csrss.exe 0xfa8002110b30 9 375 0 False 2022-02-07 16:30:13.000000 N/A Disabled
404 336 wininit.exe 0xfa800219f060 3 74 0 False 2022-02-07 16:30:13.000000 N/A Disabled
412 396 csrss.exe 0xfa80021c5b30 9 224 1 False 2022-02-07 16:30:13.000000 N/A Disabled
468 396 winlogon.exe 0xfa8002284060 5 113 1 False 2022-02-07 16:30:14.000000 N/A Disabled
``windows.pslist`` helps list the processes running while the memory dump was taken.
windows.pstree
~~~~~~~~~~~~~~
.. code-block:: shell-session
$ python3 vol.py -f MemDump.DMP windows.pstree | head -n 20
Volatility 3 Framework 2.0.1 PDB scanning finished
PID PPID ImageFileName Offset(V) Threads Handles SessionId Wow64 CreateTime ExitTime
4 0 System 0xfa8000cbc040 85 492 N/A False 2022-02-07 16:30:12.000000 N/A
* 276 4 smss.exe 0xfa8001e04040 2 29 N/A False 2022-02-07 16:30:12.000000 N/A
352 336 csrss.exe 0xfa8002110b30 9 375 0 False 2022-02-07 16:30:13.000000 N/A
404 336 wininit.exe 0xfa800219f060 3 74 0 False 2022-02-07 16:30:13.000000 N/A
* 504 404 services.exe 0xfa80022ccb30 7 190 0 False 2022-02-07 16:30:14.000000 N/A
** 960 504 svchost.exe 0xfa8001c17b30 39 1003 0 False 2022-02-07 16:30:14.000000 N/A
** 1216 504 svchost.exe 0xfa80026e0b30 18 311 0 False 2022-02-07 16:30:15.000000 N/A
** 1312 504 svchost.exe 0xfa8002740380 19 287 0 False 2022-02-07 16:30:15.000000 N/A
** 1984 504 taskhost.exe 0xfa8002eb1b30 8 129 1 False 2022-02-07 16:30:27.000000 N/A
** 804 504 svchost.exe 0xfa80024ca5f0 20 450 0 False 2022-02-07 16:30:14.000000 N/A
*** 100 804 audiodg.exe 0xfa80025b4b30 6 131 0 False 2022-02-07 16:30:14.000000 N/A
** 1568 504 SearchIndexer. 0xfa800254b480 12 616 0 False 2022-02-07 16:30:32.000000 N/A
** 744 504 svchost.exe 0xfa8002477b30 8 265 0 False 2022-02-07 16:30:14.000000 N/A
** 1096 504 svchost.exe 0xfa800260db30 14 357 0 False 2022-02-07 16:30:14.000000 N/A
** 616 504 svchost.exe 0xfa8002b86ab0 13 314 0 False 2022-02-07 16:32:16.000000 N/A
** 624 504 svchost.exe 0xfa8002410630 10 350 0 False 2022-02-07 16:30:14.000000 N/A
``windows.pstree`` helps to display the parent child relationships between processes.
.. note:: Here the the command is piped to head in-order to provide smaller output, here listing only the first 20.
windows.hashdump
~~~~~~~~~~~~~~~~
.. code-block:: shell-session
$ python3 vol.py -f MemDump.DMP windows.hashdump
Volatility 3 Framework 2.0.3
Progress: 100.00 PDB scanning finished
User rid lmhash nthash
Administrator 500 aad3b435b51404eeaad3b435b51404ee 31d6cfe0d16ae931b73c59d7e0c089c0
Guest 501 aad3b435b51404eeaad3b435b51404ee 31d6cfe0d16ae931b73c59d7e0c089c0
Frank Reynolds 1000 aad3b435b51404eeaad3b435b51404ee a88d1e18706d3aa676e01e5943d15911
HomeGroupUser$ 1002 aad3b435b51404eeaad3b435b51404ee af10ecac6ea817d2bb56e3e5c33ce1cd
Dennis 1003 aad3b435b51404eeaad3b435b51404ee cf96684bbc7877920adaa9663698bf54
``windows.hashdump`` helps to list the hashes of the users in the system.
doc/source/index.rst
+14 -3
View File
@@ -7,9 +7,10 @@ Volatility 3 is Open Source.
:doc:`List of plugins <volatility3.plugins>`
Here are some guidelines for using Volatility 3 effectively:
Below is the main documentation regarding volatility 3:
.. toctree::
:caption: Documentation
basics
development
@@ -18,12 +19,22 @@ Here are some guidelines for using Volatility 3 effectively:
volshell
glossary
Python Packages
===============
There is also some information to get you started quickly:
.. toctree::
:caption: Getting Started
getting-started-linux-tutorial
getting-started-mac-tutorial
getting-started-windows-tutorial
.. toctree::
:caption: Python Packages
volatility3
Indices and tables
==================
doc/source/simple-plugin.rst
+105 -54
View File
@@ -6,6 +6,12 @@ This guide will step through how to construct a simple plugin using Volatility 3
The example plugin we'll use is :py:class:`~volatility3.plugins.windows.dlllist.DllList`, which features the main traits
of a normal plugin, and reuses other plugins appropriately.
.. note::
This document will not include the complete code necessary for a
working plugin (such as imports, etc) since it's designed to focus on the necessary components for writing a plugin.
For complete and functioning plugins, the ``framework/plugins`` directory should be consulted.
Inherit from PluginInterface
----------------------------
@@ -30,20 +36,20 @@ to be able to run properly. Any that are defined as optional need not necessari
::
_version = (1, 0, 0)
_required_framework_version = (2, 0, 0)
@classmethod
def get_requirements(cls):
return [requirements.TranslationLayerRequirement(name = 'primary',
description = 'Memory layer for the kernel',
architectures = ["Intel32", "Intel64"]),
requirements.SymbolTableRequirement(name = "nt_symbols",
description = "Windows kernel symbols"),
requirements.PluginRequirement(name = 'pslist',
plugin = pslist.PsList,
version = (1, 0, 0)),
return [requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel',
architectures = ["Intel32", "Intel64"]),
requirements.ListRequirement(name = 'pid',
element_type = int,
description = "Process IDs to include (all other processes are excluded)",
optional = True)]
optional = True),
requirements.PluginRequirement(name = 'pslist',
plugin = pslist.PsList,
version = (2, 0, 0))]
This is a classmethod, because it is called before the specific plugin object has been instantiated (in order to know how
@@ -51,69 +57,112 @@ to instantiate the plugin). At the moment these requirements are fairly straigh
::
requirements.TranslationLayerRequirement(name = 'primary',
description = 'Memory layer for the kernel',
architectures = ["Intel32", "Intel64"]),
requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel',
architectures = ["Intel32", "Intel64"]),
This requirement indicates that the plugin will operate on a single
:py:class:`TranslationLayer <volatility3.framework.interfaces.layers.TranslationLayerInterface>`. The name of the
loaded layer will appear in the plugin's configuration under the name ``primary``. Requirement values can be
accessed within the plugin through the plugin's `config` attribute (for example ``self.config['pid']``).
This requirement specifies the need for a particular submodule. Each module requires a
:py:class:`TranslationLayer <volatility3.framework.interfaces.layers.TranslationLayerInterface>` and a
:py:class:`SymbolTable <volatility3.framework.interfaces.symbols.SymbolTableInterface>`, which are fulfilled by two
subrequirements: a
:py:class:`~volatility3.framework.configuration.requirements.TranslationLayerRequirement` and a
:py:class:`~volatility3.framework.configuration.requirements.SymbolTableRequirement`. At the moment, the automagic
only fills `ModuleRequirements` with kernels, and so has relatively few parameters. It requires the architecture for
the underlying TranslationLayer, and the offset of the module within that layer.
.. note:: The name itself is dynamic depending on the other layers already present in the Context. Always use the value
from the configuration rather than attempting to guess what the layer will be called.
The name of the module will be stored in the ``kernel`` configuration option, and the module object itself
can be accessed from the ``context.modules`` collection. This requirement is a Complex Requirement and therefore will
not be requested directly from the user.
Finally, this defines that the translation layer must be on the Intel Architecture. At the moment, this acts as a filter,
failing to be satisfied by memory images that do not match the architecture required.
Most plugins will only operate on a single layer, but it is entirely possible for a plugin to request two different
layers, for example a plugin that carries out some form of difference or statistics against multiple memory images.
.. note::
This requirement (and the next two) are known as Complex Requirements, and user interfaces will likely not directly
request a value for this from a user. The value stored in the configuration tree for a
:py:class:`~volatility3.framework.configuration.requirements.TranslationLayerRequirement` is
the string name of a layer present in the context's memory that satisfies the requirement.
In previous versions of volatility 3, there was no `ModuleRequirement`, and instead two requirements were defined
a :py:class:`TranslationLayer <volatility3.framework.interfaces.layers.TranslationLayerInterface>` and a `SymbolTableRequirement`. These still exist, and can be used, most plugins just
define a single `ModuleRequirement` for the kernel, which the automagic will populate. The `ModuleRequirement` has
two automatic sub-requirements, a `TranslationLayerRequirement` and a `SymbolTableRequirement`, but the module also
includes the offset of the module, and will allow future expansion to specify specific modules when application
level plugins become more common. Below are how the requirements would be specified:
::
::
requirements.SymbolTableRequirement(name = "nt_symbols",
description = "Windows kernel symbols"),
requirements.TranslationLayerRequirement(name = 'primary',
description = 'Memory layer for the kernel',
architectures = ["Intel32", "Intel64"]),
This requirement specifies the need for a particular
:py:class:`SymbolTable <volatility3.framework.interfaces.symbols.SymbolTableInterface>`
to be loaded. This gets populated by various
:py:class:`Automagic <volatility3.framework.interfaces.automagic.AutoMagicInterface>` as the nearest sibling to a particular
:py:class:`~volatility3.framework.configuration.requirements.TranslationLayerRequirement`.
This means that if the :py:class:`~volatility3.framework.configuration.requirements.TranslationLayerRequirement`
is satisfied and the :py:class:`Automagic <volatility3.framework.interfaces.automagic.AutoMagicInterface>` can determine
the appropriate :py:class:`SymbolTable <volatility3.framework.interfaces.symbols.SymbolTableInterface>`, the
name of the :py:class:`SymbolTable <volatility3.framework.interfaces.symbols.SymbolTableInterface>` will be stored in the configuration.
This requirement indicates that the plugin will operate on a single
:py:class:`TranslationLayer <volatility3.framework.interfaces.layers.TranslationLayerInterface>`. The name of the
loaded layer will appear in the plugin's configuration under the name ``primary``. Requirement values can be
accessed within the plugin through the plugin's `config` attribute (for example ``self.config['pid']``).
This requirement is also a Complex Requirement and therefore will not be requested directly from the user.
.. note:: The name itself is dynamic depending on the other layers already present in the Context. Always use the value
from the configuration rather than attempting to guess what the layer will be called.
::
Finally, this defines that the translation layer must be on the Intel Architecture. At the moment, this acts as a filter,
failing to be satisfied by memory images that do not match the architecture required.
requirements.PluginRequirement(name = 'pslist',
plugin = pslist.PsList,
version = (1, 0, 0)),
Most plugins will only operate on a single layer, but it is entirely possible for a plugin to request two different
layers, for example a plugin that carries out some form of difference or statistics against multiple memory images.
This requirement indicates that the plugin will make use of another plugin's code, and specifies the version requirements
on that plugin. The version is specified in terms of Semantic Versioning, meaning that to be compatible, the major
versions must be identical and the minor version must be equal to or higher than the one provided. This requirement
does not make use of any data from the configuration, even if it were provided, it is merely a functional check before
running the plugin.
This requirement (and the next two) are known as Complex Requirements, and user interfaces will likely not directly
request a value for this from a user. The value stored in the configuration tree for a
:py:class:`~volatility3.framework.configuration.requirements.TranslationLayerRequirement` is
the string name of a layer present in the context's memory that satisfies the requirement.
::
requirements.SymbolTableRequirement(name = "nt_symbols",
description = "Windows kernel symbols"),
This requirement specifies the need for a particular
:py:class:`SymbolTable <volatility3.framework.interfaces.symbols.SymbolTableInterface>`
to be loaded. This gets populated by various
:py:class:`Automagic <volatility3.framework.interfaces.automagic.AutoMagicInterface>` as the nearest sibling to a particular
:py:class:`~volatility3.framework.configuration.requirements.TranslationLayerRequirement`.
This means that if the :py:class:`~volatility3.framework.configuration.requirements.TranslationLayerRequirement`
is satisfied and the :py:class:`Automagic <volatility3.framework.interfaces.automagic.AutoMagicInterface>` can determine
the appropriate :py:class:`SymbolTable <volatility3.framework.interfaces.symbols.SymbolTableInterface>`, the
name of the :py:class:`SymbolTable <volatility3.framework.interfaces.symbols.SymbolTableInterface>` will be stored in the configuration.
This requirement is also a Complex Requirement and therefore will not be requested directly from the user.
::
requirements.ListRequirement(name = 'pid',
description = 'Filter on specific process IDs',
element_type = int,
optional = True)
optional = True),
The final requirement is a List Requirement, populated by integers. The description will be presented to the user to
The next requirement is a List Requirement, populated by integers. The description will be presented to the user to
describe what the value represents. The optional flag indicates that the plugin can function without the ``pid`` value
being defined within the configuration tree at all.
::
requirements.PluginRequirement(name = 'pslist',
plugin = pslist.PsList,
version = (2, 0, 0))]
This requirement indicates that the plugin will make use of another plugin's code, and specifies the version requirements
on that plugin. The version is specified in terms of Semantic Versioning meaning that, to be compatible, the major
versions must be identical and the minor version must be equal to or higher than the one provided. This requirement
does not make use of any data from the configuration, even if it were provided, it is merely a functional check before
running the plugin. To define the version of a plugin, populate the `_version` class variable as a tuple of version
numbers `(major, minor, patch)`. So for example:
::
_version = (1, 0, 0)
The plugin may also require a specific version of the framework, and this also uses Semantic Versioning, and can be
set by defining the `_required_framework_version`. The major version should match the version of volatility the plugin
is to be used with, which at the time of writing would be 2.2.0, and so would be specified as below. If only features, for example,
from 2.0.0 are used, then the lowest applicable version number should be used to support the greatest number of
installations:
::
_required_framework_version = (2, 0, 0)
Define the `run` method
-----------------------
@@ -129,6 +178,7 @@ that will be output as part of the :py:class:`~volatility3.framework.interfaces.
def run(self):
filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None))
kernel = self.context.modules[self.config['kernel']]
return renderers.TreeGrid([("PID", int),
("Process", str),
@@ -137,8 +187,8 @@ that will be output as part of the :py:class:`~volatility3.framework.interfaces.
("Name", str),
("Path", str)],
self._generator(pslist.PsList.list_processes(self.context,
self.config['primary'],
self.config['nt_symbols'],
kernel.layer_name,
kernel.symbol_table_name,
filter_func = filter_func)))
In this instance, the plugin constructs a filter (using the PsList plugin's *classmethod* for creating filters).
@@ -157,7 +207,8 @@ the :py:class:`~volatility3.plugins.windows.pslist.PsList` plugin. That plugin
so that other plugins can call it. As such, it takes all the necessary parameters rather than accessing them
from a configuration. Since it must be portable code, it takes a context, as well as the layer name,
symbol table and optionally a filter. In this instance we unconditionally
pass it the values from the configuration for the ``primary`` and ``nt_symbols`` requirements. This will generate a list
pass it the values from the configuration for the layer and symbol table from the kernel module object, constructed from
the ``kernel`` configuration requirement. This will generate a list
of :py:class:`~volatility3.framework.symbols.windows.extensions.EPROCESS` objects, as provided by the :py:class:`~volatility.plugins.windows.pslist.PsList` plugin,
and is not covered here but is used as an example for how to share code across plugins
(both as the provider and the consumer of the shared code).
@@ -208,7 +259,7 @@ The plugin then takes the process's ``BaseDllName`` value, and calls :py:meth:`~
as defined by the symbols, are directly accessible and use the case-style of the symbol library it came from (in Windows,
attributes are CamelCase), such as ``entry.BaseDllName`` in this instance. Any attributes not defined by the symbol but added
by Volatility extensions cannot be properties (in case they overlap with the attributes defined in the symbol libraries)
and are therefore always methods and pretended with ``get_``, in this example ``BaseDllName.get_string()``.
and are therefore always methods and prepended with ``get_``, in this example ``BaseDllName.get_string()``.
Finally, ``FullDllName`` is populated. These operations read from memory, and as such, the memory image may be unable to
read the data at a particular offset. This will cause an exception to be thrown. In Volatility 3, exceptions are thrown
doc/source/symbol-tables.rst
+14 -13
View File
@@ -12,20 +12,22 @@ Volatility will automatically decompress them on use. It will also cache their
under the user's home directory, in :file:`.cache/volatility3`, along with other useful data. The cache directory currently
cannot be altered.
Symbol table JSON files live, by default, under the :file:`volatility3/symbols`, underneath an operating system directory
(currently one of :file:`windows`, :file:`mac` or :file:`linux`). The symbols directory is configurable within the framework and can
usually be set within the user interface.
Symbol table JSON files live, by default, under the :file:`volatility3/symbols` directory. The symbols directory is
configurable within the framework and can usually be set within the user interface.
These files can also be compressed into ZIP files, which Volatility will process in order to locate symbol files.
The ZIP file must be named after the appropriate operating system (such as `linux.zip`, `mac.zip` or `windows.zip`).
Inside the ZIP file, the directory structure should match the uncompressed operating system directory.
Volatility maintains a cache mapping the appropriate identifier for each symbol file against its filename. This cache
is updated by automagic called as part of the standard automagic that's run each time a plugin is run. If a large number of new
symbols file are detected, this may take some time, but can be safely interrupted and restarted and will not need to run again
as long as the symbol files stay in the same location.
Windows symbol tables
---------------------
For Windows systems, Volatility accepts a string made up of the GUID and Age of the required PDB file. It then
searches all files under the configured symbol directories under the windows subdirectory. Any that match the filename
pattern of :file:`<pdb-name>/<GUID>-<AGE>.json` (or any compressed variant) will be used. If such a symbol table cannot be found, then
searches all files under the configured symbol directories under the windows subdirectory. Any that contain metadata
which matches the pdb name and GUID/age (or any compressed variant) will be used. If such a symbol table cannot be found, then
the associated PDB file will be downloaded from Microsoft's Symbol Server and converted into the appropriate JSON
format, and will be saved in the correct location.
@@ -38,14 +40,13 @@ following command:
The :envvar:`PYTHONPATH` environment variable is not required if the Volatility library is installed in the system's library path
or a virtual environment.
Mac/Linux symbol tables
-----------------------
Mac or Linux symbol tables
--------------------------
For Mac/Linux systems, both use the same mechanism for identification. JSON files live under the symbol directories,
under either the :file:`linux` or :file:`mac` directories. The generated files contain an identifying string (the operating system
For Mac/Linux systems, both use the same mechanism for identification. The generated files contain an identifying string (the operating system
banner), which Volatility's automagic can detect. Volatility caches the mapping between the strings and the symbol
tables they come from, meaning the precise file names don't matter and can be organized under any necessary hierarchy
under the operating system directory.
under the symbols directory.
Linux and Mac symbol tables can be generated from a DWARF file using a tool called `dwarf2json <https://github.com/volatilityfoundation/dwarf2json>`_.
Currently a kernel with debugging symbols is the only suitable means for recovering all the information required by
@@ -93,4 +94,4 @@ file, the banners must match exactly (down to the compilation date).
* Copy the `.json` file to the symbols directory into `[symbols directory]/linux`
* For Mac change `linux` to `mac`
* For Mac change `linux` to `mac`
doc/source/using-as-a-library.rst
+10 -3
View File
@@ -54,6 +54,12 @@ also be included, which can be found in `volatility3.constants.PLUGINS_PATH`.
volatility3.plugins.__path__ = <new_plugin_path> + constants.PLUGINS_PATH
failures = framework.import_files(volatility3.plugins, True)
.. note::
Volatility uses the `volatility3.plugins` namespace for all plugins (including those in `volatility3.framework.plugins`).
Please ensure you only use `volatility3.plugins` and only ever import plugins from this namespace.
This ensures the ability of users to override core plugins without needing write access to the framework directory.
Once the plugins have been imported, we can interrogate which plugins are available. The
:py:func:`~volatility3.framework.list_plugins` call will
return a dictionary of plugin names and the plugin classes.
@@ -67,9 +73,10 @@ return a dictionary of plugin names and the plugin classes.
Determine what configuration options a plugin requires
------------------------------------------------------
For each plugin class, we can call the classmethod `requirements` on it, which will return a list of objects that
adhere to the :py:class:`~volatility3.framework.interfaces.configuration.RequirementInterface` method. The various
types of Requirement are split roughly in two,
For each plugin class, we can call the classmethod
:py:func:`~volatility3.framework.interfaces.configuration.ConfigurableInterface.get_requirements` on it, which will
return a list of objects that adhere to the :py:class:`~volatility3.framework.interfaces.configuration.RequirementInterface`
method. The various types of Requirement are split roughly in two,
:py:class:`~volatility3.framework.interfaces.configuration.SimpleTypeRequirement` (such as integers, booleans, floats
and strings) and more complex requirements (such as lists, choices, multiple requirements, translation layer
requirements or symbol table requirements). A requirement just specifies a type of data and a name, and must be
doc/source/vol-cli.rst
+23 -1
View File
@@ -9,7 +9,11 @@ Synopsis
**volatility** [-h] [-c CONFIG] [--parallelism [{processes,threads,off}]]
[-e EXTEND] [-p PLUGIN_DIRS] [-s SYMBOL_DIRS] [-v] [-l LOG]
[-o OUTPUT_DIR] [-q] [-r RENDERER] [-f FILE]
[--write-config] [--single-location SINGLE_LOCATION]
[--write-config] [--save-config SAVE_CONFIG]
[--clear-cache] [--cache-path CACHE_PATH]
[--offline]
[--single-location SINGLE_LOCATION]
[--stackers [STACKERS ...]]
[--single-swap-locations SINGLE_SWAP_LOCATIONS]
<plugin> ...
@@ -98,6 +102,10 @@ Options
attempt to build upon, and can be considered the input for the program.
--write-config
*Deprecated*
Use of `--write-config` has been deprecated, replaced by `--save-config`
--save-config
This flag specifies that volatility should write or overwrite a file
called config.json in the current directory. The file will contain
the necessary JSON configuration to recreate the environment that the
@@ -105,11 +113,25 @@ Options
other plugins, but there's no guarantee that plugins use the same
configuration options.
--clear-cache
Clears out all short-term cached items.
--cache-path
Change the default path used to store the cache.
--offline
Do not search online for additional JSON files.
Run offline mode (defaults to false) and for
remote windows symbol tables, linux/mac banner repositories.
--single-location SINGLE_LOCATION
This specifies a URL which will be downloaded if necessary, and built
upon by the automagic and, since most plugins require a single memory
image, can be considered the input for the program.
--stackers STACKERS
Creates the list of stackers to use based on the config option.
--single-swap-locations SINGLE_SWAP_LOCATIONS
A comma-separated list of swap files to be considered as part of the
memory image specified by the single-location or file parameters.
requirements-dev.txt
+22
View File
@@ -0,0 +1,22 @@
# The following packages are required for core functionality.
pefile>=2023.2.7
# The following packages are optional.
# If certain packages are not necessary, place a comment (#) at the start of the line.
# This is required for the yara plugins
yara-python>=3.8.0
# This is required for several plugins that perform malware analysis and disassemble code.
# It can also improve accuracy of Windows 8 and later memory samples.
capstone>=3.0.5
# This is required by plugins that decrypt passwords, password hashes, etc.
pycryptodome
# This can improve error messages regarding improperly configured ISF files,
# but is only recommended for development
jsonschema>=2.3.0
# This is required for memory acquisition via leechcore/pcileech.
leechcorepyc>=2.4.0
requirements-minimal.txt
+1 -1
View File
@@ -1,2 +1,2 @@
# These packages are required for core functionality.
pefile>=2017.8.1 #foo
pefile>=2023.2.7 #foo
requirements.txt
+4 -7
View File
@@ -1,5 +1,5 @@
# The following packages are required for core functionality.
pefile>=2017.8.1
pefile>=2023.2.7
# The following packages are optional.
# If certain packages are not necessary, place a comment (#) at the start of the line.
@@ -14,12 +14,9 @@ capstone>=3.0.5
# This is required by plugins that decrypt passwords, password hashes, etc.
pycryptodome
# This can improve error messages regarding improperly configured ISF files.
jsonschema>=2.3.0
# This is required for memory acquisition via leechcore/pcileech.
leechcorepyc>=2.4.0
# This is required for analyzing Linux samples compressed using AVMLs native
# compression format. It is not required for AVML's standard LiME compression.
python-snappy==0.6.0
# This is required for memory analysis on a Amazon/MinIO S3 and Google Cloud object storage
gcsfs>=2023.1.0
s3fs>=2023.1.0
setup.py
+33 -30
View File
@@ -6,9 +6,10 @@ import setuptools
from volatility3.framework import constants
with open("README.md", "r", encoding = "utf-8") as fh:
with open("README.md", "r", encoding="utf-8") as fh:
long_description = fh.read()
def get_install_requires():
requirements = []
with open("requirements-minimal.txt", "r", encoding="utf-8") as fh:
@@ -19,32 +20,34 @@ def get_install_requires():
requirements.append(stripped_line)
return requirements
setuptools.setup(name = "volatility3",
description = "Memory forensics framework",
version = constants.PACKAGE_VERSION,
license = "VSL",
keywords = "volatility memory forensics framework windows linux volshell",
author = "Volatility Foundation",
long_description = long_description,
long_description_content_type = "text/markdown",
author_email = "volatility@volatilityfoundation.org",
url = "https://github.com/volatilityfoundation/volatility3/",
project_urls = {
"Bug Tracker": "https://github.com/volatilityfoundation/volatility3/issues",
"Documentation": "https://volatility3.readthedocs.io/",
"Source Code": "https://github.com/volatilityfoundation/volatility3",
},
python_requires = '>=3.6.0',
include_package_data = True,
exclude_package_data = {
'': ['development', 'development.*'],
'development': ['*']
},
packages = setuptools.find_packages(exclude = ["development", "development.*"]),
entry_points = {
'console_scripts': [
'vol = volatility3.cli:main',
'volshell = volatility3.cli.volshell:main',
],
},
install_requires = get_install_requires())
setuptools.setup(
name="volatility3",
description="Memory forensics framework",
version=constants.PACKAGE_VERSION,
license="VSL",
keywords="volatility memory forensics framework windows linux volshell",
author="Volatility Foundation",
long_description=long_description,
long_description_content_type="text/markdown",
author_email="volatility@volatilityfoundation.org",
url="https://github.com/volatilityfoundation/volatility3/",
project_urls={
"Bug Tracker": "https://github.com/volatilityfoundation/volatility3/issues",
"Documentation": "https://volatility3.readthedocs.io/",
"Source Code": "https://github.com/volatilityfoundation/volatility3",
},
packages=setuptools.find_namespace_packages(
include=["volatility3", "volatility3.*"]
),
package_dir={"volatility3": "volatility3"},
python_requires=">=3.7.0",
include_package_data=True,
entry_points={
"console_scripts": [
"vol = volatility3.cli:main",
"volshell = volatility3.cli.volshell:main",
],
},
install_requires=get_install_requires(),
)
test/README.md
+34
View File
@@ -0,0 +1,34 @@
# Volatility 3 Testing Framework
## Requirements
The Volatility 3 Testing Framework requires the same version of Python as Volatility3 itself. To install the current set of dependencies that the framework requires, use a command like this:
```shell
pip3 install -r requirements-testing.txt
```
NOTE: `requirements-testing.txt` can be found in this current `test/` directory.
## Quick Start: Manual Testing
1. To test Volatility 3 on an image, first download one with a command such as:
```shell
curl -sLO "https://downloads.volatilityfoundation.org/volatility3/images/win-xp-laptop-2005-06-25.img.gz"
gunzip win-xp-laptop-2005-06-25.img.gz
```
2. In many cases, more symbols are required to be downloaded to the `./volatility3/symbols` directory.
3. To manually run the tests, run a command, such as:
```shell
py.test ./test/test_volatility.py --volatility=vol.py --image win-xp-laptop-2005-06-25.img -k test_windows
```
The above command runs all available tests for windows on the `win-xp-laptop-2005-06-25.img` image. To choose a more specific set of tests, change the phrase after `-k` in this command.
## Github Actions
This framework currently tests two images (one linux image and one windows image) after every push on any branch. For more information/context, find the actions setup in `./github/workflows/test.yaml`
test/conftest.py
+59
View File
@@ -0,0 +1,59 @@
# This file is used to augment the test configuration
import os
import pytest
def pytest_addoption(parser):
parser.addoption(
"--volatility",
action="store",
default=None,
required=True,
help="path to the volatility script",
)
parser.addoption(
"--python",
action="store",
default="python3",
help="The name of the interpreter to use when running the volatility script",
)
parser.addoption(
"--image", action="append", default=[], help="path to an image to test"
)
parser.addoption(
"--image-dir",
action="append",
default=[],
help="path to a directory containing images to test",
)
def pytest_generate_tests(metafunc):
"""Parameterize tests based on image names"""
images = metafunc.config.getoption("image")
for image_dir in metafunc.config.getoption("image_dir"):
images = images + [
os.path.join(image_dir, dir) for dir in os.listdir(image_dir)
]
# tests with "image" parameter are run against images
if "image" in metafunc.fixturenames:
metafunc.parametrize(
"image", images, ids=[os.path.basename(image) for image in images]
)
# Fixtures
@pytest.fixture
def volatility(request):
return request.config.getoption("--volatility")
@pytest.fixture
def python(request):
return request.config.getoption("--python")
test/known_files.json
+19
View File
@@ -0,0 +1,19 @@
{
"windows_dumpfiles": {
"win-xp-laptop-2005-06-25.img": {
"0x82220e78": [
"9bdd5532286f1660f3778e68bc36efe6",
"e3bc1e9e7370e3b5a661ebe591ecf4ec"
],
"0x82350bf8": [
"e5c5e8d97b6280745b41f6572c85d1f0",
"8589f1463422884dbf1411aaad278465"
],
"0x81eaf418": [
"f7a1ae2060a58f8470b97affdb46dccf",
"54fd611021fa784912530b8007545986"
],
"0x820588e8": "458efbc8fdb859488a6ab2b200cce809"
}
}
}
test/requirements-testing.txt
+10
View File
@@ -0,0 +1,10 @@
# These packages are required for core functionality.
pefile>=2017.8.1 #foo
# The following packages are optional.
# If certain packages are not necessary, place a comment (#) at the start of the line.
# This is required for the yara plugins
yara-python>=3.8.0
pytest>=7.0.0
test/test_volatility.py
+476
View File
@@ -0,0 +1,476 @@
# volatility3 tests
#
#
# IMPORTS
#
import os
import subprocess
import sys
import shutil
import tempfile
import hashlib
import ntpath
import json
#
# HELPER FUNCTIONS
#
def runvol(args, volatility, python):
volpy = volatility
python_cmd = python
cmd = [python_cmd, volpy] + args
print(" ".join(cmd))
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
print("stdout:")
sys.stdout.write(str(stdout))
print("")
print("stderr:")
sys.stdout.write(str(stderr))
print("")
return p.returncode, stdout, stderr
def runvol_plugin(plugin, img, volatility, python, pluginargs=[], globalargs=[]):
args = (
globalargs
+ [
"--single-location",
img,
"-q",
plugin,
]
+ pluginargs
)
return runvol(args, volatility, python)
#
# TESTS
#
# WINDOWS
def test_windows_pslist(image, volatility, python):
rc, out, err = runvol_plugin("windows.pslist.PsList", image, volatility, python)
out = out.lower()
assert out.find(b"system") != -1
assert out.find(b"csrss.exe") != -1
assert out.find(b"svchost.exe") != -1
assert out.count(b"\n") > 10
assert rc == 0
rc, out, err = runvol_plugin(
"windows.pslist.PsList", image, volatility, python, pluginargs=["--pid", "4"]
)
out = out.lower()
assert out.find(b"system") != -1
assert out.count(b"\n") < 10
assert rc == 0
def test_windows_psscan(image, volatility, python):
rc, out, err = runvol_plugin("windows.psscan.PsScan", image, volatility, python)
out = out.lower()
assert out.find(b"system") != -1
assert out.find(b"csrss.exe") != -1
assert out.find(b"svchost.exe") != -1
assert out.count(b"\n") > 10
assert rc == 0
def test_windows_dlllist(image, volatility, python):
rc, out, err = runvol_plugin("windows.dlllist.DllList", image, volatility, python)
out = out.lower()
assert out.count(b"\n") > 10
assert rc == 0
def test_windows_modules(image, volatility, python):
rc, out, err = runvol_plugin("windows.modules.Modules", image, volatility, python)
out = out.lower()
assert out.count(b"\n") > 10
assert rc == 0
def test_windows_hivelist(image, volatility, python):
rc, out, err = runvol_plugin(
"windows.registry.hivelist.HiveList", image, volatility, python
)
out = out.lower()
not_xp = out.find(b"\\systemroot\\system32\\config\\software")
if not_xp == -1:
assert (
out.find(b"\\device\\harddiskvolume1\\windows\\system32\\config\\software")
!= -1
)
assert out.count(b"\n") > 10
assert rc == 0
def test_windows_dumpfiles(image, volatility, python):
with open("./test/known_files.json") as json_file:
known_files = json.load(json_file)
failed_chksms = 0
if sys.platform == "win32":
file_name = ntpath.basename(image)
else:
file_name = os.path.basename(image)
try:
for addr in known_files["windows_dumpfiles"][file_name]:
path = tempfile.mkdtemp()
rc, out, err = runvol_plugin(
"windows.dumpfiles.DumpFiles",
image,
volatility,
python,
globalargs=["-o", path],
pluginargs=["--virtaddr", addr],
)
for file in os.listdir(path):
with open(os.path.join(path, file), "rb") as fp:
if (
hashlib.md5(fp.read()).hexdigest()
not in known_files["windows_dumpfiles"][file_name][addr]
):
failed_chksms += 1
shutil.rmtree(path)
json_file.close()
assert failed_chksms == 0
assert rc == 0
except Exception as e:
json_file.close()
print("Key Error raised on " + str(e))
assert False
def test_windows_handles(image, volatility, python):
rc, out, err = runvol_plugin(
"windows.handles.Handles", image, volatility, python, pluginargs=["--pid", "4"]
)
assert out.find(b"System Pid 4") != -1
assert (
out.find(
b"MACHINE\\SYSTEM\\CONTROLSET001\\CONTROL\\SESSION MANAGER\\MEMORY MANAGEMENT\\PREFETCHPARAMETERS"
)
!= -1
)
assert out.find(b"MACHINE\\SYSTEM\\SETUP") != -1
assert out.count(b"\n") > 500
assert rc == 0
def test_windows_svcscan(image, volatility, python):
rc, out, err = runvol_plugin("windows.svcscan.SvcScan", image, volatility, python)
assert out.find(b"Microsoft ACPI Driver") != -1
assert out.count(b"\n") > 250
assert rc == 0
def test_windows_privileges(image, volatility, python):
rc, out, err = runvol_plugin(
"windows.privileges.Privs", image, volatility, python, pluginargs=["--pid", "4"]
)
assert out.find(b"SeCreateTokenPrivilege") != -1
assert out.find(b"SeCreateGlobalPrivilege") != -1
assert out.find(b"SeAssignPrimaryTokenPrivilege") != -1
assert out.count(b"\n") > 20
assert rc == 0
def test_windows_getsids(image, volatility, python):
rc, out, err = runvol_plugin(
"windows.getsids.GetSIDs", image, volatility, python, pluginargs=["--pid", "4"]
)
assert out.find(b"Local System") != -1
assert out.find(b"Administrators") != -1
assert out.find(b"Everyone") != -1
assert out.find(b"Authenticated Users") != -1
assert rc == 0
def test_windows_envars(image, volatility, python):
rc, out, err = runvol_plugin("windows.envars.Envars", image, volatility, python)
assert out.find(b"PATH") != -1
assert out.find(b"PROCESSOR_ARCHITECTURE") != -1
assert out.find(b"USERNAME") != -1
assert out.find(b"SystemRoot") != -1
assert out.find(b"CommonProgramFiles") != -1
assert out.count(b"\n") > 500
assert rc == 0
def test_windows_callbacks(image, volatility, python):
rc, out, err = runvol_plugin(
"windows.callbacks.Callbacks", image, volatility, python
)
assert out.find(b"PspCreateProcessNotifyRoutine") != -1
assert out.find(b"KeBugCheckCallbackListHead") != -1
assert out.find(b"KeBugCheckReasonCallbackListHead") != -1
assert out.count(b"KeBugCheckReasonCallbackListHead ") > 5
assert rc == 0
def test_windows_vadwalk(image, volatility, python):
rc, out, err = runvol_plugin("windows.vadwalk.VadWalk", image, volatility, python)
assert out.find(b"Vad") != -1
assert out.find(b"VadS") != -1
assert out.find(b"Vadl") != -1
assert out.find(b"VadF") != -1
assert out.find(b"0x0") != -1
assert rc == 0
def test_windows_devicetree(image, volatility, python):
rc, out, err = runvol_plugin(
"windows.devicetree.DeviceTree", image, volatility, python
)
assert out.find(b"DEV") != -1
assert out.find(b"DRV") != -1
assert out.find(b"ATT") != -1
assert out.find(b"FILE_DEVICE_CONTROLLER") != -1
assert out.find(b"FILE_DEVICE_DISK") != -1
assert out.find(b"FILE_DEVICE_DISK_FILE_SYSTEM") != -1
assert rc == 0
# LINUX
def test_linux_pslist(image, volatility, python):
rc, out, err = runvol_plugin("linux.pslist.PsList", image, volatility, python)
out = out.lower()
assert (out.find(b"init") != -1) or (out.find(b"systemd") != -1)
assert out.find(b"watchdog") != -1
assert out.count(b"\n") > 10
assert rc == 0
def test_linux_check_idt(image, volatility, python):
rc, out, err = runvol_plugin("linux.check_idt.Check_idt", image, volatility, python)
out = out.lower()
assert out.count(b"__kernel__") >= 10
assert out.count(b"\n") > 10
assert rc == 0
def test_linux_check_syscall(image, volatility, python):
rc, out, err = runvol_plugin(
"linux.check_syscall.Check_syscall", image, volatility, python
)
out = out.lower()
assert out.find(b"sys_close") != -1
assert out.find(b"sys_open") != -1
assert out.count(b"\n") > 100
assert rc == 0
def test_linux_lsmod(image, volatility, python):
rc, out, err = runvol_plugin("linux.lsmod.Lsmod", image, volatility, python)
out = out.lower()
assert out.count(b"\n") > 10
assert rc == 0
def test_linux_lsof(image, volatility, python):
rc, out, err = runvol_plugin("linux.lsof.Lsof", image, volatility, python)
out = out.lower()
assert out.count(b"socket:") >= 10
assert out.count(b"\n") > 35
assert rc == 0
def test_linux_proc_maps(image, volatility, python):
rc, out, err = runvol_plugin("linux.proc.Maps", image, volatility, python)
out = out.lower()
assert out.count(b"anonymous mapping") >= 10
assert out.count(b"\n") > 100
assert rc == 0
def test_linux_tty_check(image, volatility, python):
rc, out, err = runvol_plugin("linux.tty_check.tty_check", image, volatility, python)
out = out.lower()
assert out.find(b"__kernel__") != -1
assert out.count(b"\n") >= 5
assert rc == 0
# MAC
def test_mac_pslist(image, volatility, python):
rc, out, err = runvol_plugin("mac.pslist.PsList", image, volatility, python)
out = out.lower()
assert (out.find(b"kernel_task") != -1) or (out.find(b"launchd") != -1)
assert out.count(b"\n") > 10
assert rc == 0
def test_mac_check_syscall(image, volatility, python):
rc, out, err = runvol_plugin(
"mac.check_syscall.Check_syscall", image, volatility, python
)
out = out.lower()
assert out.find(b"chmod") != -1
assert out.find(b"chown") != -1
assert out.find(b"nosys") != -1
assert out.count(b"\n") > 100
assert rc == 0
def test_mac_check_sysctl(image, volatility, python):
rc, out, err = runvol_plugin(
"mac.check_sysctl.Check_sysctl", image, volatility, python
)
out = out.lower()
assert out.find(b"__kernel__") != -1
assert out.count(b"\n") > 250
assert rc == 0
def test_mac_check_trap_table(image, volatility, python):
rc, out, err = runvol_plugin(
"mac.check_trap_table.Check_trap_table", image, volatility, python
)
out = out.lower()
assert out.count(b"kern_invalid") >= 10
assert out.count(b"\n") > 50
assert rc == 0
def test_mac_ifconfig(image, volatility, python):
rc, out, err = runvol_plugin("mac.ifconfig.Ifconfig", image, volatility, python)
out = out.lower()
assert out.find(b"127.0.0.1") != -1
assert out.find(b"false") != -1
assert out.count(b"\n") > 9
assert rc == 0
def test_mac_lsmod(image, volatility, python):
rc, out, err = runvol_plugin("mac.lsmod.Lsmod", image, volatility, python)
out = out.lower()
assert out.find(b"com.apple") != -1
assert out.count(b"\n") > 10
assert rc == 0
def test_mac_lsof(image, volatility, python):
rc, out, err = runvol_plugin("mac.lsof.Lsof", image, volatility, python)
out = out.lower()
assert out.count(b"\n") > 50
assert rc == 0
def test_mac_malfind(image, volatility, python):
rc, out, err = runvol_plugin("mac.malfind.Malfind", image, volatility, python)
out = out.lower()
assert out.count(b"\n") > 20
assert rc == 0
def test_mac_mount(image, volatility, python):
rc, out, err = runvol_plugin("mac.mount.Mount", image, volatility, python)
out = out.lower()
assert out.find(b"/dev") != -1
assert out.count(b"\n") > 7
assert rc == 0
def test_mac_netstat(image, volatility, python):
rc, out, err = runvol_plugin("mac.netstat.Netstat", image, volatility, python)
assert out.find(b"TCP") != -1
assert out.find(b"UDP") != -1
assert out.find(b"UNIX") != -1
assert out.count(b"\n") > 10
assert rc == 0
def test_mac_proc_maps(image, volatility, python):
rc, out, err = runvol_plugin("mac.proc_maps.Maps", image, volatility, python)
out = out.lower()
assert out.find(b"[heap]") != -1
assert out.count(b"\n") > 100
assert rc == 0
def test_mac_psaux(image, volatility, python):
rc, out, err = runvol_plugin("mac.psaux.Psaux", image, volatility, python)
out = out.lower()
assert out.find(b"executable_path") != -1
assert out.count(b"\n") > 50
assert rc == 0
def test_mac_socket_filters(image, volatility, python):
rc, out, err = runvol_plugin(
"mac.socket_filters.Socket_filters", image, volatility, python
)
out = out.lower()
assert out.count(b"\n") > 9
assert rc == 0
def test_mac_timers(image, volatility, python):
rc, out, err = runvol_plugin("mac.timers.Timers", image, volatility, python)
out = out.lower()
assert out.count(b"\n") > 6
assert rc == 0
def test_mac_trustedbsd(image, volatility, python):
rc, out, err = runvol_plugin("mac.trustedbsd.Trustedbsd", image, volatility, python)
out = out.lower()
assert out.count(b"\n") > 10
assert rc == 0
vol.py
+1 -1
View File
@@ -6,5 +6,5 @@
import volatility3.cli
if __name__ == '__main__':
if __name__ == "__main__":
volatility3.cli.main()
vol.spec
+1 -1
View File
@@ -26,7 +26,7 @@ except ImportError:
# Volatility must be findable in sys.path in order for collect_submodules to work
# This adds the current working directory, which should usually do the trick
sys.path.append(os.getcwd())
sys.path.append(os.path.dirname(os.path.abspath(SPEC)))
vol_analysis = Analysis(['vol.py'],
pathex = [],
volatility3/__init__.py
+10 -5
View File
@@ -32,14 +32,19 @@ class WarningFindSpec(abc.MetaPathFinder):
used."""
@staticmethod
def find_spec(fullname: str, path: Optional[List[str]], target: None = None, **kwargs) -> None:
def find_spec(
fullname: str, path: Optional[List[str]], target: None = None, **kwargs
) -> None:
"""Mock find_spec method that just checks the name, this must go
first."""
if fullname.startswith("volatility3.framework.plugins."):
warning = "Please do not use the volatility3.framework.plugins namespace directly, only use volatility3.plugins"
# Pyinstaller uses walk_packages to import, but needs to read the modules to figure out dependencies
# As such, we only print the warning when directly imported rather than from within walk_packages
if inspect.stack()[-2].function != 'walk_packages':
warning = f"Import {fullname}: Please do not use the volatility3.framework.plugins namespace directly, only use volatility3.plugins"
# Pyinstaller uses walk_packages/_collect_submodules to import, but needs to read the modules to figure out dependencies
# As such, we only print the warning when directly imported rather than from within walk_packages/_collect_submodules
if inspect.stack()[-2].function not in [
"walk_packages",
"_collect_submodules",
] and inspect.stack()[-3].function not in ["_collect_submodules"]:
raise Warning(warning)
volatility3/cli/__init__.py
+345 -191
View File
@@ -26,7 +26,15 @@ import volatility3.plugins
import volatility3.symbols
from volatility3 import framework
from volatility3.cli import text_renderer, volargparse
from volatility3.framework import automagic, configuration, constants, contexts, exceptions, interfaces, plugins
from volatility3.framework import (
automagic,
configuration,
constants,
contexts,
exceptions,
interfaces,
plugins,
)
from volatility3.framework.automagic import stacker
from volatility3.framework.configuration import requirements
@@ -36,7 +44,7 @@ rootlog = logging.getLogger()
vollog = logging.getLogger(__name__)
console = logging.StreamHandler()
console.setLevel(logging.WARNING)
formatter = logging.Formatter('%(levelname)-8s %(name)-12s: %(message)s')
formatter = logging.Formatter("%(levelname)-8s %(name)-12s: %(message)s")
# Trim the console down by default
console.setFormatter(formatter)
@@ -59,7 +67,7 @@ class PrintedProgress(object):
message = f"\rProgress: {round(progress, 2): 7.2f}\t\t{description or ''}"
message_len = len(message)
self._max_message_len = max([self._max_message_len, message_len])
sys.stderr.write(message + (' ' * (self._max_message_len - message_len)) + '\r')
sys.stderr.write(message + (" " * (self._max_message_len - message_len)) + "\r")
class MuteProgress(PrintedProgress):
@@ -72,7 +80,7 @@ class MuteProgress(PrintedProgress):
class CommandLine:
"""Constructs a command-line interface object for users to run plugins."""
CLI_NAME = 'volatility'
CLI_NAME = "volatility"
def __init__(self):
self.setup_logging()
@@ -90,98 +98,147 @@ class CommandLine:
volatility3.framework.require_interface_version(2, 0, 0)
renderers = dict([(x.name.lower(), x) for x in framework.class_subclasses(text_renderer.CLIRenderer)])
renderers = dict(
[
(x.name.lower(), x)
for x in framework.class_subclasses(text_renderer.CLIRenderer)
]
)
# Load up system defaults
delayed_logs, default_config = self.load_system_defaults('vol.json')
parser = volargparse.HelpfulArgParser(add_help = False,
prog = self.CLI_NAME,
description = "An open-source memory forensics framework")
parser = volargparse.HelpfulArgParser(
add_help=False,
prog=self.CLI_NAME,
description="An open-source memory forensics framework",
)
parser.add_argument(
"-h",
"--help",
action = "help",
default = argparse.SUPPRESS,
help = "Show this help message and exit, for specific plugin options use '{} <pluginname> --help'".format(
parser.prog))
parser.add_argument("-c",
"--config",
help = "Load the configuration from a json file",
default = None,
type = str)
parser.add_argument("--parallelism",
help = "Enables parallelism (defaults to off if no argument given)",
nargs = '?',
choices = ['processes', 'threads', 'off'],
const = 'processes',
default = None,
type = str)
parser.add_argument("-e",
"--extend",
help = "Extend the configuration with a new (or changed) setting",
default = None,
action = 'append')
parser.add_argument("-p",
"--plugin-dirs",
help = "Semi-colon separated list of paths to find plugins",
default = "",
type = str)
parser.add_argument("-s",
"--symbol-dirs",
help = "Semi-colon separated list of paths to find symbols",
default = "",
type = str)
parser.add_argument("-v", "--verbosity", help = "Increase output verbosity", default = 0, action = "count")
parser.add_argument("-l",
"--log",
help = "Log output to a file as well as the console",
default = None,
type = str)
parser.add_argument("-o",
"--output-dir",
help = "Directory in which to output any generated files",
default = os.getcwd(),
type = str)
parser.add_argument("-q", "--quiet", help = "Remove progress feedback", default = False, action = 'store_true')
parser.add_argument("-r",
"--renderer",
metavar = 'RENDERER',
help = f"Determines how to render the output ({', '.join(list(renderers))})",
default = "quick",
choices = list(renderers))
parser.add_argument("-f",
"--file",
metavar = 'FILE',
default = None,
type = str,
help = "Shorthand for --single-location=file:// if single-location is not defined")
parser.add_argument("--write-config",
help = "Write configuration JSON file out to config.json",
default = False,
action = 'store_true')
parser.add_argument("--save-config",
help = "Save configuration JSON file to a file",
default = None,
type = str)
parser.add_argument("--clear-cache",
help = "Clears out all short-term cached items",
default = False,
action = 'store_true')
parser.add_argument("--cache-path",
help = f"Change the default path ({constants.CACHE_PATH}) used to store the cache",
default = constants.CACHE_PATH,
type = str)
parser.add_argument("--offline",
help = "Do not search online for additional JSON files",
default = False,
action = 'store_true')
action="help",
default=argparse.SUPPRESS,
help="Show this help message and exit, for specific plugin options use '{} <pluginname> --help'".format(
parser.prog
),
)
parser.add_argument(
"-c",
"--config",
help="Load the configuration from a json file",
default=None,
type=str,
)
parser.add_argument(
"--parallelism",
help="Enables parallelism (defaults to off if no argument given)",
nargs="?",
choices=["processes", "threads", "off"],
const="processes",
default=None,
type=str,
)
parser.add_argument(
"-e",
"--extend",
help="Extend the configuration with a new (or changed) setting",
default=None,
action="append",
)
parser.add_argument(
"-p",
"--plugin-dirs",
help="Semi-colon separated list of paths to find plugins",
default="",
type=str,
)
parser.add_argument(
"-s",
"--symbol-dirs",
help="Semi-colon separated list of paths to find symbols",
default="",
type=str,
)
parser.add_argument(
"-v",
"--verbosity",
help="Increase output verbosity",
default=0,
action="count",
)
parser.add_argument(
"-l",
"--log",
help="Log output to a file as well as the console",
default=None,
type=str,
)
parser.add_argument(
"-o",
"--output-dir",
help="Directory in which to output any generated files",
default=os.getcwd(),
type=str,
)
parser.add_argument(
"-q",
"--quiet",
help="Remove progress feedback",
default=False,
action="store_true",
)
parser.add_argument(
"-r",
"--renderer",
metavar="RENDERER",
help=f"Determines how to render the output ({', '.join(list(renderers))})",
default="quick",
choices=list(renderers),
)
parser.add_argument(
"-f",
"--file",
metavar="FILE",
default=None,
type=str,
help="Shorthand for --single-location=file:// if single-location is not defined",
)
parser.add_argument(
"--write-config",
help="Write configuration JSON file out to config.json",
default=False,
action="store_true",
)
parser.add_argument(
"--save-config",
help="Save configuration JSON file to a file",
default=None,
type=str,
)
parser.add_argument(
"--clear-cache",
help="Clears out all short-term cached items",
default=False,
action="store_true",
)
parser.add_argument(
"--cache-path",
help=f"Change the default path ({constants.CACHE_PATH}) used to store the cache",
default=constants.CACHE_PATH,
type=str,
)
parser.add_argument(
"--offline",
help="Do not search online for additional JSON files",
default=False,
action="store_true",
)
parser.set_defaults(**default_config)
# We have to filter out help, otherwise parse_known_args will trigger the help message before having
# processed the plugin choice or had the plugin subparser added.
known_args = [arg for arg in sys.argv if arg != '--help' and arg != '-h']
known_args = [arg for arg in sys.argv if arg != "--help" and arg != "-h"]
partial_args, _ = parser.parse_known_args(known_args)
banner_output = sys.stdout
@@ -193,8 +250,10 @@ class CommandLine:
if partial_args.log:
file_logger = logging.FileHandler(partial_args.log)
file_logger.setLevel(1)
file_formatter = logging.Formatter(datefmt = '%y-%m-%d %H:%M:%S',
fmt = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
file_formatter = logging.Formatter(
datefmt="%y-%m-%d %H:%M:%S",
fmt="%(asctime)s %(name)-12s %(levelname)-8s %(message)s",
)
file_logger.setFormatter(file_formatter)
rootlog.addHandler(file_logger)
vollog.info("Logging started")
@@ -210,23 +269,25 @@ class CommandLine:
### Alter constants if necessary
if partial_args.plugin_dirs:
volatility3.plugins.__path__ = [os.path.abspath(p)
for p in partial_args.plugin_dirs.split(";")] + constants.PLUGINS_PATH
volatility3.plugins.__path__ = [
os.path.abspath(p) for p in partial_args.plugin_dirs.split(";")
] + constants.PLUGINS_PATH
if partial_args.symbol_dirs:
volatility3.symbols.__path__ = [os.path.abspath(p)
for p in partial_args.symbol_dirs.split(";")] + constants.SYMBOL_BASEPATHS
volatility3.symbols.__path__ = [
os.path.abspath(p) for p in partial_args.symbol_dirs.split(";")
] + constants.SYMBOL_BASEPATHS
if partial_args.cache_path:
constants.CACHE_PATH = partial_args.cache_path
vollog.info(f"Volatility plugins path: {volatility3.plugins.__path__}")
vollog.info(f"Volatility symbols path: {volatility3.symbols.__path__}")
# Set the PARALLELISM
if partial_args.parallelism == 'processes':
if partial_args.parallelism == "processes":
constants.PARALLELISM = constants.Parallelism.Multiprocessing
elif partial_args.parallelism == 'threads':
elif partial_args.parallelism == "threads":
constants.PARALLELISM = constants.Parallelism.Threading
else:
constants.PARALLELISM = constants.Parallelism.Off
@@ -239,11 +300,14 @@ class CommandLine:
# Do the initialization
ctx = contexts.Context() # Construct a blank context
failures = framework.import_files(volatility3.plugins,
True) # Will not log as console's default level is WARNING
failures = framework.import_files(
volatility3.plugins, True
) # Will not log as console's default level is WARNING
if failures:
parser.epilog = "The following plugins could not be loaded (use -vv to see why): " + \
", ".join(sorted(failures))
parser.epilog = (
"The following plugins could not be loaded (use -vv to see why): "
+ ", ".join(sorted(failures))
)
vollog.info(parser.epilog)
automagics = automagic.available(ctx)
@@ -258,13 +322,18 @@ class CommandLine:
if isinstance(amagic, interfaces.configuration.ConfigurableInterface):
self.populate_requirements_argparse(parser, amagic.__class__)
subparser = parser.add_subparsers(title = "Plugins",
dest = "plugin",
description = "For plugin specific options, run '{} <plugin> --help'".format(
self.CLI_NAME),
action = volargparse.HelpfulSubparserAction)
subparser = parser.add_subparsers(
title="Plugins",
dest="plugin",
description="For plugin specific options, run '{} <plugin> --help'".format(
self.CLI_NAME
),
action=volargparse.HelpfulSubparserAction,
)
for plugin in sorted(plugin_list):
plugin_parser = subparser.add_parser(plugin, help = plugin_list[plugin].__doc__)
plugin_parser = subparser.add_parser(
plugin, help=plugin_list[plugin].__doc__
)
self.populate_requirements_argparse(plugin_parser, plugin_list[plugin])
###
@@ -277,12 +346,16 @@ class CommandLine:
if args.plugin is None:
parser.error("Please select a plugin to run")
vollog.log(constants.LOGLEVEL_VVV, f"Cache directory used: {constants.CACHE_PATH}")
vollog.log(
constants.LOGLEVEL_VVV, f"Cache directory used: {constants.CACHE_PATH}"
)
plugin = plugin_list[args.plugin]
chosen_configurables_list[args.plugin] = plugin
base_config_path = "plugins"
plugin_config_path = interfaces.configuration.path_join(base_config_path, plugin.__name__)
plugin_config_path = interfaces.configuration.path_join(
base_config_path, plugin.__name__
)
# Special case the -f argument because people use is so frequently
# It has to go here so it can be overridden by single-location if it's defined
@@ -290,8 +363,10 @@ class CommandLine:
###
if args.file:
try:
single_location = self.location_from_file(args.file)
ctx.config['automagic.LayerStacker.single_location'] = single_location
single_location = requirements.URIRequirement.location_from_file(
args.file
)
ctx.config["automagic.LayerStacker.single_location"] = single_location
except ValueError as excp:
parser.error(str(excp))
@@ -299,26 +374,37 @@ class CommandLine:
if args.config:
with open(args.config, "r") as f:
json_val = json.load(f)
ctx.config.splice(plugin_config_path, interfaces.configuration.HierarchicalDict(json_val))
ctx.config.splice(
plugin_config_path,
interfaces.configuration.HierarchicalDict(json_val),
)
# It should be up to the UI to determine which automagics to run, so this is before BACK TO THE FRAMEWORK
automagics = automagic.choose_automagic(automagics, plugin)
for amagic in automagics:
chosen_configurables_list[amagic.__class__.__name__] = amagic
if ctx.config.get('automagic.LayerStacker.stackers', None) is None:
ctx.config['automagic.LayerStacker.stackers'] = stacker.choose_os_stackers(plugin)
if ctx.config.get("automagic.LayerStacker.stackers", None) is None:
ctx.config["automagic.LayerStacker.stackers"] = stacker.choose_os_stackers(
plugin
)
self.output_dir = args.output_dir
if not os.path.exists(self.output_dir):
parser.error(f"The output directory specified does not exist: {self.output_dir}")
parser.error(
f"The output directory specified does not exist: {self.output_dir}"
)
self.populate_config(ctx, chosen_configurables_list, args, plugin_config_path)
if args.extend:
for extension in args.extend:
if '=' not in extension:
raise ValueError("Invalid extension (extensions must be of the format \"conf.path.value='value'\")")
address, value = extension[:extension.find('=')], json.loads(extension[extension.find('=') + 1:])
if "=" not in extension:
raise ValueError(
"Invalid extension (extensions must be of the format \"conf.path.value='value'\")"
)
address, value = extension[: extension.find("=")], json.loads(
extension[extension.find("=") + 1 :]
)
ctx.config[address] = value
###
@@ -330,28 +416,46 @@ class CommandLine:
if args.quiet:
progress_callback = MuteProgress()
constructed = plugins.construct_plugin(ctx, automagics, plugin, base_config_path, progress_callback,
self.file_handler_class_factory())
constructed = plugins.construct_plugin(
ctx,
automagics,
plugin,
base_config_path,
progress_callback,
self.file_handler_class_factory(),
)
if args.write_config:
vollog.warning('Use of --write-config has been deprecated, replaced by --save-config <filename>')
args.save_config = 'config.json'
vollog.warning(
"Use of --write-config has been deprecated, replaced by --save-config <filename>"
)
args.save_config = "config.json"
if args.save_config:
vollog.debug("Writing out configuration data to {args.save_config}")
if os.path.exists(os.path.abspath(args.save_config)):
parser.error(f"Cannot write configuration: file {args.save_config} already exists")
parser.error(
f"Cannot write configuration: file {args.save_config} already exists"
)
with open(args.save_config, "w") as f:
json.dump(dict(constructed.build_configuration()), f, sort_keys = True, indent = 2)
json.dump(
dict(constructed.build_configuration()),
f,
sort_keys=True,
indent=2,
)
f.write("\n")
except exceptions.UnsatisfiedException as excp:
self.process_unsatisfied_exceptions(excp)
parser.exit(1, f"Unable to validate the plugin requirements: {[x for x in excp.unsatisfied]}\n")
parser.exit(
1,
f"Unable to validate the plugin requirements: {[x for x in excp.unsatisfied]}\n",
)
try:
# Construct and run the plugin
if constructed:
renderers[args.renderer]().render(constructed.run())
except (exceptions.VolatilityException) as excp:
except exceptions.VolatilityException as excp:
self.process_exceptions(excp)
@classmethod
@@ -364,17 +468,10 @@ class CommandLine:
Returns:
The URL for the location of the file
"""
# We want to work in URLs, but we need to accept absolute and relative files (including on windows)
single_location = parse.urlparse(filename, '')
if single_location.scheme == '' or len(single_location.scheme) == 1:
single_location = parse.urlparse(parse.urljoin('file:', request.pathname2url(os.path.abspath(filename))))
if single_location.scheme == 'file':
if not os.path.exists(request.url2pathname(single_location.path)):
filename = request.url2pathname(single_location.path)
if not filename:
raise ValueError("File URL looks incorrect (potentially missing /)")
raise ValueError(f"File does not exist: {filename}")
return parse.urlunparse(single_location)
vollog.debug(
f"{__name__}.location_from_file has been deprecated and moved to requirements.URIRequirement.location_from_file"
)
return requirements.URIRequirement.location_from_file(filename)
def load_system_defaults(self, filename: str) -> Tuple[List[Tuple[int, str]], Dict[str, Any]]:
"""Modify the main configuration based on the default configuration override"""
@@ -409,7 +506,7 @@ class CommandLine:
sys.stderr.flush()
# Log the full exception at a high level for easy access
fulltrace = traceback.TracebackException.from_exception(excp).format(chain = True)
fulltrace = traceback.TracebackException.from_exception(excp).format(chain=True)
vollog.debug("".join(fulltrace))
if isinstance(excp, exceptions.InvalidAddressException):
@@ -418,22 +515,24 @@ class CommandLine:
detail = f"Swap error {hex(excp.invalid_address)} in layer {excp.layer_name} ({excp})"
caused_by = [
"No suitable swap file having been provided (locate and provide the correct swap file)",
"An intentionally invalid page (operating system protection)"
"An intentionally invalid page (operating system protection)",
]
elif isinstance(excp, exceptions.PagedInvalidAddressException):
detail = f"Page error {hex(excp.invalid_address)} in layer {excp.layer_name} ({excp})"
caused_by = [
"Memory smear during acquisition (try re-acquiring if possible)",
"An intentionally invalid page lookup (operating system protection)",
"A bug in the plugin/volatility3 (re-run with -vvv and file a bug)"
"A bug in the plugin/volatility3 (re-run with -vvv and file a bug)",
]
else:
detail = f"{hex(excp.invalid_address)} in layer {excp.layer_name} ({excp})"
detail = (
f"{hex(excp.invalid_address)} in layer {excp.layer_name} ({excp})"
)
caused_by = [
"The base memory file being incomplete (try re-acquiring if possible)",
"Memory smear during acquisition (try re-acquiring if possible)",
"An intentionally invalid page lookup (operating system protection)",
"A bug in the plugin/volatility3 (re-run with -vvv and file a bug)"
"A bug in the plugin/volatility3 (re-run with -vvv and file a bug)",
]
elif isinstance(excp, exceptions.SymbolError):
general = "Volatility experienced a symbol-related issue:"
@@ -447,22 +546,28 @@ class CommandLine:
general = "Volatility experienced an issue related to a symbol table:"
detail = f"{excp}"
caused_by = [
"An invalid symbol table", "A plugin requesting a bad symbol",
"A plugin requesting a symbol from the wrong table"
"An invalid symbol table",
"A plugin requesting a bad symbol",
"A plugin requesting a symbol from the wrong table",
]
elif isinstance(excp, exceptions.LayerException):
general = f"Volatility experienced a layer-related issue: {excp.layer_name}"
detail = f"{excp}"
caused_by = ["A faulty layer implementation (re-run with -vvv and file a bug)"]
caused_by = [
"A faulty layer implementation (re-run with -vvv and file a bug)"
]
elif isinstance(excp, exceptions.MissingModuleException):
general = f"Volatility could not import a necessary module: {excp.module}"
detail = f"{excp}"
caused_by = ["A required python module is not installed (install the module and re-run)"]
caused_by = [
"A required python module is not installed (install the module and re-run)"
]
else:
general = "Volatility encountered an unexpected situation."
detail = ""
caused_by = [
"Please re-run using with -vvv and file a bug with the output", f"at {constants.BUG_URL}"
"Please re-run using with -vvv and file a bug with the output",
f"at {constants.BUG_URL}",
]
# Code that actually renders the exception
@@ -482,27 +587,43 @@ class CommandLine:
symbols_failed = False
for config_path in excp.unsatisfied:
translation_failed = translation_failed or isinstance(
excp.unsatisfied[config_path], configuration.requirements.TranslationLayerRequirement)
symbols_failed = symbols_failed or isinstance(excp.unsatisfied[config_path],
configuration.requirements.SymbolTableRequirement)
excp.unsatisfied[config_path],
configuration.requirements.TranslationLayerRequirement,
)
symbols_failed = symbols_failed or isinstance(
excp.unsatisfied[config_path],
configuration.requirements.SymbolTableRequirement,
)
print(f"Unsatisfied requirement {config_path}: {excp.unsatisfied[config_path].description}")
print(
f"Unsatisfied requirement {config_path}: {excp.unsatisfied[config_path].description}"
)
if translation_failed:
print("\nA translation layer requirement was not fulfilled. Please verify that:\n"
"\tA file was provided to create this layer (by -f, --single-location or by config)\n"
"\tThe file exists and is readable\n"
"\tThe file is a valid memory image and was acquired cleanly")
print(
"\nA translation layer requirement was not fulfilled. Please verify that:\n"
"\tA file was provided to create this layer (by -f, --single-location or by config)\n"
"\tThe file exists and is readable\n"
"\tThe file is a valid memory image and was acquired cleanly"
)
if symbols_failed:
print("\nA symbol table requirement was not fulfilled. Please verify that:\n"
"\tThe associated translation layer requirement was fulfilled\n"
"\tYou have the correct symbol file for the requirement\n"
"\tThe symbol file is under the correct directory or zip file\n"
"\tThe symbol file is named appropriately or contains the correct banner\n")
print(
"\nA symbol table requirement was not fulfilled. Please verify that:\n"
"\tThe associated translation layer requirement was fulfilled\n"
"\tYou have the correct symbol file for the requirement\n"
"\tThe symbol file is under the correct directory or zip file\n"
"\tThe symbol file is named appropriately or contains the correct banner\n"
)
def populate_config(self, context: interfaces.context.ContextInterface,
configurables_list: Dict[str, Type[interfaces.configuration.ConfigurableInterface]],
args: argparse.Namespace, plugin_config_path: str) -> None:
def populate_config(
self,
context: interfaces.context.ContextInterface,
configurables_list: Dict[
str, Type[interfaces.configuration.ConfigurableInterface]
],
args: argparse.Namespace,
plugin_config_path: str,
) -> None:
"""Populate the context config based on the returned args.
We have already determined these elements must be descended from ConfigurableInterface
@@ -524,34 +645,42 @@ class CommandLine:
if not scheme or len(scheme) <= 1:
if not os.path.exists(value):
raise FileNotFoundError(
f"Non-existent file {value} passed to URIRequirement")
f"Non-existent file {value} passed to URIRequirement"
)
value = f"file://{request.pathname2url(os.path.abspath(value))}"
if isinstance(requirement, requirements.ListRequirement):
if not isinstance(value, list):
raise TypeError("Configuration for ListRequirement was not a list: {}".format(
requirement.name))
raise TypeError(
"Configuration for ListRequirement was not a list: {}".format(
requirement.name
)
)
value = [requirement.element_type(x) for x in value]
if not inspect.isclass(configurables_list[configurable]):
config_path = configurables_list[configurable].config_path
else:
# We must be the plugin, so name it appropriately:
config_path = plugin_config_path
extended_path = interfaces.configuration.path_join(config_path, requirement.name)
extended_path = interfaces.configuration.path_join(
config_path, requirement.name
)
context.config[extended_path] = value
def file_handler_class_factory(self, direct = True):
def file_handler_class_factory(self, direct=True):
output_dir = self.output_dir
class CLIFileHandler(interfaces.plugins.FileHandlerInterface):
def _get_final_filename(self):
"""Gets the final filename"""
if output_dir is None:
raise TypeError("Output directory is not a string")
os.makedirs(output_dir, exist_ok = True)
os.makedirs(output_dir, exist_ok=True)
pref_name_array = self.preferred_filename.split('.')
filename, extension = os.path.join(output_dir, '.'.join(pref_name_array[:-1])), pref_name_array[-1]
pref_name_array = self.preferred_filename.split(".")
filename, extension = (
os.path.join(output_dir, ".".join(pref_name_array[:-1])),
pref_name_array[-1],
)
output_filename = f"{filename}.{extension}"
counter = 1
@@ -561,7 +690,6 @@ class CommandLine:
return output_filename
class CLIMemFileHandler(io.BytesIO, CLIFileHandler):
def __init__(self, filename: str):
io.BytesIO.__init__(self)
CLIFileHandler.__init__(self, filename)
@@ -569,7 +697,7 @@ class CommandLine:
def close(self):
# Don't overcommit
if self.closed:
return
return None
self.seek(0)
@@ -578,18 +706,26 @@ class CommandLine:
with open(output_filename, "wb") as current_file:
current_file.write(self.read())
self._committed = True
vollog.log(logging.INFO, f"Saved stored plugin file: {output_filename}")
vollog.log(
logging.INFO, f"Saved stored plugin file: {output_filename}"
)
super().close()
class CLIDirectFileHandler(CLIFileHandler):
def __init__(self, filename: str):
fd, self._name = tempfile.mkstemp(suffix = '.vol3', prefix = 'tmp_', dir = output_dir)
self._file = io.open(fd, mode = 'w+b')
fd, self._name = tempfile.mkstemp(
suffix=".vol3", prefix="tmp_", dir=output_dir
)
self._file = io.open(fd, mode="w+b")
CLIFileHandler.__init__(self, filename)
for item in dir(self._file):
if not item.startswith('_') and item not in ('closed', 'close', 'mode', 'name'):
if not item.startswith("_") and item not in (
"closed",
"close",
"mode",
"name",
):
setattr(self, item, getattr(self._file, item))
def __getattr__(self, item):
@@ -611,7 +747,7 @@ class CommandLine:
"""Closes and commits the file (by moving the temporary file to the correct name"""
# Don't overcommit
if self._file.closed:
return
return None
self._file.close()
output_filename = self._get_final_filename()
@@ -622,8 +758,11 @@ class CommandLine:
else:
return CLIMemFileHandler
def populate_requirements_argparse(self, parser: Union[argparse.ArgumentParser, argparse._ArgumentGroup],
configurable: Type[interfaces.configuration.ConfigurableInterface]):
def populate_requirements_argparse(
self,
parser: Union[argparse.ArgumentParser, argparse._ArgumentGroup],
configurable: Type[interfaces.configuration.ConfigurableInterface],
):
"""Adds the plugin's simple requirements to the provided parser.
Args:
@@ -631,15 +770,22 @@ class CommandLine:
configurable: The plugin object to pull the requirements from
"""
if not issubclass(configurable, interfaces.configuration.ConfigurableInterface):
raise TypeError(f"Expected ConfigurableInterface type, not: {type(configurable)}")
raise TypeError(
f"Expected ConfigurableInterface type, not: {type(configurable)}"
)
# Construct an argparse group
for requirement in configurable.get_requirements():
additional: Dict[str, Any] = {}
if not isinstance(requirement, interfaces.configuration.RequirementInterface):
raise TypeError("Plugin contains requirements that are not RequirementInterfaces: {}".format(
configurable.__name__))
if not isinstance(
requirement, interfaces.configuration.RequirementInterface
):
raise TypeError(
"Plugin contains requirements that are not RequirementInterfaces: {}".format(
configurable.__name__
)
)
if isinstance(requirement, interfaces.configuration.SimpleTypeRequirement):
additional["type"] = requirement.instance_type
if isinstance(requirement, requirements.IntRequirement):
@@ -648,21 +794,29 @@ class CommandLine:
additional["action"] = "store_true"
if "type" in additional:
del additional["type"]
elif isinstance(requirement, volatility3.framework.configuration.requirements.ListRequirement):
elif isinstance(
requirement,
volatility3.framework.configuration.requirements.ListRequirement,
):
additional["type"] = requirement.element_type
nargs = '*' if requirement.optional else '+'
nargs = "*" if requirement.optional else "+"
additional["nargs"] = nargs
elif isinstance(requirement, volatility3.framework.configuration.requirements.ChoiceRequirement):
elif isinstance(
requirement,
volatility3.framework.configuration.requirements.ChoiceRequirement,
):
additional["type"] = str
additional["choices"] = requirement.choices
else:
continue
parser.add_argument("--" + requirement.name.replace('_', '-'),
help = requirement.description,
default = requirement.default,
dest = requirement.name,
required = not requirement.optional,
**additional)
parser.add_argument(
"--" + requirement.name.replace("_", "-"),
help=requirement.description,
default=requirement.default,
dest=requirement.name,
required=not requirement.optional,
**additional,
)
def main():
volatility3/cli/text_renderer.py
+119 -54
View File
@@ -44,9 +44,9 @@ def hex_bytes_as_text(value: bytes) -> str:
ascii.append(chr(byte) if 0x20 < byte <= 0x7E else ".")
if (count % 8) == 7:
output += "\n"
output += " ".join(hex[count - 7:count + 1])
output += " ".join(hex[count - 7 : count + 1])
output += "\t"
output += "".join(ascii[count - 7:count + 1])
output += "".join(ascii[count - 7 : count + 1])
count += 1
return output
@@ -58,10 +58,16 @@ def multitypedata_as_text(value: format_hints.MultiTypeData) -> str:
"""
if value.show_hex:
return hex_bytes_as_text(value)
string_representation = str(value, encoding = value.encoding, errors = 'replace')
if value.split_nulls and ((len(value) / 2 - 1) <= len(string_representation) <= (len(value) / 2)):
string_representation = str(value, encoding=value.encoding, errors="replace")
if value.split_nulls and (
(len(value) / 2 - 1) <= len(string_representation) <= (len(value) / 2)
):
return "\n".join(string_representation.split("\x00"))
if len(string_representation) - 1 <= len(string_representation.split("\x00")[0]) <= len(string_representation):
if (
len(string_representation) - 1
<= len(string_representation.split("\x00")[0])
<= len(string_representation)
):
return string_representation.split("\x00")[0]
return hex_bytes_as_text(value)
@@ -87,9 +93,11 @@ def quoted_optional(func: Callable) -> Callable:
return ""
if isinstance(x, format_hints.MultiTypeData) and x.converted_int:
return f"{result}"
if isinstance(x, int) and not isinstance(x, (format_hints.Hex, format_hints.Bin)):
if isinstance(x, int) and not isinstance(
x, (format_hints.Hex, format_hints.Bin)
):
return f"{result}"
return f"\"{result}\""
return f'"{result}"'
return wrapped
@@ -106,14 +114,16 @@ def display_disassembly(disasm: interfaces.renderers.Disassembly) -> str:
if CAPSTONE_PRESENT:
disasm_types = {
'intel': capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_32),
'intel64': capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_64),
'arm': capstone.Cs(capstone.CS_ARCH_ARM, capstone.CS_MODE_ARM),
'arm64': capstone.Cs(capstone.CS_ARCH_ARM64, capstone.CS_MODE_ARM)
"intel": capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_32),
"intel64": capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_64),
"arm": capstone.Cs(capstone.CS_ARCH_ARM, capstone.CS_MODE_ARM),
"arm64": capstone.Cs(capstone.CS_ARCH_ARM64, capstone.CS_MODE_ARM),
}
output = ""
if disasm.architecture is not None:
for i in disasm_types[disasm.architecture].disasm(disasm.data, disasm.offset):
for i in disasm_types[disasm.architecture].disasm(
disasm.data, disasm.offset
):
output += f"\n0x{i.address:x}:\t{i.mnemonic}\t{i.op_str}"
return output
return QuickTextRenderer._type_renderers[bytes](disasm.data)
@@ -121,6 +131,7 @@ def display_disassembly(disasm: interfaces.renderers.Disassembly) -> str:
class CLIRenderer(interfaces.renderers.Renderer):
"""Class to add specific requirements for CLI renderers."""
name = "unnamed"
structured_output = False
@@ -134,7 +145,7 @@ class QuickTextRenderer(CLIRenderer):
interfaces.renderers.Disassembly: optional(display_disassembly),
bytes: optional(lambda x: " ".join([f"{b:02x}" for b in x])),
datetime.datetime: optional(lambda x: x.strftime("%Y-%m-%d %H:%M:%S.%f %Z")),
'default': optional(lambda x: f"{x}")
"default": optional(lambda x: f"{x}"),
}
name = "quick"
@@ -163,11 +174,16 @@ class QuickTextRenderer(CLIRenderer):
def visitor(node: interfaces.renderers.TreeNode, accumulator):
accumulator.write("\n")
# Nodes always have a path value, giving them a path_depth of at least 1, we use max just in case
accumulator.write("*" * max(0, node.path_depth - 1) + ("" if (node.path_depth <= 1) else " "))
accumulator.write(
"*" * max(0, node.path_depth - 1)
+ ("" if (node.path_depth <= 1) else " ")
)
line = []
for column_index in range(len(grid.columns)):
column = grid.columns[column_index]
renderer = self._type_renderers.get(column.type, self._type_renderers['default'])
renderer = self._type_renderers.get(
column.type, self._type_renderers["default"]
)
line.append(renderer(node.values[column_index]))
accumulator.write("{}".format("\t".join(line)))
accumulator.flush()
@@ -176,13 +192,14 @@ class QuickTextRenderer(CLIRenderer):
if not grid.populated:
grid.populate(visitor, outfd)
else:
grid.visit(node = None, function = visitor, initial_accumulator = outfd)
grid.visit(node=None, function=visitor, initial_accumulator=outfd)
outfd.write("\n")
class NoneRenderer(CLIRenderer):
"""Outputs no results"""
name = "none"
def get_render_options(self):
@@ -202,7 +219,7 @@ class CSVRenderer(CLIRenderer):
interfaces.renderers.Disassembly: optional(display_disassembly),
bytes: optional(lambda x: " ".join([f"{b:02x}" for b in x])),
datetime.datetime: optional(lambda x: x.strftime("%Y-%m-%d %H:%M:%S.%f %Z")),
'default': optional(lambda x: f"{x}")
"default": optional(lambda x: f"{x}"),
}
name = "csv"
@@ -219,28 +236,32 @@ class CSVRenderer(CLIRenderer):
"""
outfd = sys.stdout
header_list = ['TreeDepth']
header_list = ["TreeDepth"]
for column in grid.columns:
# Ignore the type because namedtuples don't realize they have accessible attributes
header_list.append(f"{column.name}")
writer = csv.DictWriter(outfd, header_list)
writer = csv.DictWriter(
outfd, header_list, lineterminator="\n", escapechar="\\"
)
writer.writeheader()
def visitor(node: interfaces.renderers.TreeNode, accumulator):
# Nodes always have a path value, giving them a path_depth of at least 1, we use max just in case
row = {'TreeDepth': str(max(0, node.path_depth - 1))}
row = {"TreeDepth": str(max(0, node.path_depth - 1))}
for column_index in range(len(grid.columns)):
column = grid.columns[column_index]
renderer = self._type_renderers.get(column.type, self._type_renderers['default'])
row[f'{column.name}'] = renderer(node.values[column_index])
renderer = self._type_renderers.get(
column.type, self._type_renderers["default"]
)
row[f"{column.name}"] = renderer(node.values[column_index])
accumulator.writerow(row)
return accumulator
if not grid.populated:
grid.populate(visitor, writer)
else:
grid.visit(node = None, function = visitor, initial_accumulator = writer)
grid.visit(node=None, function=visitor, initial_accumulator=writer)
outfd.write("\n")
@@ -270,23 +291,34 @@ class PrettyTextRenderer(CLIRenderer):
display_alignment = ">"
column_separator = " | "
tree_indent_column = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(20))
max_column_widths = dict([(column.name, len(column.name)) for column in grid.columns])
tree_indent_column = "".join(
random.choice(string.ascii_uppercase + string.digits) for _ in range(20)
)
max_column_widths = dict(
[(column.name, len(column.name)) for column in grid.columns]
)
def visitor(
node: interfaces.renderers.TreeNode,
accumulator: List[Tuple[int, Dict[interfaces.renderers.Column, bytes]]]
node: interfaces.renderers.TreeNode,
accumulator: List[Tuple[int, Dict[interfaces.renderers.Column, bytes]]],
) -> List[Tuple[int, Dict[interfaces.renderers.Column, bytes]]]:
# Nodes always have a path value, giving them a path_depth of at least 1, we use max just in case
max_column_widths[tree_indent_column] = max(max_column_widths.get(tree_indent_column, 0), node.path_depth)
max_column_widths[tree_indent_column] = max(
max_column_widths.get(tree_indent_column, 0), node.path_depth
)
line = {}
for column_index in range(len(grid.columns)):
column = grid.columns[column_index]
renderer = self._type_renderers.get(column.type, self._type_renderers['default'])
renderer = self._type_renderers.get(
column.type, self._type_renderers["default"]
)
data = renderer(node.values[column_index])
field_width = max([len(self.tab_stop(x)) for x in f"{data}".split("\n")])
max_column_widths[column.name] = max(max_column_widths.get(column.name, len(column.name)),
field_width)
field_width = max(
[len(self.tab_stop(x)) for x in f"{data}".split("\n")]
)
max_column_widths[column.name] = max(
max_column_widths.get(column.name, len(column.name)), field_width
)
line[column] = data.split("\n")
accumulator.append((node.path_depth, line))
return accumulator
@@ -295,33 +327,57 @@ class PrettyTextRenderer(CLIRenderer):
if not grid.populated:
grid.populate(visitor, final_output)
else:
grid.visit(node = None, function = visitor, initial_accumulator = final_output)
grid.visit(node=None, function=visitor, initial_accumulator=final_output)
# Always align the tree to the left
format_string_list = ["{0:<" + str(max_column_widths.get(tree_indent_column, 0)) + "s}"]
format_string_list = [
"{0:<" + str(max_column_widths.get(tree_indent_column, 0)) + "s}"
]
for column_index in range(len(grid.columns)):
column = grid.columns[column_index]
format_string_list.append("{" + str(column_index + 1) + ":" + display_alignment +
str(max_column_widths[column.name]) + "s}")
format_string_list.append(
"{"
+ str(column_index + 1)
+ ":"
+ display_alignment
+ str(max_column_widths[column.name])
+ "s}"
)
format_string = column_separator.join(format_string_list) + "\n"
column_titles = [""] + [column.name for column in grid.columns]
outfd.write(format_string.format(*column_titles))
for (depth, line) in final_output:
for depth, line in final_output:
nums_line = max([len(line[column]) for column in line])
for column in line:
line[column] = line[column] + ([""] * (nums_line - len(line[column])))
for index in range(nums_line):
if index == 0:
outfd.write(format_string.format("*" * depth, *[self.tab_stop(line[column][index]) for column in grid.columns]))
outfd.write(
format_string.format(
"*" * depth,
*[
self.tab_stop(line[column][index])
for column in grid.columns
],
)
)
else:
outfd.write(format_string.format(" " * depth, *[self.tab_stop(line[column][index]) for column in grid.columns]))
outfd.write(
format_string.format(
" " * depth,
*[
self.tab_stop(line[column][index])
for column in grid.columns
],
)
)
def tab_stop(self, line: str) -> str:
tab_width = 8
while line.find('\t') >= 0:
i = line.find('\t')
while line.find("\t") >= 0:
i = line.find("\t")
pad = " " * (tab_width - (i % tab_width))
line = line.replace("\t", pad, 1)
return line
@@ -333,11 +389,15 @@ class JsonRenderer(CLIRenderer):
interfaces.renderers.Disassembly: quoted_optional(display_disassembly),
format_hints.MultiTypeData: quoted_optional(multitypedata_as_text),
bytes: optional(lambda x: " ".join([f"{b:02x}" for b in x])),
datetime.datetime: lambda x: x.isoformat() if not isinstance(x, interfaces.renderers.BaseAbsentValue) else None,
'default': lambda x: x
datetime.datetime: lambda x: (
x.isoformat()
if not isinstance(x, interfaces.renderers.BaseAbsentValue)
else None
),
"default": lambda x: x,
}
name = 'JSON'
name = "JSON"
structured_output = True
def get_render_options(self) -> List[interfaces.renderers.RenderOption]:
@@ -345,30 +405,35 @@ class JsonRenderer(CLIRenderer):
def output_result(self, outfd, result):
"""Outputs the JSON data to a file in a particular format"""
outfd.write("{}\n".format(json.dumps(result, indent = 2, sort_keys = True)))
outfd.write("{}\n".format(json.dumps(result, indent=2, sort_keys=True)))
def render(self, grid: interfaces.renderers.TreeGrid):
outfd = sys.stdout
outfd.write("\n")
final_output: Tuple[Dict[str, List[interfaces.renderers.TreeNode]], List[interfaces.renderers.TreeNode]] = (
{}, [])
final_output: Tuple[
Dict[str, List[interfaces.renderers.TreeNode]],
List[interfaces.renderers.TreeNode],
] = ({}, [])
def visitor(
node: interfaces.renderers.TreeNode, accumulator: Tuple[Dict[str, Dict[str, Any]], List[Dict[str, Any]]]
node: interfaces.renderers.TreeNode,
accumulator: Tuple[Dict[str, Dict[str, Any]], List[Dict[str, Any]]],
) -> Tuple[Dict[str, Dict[str, Any]], List[Dict[str, Any]]]:
# Nodes always have a path value, giving them a path_depth of at least 1, we use max just in case
acc_map, final_tree = accumulator
node_dict: Dict[str, Any] = {'__children': []}
node_dict: Dict[str, Any] = {"__children": []}
for column_index in range(len(grid.columns)):
column = grid.columns[column_index]
renderer = self._type_renderers.get(column.type, self._type_renderers['default'])
renderer = self._type_renderers.get(
column.type, self._type_renderers["default"]
)
data = renderer(list(node.values)[column_index])
if isinstance(data, interfaces.renderers.BaseAbsentValue):
data = None
node_dict[column.name] = data
if node.parent:
acc_map[node.parent.path]['__children'].append(node_dict)
acc_map[node.parent.path]["__children"].append(node_dict)
else:
final_tree.append(node_dict)
acc_map[node.path] = node_dict
@@ -378,16 +443,16 @@ class JsonRenderer(CLIRenderer):
if not grid.populated:
grid.populate(visitor, final_output)
else:
grid.visit(node = None, function = visitor, initial_accumulator = final_output)
grid.visit(node=None, function=visitor, initial_accumulator=final_output)
self.output_result(outfd, final_output[1])
class JsonLinesRenderer(JsonRenderer):
name = 'JSONL'
name = "JSONL"
def output_result(self, outfd, result):
"""Outputs the JSON results as JSON lines"""
for line in result:
outfd.write(json.dumps(line, sort_keys = True))
outfd.write(json.dumps(line, sort_keys=True))
outfd.write("\n")
volatility3/cli/volargparse.py
+21 -14
View File
@@ -24,13 +24,14 @@ class HelpfulSubparserAction(argparse._SubParsersAction):
# We don't want the action self-check to kick in, so we remove the choices list, the check happens in __call__
self.choices = None
def __call__(self,
parser: argparse.ArgumentParser,
namespace: argparse.Namespace,
values: Union[str, Sequence[Any], None],
option_string: Optional[str] = None) -> None:
parser_name = ''
def __call__(
self,
parser: argparse.ArgumentParser,
namespace: argparse.Namespace,
values: Union[str, Sequence[Any], None],
option_string: Optional[str] = None,
) -> None:
parser_name = ""
arg_strings = [] # type: List[str]
if values is not None:
for value in values:
@@ -43,7 +44,9 @@ class HelpfulSubparserAction(argparse._SubParsersAction):
if self.dest != argparse.SUPPRESS:
setattr(namespace, self.dest, parser_name)
matched_parsers = [name for name in self._name_parser_map if parser_name in name]
matched_parsers = [
name for name in self._name_parser_map if parser_name in name
]
if len(matched_parsers) < 1:
msg = f"invalid choice {parser_name} (choose from {', '.join(self._name_parser_map)})"
@@ -52,7 +55,7 @@ class HelpfulSubparserAction(argparse._SubParsersAction):
msg = f"plugin {parser_name} matches multiple plugins ({', '.join(matched_parsers)})"
raise argparse.ArgumentError(self, msg)
parser = self._name_parser_map[matched_parsers[0]]
setattr(namespace, 'plugin', matched_parsers[0])
setattr(namespace, "plugin", matched_parsers[0])
# parse all the remaining options into the namespace
# store any unrecognized options on the object, so that the top
@@ -71,7 +74,6 @@ class HelpfulSubparserAction(argparse._SubParsersAction):
class HelpfulArgParser(argparse.ArgumentParser):
def _match_argument(self, action, arg_strings_pattern) -> int:
# match the pattern for this action to the arg strings
nargs_pattern = self._get_nargs_pattern(action)
@@ -80,13 +82,18 @@ class HelpfulArgParser(argparse.ArgumentParser):
# raise an exception if we weren't able to find a match
if match is None:
nargs_errors = {
None: gettext.gettext('expected one argument'),
argparse.OPTIONAL: gettext.gettext('expected at most one argument'),
argparse.ONE_OR_MORE: gettext.gettext('expected at least one argument'),
None: gettext.gettext("expected one argument"),
argparse.OPTIONAL: gettext.gettext("expected at most one argument"),
argparse.ONE_OR_MORE: gettext.gettext("expected at least one argument"),
}
msg = nargs_errors.get(action.nargs)
if msg is None:
msg = gettext.ngettext('expected %s argument', 'expected %s arguments', action.nargs) % action.nargs
msg = (
gettext.ngettext(
"expected %s argument", "expected %s arguments", action.nargs
)
% action.nargs
)
if action.choices:
msg = f"{msg} (from: {', '.join(action.choices)})"
raise argparse.ArgumentError(action, msg)
volatility3/cli/volshell/__init__.py
+197 -102
View File
@@ -12,7 +12,14 @@ import volatility3.plugins
import volatility3.symbols
from volatility3 import cli, framework
from volatility3.cli.volshell import generic, linux, mac, windows
from volatility3.framework import automagic, constants, contexts, exceptions, interfaces, plugins
from volatility3.framework import (
automagic,
constants,
contexts,
exceptions,
interfaces,
plugins,
)
# Make sure we log everything
@@ -21,7 +28,7 @@ vollog = logging.getLogger()
vollog.setLevel(0)
console = logging.StreamHandler()
console.setLevel(logging.WARNING)
formatter = logging.Formatter('%(levelname)-8s %(name)-12s: %(message)s')
formatter = logging.Formatter("%(levelname)-8s %(name)-12s: %(message)s")
# Trim the console down by default
console.setFormatter(formatter)
vollog.addHandler(console)
@@ -38,107 +45,157 @@ class VolShell(cli.CommandLine):
def __init__(self):
super().__init__()
self.output_dir = None
def run(self):
"""Executes the command line module, taking the system arguments,
determining the plugin to run and then running it."""
sys.stdout.write(f"Volshell (Volatility 3 Framework) {constants.PACKAGE_VERSION}\n")
sys.stdout.write(
f"Volshell (Volatility 3 Framework) {constants.PACKAGE_VERSION}\n"
)
framework.require_interface_version(2, 0, 0)
# Load up system defaults
delayed_logs, default_config = self.load_system_defaults('volshell.json')
parser = argparse.ArgumentParser(prog = self.CLI_NAME,
description = "A tool for interactivate forensic analysis of memory images")
parser.add_argument("-c",
"--config",
help = "Load the configuration from a json file",
default = None,
type = str)
parser.add_argument("-e",
"--extend",
help = "Extend the configuration with a new (or changed) setting",
default = None,
action = 'append')
parser.add_argument("-p",
"--plugin-dirs",
help = "Semi-colon separated list of paths to find plugins",
default = "",
type = str)
parser.add_argument("-s",
"--symbol-dirs",
help = "Semi-colon separated list of paths to find symbols",
default = "",
type = str)
parser.add_argument("-v", "--verbosity", help = "Increase output verbosity", default = 0, action = "count")
parser.add_argument("--log",
help = "Log output to a file as well as the console",
default = None,
type = str)
parser.add_argument("-o",
"--output-dir",
help = "Directory in which to output any generated files",
default = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')),
type = str)
parser.add_argument("-q", "--quiet", help = "Remove progress feedback", default = False, action = 'store_true')
parser.add_argument("-f",
"--file",
metavar = 'FILE',
default = None,
type = str,
help = "Shorthand for --single-location=file:// if single-location is not defined")
parser.add_argument("--write-config",
help = "Write configuration JSON file out to config.json",
default = False,
action = 'store_true')
parser.add_argument("--save-config",
help = "Save configuration JSON file to a file",
default = None,
type = str)
parser.add_argument("--clear-cache",
help = "Clears out all short-term cached items",
default = False,
action = 'store_true')
parser.add_argument("--cache-path",
help = f"Change the default path ({constants.CACHE_PATH}) used to store the cache",
default = constants.CACHE_PATH,
type = str)
parser = argparse.ArgumentParser(
prog=self.CLI_NAME,
description="A tool for interactivate forensic analysis of memory images",
)
parser.add_argument(
"-c",
"--config",
help="Load the configuration from a json file",
default=None,
type=str,
)
parser.add_argument(
"-e",
"--extend",
help="Extend the configuration with a new (or changed) setting",
default=None,
action="append",
)
parser.add_argument(
"-p",
"--plugin-dirs",
help="Semi-colon separated list of paths to find plugins",
default="",
type=str,
)
parser.add_argument(
"-s",
"--symbol-dirs",
help="Semi-colon separated list of paths to find symbols",
default="",
type=str,
)
parser.add_argument(
"-v",
"--verbosity",
help="Increase output verbosity",
default=0,
action="count",
)
parser.add_argument(
"-o",
"--output-dir",
help="Directory in which to output any generated files",
default=os.path.abspath(
os.path.join(os.path.dirname(__file__), "..", "..")
),
type=str,
)
parser.add_argument(
"-q",
"--quiet",
help="Remove progress feedback",
default=False,
action="store_true",
)
parser.add_argument(
"--log",
help="Log output to a file as well as the console",
default=None,
type=str,
)
parser.add_argument(
"-f",
"--file",
metavar="FILE",
default=None,
type=str,
help="Shorthand for --single-location=file:// if single-location is not defined",
)
parser.add_argument(
"--write-config",
help="Write configuration JSON file out to config.json",
default=False,
action="store_true",
)
parser.add_argument(
"--save-config",
help="Save configuration JSON file to a file",
default=None,
type=str,
)
parser.add_argument(
"--clear-cache",
help="Clears out all short-term cached items",
default=False,
action="store_true",
)
parser.add_argument(
"--cache-path",
help=f"Change the default path ({constants.CACHE_PATH}) used to store the cache",
default=constants.CACHE_PATH,
type=str,
)
parser.add_argument("--offline",
help = "Do not search online for additional JSON files",
default = False,
action = 'store_true')
# Volshell specific flags
os_specific = parser.add_mutually_exclusive_group(required = False)
os_specific.add_argument("-w",
"--windows",
default = False,
action = "store_true",
help = "Run a Windows volshell")
os_specific.add_argument("-l", "--linux", default = False, action = "store_true", help = "Run a Linux volshell")
os_specific.add_argument("-m", "--mac", default = False, action = "store_true", help = "Run a Mac volshell")
os_specific = parser.add_mutually_exclusive_group(required=False)
os_specific.add_argument(
"-w",
"--windows",
default=False,
action="store_true",
help="Run a Windows volshell",
)
os_specific.add_argument(
"-l",
"--linux",
default=False,
action="store_true",
help="Run a Linux volshell",
)
os_specific.add_argument(
"-m", "--mac", default=False, action="store_true", help="Run a Mac volshell"
)
parser.set_defaults(**default_config)
# We have to filter out help, otherwise parse_known_args will trigger the help message before having
# processed the plugin choice or had the plugin subparser added.
known_args = [arg for arg in sys.argv if arg != '--help' and arg != '-h']
known_args = [arg for arg in sys.argv if arg != "--help" and arg != "-h"]
partial_args, _ = parser.parse_known_args(known_args)
### Start up logging
if partial_args.log:
file_logger = logging.FileHandler(partial_args.log)
file_logger.setLevel(1)
file_formatter = logging.Formatter(datefmt = '%y-%m-%d %H:%M:%S',
fmt = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
file_logger.setLevel(0)
file_formatter = logging.Formatter(
datefmt="%y-%m-%d %H:%M:%S",
fmt="%(asctime)s %(name)-12s %(levelname)-8s %(message)s",
)
file_logger.setFormatter(file_formatter)
rootlog.addHandler(file_logger)
vollog.addHandler(file_logger)
vollog.info("Logging started")
if partial_args.verbosity < 3:
if partial_args.verbosity < 1:
sys.tracebacklimit = None
console.setLevel(30 - (partial_args.verbosity * 10))
else:
console.setLevel(10 - (partial_args.verbosity - 2))
@@ -148,12 +205,14 @@ class VolShell(cli.CommandLine):
### Alter constants if necessary
if partial_args.plugin_dirs:
volatility3.plugins.__path__ = [os.path.abspath(p)
for p in partial_args.plugin_dirs.split(";")] + constants.PLUGINS_PATH
volatility3.plugins.__path__ = [
os.path.abspath(p) for p in partial_args.plugin_dirs.split(";")
] + constants.PLUGINS_PATH
if partial_args.symbol_dirs:
volatility3.symbols.__path__ = [os.path.abspath(p)
for p in partial_args.symbol_dirs.split(";")] + constants.SYMBOL_BASEPATHS
volatility3.symbols.__path__ = [
os.path.abspath(p) for p in partial_args.symbol_dirs.split(";")
] + constants.SYMBOL_BASEPATHS
if partial_args.cache_path:
constants.CACHE_PATH = partial_args.cache_path
@@ -161,7 +220,6 @@ class VolShell(cli.CommandLine):
vollog.info(f"Volatility plugins path: {volatility3.plugins.__path__}")
vollog.info(f"Volatility symbols path: {volatility3.symbols.__path__}")
if partial_args.clear_cache:
framework.clear_cache()
@@ -170,11 +228,14 @@ class VolShell(cli.CommandLine):
# Do the initialization
ctx = contexts.Context() # Construct a blank context
failures = framework.import_files(volatility3.plugins,
True) # Will not log as console's default level is WARNING
failures = framework.import_files(
volatility3.plugins, True
) # Will not log as console's default level is WARNING
if failures:
parser.epilog = "The following plugins could not be loaded (use -vv to see why): " + \
", ".join(sorted(failures))
parser.epilog = (
"The following plugins could not be loaded (use -vv to see why): "
+ ", ".join(sorted(failures))
)
vollog.info(parser.epilog)
automagics = automagic.available(ctx)
@@ -192,11 +253,17 @@ class VolShell(cli.CommandLine):
configurables_list[amagic.__class__.__name__] = amagic
# We don't list plugin arguments, because they can be provided within python
volshell_plugin_list = {'generic': generic.Volshell, 'windows': windows.Volshell}
volshell_plugin_list = {
"generic": generic.Volshell,
"windows": windows.Volshell,
}
for plugin in volshell_plugin_list:
subparser = parser.add_argument_group(title = plugin.capitalize(),
description = "Configuration options based on {} options".format(
plugin.capitalize()))
subparser = parser.add_argument_group(
title=plugin.capitalize(),
description="Configuration options based on {} options".format(
plugin.capitalize()
),
)
self.populate_requirements_argparse(subparser, volshell_plugin_list[plugin])
configurables_list[plugin] = volshell_plugin_list[plugin]
@@ -208,7 +275,9 @@ class VolShell(cli.CommandLine):
# Run the argparser
args = parser.parse_args()
vollog.log(constants.LOGLEVEL_VVV, f"Cache directory used: {constants.CACHE_PATH}")
vollog.log(
constants.LOGLEVEL_VVV, f"Cache directory used: {constants.CACHE_PATH}"
)
plugin = generic.Volshell
if args.windows:
@@ -219,7 +288,9 @@ class VolShell(cli.CommandLine):
plugin = mac.Volshell
base_config_path = "plugins"
plugin_config_path = interfaces.configuration.path_join(base_config_path, plugin.__name__)
plugin_config_path = interfaces.configuration.path_join(
base_config_path, plugin.__name__
)
# Special case the -f argument because people use is so frequently
# It has to go here so it can be overridden by single-location if it's defined
@@ -228,7 +299,7 @@ class VolShell(cli.CommandLine):
if args.file:
try:
single_location = self.location_from_file(args.file)
ctx.config['automagic.LayerStacker.single_location'] = single_location
ctx.config["automagic.LayerStacker.single_location"] = single_location
except ValueError as excp:
parser.error(str(excp))
@@ -236,15 +307,22 @@ class VolShell(cli.CommandLine):
if args.config:
with open(args.config, "r") as f:
json_val = json.load(f)
ctx.config.splice(plugin_config_path, interfaces.configuration.HierarchicalDict(json_val))
ctx.config.splice(
plugin_config_path,
interfaces.configuration.HierarchicalDict(json_val),
)
self.populate_config(ctx, configurables_list, args, plugin_config_path)
if args.extend:
for extension in args.extend:
if '=' not in extension:
raise ValueError("Invalid extension (extensions must be of the format \"conf.path.value='value'\")")
address, value = extension[:extension.find('=')], json.loads(extension[extension.find('=') + 1:])
if "=" not in extension:
raise ValueError(
"Invalid extension (extensions must be of the format \"conf.path.value='value'\")"
)
address, value = extension[: extension.find("=")], json.loads(
extension[extension.find("=") + 1 :]
)
ctx.config[address] = value
# It should be up to the UI to determine which automagics to run, so this is before BACK TO THE FRAMEWORK
@@ -259,22 +337,40 @@ class VolShell(cli.CommandLine):
if args.quiet:
progress_callback = cli.MuteProgress()
constructed = plugins.construct_plugin(ctx, automagics, plugin, base_config_path, progress_callback,
self.file_handler_class_factory())
constructed = plugins.construct_plugin(
ctx,
automagics,
plugin,
base_config_path,
progress_callback,
self.file_handler_class_factory(),
)
if args.write_config:
vollog.warning('Use of --write-config has been deprecated, replaced by --save-config <filename>')
args.save_config = 'config.json'
vollog.warning(
"Use of --write-config has been deprecated, replaced by --save-config <filename>"
)
args.save_config = "config.json"
if args.save_config:
vollog.debug("Writing out configuration data to {args.save_config}")
if os.path.exists(os.path.abspath(args.save_config)):
parser.error(f"Cannot write configuration: file {args.save_config} already exists")
parser.error(
f"Cannot write configuration: file {args.save_config} already exists"
)
with open(args.save_config, "w") as f:
json.dump(dict(constructed.build_configuration()), f, sort_keys = True, indent = 2)
json.dump(
dict(constructed.build_configuration()),
f,
sort_keys=True,
indent=2,
)
f.write("\n")
except exceptions.UnsatisfiedException as excp:
self.process_unsatisfied_exceptions(excp)
parser.exit(1, f"Unable to validate the plugin requirements: {[x for x in excp.unsatisfied]}\n")
parser.exit(
1,
f"Unable to validate the plugin requirements: {[x for x in excp.unsatisfied]}\n",
)
try:
# Construct and run the plugin
@@ -282,7 +378,6 @@ class VolShell(cli.CommandLine):
constructed.run()
except exceptions.VolatilityException as excp:
self.process_exceptions(excp)
parser.exit(1, f"Unable to validate the plugin requirements: {[x for x in excp.unsatisfied]}\n")
def main():
volatility3/cli/volshell/generic.py
+215 -108
View File
@@ -26,6 +26,7 @@ except ImportError:
class Volshell(interfaces.plugins.PluginInterface):
"""Shell environment to directly interact with a memory image."""
_required_framework_version = (2, 0, 0)
def __init__(self, *args, **kwargs):
@@ -36,23 +37,29 @@ class Volshell(interfaces.plugins.PluginInterface):
self.__console = None
def random_string(self, length: int = 32) -> str:
return ''.join(random.sample(string.ascii_uppercase + string.digits, length))
return "".join(random.sample(string.ascii_uppercase + string.digits, length))
@classmethod
def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]:
reqs: List[interfaces.configuration.RequirementInterface] = []
if cls == Volshell:
reqs = [
requirements.URIRequirement(name = 'script',
description = 'File to load and execute at start',
default = None,
optional = True)
requirements.URIRequirement(
name="script",
description="File to load and execute at start",
default=None,
optional=True,
)
]
return reqs + [
requirements.TranslationLayerRequirement(name = 'primary', description = 'Memory layer for the kernel'),
requirements.TranslationLayerRequirement(
name="primary", description="Memory layer for the kernel"
),
]
def run(self, additional_locals: Dict[str, Any] = None) -> interfaces.renderers.TreeGrid:
def run(
self, additional_locals: Dict[str, Any] = None
) -> interfaces.renderers.TreeGrid:
"""Runs the interactive volshell plugin.
Returns:
@@ -66,14 +73,15 @@ class Volshell(interfaces.plugins.PluginInterface):
pass
else:
import rlcompleter
completer = rlcompleter.Completer(namespace = self._construct_locals_dict())
completer = rlcompleter.Completer(namespace=self._construct_locals_dict())
readline.set_completer(completer.complete)
readline.parse_and_bind("tab: complete")
print("Readline imported successfully")
# TODO: provide help, consider generic functions (pslist?) and/or providing windows/linux functions
mode = self.__module__.split('.')[-1]
mode = self.__module__.split(".")[-1]
mode = mode[0].upper() + mode[1:]
banner = f"""
@@ -86,13 +94,13 @@ class Volshell(interfaces.plugins.PluginInterface):
"""
sys.ps1 = f"({self.current_layer}) >>> "
self.__console = code.InteractiveConsole(locals = self._construct_locals_dict())
self.__console = code.InteractiveConsole(locals=self._construct_locals_dict())
# Since we have to do work to add the option only once for all different modes of volshell, we can't
# rely on the default having been set
if self.config.get('script', None) is not None:
self.run_script(location = self.config['script'])
if self.config.get("script", None) is not None:
self.run_script(location=self.config["script"])
self.__console.interact(banner = banner)
self.__console.interact(banner=banner)
return renderers.TreeGrid([("Terminating", str)], None)
@@ -100,7 +108,7 @@ class Volshell(interfaces.plugins.PluginInterface):
"""Describes the available commands"""
if args:
help(*args)
return
return None
variables = []
print("\nMethods:")
@@ -119,47 +127,70 @@ class Volshell(interfaces.plugins.PluginInterface):
def construct_locals(self) -> List[Tuple[List[str], Any]]:
"""Returns a dictionary listing the functions to be added to the
environment."""
return [(['dt', 'display_type'], self.display_type), (['db', 'display_bytes'], self.display_bytes),
(['dw', 'display_words'], self.display_words), (['dd',
'display_doublewords'], self.display_doublewords),
(['dq', 'display_quadwords'], self.display_quadwords), (['dis', 'disassemble'], self.disassemble),
(['cl', 'change_layer'], self.change_layer),
(['cs', 'change_symboltable'], self.change_symbol_table),
(['ck', 'change_kernel'], self.change_kernel),
(['context'], self.context), (['self'], self),
(['dpo', 'display_plugin_output'], self.display_plugin_output),
(['gt', 'generate_treegrid'], self.generate_treegrid), (['rt',
'render_treegrid'], self.render_treegrid),
(['ds', 'display_symbols'], self.display_symbols), (['hh', 'help'], self.help),
(['cc', 'create_configurable'], self.create_configurable), (['lf', 'load_file'], self.load_file),
(['rs', 'run_script'], self.run_script)]
return [
(["dt", "display_type"], self.display_type),
(["db", "display_bytes"], self.display_bytes),
(["dw", "display_words"], self.display_words),
(["dd", "display_doublewords"], self.display_doublewords),
(["dq", "display_quadwords"], self.display_quadwords),
(["dis", "disassemble"], self.disassemble),
(["cl", "change_layer"], self.change_layer),
(["cs", "change_symboltable"], self.change_symbol_table),
(["ck", "change_kernel"], self.change_kernel),
(["context"], self.context),
(["self"], self),
(["dpo", "display_plugin_output"], self.display_plugin_output),
(["gt", "generate_treegrid"], self.generate_treegrid),
(["rt", "render_treegrid"], self.render_treegrid),
(["ds", "display_symbols"], self.display_symbols),
(["hh", "help"], self.help),
(["cc", "create_configurable"], self.create_configurable),
(["lf", "load_file"], self.load_file),
(["rs", "run_script"], self.run_script),
]
def _construct_locals_dict(self) -> Dict[str, Any]:
"""Returns a dictionary of the locals """
"""Returns a dictionary of the locals"""
result = {}
for aliases, value in self.construct_locals():
for alias in aliases:
result[alias] = value
return result
def _read_data(self, offset, count = 128, layer_name = None):
def _read_data(self, offset, count=128, layer_name=None):
"""Reads the bytes necessary for the display_* methods"""
return self.context.layers[layer_name or self.current_layer].read(offset, count)
def _display_data(self, offset: int, remaining_data: bytes, format_string: str = "B", ascii: bool = True):
def _display_data(
self,
offset: int,
remaining_data: bytes,
format_string: str = "B",
ascii: bool = True,
):
"""Display a series of bytes"""
chunk_size = struct.calcsize(format_string)
data_length = len(remaining_data)
remaining_data = remaining_data[:data_length - (data_length % chunk_size)]
remaining_data = remaining_data[: data_length - (data_length % chunk_size)]
while remaining_data:
current_line, remaining_data = remaining_data[:16], remaining_data[16:]
data_blocks = [current_line[chunk_size * i:chunk_size * (i + 1)] for i in range(16 // chunk_size)]
data_blocks = [x for x in data_blocks if x != b'']
valid_data = [("{:0" + str(2 * chunk_size) + "x}").format(struct.unpack(format_string, x)[0])
for x in data_blocks]
padding_data = [" " * 2 * chunk_size for _ in range((16 - len(current_line)) // chunk_size)]
data_blocks = [
current_line[chunk_size * i : chunk_size * (i + 1)]
for i in range(16 // chunk_size)
]
data_blocks = [x for x in data_blocks if x != b""]
valid_data = [
("{:0" + str(2 * chunk_size) + "x}").format(
struct.unpack(format_string, x)[0]
)
for x in data_blocks
]
padding_data = [
" " * 2 * chunk_size
for _ in range((16 - len(current_line)) // chunk_size)
]
hex_data = " ".join(valid_data + padding_data)
ascii_data = ""
@@ -175,12 +206,14 @@ class Volshell(interfaces.plugins.PluginInterface):
@staticmethod
def _ascii_bytes(bytes):
"""Converts bytes into an ascii string"""
return "".join([chr(x) if 32 < x < 127 else '.' for x in binascii.unhexlify(bytes)])
return "".join(
[chr(x) if 32 < x < 127 else "." for x in binascii.unhexlify(bytes)]
)
@property
def current_layer(self):
if self.__current_layer is None:
self.__current_layer = self.config['primary']
self.__current_layer = self.config["primary"]
return self.__current_layer
@property
@@ -192,7 +225,7 @@ class Volshell(interfaces.plugins.PluginInterface):
@property
def current_kernel_name(self):
if self.__current_kernel_name is None:
self.__current_kernel_name = self.config.get('kernel', None)
self.__current_kernel_name = self.config.get("kernel", None)
return self.__current_kernel_name
@property
@@ -217,7 +250,9 @@ class Volshell(interfaces.plugins.PluginInterface):
if not symbol_table_name:
print("No symbol table provided, not changing current symbol table")
if symbol_table_name not in self.context.symbol_space:
print(f"Symbol table {symbol_table_name} not present in context symbol_space")
print(
f"Symbol table {symbol_table_name} not present in context symbol_space"
)
else:
self.__current_symbol_table = symbol_table_name
print(f"Current Symbol Table: {self.current_symbol_table}")
@@ -231,53 +266,66 @@ class Volshell(interfaces.plugins.PluginInterface):
self.__current_kernel_name = kernel_name
print(f"Current kernel : {self.current_kernel_name}")
def display_bytes(self, offset, count = 128, layer_name = None):
def display_bytes(self, offset, count=128, layer_name=None):
"""Displays byte values and ASCII characters"""
remaining_data = self._read_data(offset, count = count, layer_name = layer_name)
remaining_data = self._read_data(offset, count=count, layer_name=layer_name)
self._display_data(offset, remaining_data)
def display_quadwords(self, offset, count = 128, layer_name = None):
def display_quadwords(self, offset, count=128, layer_name=None):
"""Displays quad-word values (8 bytes) and corresponding ASCII characters"""
remaining_data = self._read_data(offset, count = count, layer_name = layer_name)
self._display_data(offset, remaining_data, format_string = "Q")
remaining_data = self._read_data(offset, count=count, layer_name=layer_name)
self._display_data(offset, remaining_data, format_string="Q")
def display_doublewords(self, offset, count = 128, layer_name = None):
def display_doublewords(self, offset, count=128, layer_name=None):
"""Displays double-word values (4 bytes) and corresponding ASCII characters"""
remaining_data = self._read_data(offset, count = count, layer_name = layer_name)
self._display_data(offset, remaining_data, format_string = "I")
remaining_data = self._read_data(offset, count=count, layer_name=layer_name)
self._display_data(offset, remaining_data, format_string="I")
def display_words(self, offset, count = 128, layer_name = None):
def display_words(self, offset, count=128, layer_name=None):
"""Displays word values (2 bytes) and corresponding ASCII characters"""
remaining_data = self._read_data(offset, count = count, layer_name = layer_name)
self._display_data(offset, remaining_data, format_string = "H")
remaining_data = self._read_data(offset, count=count, layer_name=layer_name)
self._display_data(offset, remaining_data, format_string="H")
def disassemble(self, offset, count = 128, layer_name = None, architecture = None):
def disassemble(self, offset, count=128, layer_name=None, architecture=None):
"""Disassembles a number of instructions from the code at offset"""
remaining_data = self._read_data(offset, count = count, layer_name = layer_name)
remaining_data = self._read_data(offset, count=count, layer_name=layer_name)
if not has_capstone:
print("Capstone not available - please install it to use the disassemble command")
print(
"Capstone not available - please install it to use the disassemble command"
)
else:
if isinstance(self.context.layers[layer_name or self.current_layer], intel.Intel32e):
architecture = 'intel64'
elif isinstance(self.context.layers[layer_name or self.current_layer], intel.Intel):
architecture = 'intel'
if isinstance(
self.context.layers[layer_name or self.current_layer], intel.Intel32e
):
architecture = "intel64"
elif isinstance(
self.context.layers[layer_name or self.current_layer], intel.Intel
):
architecture = "intel"
disasm_types = {
'intel': capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_32),
'intel64': capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_64),
'arm': capstone.Cs(capstone.CS_ARCH_ARM, capstone.CS_MODE_ARM),
'arm64': capstone.Cs(capstone.CS_ARCH_ARM64, capstone.CS_MODE_ARM)
"intel": capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_32),
"intel64": capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_64),
"arm": capstone.Cs(capstone.CS_ARCH_ARM, capstone.CS_MODE_ARM),
"arm64": capstone.Cs(capstone.CS_ARCH_ARM64, capstone.CS_MODE_ARM),
}
if architecture is not None:
for i in disasm_types[architecture].disasm(remaining_data, offset):
print(f"0x{i.address:x}:\t{i.mnemonic}\t{i.op_str}")
def display_type(self,
object: Union[str, interfaces.objects.ObjectInterface, interfaces.objects.Template],
offset: int = None):
def display_type(
self,
object: Union[
str, interfaces.objects.ObjectInterface, interfaces.objects.Template
],
offset: int = None,
):
"""Display Type describes the members of a particular object in alphabetical order"""
if not isinstance(object, (str, interfaces.objects.ObjectInterface, interfaces.objects.Template)):
if not isinstance(
object,
(str, interfaces.objects.ObjectInterface, interfaces.objects.Template),
):
print("Cannot display information about non-type object")
return
return None
if not isinstance(object, str):
# Mypy requires us to order things this way
@@ -287,20 +335,29 @@ class Volshell(interfaces.plugins.PluginInterface):
volobject = self.context.symbol_space.get_type(object)
else:
# Str and offset
volobject = self.context.object(object, layer_name = self.current_layer, offset = offset)
volobject = self.context.object(
object, layer_name=self.current_layer, offset=offset
)
if offset is not None:
volobject = self.context.object(volobject.vol.type_name, layer_name = self.current_layer, offset = offset)
volobject = self.context.object(
volobject.vol.type_name, layer_name=self.current_layer, offset=offset
)
if hasattr(volobject.vol, 'size'):
if hasattr(volobject.vol, "size"):
print(f"{volobject.vol.type_name} ({volobject.vol.size} bytes)")
elif hasattr(volobject.vol, 'data_format'):
elif hasattr(volobject.vol, "data_format"):
data_format = volobject.vol.data_format
print("{} ({} bytes, {} endian, {})".format(volobject.vol.type_name, data_format.length,
data_format.byteorder,
'signed' if data_format.signed else 'unsigned'))
print(
"{} ({} bytes, {} endian, {})".format(
volobject.vol.type_name,
data_format.length,
data_format.byteorder,
"signed" if data_format.signed else "unsigned",
)
)
if hasattr(volobject.vol, 'members'):
if hasattr(volobject.vol, "members"):
longest_member = longest_offset = longest_typename = 0
for member in volobject.vol.members:
relative_offset, member_type = volobject.vol.members[member]
@@ -308,32 +365,50 @@ class Volshell(interfaces.plugins.PluginInterface):
longest_offset = max(len(hex(relative_offset)), longest_offset)
longest_typename = max(len(member_type.vol.type_name), longest_typename)
for member in sorted(volobject.vol.members, key = lambda x: (volobject.vol.members[x][0], x)):
for member in sorted(
volobject.vol.members, key=lambda x: (volobject.vol.members[x][0], x)
):
relative_offset, member_type = volobject.vol.members[member]
len_offset = len(hex(relative_offset))
len_member = len(member)
len_typename = len(member_type.vol.type_name)
if isinstance(volobject, interfaces.objects.ObjectInterface):
# We're an instance, so also display the data
print(" " * (longest_offset - len_offset), hex(relative_offset), ": ", member,
" " * (longest_member - len_member), " ",
member_type.vol.type_name, " " * (longest_typename - len_typename), " ",
self._display_value(getattr(volobject, member)))
print(
" " * (longest_offset - len_offset),
hex(relative_offset),
": ",
member,
" " * (longest_member - len_member),
" ",
member_type.vol.type_name,
" " * (longest_typename - len_typename),
" ",
self._display_value(getattr(volobject, member)),
)
else:
print(" " * (longest_offset - len_offset), hex(relative_offset), ": ", member,
" " * (longest_member - len_member), " ", member_type.vol.type_name)
print(
" " * (longest_offset - len_offset),
hex(relative_offset),
": ",
member,
" " * (longest_member - len_member),
" ",
member_type.vol.type_name,
)
@classmethod
def _display_value(self, value: Any) -> str:
def _display_value(cls, value: Any) -> str:
if isinstance(value, objects.PrimitiveObject):
return repr(value)
elif isinstance(value, objects.Array):
return repr([self._display_value(val) for val in value])
return repr([cls._display_value(val) for val in value])
else:
return hex(value.vol.offset)
def generate_treegrid(self, plugin: Type[interfaces.plugins.PluginInterface],
**kwargs) -> Optional[interfaces.renderers.TreeGrid]:
def generate_treegrid(
self, plugin: Type[interfaces.plugins.PluginInterface], **kwargs
) -> Optional[interfaces.renderers.TreeGrid]:
"""Generates a TreeGrid based on a specific plugin passing in kwarg configuration values"""
path_join = interfaces.configuration.path_join
@@ -346,21 +421,29 @@ class Volshell(interfaces.plugins.PluginInterface):
self.config[path_join(plugin_config_suffix, plugin.__name__, name)] = value
try:
constructed = plugins.construct_plugin(self.context, [], plugin, plugin_path, None, NullFileHandler)
constructed = plugins.construct_plugin(
self.context, [], plugin, plugin_path, None, NullFileHandler
)
return constructed.run()
except exceptions.UnsatisfiedException as excp:
print(f"Unable to validate the plugin requirements: {[x for x in excp.unsatisfied]}\n")
print(
f"Unable to validate the plugin requirements: {[x for x in excp.unsatisfied]}\n"
)
return None
def render_treegrid(self,
treegrid: interfaces.renderers.TreeGrid,
renderer: Optional[interfaces.renderers.Renderer] = None) -> None:
def render_treegrid(
self,
treegrid: interfaces.renderers.TreeGrid,
renderer: Optional[interfaces.renderers.Renderer] = None,
) -> None:
"""Renders a treegrid as produced by generate_treegrid"""
if renderer is None:
renderer = text_renderer.QuickTextRenderer()
renderer.render(treegrid)
def display_plugin_output(self, plugin: Type[interfaces.plugins.PluginInterface], **kwargs) -> None:
def display_plugin_output(
self, plugin: Type[interfaces.plugins.PluginInterface], **kwargs
) -> None:
"""Displays the output for a particular plugin (with keyword arguments)"""
treegrid = self.generate_treegrid(plugin, **kwargs)
if treegrid is not None:
@@ -370,7 +453,7 @@ class Volshell(interfaces.plugins.PluginInterface):
"""Prints an alphabetical list of symbols for a symbol table"""
if symbol_table is None:
print("No symbol table provided")
return
return None
longest_offset = longest_name = 0
table = self.context.symbol_space[symbol_table]
@@ -382,7 +465,12 @@ class Volshell(interfaces.plugins.PluginInterface):
for symbol_name in sorted(table.symbols):
symbol = table.get_symbol(symbol_name)
len_offset = len(hex(symbol.address))
print(" " * (longest_offset - len_offset), hex(symbol.address), " ", symbol.name)
print(
" " * (longest_offset - len_offset),
hex(symbol.address),
" ",
symbol.name,
)
def run_script(self, location: str):
"""Runs a python script within the context of volshell"""
@@ -390,32 +478,45 @@ class Volshell(interfaces.plugins.PluginInterface):
location = "file:" + request.pathname2url(location)
print(f"Running code from {location}\n")
accessor = resources.ResourceAccessor()
with io.TextIOWrapper(accessor.open(url = location), encoding = 'utf-8') as fp:
self.__console.runsource(fp.read(), symbol = 'exec')
with accessor.open(url=location) as fp:
self.__console.runsource(
io.TextIOWrapper(fp, encoding="utf-8").read(), symbol="exec"
)
print("\nCode complete")
def load_file(self, location: str):
"""Loads a file into a Filelayer and returns the name of the layer"""
layer_name = self.context.layers.free_layer_name()
location = volshell.VolShell.location_from_file(location)
current_config_path = 'volshell.layers.' + layer_name
self.context.config[interfaces.configuration.path_join(current_config_path, "location")] = location
current_config_path = "volshell.layers." + layer_name
self.context.config[
interfaces.configuration.path_join(current_config_path, "location")
] = location
layer = physical.FileLayer(self.context, current_config_path, layer_name)
self.context.add_layer(layer)
return layer_name
def create_configurable(self, clazz: Type[interfaces.configuration.ConfigurableInterface], **kwargs):
def create_configurable(
self, clazz: Type[interfaces.configuration.ConfigurableInterface], **kwargs
):
"""Creates a configurable object, converting arguments to configuration"""
config_name = self.random_string()
config_path = 'volshell.configurable.' + config_name
config_path = "volshell.configurable." + config_name
constructor_args = {}
constructor_keywords = []
if issubclass(clazz, interfaces.layers.DataLayerInterface):
constructor_keywords = [('name', self.context.layers.free_layer_name(config_name)), ('metadata', None)]
constructor_keywords = [
("name", self.context.layers.free_layer_name(config_name)),
("metadata", None),
]
if issubclass(clazz, interfaces.symbols.SymbolTableInterface):
constructor_keywords = [('name', self.context.symbol_space.free_table_name(config_name)),
('native_types', None), ('table_mapping', None), ('class_types', None)]
constructor_keywords = [
("name", self.context.symbol_space.free_table_name(config_name)),
("native_types", None),
("table_mapping", None),
("class_types", None),
]
for argname, default in constructor_keywords:
constructor_args[argname] = kwargs.get(argname, default)
@@ -424,10 +525,16 @@ class Volshell(interfaces.plugins.PluginInterface):
for keyword in kwargs:
val = kwargs[keyword]
if not isinstance(val, interfaces.configuration.BasicTypes) and not isinstance(val, list):
if not isinstance(val, list) or all([isinstance(x, interfaces.configuration.BasicTypes) for x in val]):
raise TypeError("Configurable values must be simple types (int, bool, str, bytes)")
self.context.config[config_path + '.' + keyword] = val
if not isinstance(
val, interfaces.configuration.BasicTypes
) and not isinstance(val, list):
if not isinstance(val, list) or all(
[isinstance(x, interfaces.configuration.BasicTypes) for x in val]
):
raise TypeError(
"Configurable values must be simple types (int, bool, str, bytes)"
)
self.context.config[config_path + "." + keyword] = val
constructed = clazz(self.context, config_path, **constructor_args)
volatility3/cli/volshell/linux.py
+28 -18
View File
@@ -15,13 +15,19 @@ class Volshell(generic.Volshell):
@classmethod
def get_requirements(cls):
return ([
requirements.ModuleRequirement(name = "kernel", description = "Linux kernel module"),
requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)),
requirements.IntRequirement(name = 'pid', description = "Process ID", optional = True)
])
return [
requirements.ModuleRequirement(
name="kernel", description="Linux kernel module"
),
requirements.PluginRequirement(
name="pslist", plugin=pslist.PsList, version=(2, 0, 0)
),
requirements.IntRequirement(
name="pid", description="Process ID", optional=True
),
]
def change_task(self, pid = None):
def change_task(self, pid=None):
"""Change the current process and layer, based on a process ID"""
tasks = self.list_tasks()
for task in tasks:
@@ -29,9 +35,9 @@ class Volshell(generic.Volshell):
process_layer = task.add_process_layer()
if process_layer is not None:
self.change_layer(process_layer)
return
return None
print(f"Layer for task ID {pid} could not be constructed")
return
return None
print(f"No task with task ID {pid} found")
def list_tasks(self):
@@ -42,27 +48,31 @@ class Volshell(generic.Volshell):
def construct_locals(self) -> List[Tuple[List[str], Any]]:
result = super().construct_locals()
result += [
(['ct', 'change_task', 'cp'], self.change_task),
(['lt', 'list_tasks', 'ps'], self.list_tasks),
(['symbols'], self.context.symbol_space[self.current_symbol_table]),
(["ct", "change_task", "cp"], self.change_task),
(["lt", "list_tasks", "ps"], self.list_tasks),
(["symbols"], self.context.symbol_space[self.current_symbol_table]),
]
if self.config.get('pid', None) is not None:
self.change_task(self.config['pid'])
if self.config.get("pid", None) is not None:
self.change_task(self.config["pid"])
return result
def display_type(self,
object: Union[str, interfaces.objects.ObjectInterface, interfaces.objects.Template],
offset: int = None):
def display_type(
self,
object: Union[
str, interfaces.objects.ObjectInterface, interfaces.objects.Template
],
offset: int = None,
):
"""Display Type describes the members of a particular object in alphabetical order"""
if isinstance(object, str):
if constants.BANG not in object:
object = self.config['vmlinux'] + constants.BANG + object
object = self.current_symbol_table + constants.BANG + object
return super().display_type(object, offset)
def display_symbols(self, symbol_table: str = None):
"""Prints an alphabetical list of symbols for a symbol table"""
if symbol_table is None:
symbol_table = self.config['vmlinux']
symbol_table = self.current_symbol_table
return super().display_symbols(symbol_table)
@property
volatility3/cli/volshell/mac.py
+31 -19
View File
@@ -15,13 +15,19 @@ class Volshell(generic.Volshell):
@classmethod
def get_requirements(cls):
return ([
requirements.ModuleRequirement(name = "kernel", description = "Darwin kernel module"),
requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (3, 0, 0)),
requirements.IntRequirement(name = 'pid', description = "Process ID", optional = True)
])
return [
requirements.ModuleRequirement(
name="kernel", description="Darwin kernel module"
),
requirements.PluginRequirement(
name="pslist", plugin=pslist.PsList, version=(3, 0, 0)
),
requirements.IntRequirement(
name="pid", description="Process ID", optional=True
),
]
def change_task(self, pid = None):
def change_task(self, pid=None):
"""Change the current process and layer, based on a process ID"""
tasks = self.list_tasks()
for task in tasks:
@@ -29,34 +35,40 @@ class Volshell(generic.Volshell):
process_layer = task.add_process_layer()
if process_layer is not None:
self.change_layer(process_layer)
return
return None
print(f"Layer for task ID {pid} could not be constructed")
return
return None
print(f"No task with task ID {pid} found")
def list_tasks(self, method = None):
def list_tasks(self, method=None):
"""Returns a list of task objects from the primary layer"""
# We always use the main kernel memory and associated symbols
return list(pslist.PsList.get_list_tasks(method)(self.context, self.current_kernel_name))
return list(
pslist.PsList.get_list_tasks(method)(self.context, self.current_kernel_name)
)
def construct_locals(self) -> List[Tuple[List[str], Any]]:
result = super().construct_locals()
result += [
(['ct', 'change_task', 'cp'], self.change_task),
(['lt', 'list_tasks', 'ps'], self.list_tasks),
(['symbols'], self.context.symbol_space[self.current_symbol_table]),
(["ct", "change_task", "cp"], self.change_task),
(["lt", "list_tasks", "ps"], self.list_tasks),
(["symbols"], self.context.symbol_space[self.current_symbol_table]),
]
if self.config.get('pid', None) is not None:
self.change_task(self.config['pid'])
if self.config.get("pid", None) is not None:
self.change_task(self.config["pid"])
return result
def display_type(self,
object: Union[str, interfaces.objects.ObjectInterface, interfaces.objects.Template],
offset: int = None):
def display_type(
self,
object: Union[
str, interfaces.objects.ObjectInterface, interfaces.objects.Template
],
offset: int = None,
):
"""Display Type describes the members of a particular object in alphabetical order"""
if isinstance(object, str):
if constants.BANG not in object:
object = self.config['darwin'] + constants.BANG + object
object = self.current_symbol_table + constants.BANG + object
return super().display_type(object, offset)
def display_symbols(self, symbol_table: str = None):
volatility3/cli/volshell/windows.py
+28 -16
View File
@@ -15,41 +15,53 @@ class Volshell(generic.Volshell):
@classmethod
def get_requirements(cls):
return ([
requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel'),
requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)),
requirements.IntRequirement(name = 'pid', description = "Process ID", optional = True)
])
return [
requirements.ModuleRequirement(name="kernel", description="Windows kernel"),
requirements.PluginRequirement(
name="pslist", plugin=pslist.PsList, version=(2, 0, 0)
),
requirements.IntRequirement(
name="pid", description="Process ID", optional=True
),
]
def change_process(self, pid = None):
def change_process(self, pid=None):
"""Change the current process and layer, based on a process ID"""
processes = self.list_processes()
for process in processes:
if process.UniqueProcessId == pid:
process_layer = process.add_process_layer()
self.change_layer(process_layer)
return
return None
print(f"No process with process ID {pid} found")
def list_processes(self):
"""Returns a list of EPROCESS objects from the primary layer"""
# We always use the main kernel memory and associated symbols
return list(pslist.PsList.list_processes(self.context, self.current_layer, self.current_symbol_table))
return list(
pslist.PsList.list_processes(
self.context, self.current_layer, self.current_symbol_table
)
)
def construct_locals(self) -> List[Tuple[List[str], Any]]:
result = super().construct_locals()
result += [
(['cp', 'change_process'], self.change_process),
(['lp', 'list_processes', 'ps'], self.list_processes),
(['symbols'], self.context.symbol_space[self.current_symbol_table]),
(["cp", "change_process"], self.change_process),
(["lp", "list_processes", "ps"], self.list_processes),
(["symbols"], self.context.symbol_space[self.current_symbol_table]),
]
if self.config.get('pid', None) is not None:
self.change_process(self.config['pid'])
if self.config.get("pid", None) is not None:
self.change_process(self.config["pid"])
return result
def display_type(self,
object: Union[str, interfaces.objects.ObjectInterface, interfaces.objects.Template],
offset: int = None):
def display_type(
self,
object: Union[
str, interfaces.objects.ObjectInterface, interfaces.objects.Template
],
offset: int = None,
):
"""Display Type describes the members of a particular object in alphabetical order"""
if isinstance(object, str):
if constants.BANG not in object:
volatility3/framework/__init__.py
+77 -37
View File
@@ -7,16 +7,26 @@ import glob
import sys
import zipfile
required_python_version = (3, 6, 0)
if (sys.version_info.major != required_python_version[0] or sys.version_info.minor < required_python_version[1] or
(sys.version_info.minor == required_python_version[1] and sys.version_info.micro < required_python_version[2])):
required_python_version = (3, 7, 0)
if (
sys.version_info.major != required_python_version[0]
or sys.version_info.minor < required_python_version[1]
or (
sys.version_info.minor == required_python_version[1]
and sys.version_info.micro < required_python_version[2]
)
):
raise RuntimeError(
"Volatility framework requires python version {}.{}.{} or greater".format(*required_python_version))
"Volatility framework requires python version {}.{}.{} or greater".format(
*required_python_version
)
)
import importlib
import inspect
import logging
import os
import traceback
from typing import Any, Dict, Generator, List, Tuple, Type, TypeVar
from volatility3.framework import constants, interfaces
@@ -45,24 +55,29 @@ def require_interface_version(*args) -> None:
"""Checks the required version of a plugin."""
if len(args):
if args[0] != interface_version()[0]:
raise RuntimeError("Framework interface version {} is incompatible with required version {}".format(
interface_version()[0], args[0]))
raise RuntimeError(
"Framework interface version {} is incompatible with required version {}".format(
interface_version()[0], args[0]
)
)
if len(args) > 1:
if args[1] > interface_version()[1]:
raise RuntimeError(
"Framework interface version {} is an older revision than the required version {}".format(
".".join([str(x) for x in interface_version()[0:2]]), ".".join([str(x) for x in args[0:2]])))
".".join([str(x) for x in interface_version()[0:2]]),
".".join([str(x) for x in args[0:2]]),
)
)
class NonInheritable(object):
def __init__(self, value: Any, cls: Type) -> None:
self.default_value = value
self.cls = cls
def __get__(self, obj: Any, get_type: Type = None) -> Any:
if type == self.cls:
if hasattr(self.default_value, '__get__'):
if hasattr(self.default_value, "__get__"):
return self.default_value.__get__(obj, get_type)
return self.default_value
raise AttributeError
@@ -73,7 +88,7 @@ def hide_from_subclasses(cls: Type) -> Type:
return cls
T = TypeVar('T')
T = TypeVar("T")
def class_subclasses(cls: Type[T]) -> Generator[Type[T], None, None]:
@@ -82,7 +97,7 @@ def class_subclasses(cls: Type[T]) -> Generator[Type[T], None, None]:
raise TypeError(f"class_subclasses parameter not a valid class: {cls}")
for clazz in cls.__subclasses__():
# The typing system is not clever enough to realize that clazz has a hidden attr after the hasattr check
if not hasattr(clazz, 'hidden') or not clazz.hidden: # type: ignore
if not hasattr(clazz, "hidden") or not clazz.hidden: # type: ignore
yield clazz
for return_value in class_subclasses(clazz):
yield return_value
@@ -93,10 +108,12 @@ def import_files(base_module, ignore_errors: bool = False) -> List[str]:
failures = []
if not isinstance(base_module.__path__, list):
raise TypeError("[base_module].__path__ must be a list of paths")
vollog.log(constants.LOGLEVEL_VVVV,
f"Importing from the following paths: {', '.join(base_module.__path__)}")
vollog.log(
constants.LOGLEVEL_VVVV,
f"Importing from the following paths: {', '.join(base_module.__path__)}",
)
for path in base_module.__path__:
for root, _, files in os.walk(path, followlinks = True):
for root, _, files in os.walk(path, followlinks=True):
# TODO: Figure out how to import pycache files
if root.endswith("__pycache__"):
continue
@@ -104,35 +121,51 @@ def import_files(base_module, ignore_errors: bool = False) -> List[str]:
if zipfile.is_zipfile(os.path.join(root, filename)):
# Use the root to add this to the module path, and sub-traverse the files
new_module = base_module
premodules = root[len(path) + len(os.path.sep):].replace(os.path.sep, '.')
for component in premodules.split('.'):
premodules = root[len(path) + len(os.path.sep) :].replace(
os.path.sep, "."
)
for component in premodules.split("."):
if component:
try:
new_module = getattr(new_module, component)
except AttributeError:
failures += [new_module + '.' + component]
new_module.__path__ = [os.path.join(root, filename)] + new_module.__path__
failures += [new_module + "." + component]
new_module.__path__ = [
os.path.join(root, filename)
] + new_module.__path__
for ziproot, zipfiles in _zipwalk(os.path.join(root, filename)):
for zfile in zipfiles:
if _filter_files(zfile):
submodule = zfile[:zfile.rfind('.')].replace(os.path.sep, '.')
failures += import_file(new_module.__name__ + '.' + submodule,
os.path.join(path, ziproot, zfile))
submodule = zfile[: zfile.rfind(".")].replace(
os.path.sep, "."
)
failures += import_file(
new_module.__name__ + "." + submodule,
os.path.join(path, ziproot, zfile),
)
else:
if _filter_files(filename):
modpath = os.path.join(root[len(path) + len(os.path.sep):], filename[:filename.rfind(".")])
modpath = os.path.join(
root[len(path) + len(os.path.sep) :],
filename[: filename.rfind(".")],
)
submodule = modpath.replace(os.path.sep, ".")
failures += import_file(base_module.__name__ + '.' + submodule,
os.path.join(root, filename),
ignore_errors)
failures += import_file(
base_module.__name__ + "." + submodule,
os.path.join(root, filename),
ignore_errors,
)
return failures
def _filter_files(filename: str):
"""Ensures that a filename traversed is an importable python file"""
return (filename.endswith(".py") or filename.endswith(".pyc") or filename.endswith(
".pyo")) and not filename.startswith("__")
return (
filename.endswith(".py")
or filename.endswith(".pyc")
or filename.endswith(".pyo")
) and not filename.startswith("__")
def import_file(module: str, path: str, ignore_errors: bool = False) -> List[str]:
@@ -151,8 +184,14 @@ def import_file(module: str, path: str, ignore_errors: bool = False) -> List[str
try:
importlib.import_module(module)
except ImportError as e:
vollog.debug(str(e))
vollog.debug("Failed to import module {} based on file: {}".format(module, path))
vollog.debug(
"".join(
traceback.TracebackException.from_exception(e).format(chain=True)
)
)
vollog.debug(
"Failed to import module {} based on file: {}".format(module, path)
)
failures.append(module)
if not ignore_errors:
raise
@@ -167,7 +206,9 @@ def _zipwalk(path: str):
if not file.is_dir():
dirlist = zip_results.get(os.path.dirname(file.filename), [])
dirlist.append(os.path.basename(file.filename))
zip_results[os.path.join(path, os.path.dirname(file.filename))] = dirlist
zip_results[os.path.join(path, os.path.dirname(file.filename))] = (
dirlist
)
for value in zip_results:
yield value, zip_results[value]
@@ -177,14 +218,13 @@ def list_plugins() -> Dict[str, Type[interfaces.plugins.PluginInterface]]:
for plugin in class_subclasses(interfaces.plugins.PluginInterface):
plugin_name = plugin.__module__ + "." + plugin.__name__
if plugin_name.startswith("volatility3.plugins."):
plugin_name = plugin_name[len("volatility3.plugins."):]
plugin_name = plugin_name[len("volatility3.plugins.") :]
plugin_list[plugin_name] = plugin
return plugin_list
def clear_cache(complete = False):
glob_pattern = '*.cache'
if not complete:
glob_pattern = 'data_' + glob_pattern
for cache_filename in glob.glob(os.path.join(constants.CACHE_PATH, glob_pattern)):
os.unlink(cache_filename)
def clear_cache(complete=False):
try:
os.unlink(os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME))
except FileNotFoundError:
vollog.log(constants.LOGLEVEL_VVVV, "Attempting to clear a non-existant cache")
volatility3/framework/automagic/__init__.py
+33 -19
View File
@@ -22,7 +22,9 @@ from volatility3.framework.configuration import requirements
vollog = logging.getLogger(__name__)
def available(context: interfaces.context.ContextInterface) -> List[interfaces.automagic.AutomagicInterface]:
def available(
context: interfaces.context.ContextInterface,
) -> List[interfaces.automagic.AutomagicInterface]:
"""Returns an ordered list of all subclasses of
:class:`~volatility3.framework.interfaces.automagic.AutomagicInterface`.
@@ -34,21 +36,26 @@ def available(context: interfaces.context.ContextInterface) -> List[interfaces.a
"""
import_files(sys.modules[__name__])
config_path = constants.AUTOMAGIC_CONFIG_PATH
return sorted([
clazz(context, interfaces.configuration.path_join(config_path, clazz.__name__))
for clazz in class_subclasses(interfaces.automagic.AutomagicInterface)
],
key = lambda x: x.priority)
return sorted(
[
clazz(
context, interfaces.configuration.path_join(config_path, clazz.__name__)
)
for clazz in class_subclasses(interfaces.automagic.AutomagicInterface)
],
key=lambda x: x.priority,
)
def choose_automagic(
automagics: List[Type[interfaces.automagic.AutomagicInterface]],
plugin: Type[interfaces.plugins.PluginInterface]) -> List[Type[interfaces.automagic.AutomagicInterface]]:
automagics: List[Type[interfaces.automagic.AutomagicInterface]],
plugin: Type[interfaces.plugins.PluginInterface],
) -> List[Type[interfaces.automagic.AutomagicInterface]]:
"""Chooses which automagics to run, maintaining the order they were handed
in."""
plugin_category = "None"
plugin_categories = plugin.__module__.split('.')
plugin_categories = plugin.__module__.split(".")
lowest_index = len(plugin_categories)
for os in constants.OS_CATEGORIES:
try:
@@ -73,12 +80,16 @@ def choose_automagic(
return output
def run(automagics: List[interfaces.automagic.AutomagicInterface],
context: interfaces.context.ContextInterface,
configurable: Union[interfaces.configuration.ConfigurableInterface,
Type[interfaces.configuration.ConfigurableInterface]],
config_path: str,
progress_callback: constants.ProgressCallback = None) -> List[traceback.TracebackException]:
def run(
automagics: List[interfaces.automagic.AutomagicInterface],
context: interfaces.context.ContextInterface,
configurable: Union[
interfaces.configuration.ConfigurableInterface,
Type[interfaces.configuration.ConfigurableInterface],
],
config_path: str,
progress_callback: constants.ProgressCallback = None,
) -> List[traceback.TracebackException]:
"""Runs through the list of `automagics` in order, allowing them to make
changes to the context.
@@ -99,10 +110,13 @@ def run(automagics: List[interfaces.automagic.AutomagicInterface],
"""
for automagic in automagics:
if not isinstance(automagic, interfaces.automagic.AutomagicInterface):
raise TypeError("Automagics must only contain AutomagicInterface subclasses")
raise TypeError(
"Automagics must only contain AutomagicInterface subclasses"
)
if (not isinstance(configurable, interfaces.configuration.ConfigurableInterface)
and not issubclass(configurable, interfaces.configuration.ConfigurableInterface)):
if not isinstance(
configurable, interfaces.configuration.ConfigurableInterface
) and not issubclass(configurable, interfaces.configuration.ConfigurableInterface):
raise TypeError("Automagic operates on configurables only")
# TODO: Fix need for top level config element just because we're using a MultiRequirement to group the
@@ -112,7 +126,7 @@ def run(automagics: List[interfaces.automagic.AutomagicInterface],
configurable_class = configurable.__class__
else:
configurable_class = configurable
requirement = requirements.MultiRequirement(name = configurable_class.__name__)
requirement = requirements.MultiRequirement(name=configurable_class.__name__)
for req in configurable.get_requirements():
requirement.add_requirement(req)
volatility3/framework/automagic/construct_layers.py
+34 -14
View File
@@ -25,39 +25,59 @@ class ConstructionMagic(interfaces.automagic.AutomagicInterface):
:warning: This `automagic` should run first to allow existing configurations to have been constructed for use by later automagic
"""
priority = 0
def __call__(self,
context: interfaces.context.ContextInterface,
config_path: str,
requirement: interfaces.configuration.RequirementInterface,
progress_callback = None,
optional = False) -> List[str]:
def __call__(
self,
context: interfaces.context.ContextInterface,
config_path: str,
requirement: interfaces.configuration.RequirementInterface,
progress_callback=None,
optional=False,
) -> List[str]:
# Make sure we import the layers, so they can reconstructed
framework.import_files(sys.modules['volatility3.framework.layers'])
framework.import_files(sys.modules["volatility3.framework.layers"])
result: List[str] = []
if requirement.unsatisfied(context, config_path):
# Having called validate at the top level tells us both that we need to dig deeper
# but also ensures that TranslationLayerRequirements have got the correct subrequirements if their class is populated
subreq_config_path = interfaces.configuration.path_join(config_path, requirement.name)
subreq_config_path = interfaces.configuration.path_join(
config_path, requirement.name
)
for subreq in requirement.requirements.values():
try:
self(context, subreq_config_path, subreq, optional = optional or subreq.optional)
self(
context,
subreq_config_path,
subreq,
optional=optional or subreq.optional,
)
except Exception as e:
# We don't really care if this fails, it tends to mean the configuration isn't complete for that item
vollog.log(constants.LOGLEVEL_VVVV, f"Construction Exception occurred: {e}")
vollog.log(
constants.LOGLEVEL_VVVV, f"Construction Exception occurred: {e}"
)
invalid = subreq.unsatisfied(context, subreq_config_path)
# We want to traverse optional paths, so don't check until we've tried to validate
# We also don't want to emit a debug message when a parent is optional, hence the optional parameter
if invalid and not (optional or subreq.optional):
vollog.log(constants.LOGLEVEL_V, f"Failed on requirement: {subreq_config_path}")
result.append(interfaces.configuration.path_join(subreq_config_path, subreq.name))
vollog.log(
constants.LOGLEVEL_V,
f"Failed on requirement: {subreq_config_path}",
)
result.append(
interfaces.configuration.path_join(
subreq_config_path, subreq.name
)
)
if result:
return result
elif isinstance(requirement, interfaces.configuration.ConstructableRequirementInterface):
elif isinstance(
requirement, interfaces.configuration.ConstructableRequirementInterface
):
# We know all the subrequirements are filled, so let's populate
requirement.construct(context, config_path)
volatility3/framework/automagic/linux.py
+108 -74
View File
@@ -3,10 +3,12 @@
#
import logging
import os
from typing import Optional, Tuple, Type
from volatility3.framework import interfaces, constants
from volatility3.framework import constants, interfaces
from volatility3.framework.automagic import symbol_cache, symbol_finder
from volatility3.framework.configuration import requirements
from volatility3.framework.layers import intel, scanners
from volatility3.framework.symbols import linux
@@ -15,14 +17,26 @@ vollog = logging.getLogger(__name__)
class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface):
stack_order = 35
exclusion_list = ['mac', 'windows']
exclusion_list = ["mac", "windows"]
@classmethod
def stack(cls,
context: interfaces.context.ContextInterface,
layer_name: str,
progress_callback: constants.ProgressCallback = None) -> Optional[interfaces.layers.DataLayerInterface]:
def stack(
cls,
context: interfaces.context.ContextInterface,
layer_name: str,
progress_callback: constants.ProgressCallback = None,
) -> Optional[interfaces.layers.DataLayerInterface]:
"""Attempts to identify linux within this layer."""
# Version check the SQlite cache
required = (1, 0, 0)
if not requirements.VersionRequirement.matches_required(
required, symbol_cache.SqliteCache.version
):
vollog.info(
f"SQLiteCache version not suitable: required {required} found {symbol_cache.SqliteCache.version}"
)
return None
# Bail out by default unless we can stack properly
layer = context.layers[layer_name]
join = interfaces.configuration.path_join
@@ -32,61 +46,70 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface):
if isinstance(layer, intel.Intel):
return None
linux_banners = LinuxBannerCache.load_banners()
identifiers_path = os.path.join(
constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME
)
linux_banners = symbol_cache.SqliteCache(
identifiers_path
).get_identifier_dictionary(operating_system="linux")
# If we have no banners, don't bother scanning
if not linux_banners:
vollog.info("No Linux banners found - if this is a linux plugin, please check your symbol files location")
vollog.info(
"No Linux banners found - if this is a linux plugin, please check your symbol files location"
)
return None
mss = scanners.MultiStringScanner([x for x in linux_banners if x is not None])
for _, banner in layer.scan(context = context, scanner = mss, progress_callback = progress_callback):
for _, banner in layer.scan(
context=context, scanner=mss, progress_callback=progress_callback
):
dtb = None
vollog.debug(f"Identified banner: {repr(banner)}")
symbol_files = linux_banners.get(banner, None)
if symbol_files:
if len(symbol_files) > 1:
using = "*"
vollog.warning(f"Multiple symbol files identified (using {using}):")
for symbol_file in symbol_files:
vollog.warning(f" {using} {symbol_file}")
using = " "
isf_path = symbol_files[0]
table_name = context.symbol_space.free_table_name('LintelStacker')
table = linux.LinuxKernelIntermedSymbols(context,
'temporary.' + table_name,
name = table_name,
isf_url = isf_path)
isf_path = linux_banners.get(banner, None)
if isf_path:
table_name = context.symbol_space.free_table_name("LintelStacker")
table = linux.LinuxKernelIntermedSymbols(
context,
"temporary." + table_name,
name=table_name,
isf_url=isf_path,
)
context.symbol_space.append(table)
kaslr_shift, aslr_shift = cls.find_aslr(context,
table_name,
layer_name,
progress_callback = progress_callback)
kaslr_shift, aslr_shift = cls.find_aslr(
context, table_name, layer_name, progress_callback=progress_callback
)
layer_class: Type = intel.Intel
if 'init_top_pgt' in table.symbols:
if "init_top_pgt" in table.symbols:
layer_class = intel.Intel32e
dtb_symbol_name = 'init_top_pgt'
elif 'init_level4_pgt' in table.symbols:
dtb_symbol_name = "init_top_pgt"
elif "init_level4_pgt" in table.symbols:
layer_class = intel.Intel32e
dtb_symbol_name = 'init_level4_pgt'
dtb_symbol_name = "init_level4_pgt"
else:
dtb_symbol_name = 'swapper_pg_dir'
dtb_symbol_name = "swapper_pg_dir"
dtb = cls.virtual_to_physical_address(table.get_symbol(dtb_symbol_name).address + kaslr_shift)
dtb = cls.virtual_to_physical_address(
table.get_symbol(dtb_symbol_name).address + kaslr_shift
)
# Build the new layer
new_layer_name = context.layers.free_layer_name("IntelLayer")
config_path = join("IntelHelper", new_layer_name)
context.config[join(config_path, "memory_layer")] = layer_name
context.config[join(config_path, "page_map_offset")] = dtb
context.config[join(config_path, LinuxSymbolFinder.banner_config_key)] = str(banner, 'latin-1')
context.config[
join(config_path, LinuxSymbolFinder.banner_config_key)
] = str(banner, "latin-1")
layer = layer_class(context,
config_path = config_path,
name = new_layer_name,
metadata = {'os': 'Linux'})
layer.config['kernel_virtual_offset'] = aslr_shift
layer = layer_class(
context,
config_path=config_path,
name=new_layer_name,
metadata={"os": "Linux"},
)
layer.config["kernel_virtual_offset"] = aslr_shift
if layer and dtb:
vollog.debug(f"DTB was found at: 0x{dtb:0x}")
@@ -95,43 +118,63 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface):
return None
@classmethod
def find_aslr(cls,
context: interfaces.context.ContextInterface,
symbol_table: str,
layer_name: str,
progress_callback: constants.ProgressCallback = None) \
-> Tuple[int, int]:
def find_aslr(
cls,
context: interfaces.context.ContextInterface,
symbol_table: str,
layer_name: str,
progress_callback: constants.ProgressCallback = None,
) -> Tuple[int, int]:
"""Determines the offset of the actual DTB in physical space and its
symbol offset."""
init_task_symbol = symbol_table + constants.BANG + 'init_task'
init_task_json_address = context.symbol_space.get_symbol(init_task_symbol).address
init_task_symbol = symbol_table + constants.BANG + "init_task"
init_task_json_address = context.symbol_space.get_symbol(
init_task_symbol
).address
swapper_signature = rb"swapper(\/0|\x00\x00)\x00\x00\x00\x00\x00\x00"
module = context.module(symbol_table, layer_name, 0)
address_mask = context.symbol_space[symbol_table].config.get('symbol_mask', None)
address_mask = context.symbol_space[symbol_table].config.get(
"symbol_mask", None
)
task_symbol = module.get_type('task_struct')
comm_child_offset = task_symbol.relative_child_offset('comm')
task_symbol = module.get_type("task_struct")
comm_child_offset = task_symbol.relative_child_offset("comm")
for offset in context.layers[layer_name].scan(scanner = scanners.RegExScanner(swapper_signature),
context = context,
progress_callback = progress_callback):
for offset in context.layers[layer_name].scan(
scanner=scanners.RegExScanner(swapper_signature),
context=context,
progress_callback=progress_callback,
):
init_task_address = offset - comm_child_offset
init_task = module.object(object_type = 'task_struct', offset = init_task_address, absolute = True)
init_task = module.object(
object_type="task_struct", offset=init_task_address, absolute=True
)
if init_task.pid != 0:
continue
elif init_task.has_member('state') and init_task.state.cast('unsigned int') != 0:
elif (
init_task.has_member("state")
and init_task.state.cast("unsigned int") != 0
):
continue
# This we get for free
aslr_shift = init_task.files.cast('long unsigned int') - module.get_symbol('init_files').address
kaslr_shift = init_task_address - cls.virtual_to_physical_address(init_task_json_address)
aslr_shift = (
init_task.files.cast("long unsigned int")
- module.get_symbol("init_files").address
)
kaslr_shift = init_task_address - cls.virtual_to_physical_address(
init_task_json_address
)
if address_mask:
aslr_shift = aslr_shift & address_mask
if aslr_shift & 0xfff != 0 or kaslr_shift & 0xfff != 0:
if aslr_shift & 0xFFF != 0 or kaslr_shift & 0xFFF != 0:
continue
vollog.debug("Linux ASLR shift values determined: physical {:0x} virtual {:0x}".format(
kaslr_shift, aslr_shift))
vollog.debug(
"Linux ASLR shift values determined: physical {:0x} virtual {:0x}".format(
kaslr_shift, aslr_shift
)
)
return kaslr_shift, aslr_shift
# We don't throw an exception, because we may legitimately not have an ASLR shift, but we report it
@@ -142,25 +185,16 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface):
def virtual_to_physical_address(cls, addr: int) -> int:
"""Converts a virtual linux address to a physical one (does not account
of ASLR)"""
if addr > 0xffffffff80000000:
return addr - 0xffffffff80000000
return addr - 0xc0000000
class LinuxBannerCache(symbol_cache.SymbolBannerCache):
"""Caches the banners found in the Linux symbol files."""
os = "linux"
symbol_name = "linux_banner"
banner_path = constants.LINUX_BANNERS_PATH
exclusion_list = ['mac', 'windows']
if addr > 0xFFFFFFFF80000000:
return addr - 0xFFFFFFFF80000000
return addr - 0xC0000000
class LinuxSymbolFinder(symbol_finder.SymbolFinder):
"""Linux symbol loader based on uname signature strings."""
banner_config_key = "kernel_banner"
banner_cache = LinuxBannerCache
operating_system = "linux"
symbol_class = "volatility3.framework.symbols.linux.LinuxKernelIntermedSymbols"
find_aslr = lambda cls, *args: LinuxIntelStacker.find_aslr(*args)[1]
exclusion_list = ['mac', 'windows']
exclusion_list = ["mac", "windows"]
volatility3/framework/automagic/mac.py
+132 -77
View File
@@ -3,11 +3,13 @@
#
import logging
import os
import struct
from typing import Optional
from volatility3.framework import interfaces, constants, layers, exceptions
from volatility3.framework import constants, exceptions, interfaces, layers
from volatility3.framework.automagic import symbol_cache, symbol_finder
from volatility3.framework.configuration import requirements
from volatility3.framework.layers import intel, scanners
from volatility3.framework.symbols import mac
@@ -16,14 +18,26 @@ vollog = logging.getLogger(__name__)
class MacIntelStacker(interfaces.automagic.StackerLayerInterface):
stack_order = 35
exclusion_list = ['windows', 'linux']
exclusion_list = ["windows", "linux"]
@classmethod
def stack(cls,
context: interfaces.context.ContextInterface,
layer_name: str,
progress_callback: constants.ProgressCallback = None) -> Optional[interfaces.layers.DataLayerInterface]:
def stack(
cls,
context: interfaces.context.ContextInterface,
layer_name: str,
progress_callback: constants.ProgressCallback = None,
) -> Optional[interfaces.layers.DataLayerInterface]:
"""Attempts to identify mac within this layer."""
# Version check the SQlite cache
required = (1, 0, 0)
if not requirements.VersionRequirement.matches_required(
required, symbol_cache.SqliteCache.version
):
vollog.info(
f"SQLiteCache version not suitable: required {required} found {symbol_cache.SqliteCache.version}"
)
return None
# Bail out by default unless we can stack properly
layer = context.layers[layer_name]
new_layer = None
@@ -34,55 +48,76 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface):
if isinstance(layer, intel.Intel):
return None
mac_banners = MacBannerCache.load_banners()
identifiers_path = os.path.join(
constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME
)
mac_banners = symbol_cache.SqliteCache(
identifiers_path
).get_identifier_dictionary(operating_system="mac")
# If we have no banners, don't bother scanning
if not mac_banners:
vollog.info("No Mac banners found - if this is a mac plugin, please check your symbol files location")
vollog.info(
"No Mac banners found - if this is a mac plugin, please check your symbol files location"
)
return None
mss = scanners.MultiStringScanner([x for x in mac_banners if x])
for banner_offset, banner in layer.scan(context = context, scanner = mss,
progress_callback = progress_callback):
for banner_offset, banner in layer.scan(
context=context, scanner=mss, progress_callback=progress_callback
):
dtb = None
vollog.debug(f"Identified banner: {repr(banner)}")
symbol_files = mac_banners.get(banner, None)
if symbol_files:
isf_path = symbol_files[0]
table_name = context.symbol_space.free_table_name('MacintelStacker')
table = mac.MacKernelIntermedSymbols(context = context,
config_path = join('temporary', table_name),
name = table_name,
isf_url = isf_path)
isf_path = mac_banners.get(banner, None)
if isf_path:
table_name = context.symbol_space.free_table_name("MacintelStacker")
table = mac.MacKernelIntermedSymbols(
context=context,
config_path=join("temporary", table_name),
name=table_name,
isf_url=isf_path,
)
context.symbol_space.append(table)
kaslr_shift = cls.find_aslr(context = context,
symbol_table = table_name,
layer_name = layer_name,
compare_banner = banner,
compare_banner_offset = banner_offset,
progress_callback = progress_callback)
kaslr_shift = cls.find_aslr(
context=context,
symbol_table=table_name,
layer_name=layer_name,
compare_banner=banner,
compare_banner_offset=banner_offset,
progress_callback=progress_callback,
)
if kaslr_shift == 0:
vollog.log(constants.LOGLEVEL_VVV, f"Invalid kalsr_shift found at offset: {banner_offset}")
vollog.log(
constants.LOGLEVEL_VVV,
f"Invalid kalsr_shift found at offset: {banner_offset}",
)
continue
bootpml4_addr = cls.virtual_to_physical_address(table.get_symbol("BootPML4").address + kaslr_shift)
bootpml4_addr = cls.virtual_to_physical_address(
table.get_symbol("BootPML4").address + kaslr_shift
)
new_layer_name = context.layers.free_layer_name("MacDTBTempLayer")
config_path = join("automagic", "MacIntelHelper", new_layer_name)
context.config[join(config_path, "memory_layer")] = layer_name
context.config[join(config_path, "page_map_offset")] = bootpml4_addr
layer = layers.intel.Intel32e(context,
config_path = config_path,
name = new_layer_name,
metadata = {'os': 'Mac'})
layer = layers.intel.Intel32e(
context,
config_path=config_path,
name=new_layer_name,
metadata={"os": "Mac"},
)
idlepml4_ptr = table.get_symbol("IdlePML4").address + kaslr_shift
try:
idlepml4_str = layer.read(idlepml4_ptr, 4)
except exceptions.InvalidAddressException:
vollog.log(constants.LOGLEVEL_VVVV, f"Skipping invalid idlepml4_ptr: 0x{idlepml4_ptr:0x}")
vollog.log(
constants.LOGLEVEL_VVVV,
f"Skipping invalid idlepml4_ptr: 0x{idlepml4_ptr:0x}",
)
continue
idlepml4_addr = struct.unpack("<I", idlepml4_str)[0]
@@ -90,7 +125,10 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface):
tmp_dtb = idlepml4_addr
if tmp_dtb % 4096:
vollog.log(constants.LOGLEVEL_VVV, f"Skipping non-page aligned DTB: 0x{tmp_dtb:0x}")
vollog.log(
constants.LOGLEVEL_VVV,
f"Skipping non-page aligned DTB: 0x{tmp_dtb:0x}",
)
continue
dtb = tmp_dtb
@@ -100,13 +138,17 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface):
config_path = join("automagic", "MacIntelHelper", new_layer_name)
context.config[join(config_path, "memory_layer")] = layer_name
context.config[join(config_path, "page_map_offset")] = dtb
context.config[join(config_path, MacSymbolFinder.banner_config_key)] = str(banner, 'latin-1')
context.config[join(config_path, MacSymbolFinder.banner_config_key)] = (
str(banner, "latin-1")
)
new_layer = intel.Intel32e(context,
config_path = config_path,
name = new_layer_name,
metadata = {'os': 'mac'})
new_layer.config['kernel_virtual_offset'] = kaslr_shift
new_layer = intel.Intel32e(
context,
config_path=config_path,
name=new_layer_name,
metadata={"os": "mac"},
)
new_layer.config["kernel_virtual_offset"] = kaslr_shift
if new_layer and dtb:
vollog.debug(f"DTB was found at: 0x{dtb:0x}")
@@ -115,28 +157,40 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface):
return None
@classmethod
def find_aslr(cls,
context: interfaces.context.ContextInterface,
symbol_table: str,
layer_name: str,
compare_banner: str = "",
compare_banner_offset: int = 0,
progress_callback: constants.ProgressCallback = None) -> int:
def find_aslr(
cls,
context: interfaces.context.ContextInterface,
symbol_table: str,
layer_name: str,
compare_banner: str = "",
compare_banner_offset: int = 0,
progress_callback: constants.ProgressCallback = None,
) -> int:
"""Determines the offset of the actual DTB in physical space and its
symbol offset."""
version_symbol = symbol_table + constants.BANG + 'version'
version_symbol = symbol_table + constants.BANG + "version"
version_json_address = context.symbol_space.get_symbol(version_symbol).address
version_major_symbol = symbol_table + constants.BANG + 'version_major'
version_major_json_address = context.symbol_space.get_symbol(version_major_symbol).address
version_major_phys_offset = cls.virtual_to_physical_address(version_major_json_address)
version_major_symbol = symbol_table + constants.BANG + "version_major"
version_major_json_address = context.symbol_space.get_symbol(
version_major_symbol
).address
version_major_phys_offset = cls.virtual_to_physical_address(
version_major_json_address
)
version_minor_symbol = symbol_table + constants.BANG + 'version_minor'
version_minor_json_address = context.symbol_space.get_symbol(version_minor_symbol).address
version_minor_phys_offset = cls.virtual_to_physical_address(version_minor_json_address)
version_minor_symbol = symbol_table + constants.BANG + "version_minor"
version_minor_json_address = context.symbol_space.get_symbol(
version_minor_symbol
).address
version_minor_phys_offset = cls.virtual_to_physical_address(
version_minor_json_address
)
if not compare_banner_offset or not compare_banner:
offset_generator = cls._scan_generator(context, layer_name, progress_callback)
offset_generator = cls._scan_generator(
context, layer_name, progress_callback
)
else:
offset_generator = [(compare_banner_offset, compare_banner)]
@@ -145,24 +199,30 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface):
for offset, banner in offset_generator:
banner_major, banner_minor = [int(x) for x in banner[22:].split(b".")[0:2]]
tmp_aslr_shift = offset - cls.virtual_to_physical_address(version_json_address)
tmp_aslr_shift = offset - cls.virtual_to_physical_address(
version_json_address
)
major_string = context.layers[layer_name].read(version_major_phys_offset + tmp_aslr_shift, 4)
major_string = context.layers[layer_name].read(
version_major_phys_offset + tmp_aslr_shift, 4
)
major = struct.unpack("<I", major_string)[0]
if major != banner_major:
continue
minor_string = context.layers[layer_name].read(version_minor_phys_offset + tmp_aslr_shift, 4)
minor_string = context.layers[layer_name].read(
version_minor_phys_offset + tmp_aslr_shift, 4
)
minor = struct.unpack("<I", minor_string)[0]
if minor != banner_minor:
continue
if tmp_aslr_shift & 0xfff != 0:
if tmp_aslr_shift & 0xFFF != 0:
continue
aslr_shift = tmp_aslr_shift & 0xffffffff
aslr_shift = tmp_aslr_shift & 0xFFFFFFFF
break
vollog.log(constants.LOGLEVEL_VVVV, f"Mac find_aslr returned: {aslr_shift:0x}")
@@ -173,21 +233,24 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface):
def virtual_to_physical_address(cls, addr: int) -> int:
"""Converts a virtual mac address to a physical one (does not account
of ASLR)"""
if addr > 0xffffff8000000000:
addr = addr - 0xffffff8000000000
if addr > 0xFFFFFF8000000000:
addr = addr - 0xFFFFFF8000000000
else:
addr = addr - 0xff8000000000
addr = addr - 0xFF8000000000
return addr
@classmethod
def _scan_generator(cls, context, layer_name, progress_callback):
darwin_signature = rb"Darwin Kernel Version \d{1,3}\.\d{1,3}\.\d{1,3}: [^\x00]+\x00"
for offset in context.layers[layer_name].scan(scanner = scanners.RegExScanner(darwin_signature),
context = context,
progress_callback = progress_callback):
darwin_signature = (
rb"Darwin Kernel Version \d{1,3}\.\d{1,3}\.\d{1,3}: [^\x00]+\x00"
)
for offset in context.layers[layer_name].scan(
scanner=scanners.RegExScanner(darwin_signature),
context=context,
progress_callback=progress_callback,
):
banner = context.layers[layer_name].read(offset, 128)
idx = banner.find(b"\x00")
@@ -197,19 +260,11 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface):
yield offset, banner
class MacBannerCache(symbol_cache.SymbolBannerCache):
"""Caches the banners found in the Mac symbol files."""
os = "mac"
symbol_name = "version"
banner_path = constants.MAC_BANNERS_PATH
exclusion_list = ['windows', 'linux']
class MacSymbolFinder(symbol_finder.SymbolFinder):
"""Mac symbol loader based on uname signature strings."""
banner_config_key = 'kernel_banner'
banner_cache = MacBannerCache
banner_config_key = "kernel_banner"
operating_system = "mac"
find_aslr = MacIntelStacker.find_aslr
symbol_class = "volatility3.framework.symbols.mac.MacKernelIntermedSymbols"
exclusion_list = ['windows', 'linux']
exclusion_list = ["windows", "linux"]
volatility3/framework/automagic/module.py
+40 -17
View File
@@ -1,3 +1,7 @@
# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0
# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0
#
from volatility3.framework import interfaces, constants, configuration
@@ -6,36 +10,55 @@ class KernelModule(interfaces.automagic.AutomagicInterface):
priority = 100
def __call__(self,
context: interfaces.context.ContextInterface,
config_path: str,
requirement: interfaces.configuration.RequirementInterface,
progress_callback: constants.ProgressCallback = None) -> None:
new_config_path = interfaces.configuration.path_join(config_path, requirement.name)
def __call__(
self,
context: interfaces.context.ContextInterface,
config_path: str,
requirement: interfaces.configuration.RequirementInterface,
progress_callback: constants.ProgressCallback = None,
) -> None:
new_config_path = interfaces.configuration.path_join(
config_path, requirement.name
)
if not isinstance(requirement, configuration.requirements.ModuleRequirement):
# Check subrequirements
for req in requirement.requirements:
self(context, new_config_path, requirement.requirements[req], progress_callback)
return
self(
context,
new_config_path,
requirement.requirements[req],
progress_callback,
)
return None
if not requirement.unsatisfied(context, config_path):
return
return None
# The requirement is unfulfilled and is a ModuleRequirement
context.config[interfaces.configuration.path_join(
new_config_path, 'class')] = 'volatility3.framework.contexts.Module'
context.config[interfaces.configuration.path_join(new_config_path, "class")] = (
"volatility3.framework.contexts.Module"
)
for req in requirement.requirements:
if requirement.requirements[req].unsatisfied(context, new_config_path) and req != 'offset':
return
if (
requirement.requirements[req].unsatisfied(context, new_config_path)
and req != "offset"
):
return None
# We now just have the offset requirement, but the layer requirement has been fulfilled.
# Unfortunately we don't know the layer name requirement's exact name
for req in requirement.requirements:
if isinstance(requirement.requirements[req], configuration.requirements.TranslationLayerRequirement):
layer_kvo_config_path = interfaces.configuration.path_join(new_config_path, req,
'kernel_virtual_offset')
offset_config_path = interfaces.configuration.path_join(new_config_path, 'offset')
if isinstance(
requirement.requirements[req],
configuration.requirements.TranslationLayerRequirement,
):
layer_kvo_config_path = interfaces.configuration.path_join(
new_config_path, req, "kernel_virtual_offset"
)
offset_config_path = interfaces.configuration.path_join(
new_config_path, "offset"
)
offset = context.config[layer_kvo_config_path]
context.config[offset_config_path] = offset
volatility3/framework/automagic/pdbscan.py
+226 -124
View File
@@ -7,10 +7,11 @@ from loaded PE files.
This module contains a standalone scanner, and also a :class:`~volatility3.framework.interfaces.layers.ScannerInterface`
based scanner for use within the framework by calling :func:`~volatility3.framework.interfaces.layers.DataLayerInterface.scan`.
"""
import contextlib
import logging
import math
import os
from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union, Callable
from typing import Any, Callable, Dict, Iterable, List, Optional, Set, Tuple, Union
from volatility3.framework import constants, exceptions, interfaces, layers
from volatility3.framework.configuration import requirements
@@ -21,7 +22,9 @@ from volatility3.framework.symbols.windows.pdbutil import PDBUtility
if __name__ == "__main__":
import sys
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))))
sys.path.append(
os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
)
vollog = logging.getLogger(__name__)
@@ -42,12 +45,17 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface):
searches for a particular structure that lists the kernel module's virtual address, its size (not checked) and the
module's name. This value is then used if one was not found using the previous method.
"""
priority = 30
max_pdb_size = 0x400000
exclusion_list = ['linux', 'mac']
exclusion_list = ["linux", "mac"]
def find_virtual_layers_from_req(self, context: interfaces.context.ContextInterface, config_path: str,
requirement: interfaces.configuration.RequirementInterface) -> List[str]:
def find_virtual_layers_from_req(
self,
context: interfaces.context.ContextInterface,
config_path: str,
requirement: interfaces.configuration.RequirementInterface,
) -> List[str]:
"""Traverses the requirement tree, rooted at `requirement` looking for
virtual layers that might contain a windows PDB.
@@ -61,27 +69,36 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface):
Returns:
A list of (layer_name, scan_results)
"""
sub_config_path = interfaces.configuration.path_join(config_path, requirement.name)
sub_config_path = interfaces.configuration.path_join(
config_path, requirement.name
)
results: List[str] = []
if isinstance(requirement, requirements.TranslationLayerRequirement):
# Check for symbols in this layer
# FIXME: optionally allow a full (slow) scan
# FIXME: Determine the physical layer no matter the virtual layer
virtual_layer_name = context.config.get(sub_config_path, None)
layer_name = context.config.get(interfaces.configuration.path_join(sub_config_path, "memory_layer"), None)
layer_name = context.config.get(
interfaces.configuration.path_join(sub_config_path, "memory_layer"),
None,
)
if layer_name and virtual_layer_name:
memlayer = context.layers[virtual_layer_name]
if isinstance(memlayer, intel.Intel):
results = [virtual_layer_name]
else:
for subreq in requirement.requirements.values():
results += self.find_virtual_layers_from_req(context, sub_config_path, subreq)
results += self.find_virtual_layers_from_req(
context, sub_config_path, subreq
)
return results
def recurse_symbol_fulfiller(self,
context: interfaces.context.ContextInterface,
valid_kernel: ValidKernelType,
progress_callback: constants.ProgressCallback = None) -> None:
def recurse_symbol_fulfiller(
self,
context: interfaces.context.ContextInterface,
valid_kernel: ValidKernelType,
progress_callback: constants.ProgressCallback = None,
) -> None:
"""Fulfills the SymbolTableRequirements in `self._symbol_requirements`
found by the `recurse_symbol_requirements`.
@@ -98,22 +115,28 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface):
if valid_kernel:
# TODO: Check that the symbols for this kernel will fulfill the requirement
virtual_layer, _kvo, kernel = valid_kernel
if not isinstance(kernel['pdb_name'], str) or not isinstance(kernel['GUID'], str):
if not isinstance(kernel["pdb_name"], str) or not isinstance(
kernel["GUID"], str
):
raise TypeError("PDB name or GUID not a string value")
PDBUtility.load_windows_symbol_table(
context = context,
guid = kernel['GUID'],
age = kernel['age'],
pdb_name = kernel['pdb_name'],
symbol_table_class = "volatility3.framework.symbols.windows.WindowsKernelIntermedSymbols",
config_path = sub_config_path,
progress_callback = progress_callback)
context=context,
guid=kernel["GUID"],
age=kernel["age"],
pdb_name=kernel["pdb_name"],
symbol_table_class="volatility3.framework.symbols.windows.WindowsKernelIntermedSymbols",
config_path=sub_config_path,
progress_callback=progress_callback,
)
else:
vollog.debug("No suitable kernel pdb signature found")
def set_kernel_virtual_offset(self, context: interfaces.context.ContextInterface,
valid_kernel: ValidKernelType) -> None:
def set_kernel_virtual_offset(
self,
context: interfaces.context.ContextInterface,
valid_kernel: ValidKernelType,
) -> None:
"""Traverses the requirement tree, looking for kernel_virtual_offset
values that may need setting and sets it based on the previously
identified `valid_kernel`.
@@ -126,69 +149,98 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface):
# Set the virtual offset under the TranslationLayer it applies to
virtual_layer, kvo, kernel = valid_kernel
if kvo is not None:
kvo_path = interfaces.configuration.path_join(context.layers[virtual_layer].config_path,
'kernel_virtual_offset')
kvo_path = interfaces.configuration.path_join(
context.layers[virtual_layer].config_path, "kernel_virtual_offset"
)
context.config[kvo_path] = kvo
vollog.debug(f"Setting kernel_virtual_offset to {hex(kvo)}")
def get_physical_layer_name(self, context, vlayer):
return context.config.get(interfaces.configuration.path_join(vlayer.config_path, 'memory_layer'), None)
return context.config.get(
interfaces.configuration.path_join(vlayer.config_path, "memory_layer"), None
)
def method_slow_scan(self,
context: interfaces.context.ContextInterface,
vlayer: layers.intel.Intel,
progress_callback: constants.ProgressCallback = None) -> Optional[ValidKernelType]:
def test_virtual_kernel(physical_layer_name, virtual_layer_name: str, kernel: Dict[str, Any]) -> Optional[ValidKernelType]:
def method_slow_scan(
self,
context: interfaces.context.ContextInterface,
vlayer: layers.intel.Intel,
progress_callback: constants.ProgressCallback = None,
) -> Optional[ValidKernelType]:
def test_virtual_kernel(
physical_layer_name, virtual_layer_name: str, kernel: Dict[str, Any]
) -> Optional[ValidKernelType]:
# It seems the kernel is loaded at a fixed mapping (presumably because the memory manager hasn't started yet)
if kernel['mz_offset'] is None or not isinstance(kernel['mz_offset'], int):
if kernel["mz_offset"] is None or not isinstance(kernel["mz_offset"], int):
# Rule out kernels that couldn't find a suitable MZ header
return None
return (virtual_layer_name, kernel['mz_offset'], kernel)
return (virtual_layer_name, kernel["mz_offset"], kernel)
vollog.debug("Kernel base determination - optimized scan virtual layer")
valid_kernel = self._method_layer_pdb_scan(context, vlayer, test_virtual_kernel, True, False, progress_callback)
if valid_kernel != None:
valid_kernel = self._method_layer_pdb_scan(
context, vlayer, test_virtual_kernel, True, False, progress_callback
)
if valid_kernel is not None:
return valid_kernel
vollog.debug("Kernel base determination - slow scan virtual layer")
return self._method_layer_pdb_scan(context, vlayer, test_virtual_kernel, False, False, progress_callback)
return self._method_layer_pdb_scan(
context, vlayer, test_virtual_kernel, False, False, progress_callback
)
def method_fixed_mapping(self,
context: interfaces.context.ContextInterface,
vlayer: layers.intel.Intel,
progress_callback: constants.ProgressCallback = None) -> Optional[ValidKernelType]:
def test_physical_kernel(physical_layer_name:str , virtual_layer_name: str, kernel: Dict[str, Any]) -> Optional[ValidKernelType]:
def method_fixed_mapping(
self,
context: interfaces.context.ContextInterface,
vlayer: layers.intel.Intel,
progress_callback: constants.ProgressCallback = None,
) -> Optional[ValidKernelType]:
def test_physical_kernel(
physical_layer_name: str, virtual_layer_name: str, kernel: Dict[str, Any]
) -> Optional[ValidKernelType]:
# It seems the kernel is loaded at a fixed mapping (presumably because the memory manager hasn't started yet)
if kernel['mz_offset'] is None or not isinstance(kernel['mz_offset'], int):
if kernel["mz_offset"] is None or not isinstance(kernel["mz_offset"], int):
# Rule out kernels that couldn't find a suitable MZ header
return None
if vlayer.bits_per_register == 64:
kvo = kernel['mz_offset'] + (31 << int(math.ceil(math.log2(vlayer.maximum_address + 1)) - 5))
kvo = kernel["mz_offset"] + (
31 << int(math.ceil(math.log2(vlayer.maximum_address + 1)) - 5)
)
else:
kvo = kernel['mz_offset'] + (1 << (vlayer.bits_per_register - 1))
kvo = kernel["mz_offset"] + (1 << (vlayer.bits_per_register - 1))
try:
kvp = vlayer.mapping(kvo, 0)
if (any([(p == kernel['mz_offset'] and layer_name == physical_layer_name)
for (_, _, p, _, layer_name) in kvp])):
if any(
[
(p == kernel["mz_offset"] and layer_name == physical_layer_name)
for (_, _, p, _, layer_name) in kvp
]
):
return (virtual_layer_name, kvo, kernel)
else:
vollog.debug("Potential kernel_virtual_offset did not map to expected location: {}".format(
hex(kvo)))
vollog.debug(
"Potential kernel_virtual_offset did not map to expected location: {}".format(
hex(kvo)
)
)
except exceptions.InvalidAddressException:
vollog.debug(f"Potential kernel_virtual_offset caused a page fault: {hex(kvo)}")
vollog.debug(
f"Potential kernel_virtual_offset caused a page fault: {hex(kvo)}"
)
return None
vollog.debug("Kernel base determination - testing fixed base address")
return self._method_layer_pdb_scan(context, vlayer, test_physical_kernel, False, True, progress_callback)
return self._method_layer_pdb_scan(
context, vlayer, test_physical_kernel, False, True, progress_callback
)
def _method_layer_pdb_scan(self,
context: interfaces.context.ContextInterface,
vlayer: layers.intel.Intel,
test_kernel: Callable,
optimized: bool = False,
physical: bool = True,
progress_callback: constants.ProgressCallback = None) -> Optional[ValidKernelType]:
def _method_layer_pdb_scan(
self,
context: interfaces.context.ContextInterface,
vlayer: layers.intel.Intel,
test_kernel: Callable,
optimized: bool = False,
physical: bool = True,
progress_callback: constants.ProgressCallback = None,
) -> Optional[ValidKernelType]:
# TODO: Verify this is a windows image
valid_kernel = None
virtual_layer_name = vlayer.name
@@ -199,104 +251,145 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface):
layer_to_scan = virtual_layer_name
start_scan_address = 0
if optimized and not physical and context.layers[layer_to_scan].metadata.architecture in ["Intel64"]:
if (
optimized
and not physical
and context.layers[layer_to_scan].metadata.architecture in ["Intel64"]
):
# TODO: change this value accordingly when 5-Level paging is supported.
start_scan_address = (0x1f0 << 39)
start_scan_address = 0x1F0 << 39
kernel_pdb_names = [bytes(name + ".pdb", "utf-8") for name in constants.windows.KERNEL_MODULE_NAMES]
kernels = PDBUtility.pdbname_scan(ctx = context,
layer_name = layer_to_scan,
start = start_scan_address,
page_size = vlayer.page_size,
pdb_names = kernel_pdb_names,
progress_callback = progress_callback)
kernel_pdb_names = [
bytes(name + ".pdb", "utf-8")
for name in constants.windows.KERNEL_MODULE_NAMES
]
kernels = PDBUtility.pdbname_scan(
ctx=context,
layer_name=layer_to_scan,
start=start_scan_address,
page_size=vlayer.page_size,
pdb_names=kernel_pdb_names,
progress_callback=progress_callback,
)
for kernel in kernels:
valid_kernel = test_kernel(physical_layer_name, virtual_layer_name, kernel)
if valid_kernel is not None:
break
return valid_kernel
def _method_offset(self,
context: interfaces.context.ContextInterface,
vlayer: layers.intel.Intel,
pattern: bytes,
result_offset: int,
progress_callback: constants.ProgressCallback = None) -> Optional[ValidKernelType]:
def _method_offset(
self,
context: interfaces.context.ContextInterface,
vlayer: layers.intel.Intel,
pattern: bytes,
result_offset: int,
progress_callback: constants.ProgressCallback = None,
) -> Optional[ValidKernelType]:
"""Method for finding a suitable kernel offset based on a module
table."""
vollog.debug("Kernel base determination - searching layer module list structure")
vollog.debug(
"Kernel base determination - searching layer module list structure"
)
valid_kernel: Optional[ValidKernelType] = None
# If we're here, chances are high we're in a Win10 x64 image with kernel base randomization
physical_layer_name = self.get_physical_layer_name(context, vlayer)
physical_layer = context.layers[physical_layer_name]
# TODO: On older windows, this might be \WINDOWS\system32\nt rather than \SystemRoot\system32\nt
results = physical_layer.scan(context, scanners.BytesScanner(pattern), progress_callback = progress_callback)
results = physical_layer.scan(
context, scanners.BytesScanner(pattern), progress_callback=progress_callback
)
seen: Set[int] = set()
# Because this will launch a scan of the virtual layer, we want to be careful
for result in results:
# TODO: Identify the specific structure we're finding and document this a bit better
pointer = context.object("pdbscan!unsigned long long",
offset = (result + result_offset),
layer_name = physical_layer_name)
pointer = context.object(
"pdbscan!unsigned long long",
offset=(result + result_offset),
layer_name=physical_layer_name,
)
address = pointer & vlayer.address_mask
if address in seen:
continue
seen.add(address)
valid_kernel = self.check_kernel_offset(context, vlayer, address, progress_callback)
valid_kernel = self.check_kernel_offset(
context, vlayer, address, progress_callback
)
if valid_kernel:
break
return valid_kernel
def method_module_offset(self,
context: interfaces.context.ContextInterface,
vlayer: layers.intel.Intel,
progress_callback: constants.ProgressCallback = None) -> Optional[ValidKernelType]:
return self._method_offset(context, vlayer, b"\\SystemRoot\\system32\\nt",
-16 - int(vlayer.bits_per_register / 8), progress_callback)
def method_module_offset(
self,
context: interfaces.context.ContextInterface,
vlayer: layers.intel.Intel,
progress_callback: constants.ProgressCallback = None,
) -> Optional[ValidKernelType]:
return self._method_offset(
context,
vlayer,
b"\\SystemRoot\\system32\\nt",
-16 - int(vlayer.bits_per_register / 8),
progress_callback,
)
def method_kdbg_offset(self,
context: interfaces.context.ContextInterface,
vlayer: layers.intel.Intel,
progress_callback: constants.ProgressCallback = None) -> Optional[ValidKernelType]:
def method_kdbg_offset(
self,
context: interfaces.context.ContextInterface,
vlayer: layers.intel.Intel,
progress_callback: constants.ProgressCallback = None,
) -> Optional[ValidKernelType]:
return self._method_offset(context, vlayer, b"KDBG", 8, progress_callback)
def check_kernel_offset(self,
context: interfaces.context.ContextInterface,
vlayer: layers.intel.Intel,
address: int,
progress_callback: constants.ProgressCallback = None) -> Optional[ValidKernelType]:
def check_kernel_offset(
self,
context: interfaces.context.ContextInterface,
vlayer: layers.intel.Intel,
address: int,
progress_callback: constants.ProgressCallback = None,
) -> Optional[ValidKernelType]:
"""Scans a virtual address."""
# Scan a few megs of the virtual space at the location to see if they're potential kernels
valid_kernel: Optional[ValidKernelType] = None
kernel_pdb_names = [bytes(name + ".pdb", "utf-8") for name in constants.windows.KERNEL_MODULE_NAMES]
kernel_pdb_names = [
bytes(name + ".pdb", "utf-8")
for name in constants.windows.KERNEL_MODULE_NAMES
]
virtual_layer_name = vlayer.name
try:
if vlayer.read(address, 0x2) == b'MZ':
with contextlib.suppress(exceptions.InvalidAddressException):
if vlayer.read(address, 0x2) == b"MZ":
res = list(
PDBUtility.pdbname_scan(ctx = context,
layer_name = vlayer.name,
page_size = vlayer.page_size,
pdb_names = kernel_pdb_names,
progress_callback = progress_callback,
start = address,
end = address + self.max_pdb_size))
PDBUtility.pdbname_scan(
ctx=context,
layer_name=vlayer.name,
page_size=vlayer.page_size,
pdb_names=kernel_pdb_names,
progress_callback=progress_callback,
start=address,
end=address + self.max_pdb_size,
)
)
if res:
valid_kernel = (virtual_layer_name, address, res[0])
except exceptions.InvalidAddressException:
pass
return valid_kernel
# List of methods to be run, in order, to determine the valid kernels
methods = [method_kdbg_offset, method_module_offset, method_fixed_mapping, method_slow_scan]
methods = [
method_kdbg_offset,
method_module_offset,
method_fixed_mapping,
method_slow_scan,
]
def determine_valid_kernel(self,
context: interfaces.context.ContextInterface,
potential_layers: List[str],
progress_callback: constants.ProgressCallback = None) -> Optional[ValidKernelType]:
def determine_valid_kernel(
self,
context: interfaces.context.ContextInterface,
potential_layers: List[str],
progress_callback: constants.ProgressCallback = None,
) -> Optional[ValidKernelType]:
"""Runs through the identified potential kernels and verifies their
suitability.
@@ -325,27 +418,36 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface):
vollog.info("No suitable kernels found during pdbscan")
return valid_kernel
def __call__(self,
context: interfaces.context.ContextInterface,
config_path: str,
requirement: interfaces.configuration.RequirementInterface,
progress_callback: constants.ProgressCallback = None) -> None:
def __call__(
self,
context: interfaces.context.ContextInterface,
config_path: str,
requirement: interfaces.configuration.RequirementInterface,
progress_callback: constants.ProgressCallback = None,
) -> None:
if requirement.unsatisfied(context, config_path):
if "pdbscan" not in context.symbol_space:
context.symbol_space.append(native.NativeTable("pdbscan", native.std_ctypes))
context.symbol_space.append(
native.NativeTable("pdbscan", native.std_ctypes)
)
# TODO: check if this is a windows symbol requirement, otherwise ignore it
self._symbol_requirements = self.find_requirements(context, config_path, requirement,
requirements.SymbolTableRequirement)
potential_layers = self.find_virtual_layers_from_req(context = context,
config_path = config_path,
requirement = requirement)
self._symbol_requirements = self.find_requirements(
context, config_path, requirement, requirements.SymbolTableRequirement
)
potential_layers = self.find_virtual_layers_from_req(
context=context, config_path=config_path, requirement=requirement
)
for sub_config_path, symbol_req in self._symbol_requirements:
parent_path = interfaces.configuration.parent_path(sub_config_path)
if symbol_req.unsatisfied(context, parent_path):
valid_kernel = self.determine_valid_kernel(context, potential_layers, progress_callback)
valid_kernel = self.determine_valid_kernel(
context, potential_layers, progress_callback
)
if valid_kernel:
self.set_kernel_virtual_offset(context, valid_kernel)
self.recurse_symbol_fulfiller(context, valid_kernel, progress_callback)
self.recurse_symbol_fulfiller(
context, valid_kernel, progress_callback
)
if progress_callback is not None:
progress_callback(100, "PDB scanning finished")
volatility3/framework/automagic/stacker.py
+123 -53
View File
@@ -35,6 +35,7 @@ class LayerStacker(interfaces.automagic.AutomagicInterface):
Upon completion it will re-call the :class:`~volatility3.framework.automagic.construct_layers.ConstructionMagic`,
so that any stacked layers are actually constructed and added to the context.
"""
# Most important automagic, must happen first!
priority = 10
@@ -42,14 +43,16 @@ class LayerStacker(interfaces.automagic.AutomagicInterface):
super().__init__(*args, **kwargs)
self._cached = None
def __call__(self,
context: interfaces.context.ContextInterface,
config_path: str,
requirement: interfaces.configuration.RequirementInterface,
progress_callback: constants.ProgressCallback = None) -> Optional[List[str]]:
def __call__(
self,
context: interfaces.context.ContextInterface,
config_path: str,
requirement: interfaces.configuration.RequirementInterface,
progress_callback: constants.ProgressCallback = None,
) -> Optional[List[str]]:
"""Runs the automagic over the configurable."""
framework.import_files(sys.modules['volatility3.framework.layers'])
framework.import_files(sys.modules["volatility3.framework.layers"])
# Quick exit if we're not needed
if not requirement.unsatisfied(context, config_path):
@@ -58,10 +61,14 @@ class LayerStacker(interfaces.automagic.AutomagicInterface):
# Bow out quickly if the UI hasn't provided a single_location
unsatisfied = self.unsatisfied(self.context, self.config_path)
if unsatisfied:
vollog.info(f"Unable to run LayerStacker, unsatisfied requirement: {unsatisfied}")
vollog.info(
f"Unable to run LayerStacker, unsatisfied requirement: {unsatisfied}"
)
return list(unsatisfied)
if not self.config or not self.config.get('single_location', None):
raise ValueError("Unable to run LayerStacker, single_location parameter not provided")
if not self.config or not self.config.get("single_location", None):
raise ValueError(
"Unable to run LayerStacker, single_location parameter not provided"
)
# Search for suitable requirements
self.stack(context, config_path, requirement, progress_callback)
@@ -70,9 +77,13 @@ class LayerStacker(interfaces.automagic.AutomagicInterface):
progress_callback(100, "Stacking attempts finished")
return None
def stack(self, context: interfaces.context.ContextInterface, config_path: str,
requirement: interfaces.configuration.RequirementInterface,
progress_callback: constants.ProgressCallback) -> None:
def stack(
self,
context: interfaces.context.ContextInterface,
config_path: str,
requirement: interfaces.configuration.RequirementInterface,
progress_callback: constants.ProgressCallback,
) -> None:
"""Stacks the various layers and attaches these to a specific
requirement.
@@ -85,52 +96,79 @@ class LayerStacker(interfaces.automagic.AutomagicInterface):
# If we're cached, find Now we need to find where to apply the stack configuration
if self._cached:
top_layer_name, subconfig = self._cached
result = self.find_suitable_requirements(context, config_path, requirement, [top_layer_name])
result = self.find_suitable_requirements(
context, config_path, requirement, [top_layer_name]
)
if result:
appropriate_config_path, layer_name = result
context.config.merge(appropriate_config_path, subconfig)
context.config[appropriate_config_path] = top_layer_name
return
return None
self._cached = None
new_context = context.clone()
location = self.config.get('single_location', None)
location = self.config.get("single_location", None)
# Setup the local copy of the resource
current_layer_name = context.layers.free_layer_name("FileLayer")
current_config_path = interfaces.configuration.path_join(config_path, "stack", current_layer_name)
current_config_path = interfaces.configuration.path_join(
config_path, "stack", current_layer_name
)
# This must be specific to get us started, setup the config and run
new_context.config[interfaces.configuration.path_join(current_config_path, "location")] = location
physical_layer = physical.FileLayer(new_context, current_config_path, current_layer_name)
new_context.config[
interfaces.configuration.path_join(current_config_path, "location")
] = location
physical_layer = physical.FileLayer(
new_context, current_config_path, current_layer_name
)
new_context.add_layer(physical_layer)
stacked_layers = self.stack_layer(new_context, current_layer_name, self.create_stackers_list(),
progress_callback)
stacked_layers = self.stack_layer(
new_context,
current_layer_name,
self.create_stackers_list(),
progress_callback,
)
if stacked_layers is not None:
# Applies the stacked_layers to each requirement in the requirements list
result = self.find_suitable_requirements(new_context, config_path, requirement, stacked_layers)
result = self.find_suitable_requirements(
new_context, config_path, requirement, stacked_layers
)
if result:
path, layer = result
# splice in the new configuration into the original context
context.config.merge(path, new_context.layers[layer].build_configuration())
context.config.merge(
path, new_context.layers[layer].build_configuration()
)
# Call the construction magic now we may have new things to construct
constructor = construct_layers.ConstructionMagic(
context, interfaces.configuration.path_join(self.config_path, "ConstructionMagic"))
context,
interfaces.configuration.path_join(
self.config_path, "ConstructionMagic"
),
)
constructor(context, config_path, requirement)
# Stash the changed config items
self._cached = context.config.get(path, None), context.config.branch(path)
self._cached = context.config.get(path, None), context.config.branch(
path
)
vollog.debug(
f"physical_layer maximum_address: {physical_layer.maximum_address}"
)
vollog.debug(f"Stacked layers: {stacked_layers}")
@classmethod
def stack_layer(cls,
context: interfaces.context.ContextInterface,
initial_layer: str,
stack_set: List[Type[interfaces.automagic.StackerLayerInterface]] = None,
progress_callback: constants.ProgressCallback = None):
def stack_layer(
cls,
context: interfaces.context.ContextInterface,
initial_layer: str,
stack_set: List[Type[interfaces.automagic.StackerLayerInterface]] = None,
progress_callback: constants.ProgressCallback = None,
):
"""Stacks as many possible layers on top of the initial layer as can be done.
WARNING: This modifies the context provided and may pollute it with unnecessary layers
@@ -154,11 +192,15 @@ class LayerStacker(interfaces.automagic.AutomagicInterface):
stacked = True
stacked_layers = [initial_layer]
if stack_set is None:
stack_set = list(framework.class_subclasses(interfaces.automagic.StackerLayerInterface))
stack_set = list(
framework.class_subclasses(interfaces.automagic.StackerLayerInterface)
)
for stacker_item in stack_set:
if not issubclass(stacker_item, interfaces.automagic.StackerLayerInterface):
raise TypeError(f"Stacker {stacker_item.__name__} is not a descendent of StackerLayerInterface")
raise TypeError(
f"Stacker {stacker_item.__name__} is not a descendent of StackerLayerInterface"
)
while stacked:
stacked = False
@@ -167,17 +209,27 @@ class LayerStacker(interfaces.automagic.AutomagicInterface):
for stacker_cls in stack_set:
stacker = stacker_cls()
try:
vollog.log(constants.LOGLEVEL_VV, f"Attempting to stack using {stacker_cls.__name__}")
vollog.log(
constants.LOGLEVEL_VV,
f"Attempting to stack using {stacker_cls.__name__}",
)
new_layer = stacker.stack(context, initial_layer, progress_callback)
if new_layer:
context.layers.add_layer(new_layer)
vollog.log(constants.LOGLEVEL_VV,
f"Stacked {new_layer.name} using {stacker_cls.__name__}")
vollog.log(
constants.LOGLEVEL_VV,
f"Stacked {new_layer.name} using {stacker_cls.__name__}",
)
break
except Exception as excp:
# Stacking exceptions are likely only of interest to developers, so the lowest level of logging
fulltrace = traceback.TracebackException.from_exception(excp).format(chain = True)
vollog.log(constants.LOGLEVEL_VVV, f"Exception during stacking: {str(excp)}")
fulltrace = traceback.TracebackException.from_exception(
excp
).format(chain=True)
vollog.log(
constants.LOGLEVEL_VVV,
f"Exception during stacking: {str(excp)}",
)
vollog.log(constants.LOGLEVEL_VVVV, "\n".join(fulltrace))
else:
stacked = False
@@ -188,11 +240,15 @@ class LayerStacker(interfaces.automagic.AutomagicInterface):
stack_set.remove(stacker_cls)
return stacked_layers
def create_stackers_list(self) -> List[Type[interfaces.automagic.StackerLayerInterface]]:
def create_stackers_list(
self,
) -> List[Type[interfaces.automagic.StackerLayerInterface]]:
"""Creates the list of stackers to use based on the config option"""
stack_set = sorted(framework.class_subclasses(interfaces.automagic.StackerLayerInterface),
key = lambda x: x.stack_order)
stacker_list = self.config.get('stackers', [])
stack_set = sorted(
framework.class_subclasses(interfaces.automagic.StackerLayerInterface),
key=lambda x: x.stack_order,
)
stacker_list = self.config.get("stackers", [])
if len(stacker_list):
result = []
for stacker in stack_set:
@@ -202,9 +258,13 @@ class LayerStacker(interfaces.automagic.AutomagicInterface):
return stack_set
@classmethod
def find_suitable_requirements(cls, context: interfaces.context.ContextInterface, config_path: str,
requirement: interfaces.configuration.RequirementInterface,
stacked_layers: List[str]) -> Optional[Tuple[str, str]]:
def find_suitable_requirements(
cls,
context: interfaces.context.ContextInterface,
config_path: str,
requirement: interfaces.configuration.RequirementInterface,
stacked_layers: List[str],
) -> Optional[Tuple[str, str]]:
"""Looks for translation layer requirements and attempts to apply the
stacked layers to it. If it succeeds it returns the configuration path
and layer name where the stacked nodes were spliced into the tree.
@@ -213,7 +273,9 @@ class LayerStacker(interfaces.automagic.AutomagicInterface):
A tuple of a configuration path and layer name for the top of the stacked layers
or None if suitable requirements are not found
"""
child_config_path = interfaces.configuration.path_join(config_path, requirement.name)
child_config_path = interfaces.configuration.path_join(
config_path, requirement.name
)
if isinstance(requirement, requirements.TranslationLayerRequirement):
if requirement.unsatisfied(context, config_path):
original_setting = context.config.get(child_config_path, None)
@@ -229,7 +291,9 @@ class LayerStacker(interfaces.automagic.AutomagicInterface):
else:
return child_config_path, context.config.get(child_config_path, None)
for req_name, req in requirement.requirements.items():
result = cls.find_suitable_requirements(context, child_config_path, req, stacked_layers)
result = cls.find_suitable_requirements(
context, child_config_path, req, stacked_layers
)
if result:
return result
return None
@@ -238,23 +302,29 @@ class LayerStacker(interfaces.automagic.AutomagicInterface):
def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]:
# This is not optional for the stacker to run, so optional must be marked as False
return [
requirements.URIRequirement(name = "single_location",
description = "Specifies a base location on which to stack",
optional = True),
requirements.ListRequirement(name = "stackers", description = "List of stackers", optional = True)
requirements.URIRequirement(
name="single_location",
description="Specifies a base location on which to stack",
optional=True,
),
requirements.ListRequirement(
name="stackers", description="List of stackers", optional=True
),
]
def choose_os_stackers(plugin: Type[interfaces.plugins.PluginInterface]) -> List[str]:
"""Identifies the stackers that should be run, based on the plugin (and thus os) provided"""
plugin_first_level = plugin.__module__.split('.')[2]
plugin_first_level = plugin.__module__.split(".")[2]
# Ensure all stackers are loaded
framework.import_files(sys.modules['volatility3.framework.layers'])
framework.import_files(sys.modules["volatility3.framework.layers"])
result = []
for stacker in sorted(framework.class_subclasses(interfaces.automagic.StackerLayerInterface),
key = lambda x: x.stack_order):
for stacker in sorted(
framework.class_subclasses(interfaces.automagic.StackerLayerInterface),
key=lambda x: x.stack_order,
):
if plugin_first_level in stacker.exclusion_list:
continue
result.append(stacker.__name__)
volatility3/framework/automagic/symbol_cache.py
+519 -167
View File
@@ -2,18 +2,20 @@
# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0
#
import base64
import gc
import datetime
import json
import logging
import os
import pickle
import sqlite3
import urllib
import urllib.parse
import urllib.request
import zipfile
from typing import Dict, List, Optional
from abc import abstractmethod
from typing import Dict, Generator, Iterable, List, Optional, Tuple
from volatility3.framework import constants, exceptions, interfaces
from volatility3 import framework, schemas
from volatility3.framework import constants, interfaces
from volatility3.framework.configuration import requirements
from volatility3.framework.layers import resources
from volatility3.framework.symbols import intermed
@@ -22,189 +24,539 @@ vollog = logging.getLogger(__name__)
BannersType = Dict[bytes, List[str]]
class SymbolBannerCache(interfaces.automagic.AutomagicInterface):
"""Runs through all symbols tables and caches their banners."""
### Identifiers
class IdentifierProcessor:
operating_system = None
def __init__(self):
pass
@classmethod
@abstractmethod
def get_identifier(cls, json) -> Optional[bytes]:
"""Method to extract the identifier from a particular operating system's JSON
Returns:
identifier is valid or None if not found
"""
raise NotImplementedError(
"This base class has no get_identifier method defined"
)
class WindowsIdentifier(IdentifierProcessor):
operating_system = "windows"
separator = "|"
@classmethod
def get_identifier(cls, json) -> Optional[bytes]:
"""Returns the identifier for the file if one can be found"""
windows_metadata = json.get("metadata", {}).get("windows", {}).get("pdb", {})
if windows_metadata:
guid = windows_metadata.get("GUID", None)
age = windows_metadata.get("age", None)
database = windows_metadata.get("database", None)
if guid and age and database:
return cls.generate(database, guid, age)
return None
@classmethod
def generate(cls, pdb_name: str, guid: str, age: int) -> bytes:
return bytes(cls.separator.join([pdb_name, guid.upper(), str(age)]), "latin-1")
class MacIdentifier(IdentifierProcessor):
operating_system = "mac"
@classmethod
def get_identifier(cls, json) -> Optional[bytes]:
mac_banner = (
json.get("symbols", {}).get("version", {}).get("constant_data", None)
)
if mac_banner:
return base64.b64decode(mac_banner)
return None
class LinuxIdentifier(IdentifierProcessor):
operating_system = "linux"
@classmethod
def get_identifier(cls, json) -> Optional[bytes]:
linux_banner = (
json.get("symbols", {}).get("linux_banner", {}).get("constant_data", None)
)
if linux_banner:
return base64.b64decode(linux_banner)
return None
### CacheManagers
class CacheManagerInterface(interfaces.configuration.VersionableInterface):
def __init__(self, filename: str):
super().__init__()
self._filename = filename
self._classifiers = {}
for subclazz in framework.class_subclasses(IdentifierProcessor):
self._classifiers[subclazz.operating_system] = subclazz
def add_identifier(self, location: str, operating_system: str, identifier: str):
"""Adds an identifier to the store"""
pass
def find_location(
self, identifier: bytes, operating_system: Optional[str]
) -> Optional[str]:
"""Returns the location of the symbol file given the identifier
Args:
identifier: string that uniquely identifies a particular symbol table
operating_system: optional string to restrict identifiers to just those for a particular operating system
Returns:
The location of the symbols file that matches the identifier
"""
pass
def get_local_locations(self) -> Iterable[str]:
"""Returns a list of all the local locations"""
pass
def update(self):
"""Locates all files under the symbol directories. Updates the cache with additions, modifications and removals.
This also updates remote locations based on a cache timeout.
"""
pass
def get_identifier_dictionary(
self, operating_system: Optional[str] = None, local_only: bool = False
) -> Dict[bytes, str]:
"""Returns a dictionary of identifiers and locations
Args:
operating_system: If set, limits responses to a specific operating system
local_only: Returns only local locations
Returns:
A dictionary of identifiers mapped to a location
"""
pass
def get_identifier(self, location: str) -> Optional[bytes]:
"""Returns an identifier based on a specific location or None"""
pass
def get_identifiers(self, operating_system: Optional[str]) -> List[bytes]:
"""Returns all identifiers for a particular operating system"""
pass
def get_location_statistics(
self, location: str
) -> Optional[Tuple[int, int, int, int]]:
"""Returns ISF statistics based on the location
Returns:
A tuple of base_types, types, enums, symbols, or None is location not found
"""
def get_hash(self, location: str) -> Optional[str]:
"""Returns the hash of the JSON from within a location ISF"""
class SqliteCache(CacheManagerInterface):
_required_framework_version = (2, 0, 0)
_version = (1, 0, 0)
def __init__(self, filename: str):
super().__init__(filename)
self.cache_period = constants.SQLITE_CACHE_PERIOD
try:
self._database = self._connect_storage(filename)
except sqlite3.DatabaseError:
os.unlink(filename)
self._database = self._connect_storage(filename)
def _connect_storage(self, path: str) -> sqlite3.Connection:
database = sqlite3.connect(path)
database.row_factory = sqlite3.Row
database.cursor().execute(
f"CREATE TABLE IF NOT EXISTS database_info (schema_version INT DEFAULT {constants.CACHE_SQLITE_SCHEMA_VERSION})"
)
schema_version = (
database.cursor()
.execute("SELECT schema_version FROM database_info")
.fetchone()
)
if not schema_version:
database.cursor().execute(
f"INSERT INTO database_info VALUES ({constants.CACHE_SQLITE_SCHEMA_VERSION})"
)
elif schema_version["schema_version"] == constants.CACHE_SQLITE_SCHEMA_VERSION:
# All good, so pass and move on
pass
else:
vollog.info(
f"Previous cache schema version found: {schema_version['schema_version']}"
)
# TODO: Implement code if the schema changes
# Current this should never happen so we start over again
database.close()
os.unlink(path)
return self._connect_storage(path)
database.cursor().execute(
"CREATE TABLE IF NOT EXISTS cache (location TEXT UNIQUE NOT NULL, identifier TEXT, operating_system TEXT, hash TEXT,"
"stats_base_types INT DEFAULT 0, stats_types INT DEFAULT 0, stats_enums INT DEFAULT 0, stats_symbols INT DEFAULT 0, local BOOL, cached DATETIME)"
)
database.commit()
return database
def find_location(
self, identifier: bytes, operating_system: Optional[str]
) -> Optional[str]:
"""Returns the location of the symbol file given the identifier.
If multiple locations exist for an identifier, the last found is returned
Args:
identifier: string that uniquely identifies a particular symbol table
operating_system: optional string to restrict identifiers to just those for a particular operating system
Returns:
The location of the symbols file that matches the identifier or None
"""
statement = "SELECT location FROM cache WHERE identifier = ?"
parameters = (identifier,)
if operating_system is not None:
statement = "SELECT location FROM cache WHERE identifier = ? AND operating_system = ?"
parameters = (identifier, operating_system)
results = self._database.cursor().execute(statement, parameters).fetchall()
result = None
for row in results:
result = row["location"]
return result
def get_local_locations(self) -> Generator[str, None, None]:
result = (
self._database.cursor()
.execute("SELECT DISTINCT location FROM cache WHERE local = 1")
.fetchall()
)
for row in result:
yield row["location"]
def is_url_local(self, url: str) -> bool:
"""Determines whether an url is local or not"""
parsed = urllib.parse.urlparse(url)
return parsed.scheme in ["file", "jar"]
def get_identifier(self, location: str) -> Optional[bytes]:
results = (
self._database.cursor()
.execute("SELECT identifier FROM cache WHERE location = ?", (location,))
.fetchall()
)
for row in results:
return row["identifier"]
return None
def get_location_statistics(
self, location: str
) -> Optional[Tuple[int, int, int, int]]:
results = (
self._database.cursor()
.execute(
"SELECT stats_base_types, stats_types, stats_enums, stats_symbols FROM cache WHERE location = ?",
(location,),
)
.fetchall()
)
for row in results:
return (
row["stats_base_types"],
row["stats_types"],
row["stats_enums"],
row["stats_symbols"],
)
return None
def get_hash(self, location: str) -> Optional[str]:
results = (
self._database.cursor()
.execute("SELECT hash FROM cache WHERE location = ?", (location,))
.fetchall()
)
for row in results:
return row["hash"]
return None
def update(self, progress_callback=None):
"""Locates all files under the symbol directories. Updates the cache with additions, modifications and removals.
This also updates remote locations based on a cache timeout.
"""
on_disk_locations = set(
[
filename
for filename in intermed.IntermediateSymbolTable.file_symbol_url("")
]
)
cached_locations = set(self.get_local_locations())
new_locations = on_disk_locations.difference(cached_locations)
missing_locations = cached_locations.difference(on_disk_locations)
# Missing entries
if missing_locations:
self._database.cursor().execute(
f"DELETE FROM cache WHERE location IN ({','.join(['?'] * len(missing_locations))})",
[x for x in missing_locations],
)
self._database.commit()
cache_update = set()
files_to_timestamp = on_disk_locations.intersection(cached_locations)
if files_to_timestamp:
result = self._database.cursor().execute(
"SELECT location, cached FROM cache WHERE local = 1 "
f"AND cached < date('now', '{self.cache_period}');"
)
for row in result:
location = row["location"]
stored_timestamp = datetime.datetime.fromisoformat(row["cached"])
timestamp = stored_timestamp # Default to requiring update
# See if the file is a local URL type we can handle:
parsed = urllib.parse.urlparse(location)
pathname = None
if parsed.scheme == "file":
pathname = urllib.request.url2pathname(parsed.path)
if parsed.scheme == "jar":
inner_url = urllib.parse.urlparse(parsed.path)
if inner_url.scheme == "file":
pathname = inner_url.path.split("!")[0]
if pathname and os.path.exists(pathname):
timestamp = datetime.datetime.fromtimestamp(
os.stat(pathname).st_mtime
)
else:
vollog.log(
constants.LOGLEVEL_VVVV,
"File location in database classed as local but not file/jar URL",
)
# If we're supposed to include it, and our last check is older than (or equal to) the file timestamp
if (
row["location"] in files_to_timestamp
and stored_timestamp < timestamp
):
cache_update.add(row["location"])
idextractors = list(framework.class_subclasses(IdentifierProcessor))
# New or not recently updated
files_to_process = new_locations.union(cache_update)
number_files_to_process = len(files_to_process)
cursor = self._database.cursor()
try:
for counter, location in enumerate(files_to_process):
# Open location
progress_callback(
counter * 100 / number_files_to_process,
f"Updating caches for {number_files_to_process} files...",
)
try:
with resources.ResourceAccessor().open(location) as fp:
json_obj = json.load(fp)
hash = schemas.create_json_hash(json_obj)
identifier = None
# Get stats
stats_base_types = len(json_obj.get("base_types", {}))
stats_types = len(json_obj.get("user_types", {}))
stats_enums = len(json_obj.get("enums", {}))
stats_symbols = len(json_obj.get("symbols", {}))
operating_system = None
for idextractor in idextractors:
identifier = idextractor.get_identifier(json_obj)
if identifier is not None:
operating_system = idextractor.operating_system
break
# We don't try to validate schemas here, we do that on first use
# Store in database
cursor.execute(
"INSERT OR REPLACE INTO cache (location, identifier, operating_system, hash,"
"stats_base_types, stats_types, stats_enums, stats_symbols, "
"local, cached) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))",
(
location,
identifier,
operating_system,
hash,
stats_base_types,
stats_types,
stats_enums,
stats_symbols,
self.is_url_local(location),
),
)
if identifier is not None:
vollog.log(
constants.LOGLEVEL_VV,
f"Identified {location} as {identifier}",
)
else:
vollog.log(
constants.LOGLEVEL_VVVV,
f"No identifier found for {location}",
)
except Exception as excp:
vollog.log(constants.LOGLEVEL_VVVV, excp)
finally:
self._database.commit()
# Remote Entries
if not constants.OFFLINE and constants.REMOTE_ISF_URL:
progress_callback(0, "Reading remote ISF list")
cursor = self._database.cursor()
cursor.execute(
f"SELECT cached FROM cache WHERE local = 0 and cached < datetime('now', '{self.cache_period}')"
)
remote_identifiers = RemoteIdentifierFormat(constants.REMOTE_ISF_URL)
progress_callback(50, "Reading remote ISF list")
for operating_system in constants.OS_CATEGORIES:
identifiers = remote_identifiers.process(
{}, operating_system=operating_system
)
for identifier, location in identifiers:
identifier = identifier.rstrip()
identifier = (
identifier[:-1] if identifier.endswith(b"\x00") else identifier
) # Linux banners dumped by dwarf2json end with "\x00\n". If not stripped, the banner cannot match.
cursor.execute(
"INSERT OR REPLACE INTO cache(identifier, location, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))",
(identifier, location, operating_system, False),
)
progress_callback(100, "Reading remote ISF list")
self._database.commit()
def get_identifier_dictionary(
self, operating_system: Optional[str] = None, local_only: bool = False
) -> Dict[bytes, str]:
output = {}
additions = []
statement = "SELECT location, identifier FROM cache"
if local_only:
additions.append("local = 1")
if operating_system:
additions.append(f"operating_system = '{operating_system}'")
if additions:
statement += f" WHERE {' AND '.join(additions)}"
results = self._database.cursor().execute(statement)
for row in results:
if row["identifier"] in output and row["identifier"] and row["location"]:
vollog.debug(
f"Duplicate entry for identifier {row['identifier']}: {row['location']} and {output[row['identifier']]}"
)
output[row["identifier"]] = row["location"]
return output
def get_identifiers(self, operating_system: Optional[str]) -> List[bytes]:
if operating_system:
results = (
self._database.cursor()
.execute(
"SELECT identifier FROM cache WHERE operating_system = ?",
(operating_system,),
)
.fetchall()
)
else:
results = (
self._database.cursor()
.execute("SELECT identifier FROM cache")
.fetchall()
)
output = []
for row in results:
output.append(row["identifier"])
return output
### Automagic
class SymbolCacheMagic(interfaces.automagic.AutomagicInterface):
"""Runs through all symbol tables and caches their identifiers"""
# Since this is necessary for ConstructionMagic, we set a lower priority
# The user would run it eventually either way, but running it first means it can be used that run
priority = 0
os: Optional[str] = None
symbol_name: str = "banner_name"
banner_path: Optional[str] = None
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
identifiers_path = os.path.join(
constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME
)
self._cache = SqliteCache(identifiers_path)
@classmethod
def load_banners(cls) -> BannersType:
if not cls.banner_path:
raise ValueError("Banner_path not appropriately set")
banners: BannersType = {}
if os.path.exists(cls.banner_path):
with open(cls.banner_path, "rb") as f:
# We use pickle over JSON because we're dealing with bytes objects
banners.update(pickle.load(f))
# Remove possibilities that can't exist locally.
remove_banners = []
for banner in banners:
for path in banners[banner]:
url = urllib.parse.urlparse(path)
if url.scheme == 'file' and not os.path.exists(urllib.request.url2pathname(url.path)):
vollog.log(
constants.LOGLEVEL_VV, "Removing cached path {} for banner {}: file does not exist".format(
path, str(banner or b'', 'latin-1')))
banners[banner].remove(path)
# This is probably excessive, but it's here if we need it
if url.scheme == 'jar':
zip_file, zip_path = url.path.split("!")
zip_file = urllib.parse.urlparse(zip_file).path
if ((not os.path.exists(zip_file)) or (zip_path not in zipfile.ZipFile(zip_file).namelist())):
vollog.log(constants.LOGLEVEL_VV,
"Removing cached path {} for banner {}: file does not exist".format(path, banner))
banners[banner].remove(path)
if not banners[banner]:
remove_banners.append(banner)
for remove_banner in remove_banners:
del banners[remove_banner]
return banners
@classmethod
def save_banners(cls, banners):
with open(cls.banner_path, "wb") as f:
pickle.dump(banners, f)
def __call__(self, context, config_path, configurable, progress_callback = None):
def __call__(self, context, config_path, configurable, progress_callback=None):
"""Runs the automagic over the configurable."""
# Bomb out if we're just the generic interface
if self.os is None:
return
# We only need to be called once, so no recursion necessary
banners = self.load_banners()
cacheables = self.find_new_banner_files(banners, self.os)
new_banners = self.read_new_banners(context, config_path, cacheables, self.symbol_name, self.os,
progress_callback)
# Add in any new banners to the existing list
for new_banner in new_banners:
banner_list = banners.get(new_banner, [])
banners[new_banner] = list(set(banner_list + new_banners[new_banner]))
# Do remote banners *after* the JSON loading, so that it doesn't pull down all the remote JSON
self.remote_banners(banners, self.os)
# Rewrite the cached banners each run, since writing is faster than the banner_cache validation portion
self.save_banners(banners)
if progress_callback is not None:
progress_callback(100, f"Built {self.os} caches")
self._cache.update(progress_callback)
@classmethod
def read_new_banners(cls, context: interfaces.context.ContextInterface, config_path: str, new_urls: List[str],
symbol_name: str, operating_system: str = None,
progress_callback = None) -> Optional[Dict[bytes, List[str]]]:
"""Reads the any new banners for the OS in question"""
if operating_system is None:
return None
banners = {}
total = len(new_urls)
if total > 0:
vollog.info(f"Building {operating_system} caches...")
for current in range(total):
if progress_callback is not None:
progress_callback(current * 100 / total, f"Building {operating_system} caches")
isf_url = new_urls[current]
isf = None
try:
# Loading the symbol table will be very slow until it's been validated
isf = intermed.IntermediateSymbolTable(context, config_path, "temp", isf_url, validate = False)
# We should store the banner against the filename
# We don't bother with the hash (it'll likely take too long to validate)
# but we should check at least that the banner matches on load.
banner = isf.get_symbol(symbol_name).constant_data
vollog.log(constants.LOGLEVEL_VV, f"Caching banner {banner} for file {isf_url}")
bannerlist = banners.get(banner, [])
bannerlist.append(isf_url)
banners[banner] = bannerlist
except exceptions.SymbolError:
pass
except json.JSONDecodeError:
vollog.log(constants.LOGLEVEL_VV, f"Caching file {isf_url} failed due to JSON error")
finally:
# Get rid of the loaded file, in case it sits in memory
if isf:
del isf
gc.collect()
return banners
@classmethod
def find_new_banner_files(cls, banners: Dict[bytes, List[str]], operating_system: str) -> List[str]:
"""Gathers all files and remove existing banners"""
cacheables = list(intermed.IntermediateSymbolTable.file_symbol_url(operating_system))
for banner in banners:
for json_file in banners[banner]:
if json_file in cacheables:
cacheables.remove(json_file)
return cacheables
@classmethod
def remote_banners(cls, banners: Dict[bytes, List[str]], operating_system = None, banner_location = None):
"""Adds remote URLs to the banner list"""
if operating_system is None:
return None
if banner_location is None:
banner_location = constants.REMOTE_ISF_URL
if not constants.OFFLINE and banner_location is not None:
try:
rbf = RemoteBannerFormat(banner_location)
rbf.process(banners, operating_system)
except urllib.error.URLError:
vollog.debug(f"Unable to download remote banner list from {banner_location}")
def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]:
"""Returns a list of RequirementInterface objects required by this
object."""
return [
requirements.VersionRequirement(
name="SQLiteCache", component=SqliteCache, version=(1, 0, 0)
)
]
class RemoteBannerFormat:
class RemoteIdentifierFormat:
def __init__(self, location: str):
self._location = location
with resources.ResourceAccessor().open(url = location) as fp:
with resources.ResourceAccessor().open(url=location) as fp:
self._data = json.load(fp)
if not self._verify():
raise ValueError("Unsupported version for remote banner list format")
raise ValueError("Unsupported version for remote identifier list format")
def _verify(self) -> bool:
version = self._data.get('version', 0)
version = self._data.get("version", 0)
if version in [1]:
setattr(self, 'process', getattr(self, f'process_v{version}'))
setattr(self, "process", getattr(self, f"process_v{version}"))
return True
return False
def process(self, banners: Dict[bytes, List[str]], operating_system: Optional[str]):
raise ValueError("Banner List version not verified")
def process(
self, identifiers: Dict[bytes, List[str]], operating_system: Optional[str]
) -> Generator[Tuple[bytes, str], None, None]:
raise ValueError("Identifier List version not verified")
def process_v1(self, banners: Dict[bytes, List[str]], operating_system: Optional[str]):
def process_v1(
self,
identifiers: Optional[Dict[bytes, List[str]]],
operating_system: Optional[str],
) -> Generator[Tuple[bytes, str], None, None]:
if operating_system in self._data:
for banner in self._data[operating_system]:
binary_banner = base64.b64decode(banner)
file_list = banners.get(binary_banner, [])
for value in self._data[operating_system][banner]:
if value not in file_list:
file_list = file_list + [value]
banners[binary_banner] = file_list
if 'additional' in self._data:
for location in self._data['additional']:
for identifier in self._data[operating_system]:
binary_identifier = base64.b64decode(identifier)
for value in self._data[operating_system][identifier]:
yield binary_identifier, value
if "additional" in self._data:
for location in self._data["additional"]:
try:
subrbf = RemoteBannerFormat(location)
subrbf.process(banners, operating_system)
subrbf = RemoteIdentifierFormat(location)
yield from subrbf.process(identifiers, operating_system)
except IOError:
vollog.debug(f"Remote file not found: {location}")
return banners
return identifiers
volatility3/framework/automagic/symbol_finder.py
+90 -43
View File
@@ -3,9 +3,10 @@
#
import logging
from typing import Any, Iterable, List, Tuple, Type, Optional, Callable
import os
from typing import Any, Callable, Iterable, List, Optional, Tuple
from volatility3.framework import interfaces, constants, layers
from volatility3.framework import constants, interfaces, layers
from volatility3.framework.automagic import symbol_cache
from volatility3.framework.configuration import requirements
from volatility3.framework.layers import scanners
@@ -15,73 +16,111 @@ vollog = logging.getLogger(__name__)
class SymbolFinder(interfaces.automagic.AutomagicInterface):
"""Symbol loader based on signature strings."""
priority = 40
banner_config_key: str = "banner"
banner_cache: Optional[Type[symbol_cache.SymbolBannerCache]] = None
operating_system: Optional[str] = None
symbol_class: Optional[str] = None
find_aslr: Optional[Callable] = None
def __init__(self, context: interfaces.context.ContextInterface, config_path: str) -> None:
def __init__(
self, context: interfaces.context.ContextInterface, config_path: str
) -> None:
super().__init__(context, config_path)
self._requirements: List[Tuple[str, interfaces.configuration.RequirementInterface]] = []
self._requirements: List[
Tuple[str, interfaces.configuration.RequirementInterface]
] = []
self._banners: symbol_cache.BannersType = {}
@classmethod
def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]:
return [
requirements.VersionRequirement(
name="SQLiteCache",
component=symbol_cache.SqliteCache,
version=(1, 0, 0),
)
]
@property
def banners(self) -> symbol_cache.BannersType:
"""Creates a cached copy of the results, but only it's been
requested."""
if not self._banners:
if not self.banner_cache:
raise RuntimeError(f"Cache has not been properly defined for {self.__class__.__name__}")
self._banners = self.banner_cache.load_banners()
identifiers_path = os.path.join(
constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME
)
cache = symbol_cache.SqliteCache(identifiers_path)
self._banners = cache.get_identifier_dictionary(
operating_system=self.operating_system
)
return self._banners
def __call__(self,
context: interfaces.context.ContextInterface,
config_path: str,
requirement: interfaces.configuration.RequirementInterface,
progress_callback: constants.ProgressCallback = None) -> None:
def __call__(
self,
context: interfaces.context.ContextInterface,
config_path: str,
requirement: interfaces.configuration.RequirementInterface,
progress_callback: constants.ProgressCallback = None,
) -> None:
"""Searches for SymbolTableRequirements and attempt to populate
them."""
# Bomb out early if our details haven't been configured
if self.symbol_class is None:
return
return None
self._requirements = self.find_requirements(
context,
config_path,
requirement, (requirements.TranslationLayerRequirement, requirements.SymbolTableRequirement),
shortcut = False)
requirement,
(
requirements.TranslationLayerRequirement,
requirements.SymbolTableRequirement,
),
shortcut=False,
)
for (sub_path, requirement) in self._requirements:
for sub_path, requirement in self._requirements:
parent_path = interfaces.configuration.parent_path(sub_path)
if (isinstance(requirement, requirements.SymbolTableRequirement)
and requirement.unsatisfied(context, parent_path)):
for (tl_sub_path, tl_requirement) in self._requirements:
if isinstance(
requirement, requirements.SymbolTableRequirement
) and requirement.unsatisfied(context, parent_path):
for tl_sub_path, tl_requirement in self._requirements:
tl_parent_path = interfaces.configuration.parent_path(tl_sub_path)
# Find the TranslationLayer sibling to the SymbolTableRequirement
if (isinstance(tl_requirement, requirements.TranslationLayerRequirement)
and tl_parent_path == parent_path):
if (
isinstance(
tl_requirement, requirements.TranslationLayerRequirement
)
and tl_parent_path == parent_path
):
if context.config.get(tl_sub_path, None):
self._banner_scan(context, parent_path, requirement, context.config[tl_sub_path],
progress_callback)
self._banner_scan(
context,
parent_path,
requirement,
context.config[tl_sub_path],
progress_callback,
)
break
def _banner_scan(self,
context: interfaces.context.ContextInterface,
config_path: str,
requirement: interfaces.configuration.ConstructableRequirementInterface,
layer_name: str,
progress_callback: constants.ProgressCallback = None) -> None:
def _banner_scan(
self,
context: interfaces.context.ContextInterface,
config_path: str,
requirement: interfaces.configuration.ConstructableRequirementInterface,
layer_name: str,
progress_callback: constants.ProgressCallback = None,
) -> None:
"""Accepts a context, config_path and SymbolTableRequirement, with a
constructed layer_name and scans the layer for banners."""
# Bomb out early if there's no banners
if not self.banners:
return
return None
mss = scanners.MultiStringScanner([x for x in self.banners if x is not None])
@@ -89,36 +128,44 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface):
# Check if the Stacker has already found what we're looking for
if layer.config.get(self.banner_config_key, None):
banner_list = [(0, bytes(layer.config[self.banner_config_key],
'raw_unicode_escape'))] # type: Iterable[Any]
banner_list = [
(0, bytes(layer.config[self.banner_config_key], "raw_unicode_escape"))
] # type: Iterable[Any]
else:
# Swap to the physical layer for scanning
# Only traverse down a layer if it's an intel layer
# TODO: Fix this so it works for layers other than just Intel
if isinstance(layer, layers.intel.Intel):
layer = context.layers[layer.config['memory_layer']]
banner_list = layer.scan(context = context, scanner = mss, progress_callback = progress_callback)
layer = context.layers[layer.config["memory_layer"]]
banner_list = layer.scan(
context=context, scanner=mss, progress_callback=progress_callback
)
for _, banner in banner_list:
vollog.debug(f"Identified banner: {repr(banner)}")
symbol_files = self.banners.get(banner, None)
if symbol_files:
isf_path = symbol_files[0]
vollog.debug(f"Using symbol library: {symbol_files[0]}")
isf_path = symbol_files
vollog.debug(f"Using symbol library: {symbol_files}")
clazz = self.symbol_class
# Set the discovered options
path_join = interfaces.configuration.path_join
context.config[path_join(config_path, requirement.name, "class")] = clazz
context.config[path_join(config_path, requirement.name, "isf_url")] = isf_path
context.config[path_join(config_path, requirement.name, "symbol_mask")] = layer.address_mask
context.config[path_join(config_path, requirement.name, "class")] = (
clazz
)
context.config[path_join(config_path, requirement.name, "isf_url")] = (
isf_path
)
context.config[
path_join(config_path, requirement.name, "symbol_mask")
] = layer.address_mask
# Construct the appropriate symbol table
requirement.construct(context, config_path)
break
else:
if symbol_files:
vollog.debug(f"Symbol library path not found: {symbol_files[0]}")
# print("Kernel", banner, hex(banner_offset))
vollog.debug(f"Symbol library path not found for: {banner}")
# print("Kernel", banner, hex(banner_offset))
else:
vollog.debug("No existing banners found")
# TODO: Fallback to generic regex search?
volatility3/framework/automagic/windows.py
+217 -106
View File
@@ -41,8 +41,14 @@ class DtbSelfReferential:
"""A generic DTB test which looks for a self-referential pointer at *any*
index within the page."""
def __init__(self, layer_type: Type[layers.intel.Intel], ptr_struct: str, mask: int,
valid_range: Iterable[int], reserved_bits: int) -> None:
def __init__(
self,
layer_type: Type[layers.intel.Intel],
ptr_struct: str,
mask: int,
valid_range: Iterable[int],
reserved_bits: int,
) -> None:
self.layer_type = layer_type
self.ptr_struct = ptr_struct
self.ptr_size = struct.calcsize(ptr_struct)
@@ -51,22 +57,26 @@ class DtbSelfReferential:
self.valid_range = valid_range
self.reserved_bits = reserved_bits
def __call__(self, data: bytes, data_offset: int, page_offset: int) -> Optional[Tuple[int, int]]:
page = data[page_offset:page_offset + self.page_size]
def __call__(
self, data: bytes, data_offset: int, page_offset: int
) -> Optional[Tuple[int, int]]:
page = data[page_offset : page_offset + self.page_size]
if not page:
return None
ref_pages = set()
for ref in range(0, self.page_size, self.ptr_size):
ptr_data = page[ref:ref + self.ptr_size]
ptr, = struct.unpack(self.ptr_struct, ptr_data)
ptr_data = page[ref : ref + self.ptr_size]
(ptr,) = struct.unpack(self.ptr_struct, ptr_data)
# For both Intel-32e, bit 7 is reserved (more are reserved in PAE), so if that's ever set,
# we can move on
if (ptr & self.reserved_bits) and (ptr & 0x01):
return None
if ((ptr & self.mask) == (data_offset + page_offset)) and (data_offset + page_offset > 0):
if ((ptr & self.mask) == (data_offset + page_offset)) and (
data_offset + page_offset > 0
):
# Pointer must be valid
if (ptr & 0x01):
if ptr & 0x01:
ref_pages.add(ref)
# The DTB is extremely unlikely to refer back to itself. so the number of reference should always be exactly 1
@@ -78,62 +88,78 @@ class DtbSelfReferential:
class DtbSelfRef32bit(DtbSelfReferential):
def __init__(self):
super().__init__(layer_type = layers.intel.WindowsIntel,
ptr_struct = "I",
mask = 0xFFFFF000,
valid_range = [0x300],
reserved_bits = 0x0)
super().__init__(
layer_type=layers.intel.WindowsIntel,
ptr_struct="I",
mask=0xFFFFF000,
valid_range=[0x300],
reserved_bits=0x0,
)
class DtbSelfRef64bit(DtbSelfReferential):
def __init__(self) -> None:
super().__init__(layer_type = layers.intel.WindowsIntel32e,
ptr_struct = "Q",
mask = 0x3FFFFFFFFFF000,
valid_range = range(0x100, 0x1ff),
reserved_bits = 0x80)
super().__init__(
layer_type=layers.intel.WindowsIntel32e,
ptr_struct="Q",
mask=0x3FFFFFFFFFF000,
valid_range=range(0x100, 0x1FF),
reserved_bits=0x80,
)
class DtbSelfRef64bitOldWindows(DtbSelfReferential):
def __init__(self) -> None:
super().__init__(layer_type = layers.intel.WindowsIntel32e,
ptr_struct = "Q",
mask = 0x3FFFFFFFFFF000,
valid_range = [0x1ed],
reserved_bits = 0x80)
super().__init__(
layer_type=layers.intel.WindowsIntel32e,
ptr_struct="Q",
mask=0x3FFFFFFFFFF000,
valid_range=[0x1ED],
reserved_bits=0x80,
)
class DtbSelfRefPae(DtbSelfReferential):
def __init__(self) -> None:
super().__init__(layer_type = layers.intel.WindowsIntelPAE,
ptr_struct = "Q",
valid_range = [0x3],
mask = 0x3FFFFFFFFFF000,
reserved_bits = 0x0)
super().__init__(
layer_type=layers.intel.WindowsIntelPAE,
ptr_struct="Q",
valid_range=[0x3],
mask=0x3FFFFFFFFFF000,
reserved_bits=0x0,
)
@staticmethod
def _and_bytes(abytes, bbytes):
return bytes([a & b for a, b in zip(abytes[::-1], bbytes[::-1])][::-1])
def __call__(self, data: bytes, data_offset: int, page_offset: int) -> Optional[Tuple[int, int]]:
def __call__(
self, data: bytes, data_offset: int, page_offset: int
) -> Optional[Tuple[int, int]]:
dtb = super().__call__(data, data_offset, page_offset)
if dtb:
# Find the top page
top_pae_page = dtb[0] - 0x4000
# The top page should map to the next four pages after it
# Build what we expect the page table to be
expected_table = b''.join([struct.pack(self.ptr_struct, top_pae_page + (i * 0x1000)) for i in range(1, 5)])
expected_table = b"".join(
[
struct.pack(self.ptr_struct, top_pae_page + (i * 0x1000))
for i in range(1, 5)
]
)
# Mask off the page bits of top level page map
page_table_mask = b"\x00\xf0\xff\xff\xff\xff\xff\xff" * 4
page_table = data[top_pae_page - data_offset: top_pae_page - data_offset + (4 * self.ptr_size)]
page_table = data[
top_pae_page
- data_offset : top_pae_page
- data_offset
+ (4 * self.ptr_size)
]
# Compare them
anded_bytes = self._and_bytes(page_table, page_table_mask)
if (anded_bytes == expected_table):
if anded_bytes == expected_table:
return top_pae_page, dtb[1]
# Return None since the dtb value *isn't* None
return None
@@ -143,6 +169,7 @@ class DtbSelfRefPae(DtbSelfReferential):
class PageMapScanner(interfaces.layers.ScannerInterface):
"""Scans through all pages using DTB tests to determine a dtb offset and
architecture."""
overlap = 0x4000
thread_safe = True
tests = [DtbSelfRef64bit(), DtbSelfRefPae(), DtbSelfRef32bit()]
@@ -153,7 +180,9 @@ class PageMapScanner(interfaces.layers.ScannerInterface):
if tests:
self.tests = tests
def __call__(self, data: bytes, data_offset: int) -> Generator[Tuple[DtbSelfReferential, int], None, None]:
def __call__(
self, data: bytes, data_offset: int
) -> Generator[Tuple[DtbSelfReferential, int], None, None]:
for page_offset in range(0, len(data), 0x1000):
for test in self.tests:
result = test(data, data_offset, page_offset)
@@ -163,20 +192,29 @@ class PageMapScanner(interfaces.layers.ScannerInterface):
class WindowsIntelStacker(interfaces.automagic.StackerLayerInterface):
stack_order = 40
exclusion_list = ['mac', 'linux']
exclusion_list = ["mac", "linux"]
# Group these by region so we only run over the data once
test_sets = [("Detecting Self-referential pointer for recent windows",
[DtbSelfRef64bit()], [(0x150000, 0x150000), (0x650000, 0xa0000)]),
("Older windows fixed location self-referential pointers",
[DtbSelfRefPae(), DtbSelfRef32bit(), DtbSelfRef64bitOldWindows()], [(0x30000, 0x1000000)])
]
test_sets = [
(
"Detecting Self-referential pointer for recent windows",
[DtbSelfRef64bit()],
[(0x150000, 0x150000), (0x650000, 0xA0000)],
),
(
"Older windows fixed location self-referential pointers",
[DtbSelfRefPae(), DtbSelfRef32bit(), DtbSelfRef64bitOldWindows()],
[(0x30000, 0x1000000)],
),
]
@classmethod
def stack(cls,
context: interfaces.context.ContextInterface,
layer_name: str,
progress_callback: constants.ProgressCallback = None) -> Optional[interfaces.layers.DataLayerInterface]:
def stack(
cls,
context: interfaces.context.ContextInterface,
layer_name: str,
progress_callback: constants.ProgressCallback = None,
) -> Optional[interfaces.layers.DataLayerInterface]:
"""Attempts to determine and stack an intel layer on a physical layer
where possible.
@@ -192,37 +230,56 @@ class WindowsIntelStacker(interfaces.automagic.StackerLayerInterface):
base_layer = context.layers[layer_name]
if isinstance(base_layer, intel.Intel):
return None
if base_layer.metadata.get('os', None) not in ['Windows', 'Unknown']:
if base_layer.metadata.get("os", None) not in ["Windows", "Unknown"]:
return None
layer = config_path = None
# Check the metadata
if (base_layer.metadata.get('os', None) == 'Windows' and base_layer.metadata.get('page_map_offset')):
arch = base_layer.metadata.get('architecture', None)
if arch not in ['Intel32', 'Intel64']:
if base_layer.metadata.get("os", None) == "Windows" and base_layer.metadata.get(
"page_map_offset"
):
arch = base_layer.metadata.get("architecture", None)
if arch not in ["Intel32", "Intel64"]:
return None
# Set the layer type
layer_type: Type = intel.WindowsIntel
if arch == 'Intel64':
if arch == "Intel64":
layer_type = intel.WindowsIntel32e
elif base_layer.metadata.get('pae', False):
elif base_layer.metadata.get("pae", False):
layer_type = intel.WindowsIntelPAE
# Construct the layer
new_layer_name = context.layers.free_layer_name("IntelLayer")
config_path = interfaces.configuration.path_join("IntelHelper", new_layer_name)
context.config[interfaces.configuration.path_join(config_path, "memory_layer")] = layer_name
context.config[interfaces.configuration.path_join(
config_path, "page_map_offset")] = base_layer.metadata['page_map_offset']
layer = layer_type(context, config_path = config_path, name = new_layer_name, metadata = {'os': 'Windows'})
config_path = interfaces.configuration.path_join(
"IntelHelper", new_layer_name
)
context.config[
interfaces.configuration.path_join(config_path, "memory_layer")
] = layer_name
context.config[
interfaces.configuration.path_join(config_path, "page_map_offset")
] = base_layer.metadata["page_map_offset"]
layer = layer_type(
context,
config_path=config_path,
name=new_layer_name,
metadata={"os": "Windows"},
)
page_map_offset = context.config[
interfaces.configuration.path_join(config_path, "page_map_offset")
]
vollog.debug(f"DTB was given to us by base layer: {hex(page_map_offset)}")
return layer
# Self Referential finder
for description, tests, sections in cls.test_sets:
vollog.debug(description)
# There is a very high chance that the DTB will live in these very narrow segments, assuming we couldn't find them previously
hits = base_layer.scan(context,
PageMapScanner(tests = tests),
sections = sections,
progress_callback = progress_callback)
hits = base_layer.scan(
context,
PageMapScanner(tests=tests),
sections=sections,
progress_callback=progress_callback,
)
# Flatten the generator
def sort_by_tests(x):
@@ -233,13 +290,19 @@ class WindowsIntelStacker(interfaces.automagic.StackerLayerInterface):
"""Determines a pointer from a page_table"""
max_ptr = 0
for index in range(0, len(page_table), ptr_size):
pointer = struct.unpack(test.ptr_struct, page_table[index:index + ptr_size])[0]
pointer = struct.unpack(
test.ptr_struct, page_table[index : index + ptr_size]
)[0]
# Make sure the pointer is valid, ignore large pages which would require more calculation
if pointer & 0x1 and not pointer & 0x80:
max_ptr = max(max_ptr, (pointer ^ (pointer & 0xfff)) % test.layer_type.maximum_address)
max_ptr = max(
max_ptr,
(pointer ^ (pointer & 0xFFF))
% test.layer_type.maximum_address,
)
return max_ptr
hits = sorted(list(hits), key = sort_by_tests)
hits = sorted(list(hits), key=sort_by_tests)
for test, page_map_offset in hits:
# Turn the page tables into integers and find the largest one
@@ -248,26 +311,45 @@ class WindowsIntelStacker(interfaces.automagic.StackerLayerInterface):
max_pointer = get_max_pointer(page_table, test, ptr_size)
if max_pointer <= base_layer.maximum_address:
vollog.debug(f"{test.__class__.__name__} test succeeded at {hex(page_map_offset)}")
vollog.debug(
f"{test.__class__.__name__} test succeeded at {hex(page_map_offset)}"
)
new_layer_name = context.layers.free_layer_name("IntelLayer")
config_path = interfaces.configuration.path_join("IntelHelper", new_layer_name)
context.config[interfaces.configuration.path_join(config_path, "memory_layer")] = layer_name
config_path = interfaces.configuration.path_join(
"IntelHelper", new_layer_name
)
context.config[
interfaces.configuration.path_join(config_path, "page_map_offset")] = page_map_offset
layer = test.layer_type(context,
config_path = config_path,
name = new_layer_name,
metadata = {'os': 'Windows'})
interfaces.configuration.path_join(config_path, "memory_layer")
] = layer_name
context.config[
interfaces.configuration.path_join(
config_path, "page_map_offset"
)
] = page_map_offset
layer = test.layer_type(
context,
config_path=config_path,
name=new_layer_name,
metadata={"os": "Windows"},
)
break
else:
vollog.debug(
f"Max pointer for hit with test {test.__class__.__name__} not met: {hex(max_pointer)} > {hex(base_layer.maximum_address)}")
f"Max pointer for hit with test {test.__class__.__name__} not met: {hex(max_pointer)} > {hex(base_layer.maximum_address)}"
)
if layer is not None and config_path:
break
if layer is not None and config_path:
vollog.debug("DTB was found at: 0x{:0x}".format(context.config[interfaces.configuration.path_join(
config_path, "page_map_offset")]))
vollog.debug(
"DTB was found at: 0x{:0x}".format(
context.config[
interfaces.configuration.path_join(
config_path, "page_map_offset"
)
]
)
)
return layer
@@ -275,31 +357,40 @@ class WinSwapLayers(interfaces.automagic.AutomagicInterface):
"""Class to read swap_layers filenames from single-swap-layers, create the
layers and populate the single-layers swap_layers."""
exclusion_list = ['linux', 'mac']
exclusion_list = ["linux", "mac"]
def __call__(self,
context: interfaces.context.ContextInterface,
config_path: str,
requirement: interfaces.configuration.RequirementInterface,
progress_callback: constants.ProgressCallback = None) -> None:
def __call__(
self,
context: interfaces.context.ContextInterface,
config_path: str,
requirement: interfaces.configuration.RequirementInterface,
progress_callback: constants.ProgressCallback = None,
) -> None:
"""Finds translation layers that can have swap layers added."""
path_join = interfaces.configuration.path_join
self._translation_requirement = self.find_requirements(context,
config_path,
requirement,
requirements.TranslationLayerRequirement,
shortcut = False)
self._translation_requirement = self.find_requirements(
context,
config_path,
requirement,
requirements.TranslationLayerRequirement,
shortcut=False,
)
for trans_sub_config, trans_req in self._translation_requirement:
if not isinstance(trans_req, requirements.TranslationLayerRequirement):
# We need this so the type-checker knows we're a TranslationLayerRequirement
continue
swap_sub_config, swap_req = self.find_swap_requirement(trans_sub_config, trans_req)
swap_sub_config, swap_req = self.find_swap_requirement(
trans_sub_config, trans_req
)
counter = 0
swap_config = interfaces.configuration.parent_path(swap_sub_config)
if swap_req and swap_req.unsatisfied(context, swap_config):
# See if any of them need constructing
for swap_location in self.config.get('single_swap_locations', []):
for swap_location in self.config.get("single_swap_locations", []):
# Setup config locations/paths
current_layer_name = swap_req.name + str(counter)
current_layer_path = path_join(swap_sub_config, current_layer_name)
@@ -310,33 +401,52 @@ class WinSwapLayers(interfaces.automagic.AutomagicInterface):
# Fill in the config
if swap_location:
context.config[current_layer_path] = current_layer_name
context.config[layer_loc_path] = swap_location
context.config[layer_class_path] = 'volatility3.framework.layers.physical.FileLayer'
try:
context.config[layer_loc_path] = (
requirements.URIRequirement.location_from_file(
swap_location
)
)
except ValueError:
vollog.warning(
f"Volatility swap_location {swap_location} could not be validated - swap layer disabled"
)
continue
context.config[layer_class_path] = (
"volatility3.framework.layers.physical.FileLayer"
)
# Add the requirement
new_req = requirements.TranslationLayerRequirement(name = current_layer_name,
description = "Swap Layer",
optional = False)
new_req = requirements.TranslationLayerRequirement(
name=current_layer_name,
description="Swap Layer",
optional=False,
)
swap_req.add_requirement(new_req)
context.config[path_join(swap_sub_config, 'number_of_elements')] = counter
context.config[path_join(swap_sub_config, "number_of_elements")] = (
counter
)
context.config[swap_sub_config] = True
swap_req.construct(context, swap_config)
@staticmethod
def find_swap_requirement(config: str,
requirement: requirements.TranslationLayerRequirement) \
-> Tuple[str, Optional[requirements.LayerListRequirement]]:
def find_swap_requirement(
config: str, requirement: requirements.TranslationLayerRequirement
) -> Tuple[str, Optional[requirements.LayerListRequirement]]:
"""Takes a Translation layer and returns its swap_layer requirement."""
swap_req = None
for req_name in requirement.requirements:
req = requirement.requirements[req_name]
if isinstance(req, requirements.LayerListRequirement) and req.name == 'swap_layers':
if (
isinstance(req, requirements.LayerListRequirement)
and req.name == "swap_layers"
):
swap_req = req
continue
swap_config = interfaces.configuration.path_join(config, 'swap_layers')
swap_config = interfaces.configuration.path_join(config, "swap_layers")
return swap_config, swap_req
@classmethod
@@ -344,10 +454,11 @@ class WinSwapLayers(interfaces.automagic.AutomagicInterface):
"""Returns the requirements of this plugin."""
return [
requirements.ListRequirement(
name = "single_swap_locations",
element_type = str,
min_elements = 0,
max_elements = 16,
description = "Specifies a list of swap layer URIs for use with single-location",
optional = True)
name="single_swap_locations",
element_type=str,
min_elements=0,
max_elements=16,
description="Specifies a list of swap layer URIs for use with single-location",
optional=True,
)
]
volatility3/framework/configuration/requirements.py
+317 -133
View File
@@ -10,7 +10,9 @@ expect to be in the context (such as particular layers or symboltables).
"""
import abc
import logging
import os
from typing import Any, ClassVar, Dict, List, Optional, Tuple, Type
from urllib import parse, request
from volatility3.framework import constants, interfaces
@@ -24,23 +26,27 @@ class MultiRequirement(interfaces.configuration.RequirementInterface):
so this is a concrete implementation.
"""
def unsatisfied(self, context: interfaces.context.ContextInterface,
config_path: str) -> Dict[str, interfaces.configuration.RequirementInterface]:
def unsatisfied(
self, context: interfaces.context.ContextInterface, config_path: str
) -> Dict[str, interfaces.configuration.RequirementInterface]:
return self.unsatisfied_children(context, config_path)
class BooleanRequirement(interfaces.configuration.SimpleTypeRequirement):
"""A requirement type that contains a boolean value."""
# Note, this must be a separate class in order to differentiate between Booleans and other instance requirements
class IntRequirement(interfaces.configuration.SimpleTypeRequirement):
"""A requirement type that contains a single integer."""
instance_type: ClassVar[Type] = int
class StringRequirement(interfaces.configuration.SimpleTypeRequirement):
"""A requirement type that contains a single unicode string."""
# TODO: Maybe add string length limits?
instance_type: ClassVar[Type] = str
@@ -48,11 +54,37 @@ class StringRequirement(interfaces.configuration.SimpleTypeRequirement):
class URIRequirement(StringRequirement):
"""A requirement type that contains a single unicode string that is a valid
URI."""
# TODO: Maybe a a check that to unsatisfied that the path really is a URL?
@classmethod
def location_from_file(cls, filename: str) -> str:
"""Returns the URL location from a file parameter (which may be a URL)
Args:
filename: The path to the file (either an absolute, relative, or URL path)
Returns:
The URL for the location of the file
"""
# We want to work in URLs, but we need to accept absolute and relative files (including on windows)
single_location = parse.urlparse(filename, "")
if single_location.scheme == "" or len(single_location.scheme) == 1:
single_location = parse.urlparse(
parse.urljoin("file:", request.pathname2url(os.path.abspath(filename)))
)
if single_location.scheme == "file":
if not os.path.exists(request.url2pathname(single_location.path)):
filename = request.url2pathname(single_location.path)
if not filename:
raise ValueError("File URL looks incorrect (potentially missing /)")
raise ValueError(f"File does not exist: {filename}")
return parse.urlunparse(single_location)
class BytesRequirement(interfaces.configuration.SimpleTypeRequirement):
"""A requirement type that contains a byte string."""
instance_type: ClassVar[Type] = bytes
@@ -67,12 +99,14 @@ class ListRequirement(interfaces.configuration.RequirementInterface):
and does not allow for a dynamic number of values.
"""
def __init__(self,
element_type: Type[interfaces.configuration.SimpleTypes] = str,
max_elements: Optional[int] = 0,
min_elements: Optional[int] = None,
*args,
**kwargs) -> None:
def __init__(
self,
element_type: Type[interfaces.configuration.SimpleTypes] = str,
max_elements: Optional[int] = 0,
min_elements: Optional[int] = None,
*args,
**kwargs,
) -> None:
"""Constructs the object.
Args:
@@ -82,24 +116,33 @@ class ListRequirement(interfaces.configuration.RequirementInterface):
"""
super().__init__(*args, **kwargs)
if not issubclass(element_type, interfaces.configuration.BasicTypes):
raise TypeError("ListRequirements can only be populated with simple InstanceRequirements")
raise TypeError(
"ListRequirements can only be populated with simple InstanceRequirements"
)
self.element_type: Type = element_type
self.min_elements: int = min_elements or 0
self.max_elements: Optional[int] = max_elements
def unsatisfied(self, context: interfaces.context.ContextInterface,
config_path: str) -> Dict[str, interfaces.configuration.RequirementInterface]:
def unsatisfied(
self, context: interfaces.context.ContextInterface, config_path: str
) -> Dict[str, interfaces.configuration.RequirementInterface]:
"""Check the types on each of the returned values and their number and
then call the element type's check for each one."""
config_path = interfaces.configuration.path_join(config_path, self.name)
default = None
value = self.config_value(context, config_path, default)
if not value and self.min_elements > 0:
vollog.log(constants.LOGLEVEL_V, "ListRequirement Unsatisfied - ListRequirement has non-zero min_elements")
vollog.log(
constants.LOGLEVEL_V,
"ListRequirement Unsatisfied - ListRequirement has non-zero min_elements",
)
return {config_path: self}
if value is None and not self.optional:
# We need to differentiate between no value and an empty list
vollog.log(constants.LOGLEVEL_V, "ListRequirement Unsatisfied - Value was not specified")
vollog.log(
constants.LOGLEVEL_V,
"ListRequirement Unsatisfied - Value was not specified",
)
return {config_path: self}
elif value is None:
context.config[config_path] = []
@@ -107,13 +150,22 @@ class ListRequirement(interfaces.configuration.RequirementInterface):
# TODO: Check this is the correct response for an error
raise TypeError(f"Unexpected config value found: {repr(value)}")
if not (self.min_elements <= len(value)):
vollog.log(constants.LOGLEVEL_V, "TypeError - Too few values provided to list option.")
vollog.log(
constants.LOGLEVEL_V,
"TypeError - Too few values provided to list option.",
)
return {config_path: self}
if self.max_elements and not (len(value) < self.max_elements):
vollog.log(constants.LOGLEVEL_V, "TypeError - Too many values provided to list option.")
vollog.log(
constants.LOGLEVEL_V,
"TypeError - Too many values provided to list option.",
)
return {config_path: self}
if not all([isinstance(element, self.element_type) for element in value]):
vollog.log(constants.LOGLEVEL_V, "TypeError - At least one element in the list is not of the correct type.")
vollog.log(
constants.LOGLEVEL_V,
"TypeError - At least one element in the list is not of the correct type.",
)
return {config_path: self}
return {}
@@ -128,37 +180,48 @@ class ChoiceRequirement(interfaces.configuration.RequirementInterface):
choices: A list of possible string options that can be chosen from
"""
super().__init__(*args, **kwargs)
if not isinstance(choices, list) or any([not isinstance(choice, str) for choice in choices]):
if not isinstance(choices, list) or any(
[not isinstance(choice, str) for choice in choices]
):
raise TypeError("ChoiceRequirement takes a list of strings as choices")
self.choices = choices
def unsatisfied(self, context: interfaces.context.ContextInterface,
config_path: str) -> Dict[str, interfaces.configuration.RequirementInterface]:
def unsatisfied(
self, context: interfaces.context.ContextInterface, config_path: str
) -> Dict[str, interfaces.configuration.RequirementInterface]:
"""Validates the provided value to ensure it is one of the available
choices."""
config_path = interfaces.configuration.path_join(config_path, self.name)
value = self.config_value(context, config_path)
if value not in self.choices:
vollog.log(constants.LOGLEVEL_V, "ValueError - Value is not within the set of available choices")
vollog.log(
constants.LOGLEVEL_V,
"ValueError - Value is not within the set of available choices",
)
return {config_path: self}
return {}
class ComplexListRequirement(MultiRequirement,
interfaces.configuration.ConfigurableRequirementInterface,
metaclass = abc.ABCMeta):
class ComplexListRequirement(
MultiRequirement,
interfaces.configuration.ConfigurableRequirementInterface,
metaclass=abc.ABCMeta,
):
"""Allows a variable length list of requirements."""
def unsatisfied(self, context: interfaces.context.ContextInterface,
config_path: str) -> Dict[str, interfaces.configuration.RequirementInterface]:
def unsatisfied(
self, context: interfaces.context.ContextInterface, config_path: str
) -> Dict[str, interfaces.configuration.RequirementInterface]:
"""Validates the provided value to ensure it is one of the available
choices."""
config_path = interfaces.configuration.path_join(config_path, self.name)
ret_list = super().unsatisfied(context, config_path)
if ret_list:
return ret_list
if (self.config_value(context, config_path, None) is None
or self.config_value(context, interfaces.configuration.path_join(config_path, 'number_of_elements'))):
if self.config_value(context, config_path, None) is None or self.config_value(
context,
interfaces.configuration.path_join(config_path, "number_of_elements"),
):
return {config_path: self}
return {}
@@ -166,13 +229,17 @@ class ComplexListRequirement(MultiRequirement,
def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]:
# This is not optional for the stacker to run, so optional must be marked as False
return [
IntRequirement("number_of_elements",
description = "Determines how many layers are in this list",
optional = False)
IntRequirement(
"number_of_elements",
description="Determines how many layers are in this list",
optional=False,
)
]
@abc.abstractmethod
def construct(self, context: interfaces.context.ContextInterface, config_path: str) -> None:
def construct(
self, context: interfaces.context.ContextInterface, config_path: str
) -> None:
"""Method for constructing within the context any required elements
from subrequirements."""
@@ -180,17 +247,22 @@ class ComplexListRequirement(MultiRequirement,
def new_requirement(self, index) -> interfaces.configuration.RequirementInterface:
"""Builds a new requirement based on the specified index."""
def build_configuration(self, context: interfaces.context.ContextInterface, config_path: str,
_: Any) -> interfaces.configuration.HierarchicalDict:
def build_configuration(
self, context: interfaces.context.ContextInterface, config_path: str, _: Any
) -> interfaces.configuration.HierarchicalDict:
result = interfaces.configuration.HierarchicalDict()
num_elem_config_path = interfaces.configuration.path_join(config_path, self.name, 'number_of_elements')
num_elem_config_path = interfaces.configuration.path_join(
config_path, self.name, "number_of_elements"
)
num_elements = context.config.get(num_elem_config_path, None)
if num_elements is not None:
result["number_of_elements"] = num_elements
for i in range(num_elements):
req = self.new_requirement(i)
self.add_requirement(req)
value_path = interfaces.configuration.path_join(config_path, self.name, req.name)
value_path = interfaces.configuration.path_join(
config_path, self.name, req.name
)
value = context.config.get(value_path, None)
if value is not None:
result.splice(req.name, context.layers[value].build_configuration())
@@ -201,11 +273,15 @@ class ComplexListRequirement(MultiRequirement,
class LayerListRequirement(ComplexListRequirement):
"""Allows a variable length list of layers that must exist."""
def construct(self, context: interfaces.context.ContextInterface, config_path: str) -> None:
def construct(
self, context: interfaces.context.ContextInterface, config_path: str
) -> None:
"""Method for constructing within the context any required elements
from subrequirements."""
new_config_path = interfaces.configuration.path_join(config_path, self.name)
num_layers_path = interfaces.configuration.path_join(new_config_path, "number_of_elements")
num_layers_path = interfaces.configuration.path_join(
new_config_path, "number_of_elements"
)
number_of_layers = context.config[num_layers_path]
if not isinstance(number_of_layers, int):
@@ -214,28 +290,36 @@ class LayerListRequirement(ComplexListRequirement):
# Build all the layers that can be built
for i in range(number_of_layers):
layer_req = self.requirements.get(self.name + str(i), None)
if layer_req is not None and isinstance(layer_req, TranslationLayerRequirement):
if layer_req is not None and isinstance(
layer_req, TranslationLayerRequirement
):
layer_req.construct(context, new_config_path)
def new_requirement(self, index) -> interfaces.configuration.RequirementInterface:
"""Constructs a new requirement based on the specified index."""
return TranslationLayerRequirement(name = self.name + str(index),
description = "Layer for swap space",
optional = False)
return TranslationLayerRequirement(
name=self.name + str(index),
description="Layer for swap space",
optional=False,
)
class TranslationLayerRequirement(interfaces.configuration.ConstructableRequirementInterface,
interfaces.configuration.ConfigurableRequirementInterface):
class TranslationLayerRequirement(
interfaces.configuration.ConstructableRequirementInterface,
interfaces.configuration.ConfigurableRequirementInterface,
):
"""Class maintaining the limitations on what sort of translation layers are
acceptable."""
def __init__(self,
name: str,
description: str = None,
default: interfaces.configuration.ConfigSimpleType = None,
optional: bool = False,
oses: List = None,
architectures: List = None) -> None:
def __init__(
self,
name: str,
description: str = None,
default: interfaces.configuration.ConfigSimpleType = None,
optional: bool = False,
oses: List = None,
architectures: List = None,
) -> None:
"""Constructs a Translation Layer Requirement.
The configuration option's value will be the name of the layer once it exists in the store
@@ -256,28 +340,46 @@ class TranslationLayerRequirement(interfaces.configuration.ConstructableRequirem
self.architectures = architectures
super().__init__(name, description, default, optional)
def unsatisfied(self, context: interfaces.context.ContextInterface,
config_path: str) -> Dict[str, interfaces.configuration.RequirementInterface]:
def unsatisfied(
self, context: interfaces.context.ContextInterface, config_path: str
) -> Dict[str, interfaces.configuration.RequirementInterface]:
"""Validate that the value is a valid layer name and that the layer
adheres to the requirements."""
config_path = interfaces.configuration.path_join(config_path, self.name)
value = self.config_value(context, config_path, None)
if isinstance(value, str):
if value not in context.layers:
vollog.log(constants.LOGLEVEL_V, f"IndexError - Layer not found in memory space: {value}")
vollog.log(
constants.LOGLEVEL_V,
f"IndexError - Layer not found in memory space: {value}",
)
return {config_path: self}
if self.oses and context.layers[value].metadata.get('os', None) not in self.oses:
vollog.log(constants.LOGLEVEL_V, f"TypeError - Layer is not the required OS: {value}")
if (
self.oses
and context.layers[value].metadata.get("os", None) not in self.oses
):
vollog.log(
constants.LOGLEVEL_V,
f"TypeError - Layer is not the required OS: {value}",
)
return {config_path: self}
if (self.architectures
and context.layers[value].metadata.get('architecture', None) not in self.architectures):
vollog.log(constants.LOGLEVEL_V, f"TypeError - Layer is not the required Architecture: {value}")
if (
self.architectures
and context.layers[value].metadata.get("architecture", None)
not in self.architectures
):
vollog.log(
constants.LOGLEVEL_V,
f"TypeError - Layer is not the required Architecture: {value}",
)
return {config_path: self}
return {}
if value is not None:
vollog.log(constants.LOGLEVEL_V,
f"TypeError - Translation Layer Requirement only accepts string labels: {repr(value)}")
vollog.log(
constants.LOGLEVEL_V,
f"TypeError - Translation Layer Requirement only accepts string labels: {repr(value)}",
)
return {config_path: self}
# TODO: check that the space in the context lives up to the requirements for arch/os etc
@@ -285,10 +387,15 @@ class TranslationLayerRequirement(interfaces.configuration.ConstructableRequirem
### NOTE: This validate method has side effects (the dependencies can change)!!!
self._validate_class(context, interfaces.configuration.parent_path(config_path))
vollog.log(constants.LOGLEVEL_V, f"IndexError - No configuration provided: {config_path}")
vollog.log(
constants.LOGLEVEL_V,
f"IndexError - No configuration provided: {config_path}",
)
return {config_path: self}
def construct(self, context: interfaces.context.ContextInterface, config_path: str) -> None:
def construct(
self, context: interfaces.context.ContextInterface, config_path: str
) -> None:
"""Constructs the appropriate layer and adds it based on the class
parameter."""
config_path = interfaces.configuration.path_join(config_path, self.name)
@@ -303,8 +410,12 @@ class TranslationLayerRequirement(interfaces.configuration.ConstructableRequirem
args = {"context": context, "config_path": config_path, "name": name}
if any(
[subreq.unsatisfied(context, config_path) for subreq in self.requirements.values() if
not subreq.optional]):
[
subreq.unsatisfied(context, config_path)
for subreq in self.requirements.values()
if not subreq.optional
]
):
return None
obj = self._construct_class(context, config_path, args)
@@ -314,42 +425,57 @@ class TranslationLayerRequirement(interfaces.configuration.ConstructableRequirem
# context.config[config_path] = obj.name
return None
def build_configuration(self, context: interfaces.context.ContextInterface, _: str,
value: Any) -> interfaces.configuration.HierarchicalDict:
def build_configuration(
self, context: interfaces.context.ContextInterface, _: str, value: Any
) -> interfaces.configuration.HierarchicalDict:
"""Builds the appropriate configuration for the specified
requirement."""
return context.layers[value].build_configuration()
class SymbolTableRequirement(interfaces.configuration.ConstructableRequirementInterface,
interfaces.configuration.ConfigurableRequirementInterface):
class SymbolTableRequirement(
interfaces.configuration.ConstructableRequirementInterface,
interfaces.configuration.ConfigurableRequirementInterface,
):
"""Class maintaining the limitations on what sort of symbol spaces are
acceptable."""
def unsatisfied(self, context: interfaces.context.ContextInterface,
config_path: str) -> Dict[str, interfaces.configuration.RequirementInterface]:
def unsatisfied(
self, context: interfaces.context.ContextInterface, config_path: str
) -> Dict[str, interfaces.configuration.RequirementInterface]:
"""Validate that the value is a valid within the symbol space of the
provided context."""
config_path = interfaces.configuration.path_join(config_path, self.name)
value = self.config_value(context, config_path, None)
if not isinstance(value, str) and value is not None:
vollog.log(constants.LOGLEVEL_V,
f"TypeError - SymbolTableRequirement only accepts string labels: {repr(value)}")
vollog.log(
constants.LOGLEVEL_V,
f"TypeError - SymbolTableRequirement only accepts string labels: {repr(value)}",
)
return {config_path: self}
if value and value in context.symbol_space:
# This is an expected situation, so return rather than raise
return {}
elif value:
vollog.log(constants.LOGLEVEL_V, "IndexError - Value not present in the symbol space: {}".format(value
or ""))
vollog.log(
constants.LOGLEVEL_V,
"IndexError - Value not present in the symbol space: {}".format(
value or ""
),
)
### NOTE: This validate method has side effects (the dependencies can change)!!!
self._validate_class(context, interfaces.configuration.parent_path(config_path))
vollog.log(constants.LOGLEVEL_V, f"Symbol table requirement not yet fulfilled: {config_path}")
vollog.log(
constants.LOGLEVEL_V,
f"Symbol table requirement not yet fulfilled: {config_path}",
)
return {config_path: self}
def construct(self, context: interfaces.context.ContextInterface, config_path: str) -> None:
def construct(
self, context: interfaces.context.ContextInterface, config_path: str
) -> None:
"""Constructs the symbol space within the context based on the
subrequirements."""
config_path = interfaces.configuration.path_join(config_path, self.name)
@@ -359,14 +485,23 @@ class SymbolTableRequirement(interfaces.configuration.ConstructableRequirementIn
args = {"context": context, "config_path": config_path, "name": name}
if any(
[subreq.unsatisfied(context, config_path) for subreq in self.requirements.values() if
not subreq.optional]):
[
subreq.unsatisfied(context, config_path)
for subreq in self.requirements.values()
if not subreq.optional
]
):
return None
# Fill out the parameter for class creation
if not isinstance(self.requirements["class"], interfaces.configuration.ClassRequirement):
raise TypeError("Class requirement is not of type ClassRequirement: {}".format(
repr(self.requirements["class"])))
if not isinstance(
self.requirements["class"], interfaces.configuration.ClassRequirement
):
raise TypeError(
"Class requirement is not of type ClassRequirement: {}".format(
repr(self.requirements["class"])
)
)
cls = self.requirements["class"].cls
if cls is None:
return None
@@ -380,23 +515,27 @@ class SymbolTableRequirement(interfaces.configuration.ConstructableRequirementIn
context.symbol_space.append(obj)
return None
def build_configuration(self, context: interfaces.context.ContextInterface, _: str,
value: Any) -> interfaces.configuration.HierarchicalDict:
def build_configuration(
self, context: interfaces.context.ContextInterface, _: str, value: Any
) -> interfaces.configuration.HierarchicalDict:
"""Builds the appropriate configuration for the specified
requirement."""
return context.symbol_space[value].build_configuration()
class VersionRequirement(interfaces.configuration.RequirementInterface):
def __init__(self,
name: str,
description: str = None,
default: bool = False,
optional: bool = False,
component: Type[interfaces.configuration.VersionableInterface] = None,
version: Optional[Tuple[int, ...]] = None) -> None:
super().__init__(name = name, description = description, default = default, optional = optional)
def __init__(
self,
name: str,
description: str = None,
default: bool = False,
optional: bool = False,
component: Type[interfaces.configuration.VersionableInterface] = None,
version: Optional[Tuple[int, ...]] = None,
) -> None:
super().__init__(
name=name, description=description, default=default, optional=optional
)
if component is None:
raise TypeError("Component cannot be None")
self._component: Type[interfaces.configuration.VersionableInterface] = component
@@ -404,73 +543,111 @@ class VersionRequirement(interfaces.configuration.RequirementInterface):
raise TypeError("Version cannot be None")
self._version = version
def unsatisfied(self, context: interfaces.context.ContextInterface,
config_path: str) -> Dict[str, interfaces.configuration.RequirementInterface]:
def unsatisfied(
self, context: interfaces.context.ContextInterface, config_path: str
) -> Dict[str, interfaces.configuration.RequirementInterface]:
# Mypy doesn't appreciate our classproperty implementation, self._plugin.version has no type
config_path = interfaces.configuration.path_join(config_path, self.name)
if len(self._version) > 0 and self._component.version[0] != self._version[0]:
if not self.matches_required(self._version, self._component.version):
return {config_path: self}
if len(self._version) > 1 and self._component.version[1] < self._version[1]:
return {config_path: self}
context.config[interfaces.configuration.path_join(config_path, self.name)] = True
context.config[interfaces.configuration.path_join(config_path, self.name)] = (
True
)
return {}
@classmethod
def matches_required(
cls, required: Tuple[int, ...], version: Tuple[int, int, int]
) -> bool:
if len(required) > 0 and version[0] != required[0]:
return False
if len(required) > 1 and version[1] < required[1]:
return False
return True
class PluginRequirement(VersionRequirement):
def __init__(self,
name: str,
description: str = None,
default: bool = False,
optional: bool = False,
plugin: Type[interfaces.plugins.PluginInterface] = None,
version: Optional[Tuple[int, ...]] = None) -> None:
super().__init__(name = name,
description = description,
default = default,
optional = optional,
component = plugin,
version = version)
def __init__(
self,
name: str,
description: str = None,
default: bool = False,
optional: bool = False,
plugin: Type[interfaces.plugins.PluginInterface] = None,
version: Optional[Tuple[int, ...]] = None,
) -> None:
super().__init__(
name=name,
description=description,
default=default,
optional=optional,
component=plugin,
version=version,
)
class ModuleRequirement(interfaces.configuration.ConstructableRequirementInterface,
interfaces.configuration.ConfigurableRequirementInterface):
def __init__(self, name: str, description: str = None, default: bool = False,
architectures: Optional[List[str]] = None, optional: bool = False):
super().__init__(name = name, description = description, default = default, optional = optional)
self.add_requirement(TranslationLayerRequirement(name = 'layer_name', architectures = architectures))
self.add_requirement(SymbolTableRequirement(name = 'symbol_table_name'))
class ModuleRequirement(
interfaces.configuration.ConstructableRequirementInterface,
interfaces.configuration.ConfigurableRequirementInterface,
):
def __init__(
self,
name: str,
description: str = None,
default: bool = False,
architectures: Optional[List[str]] = None,
optional: bool = False,
):
super().__init__(
name=name, description=description, default=default, optional=optional
)
self.add_requirement(
TranslationLayerRequirement(name="layer_name", architectures=architectures)
)
self.add_requirement(SymbolTableRequirement(name="symbol_table_name"))
@classmethod
def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]:
return [
IntRequirement(name = 'offset'),
IntRequirement(name="offset"),
]
def unsatisfied(self, context: 'interfaces.context.ContextInterface',
config_path: str) -> Dict[str, interfaces.configuration.RequirementInterface]:
def unsatisfied(
self, context: "interfaces.context.ContextInterface", config_path: str
) -> Dict[str, interfaces.configuration.RequirementInterface]:
"""Validate that the value is a valid module"""
config_path = interfaces.configuration.path_join(config_path, self.name)
value = self.config_value(context, config_path, None)
if isinstance(value, str):
if value not in context.modules:
vollog.log(constants.LOGLEVEL_V, f"IndexError - Module not found in context: {value}")
vollog.log(
constants.LOGLEVEL_V,
f"IndexError - Module not found in context: {value}",
)
return {config_path: self}
return {}
if value is not None:
vollog.log(constants.LOGLEVEL_V,
"TypeError - Module Requirement only accepts string labels: {}".format(repr(value)))
vollog.log(
constants.LOGLEVEL_V,
"TypeError - Module Requirement only accepts string labels: {}".format(
repr(value)
),
)
return {config_path: self}
result = {}
for subreq in self._requirements:
req_unsatisfied = self._requirements[subreq].unsatisfied(context, config_path)
req_unsatisfied = self._requirements[subreq].unsatisfied(
context, config_path
)
if req_unsatisfied:
result.update(req_unsatisfied)
if not result:
vollog.log(constants.LOGLEVEL_V, f"IndexError - No configuration provided: {config_path}")
vollog.log(
constants.LOGLEVEL_V,
f"IndexError - No configuration provided: {config_path}",
)
result = {config_path: self}
### NOTE: This validate method has side effects (the dependencies can change)!!!
@@ -479,7 +656,9 @@ class ModuleRequirement(interfaces.configuration.ConstructableRequirementInterfa
return result
def construct(self, context: interfaces.context.ContextInterface, config_path: str) -> None:
def construct(
self, context: interfaces.context.ContextInterface, config_path: str
) -> None:
"""Constructs the appropriate layer and adds it based on the class parameter."""
config_path = interfaces.configuration.path_join(config_path, self.name)
@@ -493,8 +672,12 @@ class ModuleRequirement(interfaces.configuration.ConstructableRequirementInterfa
args = {"context": context, "config_path": config_path, "name": name}
if any(
[subreq.unsatisfied(context, config_path) for subreq in self.requirements.values() if
not subreq.optional]):
[
subreq.unsatisfied(context, config_path)
for subreq in self.requirements.values()
if not subreq.optional
]
):
return None
obj = self._construct_class(context, config_path, args)
@@ -504,8 +687,9 @@ class ModuleRequirement(interfaces.configuration.ConstructableRequirementInterfa
# context.config[config_path] = obj.name
return None
def build_configuration(self, context: 'interfaces.context.ContextInterface', _: str,
value: Any) -> interfaces.configuration.HierarchicalDict:
def build_configuration(
self, context: "interfaces.context.ContextInterface", _: str, value: Any
) -> interfaces.configuration.HierarchicalDict:
"""Builds the appropriate configuration for the specified
requirement."""
return context.modules[value].build_configuration()
volatility3/framework/constants/__init__.py
+55 -18
View File
@@ -9,6 +9,7 @@ volatility This includes default scanning block sizes, etc.
import enum
import os.path
import sys
import warnings
from typing import Callable, Optional
import volatility3.framework.constants.linux
@@ -16,39 +17,46 @@ import volatility3.framework.constants.windows
PLUGINS_PATH = [
os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "plugins")),
os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "plugins"))
os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "plugins")),
]
"""Default list of paths to load plugins from (volatility3/plugins and volatility3/framework/plugins)"""
SYMBOL_BASEPATHS = [
os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "symbols")),
os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "symbols"))
os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "symbols")),
]
"""Default list of paths to load symbols from (volatility3/symbols and volatility3/framework/symbols)"""
ISF_EXTENSIONS = ['.json', '.json.xz', '.json.gz', '.json.bz2']
ISF_EXTENSIONS = [".json", ".json.xz", ".json.gz", ".json.bz2"]
"""List of accepted extensions for ISF files"""
if hasattr(sys, 'frozen') and sys.frozen:
if hasattr(sys, "frozen") and sys.frozen:
# Ensure we include the executable's directory as the base for plugins and symbols
PLUGINS_PATH = [os.path.abspath(os.path.join(os.path.dirname(sys.executable), 'plugins'))] + PLUGINS_PATH
SYMBOL_BASEPATHS = [os.path.abspath(os.path.join(os.path.dirname(sys.executable), 'symbols'))] + SYMBOL_BASEPATHS
PLUGINS_PATH = [
os.path.abspath(os.path.join(os.path.dirname(sys.executable), "plugins"))
] + PLUGINS_PATH
SYMBOL_BASEPATHS = [
os.path.abspath(os.path.join(os.path.dirname(sys.executable), "symbols"))
] + SYMBOL_BASEPATHS
BANG = "!"
"""Constant used to delimit table names from type names when referring to a symbol"""
# We use the SemVer 2.0.0 versioning scheme
VERSION_MAJOR = 2 # Number of releases of the library with a breaking change
VERSION_MINOR = 2 # Number of changes that only add to the interface
VERSION_PATCH = 0 # Number of changes that do not change the interface
VERSION_MINOR = 5 # Number of changes that only add to the interface
VERSION_PATCH = 2 # Number of changes that do not change the interface
VERSION_SUFFIX = ""
# TODO: At version 2.0.0, remove the symbol_shift feature
PACKAGE_VERSION = ".".join([str(x) for x in [VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH]]) + VERSION_SUFFIX
PACKAGE_VERSION = (
".".join([str(x) for x in [VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH]])
+ VERSION_SUFFIX
)
"""The canonical version of the volatility3 package"""
AUTOMAGIC_CONFIG_PATH = 'automagic'
AUTOMAGIC_CONFIG_PATH = "automagic"
"""The root section within the context configuration for automagic values"""
LOGLEVEL_V = 9
@@ -63,27 +71,33 @@ LOGLEVEL_VVVV = 6
CACHE_PATH = os.path.join(os.path.expanduser("~"), ".cache", "volatility3")
"""Default path to store cached data"""
if sys.platform == 'win32':
CACHE_PATH = os.path.join(os.environ.get("APPDATA", os.path.expanduser("~")), "volatility3")
os.makedirs(CACHE_PATH, exist_ok = True)
SQLITE_CACHE_PERIOD = "-3 days"
"""SQLite time modifier for how long each item is valid in the cache for"""
LINUX_BANNERS_PATH = os.path.join(CACHE_PATH, "linux_banners.cache")
""""Default location to record information about available linux banners"""
if sys.platform == "win32":
CACHE_PATH = os.path.realpath(
os.path.join(os.environ.get("APPDATA", os.path.expanduser("~")), "volatility3")
)
os.makedirs(CACHE_PATH, exist_ok=True)
MAC_BANNERS_PATH = os.path.join(CACHE_PATH, "mac_banners.cache")
""""Default location to record information about available mac banners"""
IDENTIFIERS_FILENAME = "identifier.cache"
"""Default location to record information about available identifiers"""
CACHE_SQLITE_SCHEMA_VERSION = 1
"""Version for the sqlite3 cache schema"""
BUG_URL = "https://github.com/volatilityfoundation/volatility3/issues"
ProgressCallback = Optional[Callable[[float, str], None]]
"""Type information for ProgressCallback objects"""
OS_CATEGORIES = ['windows', 'mac', 'linux']
OS_CATEGORIES = ["windows", "mac", "linux"]
class Parallelism(enum.IntEnum):
"""An enumeration listing the different types of parallelism applied to
volatility."""
Off = 0
Threading = 1
Multiprocessing = 2
@@ -101,3 +115,26 @@ OFFLINE = False
REMOTE_ISF_URL = None # 'http://localhost:8000/banners.json'
"""Remote URL to query for a list of ISF addresses"""
###
# DEPRECATED VALUES
###
_deprecated_LINUX_BANNERS_FILENAME = os.path.join(CACHE_PATH, "linux_banners.cache")
"""This value is deprecated and is no longer used within volatility"""
_deprecated_MAC_BANNERS_PATH = os.path.join(CACHE_PATH, "mac_banners.cache")
"""This value is deprecated and is no longer used within volatility"""
_deprecated_IDENTIFIERS_PATH = os.path.join(CACHE_PATH, IDENTIFIERS_FILENAME)
"""This value is deprecated in favour of CACHE_PATH joined to IDENTIFIER_FILENAME"""
def __getattr__(name):
deprecated_tag = "_deprecated_"
if name in [
x[len(deprecated_tag) :] for x in globals() if x.startswith(deprecated_tag)
]:
warnings.warn(f"{name} is deprecated", FutureWarning)
return globals()[f"{deprecated_tag}{name}"]
return None
volatility3/framework/constants/linux/__init__.py
+268 -1
View File
@@ -13,4 +13,271 @@ PAGE_SHIFT = 12
"""The value hard coded from the Linux Kernel (hence not extracted from the layer itself)"""
# include/linux/sched.h
PF_KTHREAD = 0x00200000 # I'm a kernel thread
PF_KTHREAD = 0x00200000 # I'm a kernel thread
# Standard well-defined IP protocols.
# ref: include/uapi/linux/in.h
IP_PROTOCOLS = {
0: "IP",
1: "ICMP",
2: "IGMP",
4: "IPIP",
6: "TCP",
8: "EGP",
12: "PUP",
17: "UDP",
22: "IDP",
29: "TP",
33: "DCCP",
41: "IPV6",
46: "RSVP",
47: "GRE",
50: "ESP",
51: "AH",
92: "MTP",
94: "BEETPH",
98: "ENCAP",
103: "PIM",
108: "COMP",
132: "SCTP",
136: "UDPLITE",
137: "MPLS",
143: "ETHERNET",
255: "RAW",
262: "MPTCP",
}
# IPV6 extension headers
# ref: include/uapi/linux/in6.h
IPV6_PROTOCOLS = {
0: "HOPBYHOP_OPTS",
43: "ROUTING",
44: "FRAGMENT",
58: "ICMPv6",
59: "NO_NEXT",
60: "DESTINATION_OPTS",
135: "MOBILITY",
}
# ref: include/net/tcp_states.h
TCP_STATES = (
"",
"ESTABLISHED",
"SYN_SENT",
"SYN_RECV",
"FIN_WAIT1",
"FIN_WAIT2",
"TIME_WAIT",
"CLOSE",
"CLOSE_WAIT",
"LAST_ACK",
"LISTEN",
"CLOSING",
"TCP_NEW_SYN_RECV",
)
# ref: include/linux/net.h (socket_type enum)
SOCK_TYPES = {
1: "STREAM",
2: "DGRAM",
3: "RAW",
4: "RDM",
5: "SEQPACKET",
6: "DCCP",
10: "PACKET",
}
# Address families
# ref: include/linux/socket.h
SOCK_FAMILY = (
"AF_UNSPEC",
"AF_UNIX",
"AF_INET",
"AF_AX25",
"AF_IPX",
"AF_APPLETALK",
"AF_NETROM",
"AF_BRIDGE",
"AF_ATMPVC",
"AF_X25",
"AF_INET6",
"AF_ROSE",
"AF_DECnet",
"AF_NETBEUI",
"AF_SECURITY",
"AF_KEY",
"AF_NETLINK",
"AF_PACKET",
"AF_ASH",
"AF_ECONET",
"AF_ATMSVC",
"AF_RDS",
"AF_SNA",
"AF_IRDA",
"AF_PPPOX",
"AF_WANPIPE",
"AF_LLC",
"AF_IB",
"AF_MPLS",
"AF_CAN",
"AF_TIPC",
"AF_BLUETOOTH",
"AF_IUCV",
"AF_RXRPC",
"AF_ISDN",
"AF_PHONET",
"AF_IEEE802154",
"AF_CAIF",
"AF_ALG",
"AF_NFC",
"AF_VSOCK",
"AF_KCM",
"AF_QIPCRTR",
"AF_SMC",
"AF_XDP",
)
# Socket states
# ref: include/uapi/linux/net.h
SOCKET_STATES = ("FREE", "UNCONNECTED", "CONNECTING", "CONNECTED", "DISCONNECTING")
# Netlink protocols
# ref: include/uapi/linux/netlink.h
NETLINK_PROTOCOLS = (
"NETLINK_ROUTE",
"NETLINK_UNUSED",
"NETLINK_USERSOCK",
"NETLINK_FIREWALL",
"NETLINK_SOCK_DIAG",
"NETLINK_NFLOG",
"NETLINK_XFRM",
"NETLINK_SELINUX",
"NETLINK_ISCSI",
"NETLINK_AUDIT",
"NETLINK_FIB_LOOKUP",
"NETLINK_CONNECTOR",
"NETLINK_NETFILTER",
"NETLINK_IP6_FW",
"NETLINK_DNRTMSG",
"NETLINK_KOBJECT_UEVENT",
"NETLINK_GENERIC",
"NETLINK_DM",
"NETLINK_SCSITRANSPORT",
"NETLINK_ECRYPTFS",
"NETLINK_RDMA",
"NETLINK_CRYPTO",
"NETLINK_SMC",
)
# Short list of Ethernet Protocol ID's.
# ref: include/uapi/linux/if_ether.h
# Used in AF_PACKET socket family
ETH_PROTOCOLS = {
0x0001: "ETH_P_802_3",
0x0002: "ETH_P_AX25",
0x0003: "ETH_P_ALL",
0x0004: "ETH_P_802_2",
0x0005: "ETH_P_SNAP",
0x0006: "ETH_P_DDCMP",
0x0007: "ETH_P_WAN_PPP",
0x0008: "ETH_P_PPP_MP",
0x0009: "ETH_P_LOCALTALK",
0x000C: "ETH_P_CAN",
0x000F: "ETH_P_CANFD",
0x0010: "ETH_P_PPPTALK",
0x0011: "ETH_P_TR_802_2",
0x0016: "ETH_P_CONTROL",
0x0017: "ETH_P_IRDA",
0x0018: "ETH_P_ECONET",
0x0019: "ETH_P_HDLC",
0x001A: "ETH_P_ARCNET",
0x001B: "ETH_P_DSA",
0x001C: "ETH_P_TRAILER",
0x0060: "ETH_P_LOOP",
0x00F6: "ETH_P_IEEE802154",
0x00F7: "ETH_P_CAIF",
0x00F8: "ETH_P_XDSA",
0x00F9: "ETH_P_MAP",
0x0800: "ETH_P_IP",
0x0805: "ETH_P_X25",
0x0806: "ETH_P_ARP",
0x8035: "ETH_P_RARP",
0x809B: "ETH_P_ATALK",
0x80F3: "ETH_P_AARP",
0x8100: "ETH_P_8021Q",
}
# Connection and socket states
# ref: include/net/bluetooth/bluetooth.h
BLUETOOTH_STATES = (
"",
"CONNECTED",
"OPEN",
"BOUND",
"LISTEN",
"CONNECT",
"CONNECT2",
"CONFIG",
"DISCONN",
"CLOSED",
)
# Bluetooth protocols
# ref: include/net/bluetooth/bluetooth.h
BLUETOOTH_PROTOCOLS = (
"L2CAP",
"HCI",
"SCO",
"RFCOMM",
"BNEP",
"CMTP",
"HIDP",
"AVDTP",
)
# Ref: include/uapi/linux/capability.h
CAPABILITIES = (
"chown",
"dac_override",
"dac_read_search",
"fowner",
"fsetid",
"kill",
"setgid",
"setuid",
"setpcap",
"linux_immutable",
"net_bind_service",
"net_broadcast",
"net_admin",
"net_raw",
"ipc_lock",
"ipc_owner",
"sys_module",
"sys_rawio",
"sys_chroot",
"sys_ptrace",
"sys_pacct",
"sys_admin",
"sys_boot",
"sys_nice",
"sys_resource",
"sys_time",
"sys_tty_config",
"mknod",
"lease",
"audit_write",
"audit_control",
"setfcap",
"mac_override",
"mac_admin",
"syslog",
"wake_alarm",
"block_suspend",
"audit_read",
"perfmon",
"bpf",
"checkpoint_restore",
)
ELF_MAX_EXTRACTION_SIZE = 1024 * 1024 * 1024 * 4 - 1
volatility3/framework/contexts/__init__.py
+166 -100
View File
@@ -87,12 +87,14 @@ class Context(interfaces.context.ContextInterface):
# ## Object Factory Functions
def object(self,
object_type: Union[str, interfaces.objects.Template],
layer_name: str,
offset: int,
native_layer_name: Optional[str] = None,
**arguments) -> interfaces.objects.ObjectInterface:
def object(
self,
object_type: Union[str, interfaces.objects.Template],
layer_name: str,
offset: int,
native_layer_name: Optional[str] = None,
**arguments,
) -> interfaces.objects.ObjectInterface:
"""Object factory, takes a context, symbol, offset and optional
layername.
@@ -122,18 +124,24 @@ class Context(interfaces.context.ContextInterface):
object_template = object_template.clone()
object_template.update_vol(**arguments)
return object_template(context = self,
object_info = interfaces.objects.ObjectInformation(layer_name = layer_name,
offset = offset,
native_layer_name = native_layer_name,
size = object_template.size))
return object_template(
context=self,
object_info=interfaces.objects.ObjectInformation(
layer_name=layer_name,
offset=offset,
native_layer_name=native_layer_name,
size=object_template.size,
),
)
def module(self,
module_name: str,
layer_name: str,
offset: int,
native_layer_name: Optional[str] = None,
size: Optional[int] = None) -> interfaces.context.ModuleInterface:
def module(
self,
module_name: str,
layer_name: str,
offset: int,
native_layer_name: Optional[str] = None,
size: Optional[int] = None,
) -> interfaces.context.ModuleInterface:
"""Constructs a new os-independent module.
Args:
@@ -144,17 +152,21 @@ class Context(interfaces.context.ContextInterface):
size: The size, in bytes, that the module occupies from offset location within the layer named layer_name
"""
if size:
return SizedModule.create(self,
module_name = module_name,
layer_name = layer_name,
offset = offset,
size = size,
native_layer_name = native_layer_name)
return Module.create(self,
module_name = module_name,
layer_name = layer_name,
offset = offset,
native_layer_name = native_layer_name)
return SizedModule.create(
self,
module_name=module_name,
layer_name=layer_name,
offset=offset,
size=size,
native_layer_name=native_layer_name,
)
return Module.create(
self,
module_name=module_name,
layer_name=layer_name,
offset=offset,
native_layer_name=native_layer_name,
)
def get_module_wrapper(method: str) -> Callable:
@@ -169,7 +181,13 @@ def get_module_wrapper(method: str) -> Callable:
raise ValueError(f"Cannot reference another module when calling {method}")
return getattr(self._context.symbol_space, method)(name)
for entry in ['__annotations__', '__doc__', '__module__', '__name__', '__qualname__']:
for entry in [
"__annotations__",
"__doc__",
"__module__",
"__name__",
"__qualname__",
]:
proxy_interface = getattr(interfaces.context.ModuleInterface, method)
if hasattr(proxy_interface, entry):
setattr(wrapper, entry, getattr(proxy_interface, entry))
@@ -178,26 +196,27 @@ def get_module_wrapper(method: str) -> Callable:
class Module(interfaces.context.ModuleInterface):
@classmethod
def create(cls,
context: interfaces.context.ContextInterface,
module_name: str,
layer_name: str,
offset: int,
**kwargs) -> 'Module':
def create(
cls,
context: interfaces.context.ContextInterface,
module_name: str,
layer_name: str,
offset: int,
**kwargs,
) -> "Module":
pathjoin = interfaces.configuration.path_join
# Check if config_path is None
free_module_name = context.modules.free_module_name(module_name)
config_path = kwargs.get('config_path', None)
config_path = kwargs.get("config_path", None)
if config_path is None:
config_path = pathjoin('temporary', 'modules', free_module_name)
config_path = pathjoin("temporary", "modules", free_module_name)
# Populate the configuration
context.config[pathjoin(config_path, 'layer_name')] = layer_name
context.config[pathjoin(config_path, 'offset')] = offset
context.config[pathjoin(config_path, "layer_name")] = layer_name
context.config[pathjoin(config_path, "offset")] = offset
# This is important, since the module_name may be changed in case it is already in use
if 'symbol_table_name' not in kwargs:
kwargs['symbol_table_name'] = module_name
if "symbol_table_name" not in kwargs:
kwargs["symbol_table_name"] = module_name
for arg in kwargs:
context.config[pathjoin(config_path, arg)] = kwargs.get(arg, None)
# Construct the object
@@ -207,12 +226,14 @@ class Module(interfaces.context.ModuleInterface):
# Add the module to the context modules collection
return return_val
def object(self,
object_type: str,
offset: int = None,
native_layer_name: Optional[str] = None,
absolute: bool = False,
**kwargs) -> 'interfaces.objects.ObjectInterface':
def object(
self,
object_type: str,
offset: int = None,
native_layer_name: Optional[str] = None,
absolute: bool = False,
**kwargs,
) -> "interfaces.objects.ObjectInterface":
"""Returns an object created using the symbol_table_name and layer_name
of the Module.
@@ -225,7 +246,9 @@ class Module(interfaces.context.ModuleInterface):
if constants.BANG not in object_type:
object_type = self.symbol_table_name + constants.BANG + object_type
else:
raise ValueError("Cannot reference another module when constructing an object")
raise ValueError(
"Cannot reference another module when constructing an object"
)
if offset is None:
raise TypeError("Offset must not be None for non-symbol objects")
@@ -234,19 +257,24 @@ class Module(interfaces.context.ModuleInterface):
offset += self._offset
# Ensure we don't use a layer_name other than the module's, why would anyone do that?
if 'layer_name' in kwargs:
del kwargs['layer_name']
return self._context.object(object_type = object_type,
layer_name = self._layer_name,
offset = offset,
native_layer_name = native_layer_name or self._native_layer_name,
**kwargs)
if "layer_name" in kwargs:
del kwargs["layer_name"]
return self._context.object(
object_type=object_type,
layer_name=self._layer_name,
offset=offset,
native_layer_name=native_layer_name or self._native_layer_name,
**kwargs,
)
def object_from_symbol(self,
symbol_name: str,
native_layer_name: Optional[str] = None,
absolute: bool = False,
**kwargs) -> 'interfaces.objects.ObjectInterface':
def object_from_symbol(
self,
symbol_name: str,
native_layer_name: Optional[str] = None,
absolute: bool = False,
object_type: Optional[Union[str, "interfaces.objects.ObjectInterface"]] = None,
**kwargs,
) -> "interfaces.objects.ObjectInterface":
"""Returns an object based on a specific symbol (containing type and
offset information) and the layer_name of the Module. This will throw
a ValueError if the symbol does not contain an associated type, or if
@@ -257,11 +285,14 @@ class Module(interfaces.context.ModuleInterface):
symbol_name: Name of the symbol (within the module) to construct
native_layer_name: Name of the layer in which constructed objects are made (for pointers)
absolute: whether the symbol's address is absolute or relative to the module
object_type: Override for the type from the symobl to use (or if the symbol type is missing)
"""
if constants.BANG not in symbol_name:
symbol_name = self.symbol_table_name + constants.BANG + symbol_name
else:
raise ValueError("Cannot reference another module when constructing an object")
raise ValueError(
"Cannot reference another module when constructing an object"
)
# Only set the offset if type is Symbol and we were given a name, not a template
symbol_val = self._context.symbol_space.get_symbol(symbol_name)
@@ -270,19 +301,26 @@ class Module(interfaces.context.ModuleInterface):
if not absolute:
offset += self._offset
if symbol_val.type is None:
raise TypeError(f"Symbol {symbol_val.name} has no associated type")
if object_type is None:
if symbol_val.type is None:
raise TypeError(
f"Symbol {symbol_val.name} has no associated type and no object_type specified"
)
else:
object_type = symbol_val.type
# Ensure we don't use a layer_name other than the module's, why would anyone do that?
if 'layer_name' in kwargs:
del kwargs['layer_name']
if "layer_name" in kwargs:
del kwargs["layer_name"]
# Since type may be a template, we don't just call our own module method
return self._context.object(object_type = symbol_val.type,
layer_name = self._layer_name,
offset = offset,
native_layer_name = native_layer_name or self._native_layer_name,
**kwargs)
return self._context.object(
object_type=object_type,
layer_name=self._layer_name,
offset=offset,
native_layer_name=native_layer_name or self._native_layer_name,
**kwargs,
)
def get_symbols_by_absolute_location(self, offset: int, size: int = 0) -> List[str]:
"""Returns the symbols within this module that live at the specified
@@ -290,28 +328,30 @@ class Module(interfaces.context.ModuleInterface):
if size < 0:
raise ValueError("Size must be strictly non-negative")
return list(
self._context.symbol_space.get_symbols_by_location(offset = offset - self._offset,
size = size,
table_name = self.symbol_table_name))
self._context.symbol_space.get_symbols_by_location(
offset=offset - self._offset,
size=size,
table_name=self.symbol_table_name,
)
)
@property
def symbols(self):
return self.context.symbol_space[self.symbol_table_name].symbols
get_symbol = get_module_wrapper('get_symbol')
get_type = get_module_wrapper('get_type')
get_enumeration = get_module_wrapper('get_enumeration')
has_symbol = get_module_wrapper('has_symbol')
has_type = get_module_wrapper('has_type')
has_enumeration = get_module_wrapper('has_enumeration')
get_symbol = get_module_wrapper("get_symbol")
get_type = get_module_wrapper("get_type")
get_enumeration = get_module_wrapper("get_enumeration")
has_symbol = get_module_wrapper("has_symbol")
has_type = get_module_wrapper("has_type")
has_enumeration = get_module_wrapper("has_enumeration")
class SizedModule(Module):
@property
def size(self) -> int:
"""Returns the size of the module (0 for unknown size)"""
size = self.config.get('size', 0)
size = self.config.get("size", 0)
return size or 0
@property # type: ignore # FIXME: mypy #5107
@@ -326,8 +366,12 @@ class SizedModule(Module):
layer = self._context.layers[self.layer_name]
if not isinstance(layer, interfaces.layers.TranslationLayerInterface):
raise TypeError("Hashing modules on non-TranslationLayers is not allowed")
return hashlib.md5(bytes(str(list(layer.mapping(self.offset, self.size, ignore_errors = True))),
'utf-8')).hexdigest()
return hashlib.md5(
bytes(
str(list(layer.mapping(self.offset, self.size, ignore_errors=True))),
"utf-8",
)
).hexdigest()
def get_symbols_by_absolute_location(self, offset: int, size: int = 0) -> List[str]:
"""Returns the symbols within this module that live at the specified
@@ -341,10 +385,13 @@ class ModuleCollection(interfaces.context.ModuleContainer):
"""Class to contain a collection of SizedModules and reason about their
contents."""
def __init__(self, modules: Optional[List[interfaces.context.ModuleInterface]] = None) -> None:
def __init__(
self, modules: Optional[List[interfaces.context.ModuleInterface]] = None
) -> None:
self._prefix_count = {}
super().__init__(modules)
def deduplicate(self) -> 'ModuleCollection':
def deduplicate(self) -> "ModuleCollection":
"""Returns a new deduplicated ModuleCollection featuring no repeated
modules (based on data hash)
@@ -361,20 +408,27 @@ class ModuleCollection(interfaces.context.ModuleContainer):
def free_module_name(self, prefix: str = "module") -> str:
"""Returns an unused module name"""
count = 1
if prefix not in self._prefix_count:
self._prefix_count[prefix] = 1
return prefix
count = self._prefix_count[prefix]
while prefix + str(count) in self:
count += 1
self._prefix_count[prefix] = count
return prefix + str(count)
@property
def modules(self) -> 'ModuleCollection':
def modules(self) -> "ModuleCollection":
"""A name indexed dictionary of modules using that name in this
collection."""
vollog.warning(
"This method has been deprecated in favour of the ModuleCollection acting as a dictionary itself")
"This method has been deprecated in favour of the ModuleCollection acting as a dictionary itself"
)
return self
def get_module_symbols_by_absolute_location(self, offset: int, size: int = 0) -> Iterable[Tuple[str, List[str]]]:
def get_module_symbols_by_absolute_location(
self, offset: int, size: int = 0
) -> Iterable[Tuple[str, List[str]]]:
"""Returns a tuple of (module_name, list_of_symbol_names) for each
module, where symbols live at the absolute offset in memory
provided."""
@@ -383,16 +437,28 @@ class ModuleCollection(interfaces.context.ModuleContainer):
for module_name in self._modules:
module = self._modules[module_name]
if isinstance(module, SizedModule):
if (offset <= module.offset + module.size) and (offset + size >= module.offset):
yield (module.name, module.get_symbols_by_absolute_location(offset, size))
if (offset <= module.offset + module.size) and (
offset + size >= module.offset
):
yield (
module.name,
module.get_symbols_by_absolute_location(offset, size),
)
class ConfigurableModule(Module, interfaces.configuration.ConfigurableInterface):
def __init__(self, context: interfaces.context.ContextInterface, config_path: str, name: str) -> None:
interfaces.configuration.ConfigurableInterface.__init__(self, context, config_path)
layer_name = self.config['layer_name']
offset = self.config['offset']
symbol_table_name = self.config['symbol_table_name']
interfaces.configuration.ConfigurableInterface.__init__(self, context, config_path)
Module.__init__(self, context, name, layer_name, offset, symbol_table_name, layer_name)
def __init__(
self, context: interfaces.context.ContextInterface, config_path: str, name: str
) -> None:
interfaces.configuration.ConfigurableInterface.__init__(
self, context, config_path
)
layer_name = self.config["layer_name"]
offset = self.config["offset"]
symbol_table_name = self.config["symbol_table_name"]
interfaces.configuration.ConfigurableInterface.__init__(
self, context, config_path
)
Module.__init__(
self, context, name, layer_name, offset, symbol_table_name, layer_name
)
volatility3/framework/exceptions.py
+24 -8
View File
@@ -30,7 +30,9 @@ class PluginRequirementException(VolatilityException):
class SymbolError(VolatilityException):
"""Thrown when a symbol lookup has failed."""
def __init__(self, symbol_name: Optional[str], table_name: Optional[str], *args) -> None:
def __init__(
self, symbol_name: Optional[str], table_name: Optional[str], *args
) -> None:
super().__init__(*args)
self.symbol_name = symbol_name
self.table_name = table_name
@@ -63,7 +65,14 @@ class PagedInvalidAddressException(InvalidAddressException):
that are invalid
"""
def __init__(self, layer_name: str, invalid_address: int, invalid_bits: int, entry: int, *args) -> None:
def __init__(
self,
layer_name: str,
invalid_address: int,
invalid_bits: int,
entry: int,
*args,
) -> None:
super().__init__(layer_name, invalid_address, *args)
self.invalid_bits = invalid_bits
self.entry = entry
@@ -77,8 +86,15 @@ class SwappedInvalidAddressException(PagedInvalidAddressException):
the lookup that were invalid.
"""
def __init__(self, layer_name: str, invalid_address: int, invalid_bits: int, entry: int, swap_offset: int,
*args) -> None:
def __init__(
self,
layer_name: str,
invalid_address: int,
invalid_bits: int,
entry: int,
swap_offset: int,
*args,
) -> None:
super().__init__(layer_name, invalid_address, invalid_bits, entry, *args)
self.swap_offset = swap_offset
@@ -88,14 +104,14 @@ class SymbolSpaceError(VolatilityException):
class UnsatisfiedException(VolatilityException):
def __init__(self, unsatisfied: Dict[str, interfaces.configuration.RequirementInterface]) -> None:
def __init__(
self, unsatisfied: Dict[str, interfaces.configuration.RequirementInterface]
) -> None:
super().__init__()
self.unsatisfied = unsatisfied
class MissingModuleException(VolatilityException):
def __init__(self, module: str, *args) -> None:
super().__init__(*args)
self.module = module
@@ -109,4 +125,4 @@ class OfflineException(VolatilityException):
self._url = url
def __str__(self):
return f'Volatility 3 is offline: unable to access {self._url}'
return f"Volatility 3 is offline: unable to access {self._url}"
volatility3/framework/interfaces/__init__.py
+10 -2
View File
@@ -12,5 +12,13 @@ components of volatility to write plugins.
# Import the submodules we want people to be able to use without importing them themselves
# This will also avoid namespace issues, because people can use interfaces.layers to
# avoid clashing with the layers package
from volatility3.framework.interfaces import renderers, configuration, context, layers, objects, plugins, symbols, \
automagic
from volatility3.framework.interfaces import (
renderers,
configuration,
context,
layers,
objects,
plugins,
symbols,
automagic,
)
volatility3/framework/interfaces/automagic.py
+56 -27
View File
@@ -9,15 +9,17 @@ that a user has not filled.
"""
import logging
from abc import ABCMeta
from typing import Any, List, Optional, Tuple, Union, Type
from typing import Any, List, Optional, Tuple, Type, Union
from volatility3.framework import interfaces, constants
from volatility3.framework import constants, interfaces
from volatility3.framework.configuration import requirements
vollog = logging.getLogger(__name__)
class AutomagicInterface(interfaces.configuration.ConfigurableInterface, metaclass = ABCMeta):
class AutomagicInterface(
interfaces.configuration.ConfigurableInterface, metaclass=ABCMeta
):
"""Class that defines an automagic component that can help fulfill
`Requirements`
@@ -43,32 +45,52 @@ class AutomagicInterface(interfaces.configuration.ConfigurableInterface, metacla
exclusion_list = []
"""A list of plugin categories (typically operating systems) which the plugin will not operate on"""
def __init__(self, context: interfaces.context.ContextInterface, config_path: str, *args, **kwargs) -> None:
def __init__(
self,
context: interfaces.context.ContextInterface,
config_path: str,
*args,
**kwargs
) -> None:
super().__init__(context, config_path)
for requirement in self.get_requirements():
if not isinstance(requirement, (interfaces.configuration.SimpleTypeRequirement,
requirements.ChoiceRequirement, requirements.ListRequirement)):
if not isinstance(
requirement,
(
interfaces.configuration.SimpleTypeRequirement,
requirements.ChoiceRequirement,
requirements.ListRequirement,
requirements.VersionRequirement,
),
):
raise TypeError(
"Automagic requirements must be a SimpleTypeRequirement, ChoiceRequirement or ListRequirement")
"Automagic requirements must be a SimpleTypeRequirement, ChoiceRequirement, ListRequirement or VersionRequirement"
)
def __call__(self,
context: interfaces.context.ContextInterface,
config_path: str,
requirement: interfaces.configuration.RequirementInterface,
progress_callback: constants.ProgressCallback = None) -> Optional[List[Any]]:
def __call__(
self,
context: interfaces.context.ContextInterface,
config_path: str,
requirement: interfaces.configuration.RequirementInterface,
progress_callback: constants.ProgressCallback = None,
) -> Optional[List[Any]]:
"""Runs the automagic over the configurable."""
return []
# TODO: requirement_type can be made UnionType[Type[T], Tuple[Type[T], ...]]
# once mypy properly supports Tuples in instance
def find_requirements(self,
context: interfaces.context.ContextInterface,
config_path: str,
requirement_root: interfaces.configuration.RequirementInterface,
requirement_type: Union[Tuple[Type[interfaces.configuration.RequirementInterface], ...],
Type[interfaces.configuration.RequirementInterface]],
shortcut: bool = True) -> List[Tuple[str, interfaces.configuration.RequirementInterface]]:
def find_requirements(
self,
context: interfaces.context.ContextInterface,
config_path: str,
requirement_root: interfaces.configuration.RequirementInterface,
requirement_type: Union[
Tuple[Type[interfaces.configuration.RequirementInterface], ...],
Type[interfaces.configuration.RequirementInterface],
],
shortcut: bool = True,
) -> List[Tuple[str, interfaces.configuration.RequirementInterface]]:
"""Determines if there is actually an unfulfilled `Requirement`
waiting.
@@ -84,7 +106,9 @@ class AutomagicInterface(interfaces.configuration.ConfigurableInterface, metacla
Returns:
A list of tuples containing the config_path, sub_config_path and requirement identifying the unsatisfied `Requirements`
"""
sub_config_path = interfaces.configuration.path_join(config_path, requirement_root.name)
sub_config_path = interfaces.configuration.path_join(
config_path, requirement_root.name
)
results: List[Tuple[str, interfaces.configuration.RequirementInterface]] = []
recurse = not shortcut
if isinstance(requirement_root, requirement_type):
@@ -94,11 +118,13 @@ class AutomagicInterface(interfaces.configuration.ConfigurableInterface, metacla
recurse = True
if recurse:
for subreq in requirement_root.requirements.values():
results += self.find_requirements(context, sub_config_path, subreq, requirement_type, shortcut)
results += self.find_requirements(
context, sub_config_path, subreq, requirement_type, shortcut
)
return results
class StackerLayerInterface(metaclass = ABCMeta):
class StackerLayerInterface(metaclass=ABCMeta):
"""Class that takes a lower layer and attempts to build on it.
stack_order determines the order (from low to high) that stacking
@@ -112,10 +138,12 @@ class StackerLayerInterface(metaclass = ABCMeta):
"""The list operating systems/first-level plugin hierarchy that should exclude this stacker"""
@classmethod
def stack(self,
context: interfaces.context.ContextInterface,
layer_name: str,
progress_callback: constants.ProgressCallback = None) -> Optional[interfaces.layers.DataLayerInterface]:
def stack(
cls,
context: interfaces.context.ContextInterface,
layer_name: str,
progress_callback: constants.ProgressCallback = None,
) -> Optional[interfaces.layers.DataLayerInterface]:
"""Method to determine whether this builder can operate on the named
layer. If so, modify the context appropriately.
@@ -134,4 +162,5 @@ class StackerLayerInterface(metaclass = ABCMeta):
@classmethod
def stacker_slow_warning(cls):
vollog.warning(
"Reads to this layer are slow, it's recommended to use the layerwriter plugin once to produce a raw file")
"Reads to this layer are slow, it's recommended to use the layerwriter plugin once to produce a raw file"
)
volatility3/framework/interfaces/configuration.py
+142 -68
View File
@@ -23,7 +23,19 @@ import random
import string
import sys
from abc import ABCMeta, abstractmethod
from typing import Any, ClassVar, Dict, Generator, Iterator, List, Optional, Type, Union, Tuple, Set
from typing import (
Any,
ClassVar,
Dict,
Generator,
Iterator,
List,
Optional,
Type,
Union,
Tuple,
Set,
)
from volatility3 import classproperty, framework
from volatility3.framework import constants, interfaces
@@ -68,9 +80,11 @@ class HierarchicalDict(collections.abc.Mapping):
"""The core of configuration data, it is a mapping class that stores keys
within itself, and also stores lower hierarchies."""
def __init__(self,
initial_dict: Dict[str, 'SimpleTypeRequirement'] = None,
separator: str = CONFIG_SEPARATOR) -> None:
def __init__(
self,
initial_dict: Dict[str, "SimpleTypeRequirement"] = None,
separator: str = CONFIG_SEPARATOR,
) -> None:
"""
Args:
initial_dict: A dictionary to populate the HierarchicalDict with initially
@@ -80,7 +94,7 @@ class HierarchicalDict(collections.abc.Mapping):
raise TypeError(f"Separator must be a one character string: {separator}")
self._separator = separator
self._data: Dict[str, ConfigSimpleType] = {}
self._subdict: Dict[str, 'HierarchicalDict'] = {}
self._subdict: Dict[str, "HierarchicalDict"] = {}
if isinstance(initial_dict, str):
initial_dict = json.loads(initial_dict)
if isinstance(initial_dict, dict):
@@ -88,7 +102,8 @@ class HierarchicalDict(collections.abc.Mapping):
self[k] = v
elif initial_dict is not None:
raise TypeError(
f"Initial_dict must be a dictionary or JSON string containing a dictionary: {initial_dict}")
f"Initial_dict must be a dictionary or JSON string containing a dictionary: {initial_dict}"
)
def __eq__(self, other):
"""Define equality between HierarchicalDicts"""
@@ -109,7 +124,7 @@ class HierarchicalDict(collections.abc.Mapping):
"""Returns the first division of a key based on the dict separator, or
the full key if the separator is not present."""
if self.separator in key:
return key[:key.index(self.separator)]
return key[: key.index(self.separator)]
else:
return key
@@ -117,8 +132,8 @@ class HierarchicalDict(collections.abc.Mapping):
"""Returns all but the first division of a key based on the dict
separator, or None if the separator is not in the key."""
if self.separator in key:
return key[key.index(self.separator) + 1:]
return ''
return key[key.index(self.separator) + 1 :]
return ""
def __iter__(self) -> Iterator[Any]:
"""Returns an iterator object that supports the iterator protocol."""
@@ -156,7 +171,9 @@ class HierarchicalDict(collections.abc.Mapping):
def _setitem(self, key: str, value: Any, is_data: bool = True) -> None:
"""Set an item or appends a whole subtree at a key location."""
if self.separator in key:
subdict = self._subdict.get(self._key_head(key), HierarchicalDict(separator = self.separator))
subdict = self._subdict.get(
self._key_head(key), HierarchicalDict(separator=self.separator)
)
subdict._setitem(self._key_tail(key), value, is_data)
self._subdict[self._key_head(key)] = subdict
else:
@@ -166,7 +183,9 @@ class HierarchicalDict(collections.abc.Mapping):
if not isinstance(value, HierarchicalDict):
raise TypeError(
"HierarchicalDicts can only store HierarchicalDicts within their structure: {}".format(
type(value)))
type(value)
)
)
self._subdict[key] = value
def _sanitize_value(self, value: Any) -> ConfigSimpleType:
@@ -185,7 +204,9 @@ class HierarchicalDict(collections.abc.Mapping):
for element in value:
element_value = self._sanitize_value(element)
if isinstance(element_value, list):
raise TypeError("Configuration list types cannot contain list types")
raise TypeError(
"Configuration list types cannot contain list types"
)
if element_value is not None:
new_list.append(element_value)
return new_list
@@ -220,7 +241,7 @@ class HierarchicalDict(collections.abc.Mapping):
"""Returns the length of all items."""
return len(self._data) + sum([len(subdict) for subdict in self._subdict])
def branch(self, key: str) -> 'HierarchicalDict':
def branch(self, key: str) -> "HierarchicalDict":
"""Returns the HierarchicalDict housed under the key.
This differs from the data property, in that it is directed by the `key`, and all layers under that key are
@@ -241,10 +262,12 @@ class HierarchicalDict(collections.abc.Mapping):
else:
return self._subdict[key]
except KeyError:
self._setitem(key = key, value = HierarchicalDict(separator = self.separator), is_data = False)
self._setitem(
key=key, value=HierarchicalDict(separator=self.separator), is_data=False
)
return HierarchicalDict()
def splice(self, key: str, value: 'HierarchicalDict') -> None:
def splice(self, key: str, value: "HierarchicalDict") -> None:
"""Splices an existing HierarchicalDictionary under a specific key.
This can be thought of as an inverse of :func:`branch`, although
@@ -255,7 +278,9 @@ class HierarchicalDict(collections.abc.Mapping):
raise TypeError("Splice requires a string key and HierarchicalDict value")
self._setitem(key, value, False)
def merge(self, key: str, value: 'HierarchicalDict', overwrite: bool = False) -> None:
def merge(
self, key: str, value: "HierarchicalDict", overwrite: bool = False
) -> None:
"""Acts similarly to splice, but maintains previous values.
If overwrite is true, then entries in the new value are used over those that exist within key already
@@ -274,7 +299,7 @@ class HierarchicalDict(collections.abc.Mapping):
else:
self[key + self._separator + item] = value[item]
def clone(self) -> 'HierarchicalDict':
def clone(self) -> "HierarchicalDict":
"""Duplicates the configuration, allowing changes without affecting the
original.
@@ -285,10 +310,12 @@ class HierarchicalDict(collections.abc.Mapping):
def __str__(self) -> str:
"""Turns the Hierarchical dict into a string representation."""
return json.dumps(dict([(key, self[key]) for key in sorted(self.generator())]), indent = 2)
return json.dumps(
dict([(key, self[key]) for key in sorted(self.generator())]), indent=2
)
class RequirementInterface(metaclass = ABCMeta):
class RequirementInterface(metaclass=ABCMeta):
"""Class that defines a requirement.
A requirement is a means for plugins and other framework components to request specific configuration data.
@@ -300,11 +327,13 @@ class RequirementInterface(metaclass = ABCMeta):
as :class:`TranslationLayerRequirement`, :class:`SymbolTableRequirement` and :class:`ClassRequirement`
"""
def __init__(self,
name: str,
description: str = None,
default: ConfigSimpleType = None,
optional: bool = False) -> None:
def __init__(
self,
name: str,
description: str = None,
default: ConfigSimpleType = None,
optional: bool = False,
) -> None:
"""
Args:
@@ -315,7 +344,9 @@ class RequirementInterface(metaclass = ABCMeta):
"""
super().__init__()
if CONFIG_SEPARATOR in name:
raise ValueError(f"Name cannot contain the config-hierarchy divider ({CONFIG_SEPARATOR})")
raise ValueError(
f"Name cannot contain the config-hierarchy divider ({CONFIG_SEPARATOR})"
)
self._name = name
self._description = description or ""
self._default = default
@@ -363,10 +394,12 @@ class RequirementInterface(metaclass = ABCMeta):
"""Sets the optional value for a requirement."""
self._optional = bool(value)
def config_value(self,
context: 'interfaces.context.ContextInterface',
config_path: str,
default: ConfigSimpleType = None) -> ConfigSimpleType:
def config_value(
self,
context: "interfaces.context.ContextInterface",
config_path: str,
default: ConfigSimpleType = None,
) -> ConfigSimpleType:
"""Returns the value for this Requirement from its config path.
Args:
@@ -378,12 +411,12 @@ class RequirementInterface(metaclass = ABCMeta):
# Child operations
@property
def requirements(self) -> Dict[str, 'RequirementInterface']:
def requirements(self) -> Dict[str, "RequirementInterface"]:
"""Returns a dictionary of all the child requirements, indexed by
name."""
return self._requirements.copy()
def add_requirement(self, requirement: 'RequirementInterface') -> None:
def add_requirement(self, requirement: "RequirementInterface") -> None:
"""Adds a child to the list of requirements.
Args:
@@ -391,7 +424,7 @@ class RequirementInterface(metaclass = ABCMeta):
"""
self._requirements[requirement.name] = requirement
def remove_requirement(self, requirement: 'RequirementInterface') -> None:
def remove_requirement(self, requirement: "RequirementInterface") -> None:
"""Removes a child from the list of requirements.
Args:
@@ -399,8 +432,9 @@ class RequirementInterface(metaclass = ABCMeta):
"""
del self._requirements[requirement.name]
def unsatisfied_children(self, context: 'interfaces.context.ContextInterface',
config_path: str) -> Dict[str, 'RequirementInterface']:
def unsatisfied_children(
self, context: "interfaces.context.ContextInterface", config_path: str
) -> Dict[str, "RequirementInterface"]:
"""Method that will validate all child requirements.
Args:
@@ -413,14 +447,17 @@ class RequirementInterface(metaclass = ABCMeta):
result = {}
for requirement in self.requirements.values():
if not requirement.optional:
subresult = requirement.unsatisfied(context, path_join(config_path, self._name))
subresult = requirement.unsatisfied(
context, path_join(config_path, self._name)
)
result.update(subresult)
return result
# Validation routines
@abstractmethod
def unsatisfied(self, context: 'interfaces.context.ContextInterface',
config_path: str) -> Dict[str, 'RequirementInterface']:
def unsatisfied(
self, context: "interfaces.context.ContextInterface", config_path: str
) -> Dict[str, "RequirementInterface"]:
"""Method to validate the value stored at config_path for the
configuration object against a context.
@@ -438,6 +475,7 @@ class RequirementInterface(metaclass = ABCMeta):
class SimpleTypeRequirement(RequirementInterface):
"""Class to represent a single simple type (such as a boolean, a string, an
integer or a series of bytes)"""
instance_type: ClassVar[Type] = bool
def add_requirement(self, requirement: RequirementInterface):
@@ -450,8 +488,9 @@ class SimpleTypeRequirement(RequirementInterface):
children."""
raise TypeError("Instance Requirements cannot have subrequirements")
def unsatisfied(self, context: 'interfaces.context.ContextInterface',
config_path: str) -> Dict[str, RequirementInterface]:
def unsatisfied(
self, context: "interfaces.context.ContextInterface", config_path: str
) -> Dict[str, RequirementInterface]:
"""Validates the instance requirement based upon its
`instance_type`."""
config_path = path_join(config_path, self.name)
@@ -460,8 +499,10 @@ class SimpleTypeRequirement(RequirementInterface):
if not isinstance(value, self.instance_type):
vollog.log(
constants.LOGLEVEL_V,
"TypeError - {} requirements only accept {} type: {}".format(self.name, self.instance_type.__name__,
repr(value)))
"TypeError - {} requirements only accept {} type: {}".format(
self.name, self.instance_type.__name__, repr(value)
),
)
return {config_path: self}
return {}
@@ -489,8 +530,9 @@ class ClassRequirement(RequirementInterface):
class name."""
return self._cls
def unsatisfied(self, context: 'interfaces.context.ContextInterface',
config_path: str) -> Dict[str, RequirementInterface]:
def unsatisfied(
self, context: "interfaces.context.ContextInterface", config_path: str
) -> Dict[str, RequirementInterface]:
"""Checks to see if a class can be recovered."""
config_path = path_join(config_path, self.name)
@@ -499,8 +541,8 @@ class ClassRequirement(RequirementInterface):
if value is not None and isinstance(value, str):
if "." in value:
# TODO: consider importing the prefix
module = sys.modules.get(value[:value.rindex(".")], None)
class_name = value[value.rindex(".") + 1:]
module = sys.modules.get(value[: value.rindex(".")], None)
class_name = value[value.rindex(".") + 1 :]
if hasattr(module, class_name):
self._cls = getattr(module, class_name)
else:
@@ -523,12 +565,14 @@ class ConstructableRequirementInterface(RequirementInterface):
must happen after the class configuration value has been provided).
These values are then provided to the object's constructor by name
as arguments (as well as the standard `context` and `config_path`
arguments.
arguments).
"""
def __init__(self, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
self.add_requirement(ClassRequirement("class", "Class of the constructable requirement"))
self.add_requirement(
ClassRequirement("class", "Class of the constructable requirement")
)
self._current_class_requirements: Set[Any] = set()
def __eq__(self, other):
@@ -537,7 +581,9 @@ class ConstructableRequirementInterface(RequirementInterface):
return super().__eq__(other)
@abstractmethod
def construct(self, context: 'interfaces.context.ContextInterface', config_path: str) -> None:
def construct(
self, context: "interfaces.context.ContextInterface", config_path: str
) -> None:
"""Method for constructing within the context any required elements
from subrequirements.
@@ -546,7 +592,9 @@ class ConstructableRequirementInterface(RequirementInterface):
config_path: The configuration path for the specific instance of this constructable
"""
def _validate_class(self, context: 'interfaces.context.ContextInterface', config_path: str) -> None:
def _validate_class(
self, context: "interfaces.context.ContextInterface", config_path: str
) -> None:
"""Method to check if the class Requirement is valid and if so populate
the other requirements (but no need to validate, since we're invalid
already)
@@ -555,9 +603,11 @@ class ConstructableRequirementInterface(RequirementInterface):
context: The context object containing the configuration data for the constructable
config_path: The configuration path for the specific instance of this constructable
"""
class_req = self.requirements['class']
class_req = self.requirements["class"]
subreq_config_path = path_join(config_path, self.name)
if not class_req.unsatisfied(context, subreq_config_path) and isinstance(class_req, ClassRequirement):
if not class_req.unsatisfied(context, subreq_config_path) and isinstance(
class_req, ClassRequirement
):
# We have a class, and since it's validated we can construct our requirements from it
if issubclass(class_req.cls, ConfigurableInterface):
# In case the class has changed, clear out the old requirements
@@ -569,10 +619,12 @@ class ConstructableRequirementInterface(RequirementInterface):
self._current_class_requirements.add(requirement.name)
self.add_requirement(requirement)
def _construct_class(self,
context: 'interfaces.context.ContextInterface',
config_path: str,
requirement_dict: Dict[str, object] = None) -> Optional['interfaces.objects.ObjectInterface']:
def _construct_class(
self,
context: "interfaces.context.ContextInterface",
config_path: str,
requirement_dict: Dict[str, object] = None,
) -> Optional["interfaces.objects.ObjectInterface"]:
"""Constructs the class, handing args and the subrequirements as
parameters to __init__"""
if self.requirements["class"].unsatisfied(context, config_path):
@@ -605,16 +657,22 @@ class ConstructableRequirementInterface(RequirementInterface):
class ConfigurableRequirementInterface(RequirementInterface):
"""Simple Abstract class to provide build_required_config."""
def build_configuration(self, context: 'interfaces.context.ContextInterface', config_path: str,
value: Any) -> HierarchicalDict:
def build_configuration(
self,
context: "interfaces.context.ContextInterface",
config_path: str,
value: Any,
) -> HierarchicalDict:
"""Proxies to a ConfigurableInterface if necessary."""
class ConfigurableInterface(metaclass = ABCMeta):
class ConfigurableInterface(metaclass=ABCMeta):
"""Class to allow objects to have requirements and read configuration data
from the context config tree."""
def __init__(self, context: 'interfaces.context.ContextInterface', config_path: str) -> None:
def __init__(
self, context: "interfaces.context.ContextInterface", config_path: str
) -> None:
"""Basic initializer that allows configurables to access their own
config settings."""
super().__init__()
@@ -623,7 +681,7 @@ class ConfigurableInterface(metaclass = ABCMeta):
self._config_cache: Optional[HierarchicalDict] = None
@property
def context(self) -> 'interfaces.context.ContextInterface':
def context(self) -> "interfaces.context.ContextInterface":
"""The context object that this configurable belongs to/configuration
is stored in."""
return self._context
@@ -660,11 +718,16 @@ class ConfigurableInterface(metaclass = ABCMeta):
for req in self.get_requirements():
value = self.config.get(req.name, None)
# Do not include the name of constructed classes
if value is not None and not isinstance(req, ConstructableRequirementInterface):
if value is not None and not isinstance(
req, ConstructableRequirementInterface
):
result[req.name] = value
if isinstance(req, ConfigurableRequirementInterface):
if value is not None:
result.splice(req.name, req.build_configuration(self.context, self.config_path, value))
result.splice(
req.name,
req.build_configuration(self.context, self.config_path, value),
)
return result
@classmethod
@@ -674,8 +737,9 @@ class ConfigurableInterface(metaclass = ABCMeta):
return []
@classmethod
def unsatisfied(cls, context: 'interfaces.context.ContextInterface',
config_path: str) -> Dict[str, RequirementInterface]:
def unsatisfied(
cls, context: "interfaces.context.ContextInterface", config_path: str
) -> Dict[str, RequirementInterface]:
"""Returns a list of the names of all unsatisfied requirements.
Since a satisfied set of requirements will return [], it can be used in tests as follows:
@@ -694,7 +758,12 @@ class ConfigurableInterface(metaclass = ABCMeta):
return result
@classmethod
def make_subconfig(cls, context: 'interfaces.context.ContextInterface', base_config_path: str, **kwargs) -> str:
def make_subconfig(
cls,
context: "interfaces.context.ContextInterface",
base_config_path: str,
**kwargs,
) -> str:
"""Convenience function to allow constructing a new randomly generated
sub-configuration path, containing each element from kwargs.
@@ -706,8 +775,10 @@ class ConfigurableInterface(metaclass = ABCMeta):
Returns:
str: The newly generated full configuration path
"""
random_config_dict = ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.digits)
for _ in range(8))
random_config_dict = "".join(
random.SystemRandom().choice(string.ascii_uppercase + string.digits)
for _ in range(8)
)
new_config_path = path_join(base_config_path, random_config_dict)
# TODO: Check that the new_config_path is empty, although it's not critical if it's not since the values are merged in
@@ -716,7 +787,9 @@ class ConfigurableInterface(metaclass = ABCMeta):
# constructor anyway, however, to prevent bad types getting into the config tree we just verify that v is a simple type
for k, v in kwargs.items():
if not isinstance(v, (int, str, bool, float, bytes)):
raise TypeError("Config values passed to make_subconfig can only be simple types")
raise TypeError(
"Config values passed to make_subconfig can only be simple types"
)
context.config[path_join(new_config_path, k)] = v
return new_config_path
@@ -729,6 +802,7 @@ class VersionableInterface:
All version number should use semantic versioning
"""
_version: Tuple[int, int, int] = (0, 0, 0)
_required_framework_version: Tuple[int, int, int] = (0, 0, 0)
volatility3/framework/interfaces/context.py
+66 -49
View File
@@ -19,7 +19,7 @@ from typing import Optional, Union, Dict, List, Iterable
from volatility3.framework import interfaces, exceptions
class ContextInterface(metaclass = ABCMeta):
class ContextInterface(metaclass=ABCMeta):
"""All context-like objects must adhere to the following interface.
This interface is present to avoid import dependency cycles.
@@ -32,12 +32,12 @@ class ContextInterface(metaclass = ABCMeta):
@property
@abstractmethod
def config(self) -> 'interfaces.configuration.HierarchicalDict':
def config(self) -> "interfaces.configuration.HierarchicalDict":
"""Returns the configuration object for this context."""
@property
@abstractmethod
def symbol_space(self) -> 'interfaces.symbols.SymbolSpaceInterface':
def symbol_space(self) -> "interfaces.symbols.SymbolSpaceInterface":
"""Returns the symbol_space for the context.
This object must support the :class:`~volatility3.framework.interfaces.symbols.SymbolSpaceInterface`
@@ -47,11 +47,11 @@ class ContextInterface(metaclass = ABCMeta):
@property
@abstractmethod
def modules(self) -> 'ModuleContainer':
def modules(self) -> "ModuleContainer":
"""Returns the memory object for the context."""
raise NotImplementedError("ModuleContainer has not been implemented.")
def add_module(self, module: 'interfaces.context.ModuleInterface'):
def add_module(self, module: "interfaces.context.ModuleInterface"):
"""Adds a named module to the context.
Args:
@@ -65,11 +65,11 @@ class ContextInterface(metaclass = ABCMeta):
@property
@abstractmethod
def layers(self) -> 'interfaces.layers.LayerContainer':
def layers(self) -> "interfaces.layers.LayerContainer":
"""Returns the memory object for the context."""
raise NotImplementedError("LayerContainer has not been implemented.")
def add_layer(self, layer: 'interfaces.layers.DataLayerInterface'):
def add_layer(self, layer: "interfaces.layers.DataLayerInterface"):
"""Adds a named translation layer to the context memory.
Args:
@@ -80,12 +80,14 @@ class ContextInterface(metaclass = ABCMeta):
# ## Object Factory Functions
@abstractmethod
def object(self,
object_type: Union[str, 'interfaces.objects.Template'],
layer_name: str,
offset: int,
native_layer_name: str = None,
**arguments):
def object(
self,
object_type: Union[str, "interfaces.objects.Template"],
layer_name: str,
offset: int,
native_layer_name: str = None,
**arguments,
) -> "interfaces.objects.ObjectInterface":
"""Object factory, takes a context, symbol, offset and optional
layer_name.
@@ -102,7 +104,7 @@ class ContextInterface(metaclass = ABCMeta):
A fully constructed object
"""
def clone(self) -> 'ContextInterface':
def clone(self) -> "ContextInterface":
"""Produce a clone of the context (and configuration), allowing
modifications to be made without affecting any mutable objects in the
original.
@@ -112,12 +114,14 @@ class ContextInterface(metaclass = ABCMeta):
"""
return copy.deepcopy(self)
def module(self,
module_name: str,
layer_name: str,
offset: int,
native_layer_name: Optional[str] = None,
size: Optional[int] = None) -> 'ModuleInterface':
def module(
self,
module_name: str,
layer_name: str,
offset: int,
native_layer_name: Optional[str] = None,
size: Optional[int] = None,
) -> "ModuleInterface":
"""Create a module object.
A module object is associated with a symbol table, and acts like a context, but offsets locations by a known value
@@ -142,10 +146,7 @@ class ModuleInterface(interfaces.configuration.ConfigurableInterface):
This object is OS-independent.
"""
def __init__(self,
context: ContextInterface,
config_path: str,
name: str) -> None:
def __init__(self, context: ContextInterface, config_path: str, name: str) -> None:
"""Constructs a new os-independent module.
Args:
@@ -158,35 +159,43 @@ class ModuleInterface(interfaces.configuration.ConfigurableInterface):
@property
def _layer_name(self) -> str:
return self.config['layer_name']
return self.config["layer_name"]
@property
def _offset(self) -> int:
return self.config['offset']
return self.config["offset"]
@property
def _native_layer_name(self) -> str:
return self.config.get('native_layer_name', self._layer_name)
return self.config.get("native_layer_name", self._layer_name)
@property
def _symbol_table_name(self) -> str:
return self.config.get('symbol_table_name', self._module_name)
return self.config.get("symbol_table_name", self._module_name)
def build_configuration(self) -> 'interfaces.configuration.HierarchicalDict':
def build_configuration(self) -> "interfaces.configuration.HierarchicalDict":
"""Builds the configuration dictionary for this specific Module"""
config = super().build_configuration()
config['offset'] = self.config['offset']
subconfigs = {'symbol_table_name': self.context.symbol_space[self.symbol_table_name].build_configuration(),
'layer_name': self.context.layers[self.layer_name].build_configuration()}
config["offset"] = self.config["offset"]
subconfigs = {
"symbol_table_name": self.context.symbol_space[
self.symbol_table_name
].build_configuration(),
"layer_name": self.context.layers[self.layer_name].build_configuration(),
}
if self.layer_name != self._native_layer_name:
subconfigs['native_layer_name'] = self.context.layers[self._native_layer_name].build_configuration()
subconfigs["native_layer_name"] = self.context.layers[
self._native_layer_name
].build_configuration()
for subconfig in subconfigs:
for req in subconfigs[subconfig]:
config[interfaces.configuration.path_join(subconfig, req)] = subconfigs[subconfig][req]
config[interfaces.configuration.path_join(subconfig, req)] = subconfigs[
subconfig
][req]
return config
@@ -217,12 +226,14 @@ class ModuleInterface(interfaces.configuration.ConfigurableInterface):
return self._symbol_table_name
@abstractmethod
def object(self,
object_type: str,
offset: int = None,
native_layer_name: Optional[str] = None,
absolute: bool = False,
**kwargs) -> 'interfaces.objects.ObjectInterface':
def object(
self,
object_type: str,
offset: int = None,
native_layer_name: Optional[str] = None,
absolute: bool = False,
**kwargs,
) -> "interfaces.objects.ObjectInterface":
"""Returns an object created using the symbol_table_name and layer_name
of the Module.
@@ -237,11 +248,14 @@ class ModuleInterface(interfaces.configuration.ConfigurableInterface):
"""
@abstractmethod
def object_from_symbol(self,
symbol_name: str,
native_layer_name: Optional[str] = None,
absolute: bool = False,
**kwargs) -> 'interfaces.objects.ObjectInterface':
def object_from_symbol(
self,
symbol_name: str,
native_layer_name: Optional[str] = None,
absolute: bool = False,
object_type: Optional[Union[str, "interfaces.objects.ObjectInterface"]] = None,
**kwargs,
) -> "interfaces.objects.ObjectInterface":
"""Returns an object created using the symbol_table_name and layer_name
of the Module.
@@ -249,6 +263,7 @@ class ModuleInterface(interfaces.configuration.ConfigurableInterface):
symbol_name: The name of a symbol (that must be present in the module's symbol table). The symbol's associated type will be used to construct an object at the symbol's offset.
native_layer_name: The native layer for objects that reference a different layer (if not the default provided during module construction)
absolute: A boolean specifying whether the offset is absolute within the layer, or relative to the start of the module
object_type: Override for the type from the symobl to use (or if the symbol type is missing)
Returns:
The constructed object
@@ -259,13 +274,13 @@ class ModuleInterface(interfaces.configuration.ConfigurableInterface):
symbol = self.get_symbol(name)
return self.offset + symbol.address
def get_type(self, name: str) -> 'interfaces.objects.Template':
def get_type(self, name: str) -> "interfaces.objects.Template":
"""Returns a type from the module's symbol table."""
def get_symbol(self, name: str) -> 'interfaces.symbols.SymbolInterface':
def get_symbol(self, name: str) -> "interfaces.symbols.SymbolInterface":
"""Returns a symbol object from the module's symbol table."""
def get_enumeration(self, name: str) -> 'interfaces.objects.Template':
def get_enumeration(self, name: str) -> "interfaces.objects.Template":
"""Returns an enumeration from the module's symbol table."""
def has_type(self, name: str) -> bool:
@@ -306,7 +321,9 @@ class ModuleContainer(collections.abc.Mapping):
module: the module to add to the list of modules (based on module.name)
"""
if module.name in self._modules:
raise exceptions.VolatilityException(f"Module already exists: {module.name}")
raise exceptions.VolatilityException(
f"Module already exists: {module.name}"
)
self._modules[module.name] = module
def __delitem__(self, name: str) -> None:
volatility3/framework/interfaces/layers.py
+170 -76
View File
@@ -22,11 +22,13 @@ from volatility3.framework import constants, exceptions, interfaces
vollog = logging.getLogger(__name__)
ProgressValue = Union['DummyProgress', multiprocessing.managers.ValueProxy]
ProgressValue = Union["DummyProgress", multiprocessing.managers.ValueProxy]
IteratorValue = Tuple[List[Tuple[str, int, int]], int]
class ScannerInterface(interfaces.configuration.VersionableInterface, metaclass = ABCMeta):
class ScannerInterface(
interfaces.configuration.VersionableInterface, metaclass=ABCMeta
):
"""Class for layer scanners that return locations of particular values from
within the data.
@@ -52,6 +54,7 @@ class ScannerInterface(interfaces.configuration.VersionableInterface, metaclass
in either their own class or the context. This will allow the scanner to be run
in parallel against multiple blocks.
"""
thread_safe = False
_required_framework_version = (2, 0, 0)
@@ -64,11 +67,11 @@ class ScannerInterface(interfaces.configuration.VersionableInterface, metaclass
self._layer_name: Optional[str] = None
@property
def context(self) -> Optional['interfaces.context.ContextInterface']:
def context(self) -> Optional["interfaces.context.ContextInterface"]:
return self._context
@context.setter
def context(self, ctx: 'interfaces.context.ContextInterface') -> None:
def context(self, ctx: "interfaces.context.ContextInterface") -> None:
"""Stores the context locally in case the scanner needs to access the
layer."""
self._context = ctx
@@ -94,20 +97,24 @@ class ScannerInterface(interfaces.configuration.VersionableInterface, metaclass
"""
class DataLayerInterface(interfaces.configuration.ConfigurableInterface, metaclass = ABCMeta):
class DataLayerInterface(
interfaces.configuration.ConfigurableInterface, metaclass=ABCMeta
):
"""A Layer that directly holds data (and does not translate it).
This is effectively a leaf node in a layer tree. It directly
accesses a data source and exposes it within volatility.
"""
_direct_metadata: Mapping = {'architecture': 'Unknown', 'os': 'Unknown'}
_direct_metadata: Mapping = {"architecture": "Unknown", "os": "Unknown"}
def __init__(self,
context: 'interfaces.context.ContextInterface',
config_path: str,
name: str,
metadata: Optional[Dict[str, Any]] = None) -> None:
def __init__(
self,
context: "interfaces.context.ContextInterface",
config_path: str,
name: str,
metadata: Optional[Dict[str, Any]] = None,
) -> None:
super().__init__(context, config_path)
self._name = name
self._metadata = metadata or {}
@@ -199,11 +206,13 @@ class DataLayerInterface(interfaces.configuration.ConfigurableInterface, metacla
# ## General scanning methods
def scan(self,
context: interfaces.context.ContextInterface,
scanner: ScannerInterface,
progress_callback: constants.ProgressCallback = None,
sections: Iterable[Tuple[int, int]] = None) -> Iterable[Any]:
def scan(
self,
context: interfaces.context.ContextInterface,
scanner: ScannerInterface,
progress_callback: constants.ProgressCallback = None,
sections: Iterable[Tuple[int, int]] = None,
) -> Iterable[Any]:
"""Scans a Translation layer by chunk.
Note: this will skip missing/unmappable chunks of memory
@@ -224,7 +233,9 @@ class DataLayerInterface(interfaces.configuration.ConfigurableInterface, metacla
scanner.layer_name = self.name
if sections is None:
sections = [(self.minimum_address, self.maximum_address - self.minimum_address)]
sections = [
(self.minimum_address, self.maximum_address - self.minimum_address)
]
sections = list(self._coalesce_sections(sections))
@@ -232,13 +243,18 @@ class DataLayerInterface(interfaces.configuration.ConfigurableInterface, metacla
progress: ProgressValue = DummyProgress()
scan_iterator = functools.partial(self._scan_iterator, scanner, sections)
scan_metric = self._scan_metric(scanner, sections)
if not scanner.thread_safe or constants.PARALLELISM == constants.Parallelism.Off:
if (
not scanner.thread_safe
or constants.PARALLELISM == constants.Parallelism.Off
):
progress = DummyProgress()
scan_chunk = functools.partial(self._scan_chunk, scanner, progress)
for value in scan_iterator():
if progress_callback:
progress_callback(scan_metric(progress.value),
f"Scanning {self.name} using {scanner.__class__.__name__}")
progress_callback(
scan_metric(progress.value),
f"Scanning {self.name} using {scanner.__class__.__name__}",
)
yield from scan_chunk(value)
else:
progress = multiprocessing.Manager().Value("Q", 0)
@@ -252,8 +268,10 @@ class DataLayerInterface(interfaces.configuration.ConfigurableInterface, metacla
while not result.ready():
if progress_callback:
# Run the progress_callback
progress_callback(scan_metric(progress.value),
f"Scanning {self.name} using {scanner.__class__.__name__}")
progress_callback(
scan_metric(progress.value),
f"Scanning {self.name} using {scanner.__class__.__name__}",
)
# Ensures we don't burn CPU cycles going round in a ready waiting loop
# without delaying the user too long between progress updates/results
result.wait(0.1)
@@ -262,15 +280,21 @@ class DataLayerInterface(interfaces.configuration.ConfigurableInterface, metacla
except Exception as e:
# We don't care the kind of exception, so catch and report on everything, yielding nothing further
vollog.debug(f"Scan Failure: {str(e)}")
vollog.log(constants.LOGLEVEL_VVV,
"\n".join(traceback.TracebackException.from_exception(e).format(chain = True)))
vollog.log(
constants.LOGLEVEL_VVV,
"\n".join(
traceback.TracebackException.from_exception(e).format(chain=True)
),
)
def _coalesce_sections(self, sections: Iterable[Tuple[int, int]]) -> Iterable[Tuple[int, int]]:
def _coalesce_sections(
self, sections: Iterable[Tuple[int, int]]
) -> Iterable[Tuple[int, int]]:
"""Take a list of (start, length) sections and coalesce any adjacent
sections."""
result: List[Tuple[int, int]] = []
position = 0
for (start, length) in sorted(sections):
for start, length in sorted(sections):
if result and start <= position:
initial_start, _ = result.pop()
result.append((initial_start, (start + length) - initial_start))
@@ -283,7 +307,10 @@ class DataLayerInterface(interfaces.configuration.ConfigurableInterface, metacla
if first_start + first_length < self.minimum_address:
result = result[1:]
elif first_start < self.minimum_address:
result[0] = (self.minimum_address, (first_start + first_length) - self.minimum_address)
result[0] = (
self.minimum_address,
(first_start + first_length) - self.minimum_address,
)
while result and result[-1] > (self.maximum_address, 0):
last_start, last_length = result[-1]
if last_start > self.maximum_address:
@@ -292,8 +319,9 @@ class DataLayerInterface(interfaces.configuration.ConfigurableInterface, metacla
result[1] = (last_start, self.maximum_address - last_start)
return result
def _scan_iterator(self, scanner: 'ScannerInterface', sections: Iterable[Tuple[int,
int]]) -> Iterable[IteratorValue]:
def _scan_iterator(
self, scanner: "ScannerInterface", sections: Iterable[Tuple[int, int]]
) -> Iterable[IteratorValue]:
"""Iterator that indicates which blocks in the layer are to be read by
for the scanning.
@@ -303,11 +331,16 @@ class DataLayerInterface(interfaces.configuration.ConfigurableInterface, metacla
assumed to have no holes
"""
for section_start, section_length in sections:
offset, mapped_offset, length, layer_name = section_start, section_start, section_length, self.name
offset, mapped_offset, length, layer_name = (
section_start,
section_start,
section_length,
self.name,
)
while length > 0:
chunk_size = min(length, scanner.chunk_size + scanner.overlap)
yield [(layer_name, mapped_offset, chunk_size)], offset + chunk_size
# It we've got more than the scanner's chunk_size, only move up by the chunk_size
# If we've got more than the scanner's chunk_size, only move up by the chunk_size
if chunk_size > scanner.chunk_size:
chunk_size -= scanner.overlap
length -= chunk_size
@@ -315,16 +348,23 @@ class DataLayerInterface(interfaces.configuration.ConfigurableInterface, metacla
offset += chunk_size
# We ignore the type due to the iterator_value, actually it only needs to match the output from _scan_iterator
def _scan_chunk(self, scanner: 'ScannerInterface', progress: 'ProgressValue',
iterator_value: IteratorValue) -> List[Any]:
def _scan_chunk(
self,
scanner: "ScannerInterface",
progress: "ProgressValue",
iterator_value: IteratorValue,
) -> List[Any]:
data_to_scan, chunk_end = iterator_value
data = b''
data = b""
for layer_name, address, chunk_size in data_to_scan:
try:
data += self.context.layers[layer_name].read(address, chunk_size)
except exceptions.InvalidAddressException:
vollog.debug("Invalid address in layer {} found scanning {} at address {:x}".format(
layer_name, self.name, address))
vollog.debug(
"Invalid address in layer {} found scanning {} at address {:x}".format(
layer_name, self.name, address
)
)
if len(data) > scanner.chunk_size + scanner.overlap:
vollog.debug(f"Scan chunk too large: {hex(len(data))}")
@@ -332,8 +372,9 @@ class DataLayerInterface(interfaces.configuration.ConfigurableInterface, metacla
progress.value = chunk_end
return list(scanner(data, chunk_end - len(data)))
def _scan_metric(self, _scanner: 'ScannerInterface', sections: List[Tuple[int, int]]) -> Callable[[int], float]:
def _scan_metric(
self, _scanner: "ScannerInterface", sections: List[Tuple[int, int]]
) -> Callable[[int], float]:
if not sections:
raise ValueError("Sections have no size, nothing to scan")
last_section, last_length = sections[-1]
@@ -357,11 +398,15 @@ class DataLayerInterface(interfaces.configuration.ConfigurableInterface, metacla
@property
def metadata(self) -> Mapping:
"""Returns a ReadOnly copy of the metadata published by this layer."""
maps = [self.context.layers[layer_name].metadata for layer_name in self.dependencies]
return interfaces.objects.ReadOnlyMapping(collections.ChainMap(self._metadata, self._direct_metadata, *maps))
maps = [
self.context.layers[layer_name].metadata for layer_name in self.dependencies
]
return interfaces.objects.ReadOnlyMapping(
collections.ChainMap(self._metadata, self._direct_metadata, *maps)
)
class TranslationLayerInterface(DataLayerInterface, metaclass = ABCMeta):
class TranslationLayerInterface(DataLayerInterface, metaclass=ABCMeta):
"""Provides a layer that translates or transforms another layer or layers.
Translation layers always depend on another layer (typically
@@ -370,10 +415,9 @@ class TranslationLayerInterface(DataLayerInterface, metaclass = ABCMeta):
"""
@abstractmethod
def mapping(self,
offset: int,
length: int,
ignore_errors: bool = False) -> Iterable[Tuple[int, int, int, int, str]]:
def mapping(
self, offset: int, length: int, ignore_errors: bool = False
) -> Iterable[Tuple[int, int, int, int, str]]:
"""Returns a sorted iterable of (offset, sublength, mapped_offset, mapped_length, layer)
mappings.
@@ -390,7 +434,9 @@ class TranslationLayerInterface(DataLayerInterface, metaclass = ABCMeta):
"""Returns a list of layer names that this layer translates onto."""
return []
def _decode_data(self, data: bytes, mapped_offset: int, offset: int, output_length: int) -> bytes:
def _decode_data(
self, data: bytes, mapped_offset: int, offset: int, output_length: int
) -> bytes:
"""Decodes any necessary data. Note, additional data may need to be read from the lower layer, such as lookup
tables or similar. The data provided to this layer is purely that data which encompasses the requested data
range.
@@ -405,7 +451,9 @@ class TranslationLayerInterface(DataLayerInterface, metaclass = ABCMeta):
The data to be read from the underlying layer."""
return data
def _encode_data(self, layer_name: str, mapped_offset: int, offset: int, value: bytes) -> bytes:
def _encode_data(
self, layer_name: str, mapped_offset: int, offset: int, value: bytes
) -> bytes:
"""Encodes any necessary data.
Args:
@@ -420,28 +468,41 @@ class TranslationLayerInterface(DataLayerInterface, metaclass = ABCMeta):
# ## Read/Write functions for mapped pages
@functools.lru_cache(maxsize = 512)
@functools.lru_cache(maxsize=512)
def read(self, offset: int, length: int, pad: bool = False) -> bytes:
"""Reads an offset for length bytes and returns 'bytes' (not 'str') of
length size."""
current_offset = offset
output: bytes = b''
for (layer_offset, sublength, mapped_offset, mapped_length, layer) in self.mapping(offset,
length,
ignore_errors = pad):
output: bytes = b""
for (
layer_offset,
sublength,
mapped_offset,
mapped_length,
layer,
) in self.mapping(offset, length, ignore_errors=pad):
if not pad and layer_offset > current_offset:
raise exceptions.InvalidAddressException(
self.name, current_offset, f"Layer {self.name} cannot map offset: {current_offset}")
self.name,
current_offset,
f"Layer {self.name} cannot map offset: {current_offset}",
)
elif layer_offset > current_offset:
output += b"\x00" * (layer_offset - current_offset)
current_offset = layer_offset
# The layer_offset can be less than the current_offset in non-linearly mapped layers
# it does not suggest an overlap, but that the data is in an encoded block
if mapped_length > 0:
unprocessed_data = self._context.layers.read(layer, mapped_offset, mapped_length, pad)
processed_data = self._decode_data(unprocessed_data, mapped_offset, layer_offset, sublength)
unprocessed_data = self._context.layers.read(
layer, mapped_offset, mapped_length, pad
)
processed_data = self._decode_data(
unprocessed_data, mapped_offset, layer_offset, sublength
)
if len(processed_data) != sublength:
raise ValueError("ProcessedData length does not match expected length of chunk")
raise ValueError(
"ProcessedData length does not match expected length of chunk"
)
output += processed_data
current_offset += sublength
return output + (b"\x00" * (length - len(output)))
@@ -451,21 +512,36 @@ class TranslationLayerInterface(DataLayerInterface, metaclass = ABCMeta):
underlying mapping."""
current_offset = offset
length = len(value)
for (layer_offset, sublength, mapped_offset, mapped_length, layer) in self.mapping(offset, length):
for (
layer_offset,
sublength,
mapped_offset,
mapped_length,
layer,
) in self.mapping(offset, length):
if layer_offset > current_offset:
raise exceptions.InvalidAddressException(
self.name, current_offset, f"Layer {self.name} cannot map offset: {current_offset}")
self.name,
current_offset,
f"Layer {self.name} cannot map offset: {current_offset}",
)
value_chunk = value[layer_offset - offset:layer_offset - offset + sublength]
new_data = self._encode_data(layer, mapped_offset, layer_offset, value_chunk)
value_chunk = value[
layer_offset - offset : layer_offset - offset + sublength
]
new_data = self._encode_data(
layer, mapped_offset, layer_offset, value_chunk
)
self._context.layers.write(layer, mapped_offset, new_data)
current_offset += len(new_data)
def _scan_iterator(self,
scanner: 'ScannerInterface',
sections: Iterable[Tuple[int, int]],
linear: bool = False) -> Iterable[IteratorValue]:
def _scan_iterator(
self,
scanner: "ScannerInterface",
sections: Iterable[Tuple[int, int]],
linear: bool = False,
) -> Iterable[IteratorValue]:
"""Iterator that indicates which blocks in the layer are to be read by
for the scanning.
@@ -474,7 +550,7 @@ class TranslationLayerInterface(DataLayerInterface, metaclass = ABCMeta):
scanner.chunk_size + scanner.overlap DataLayers by default are
assumed to have no holes
"""
for (section_start, section_length) in sections:
for section_start, section_length in sections:
output: List[Tuple[str, int, int]] = []
# Hold the offsets of each chunk (including how much has been filled)
@@ -483,7 +559,9 @@ class TranslationLayerInterface(DataLayerInterface, metaclass = ABCMeta):
# For each section, find out which bits of its exists and where they map to
# This is faster than cutting the entire space into scan_chunk sized blocks and then
# finding out what exists (particularly if most of the space isn't mapped)
for mapped in self.mapping(section_start, section_length, ignore_errors = True):
for mapped in self.mapping(
section_start, section_length, ignore_errors=True
):
offset, sublength, mapped_offset, mapped_length, layer_name = mapped
# Setup the variables for this block
@@ -506,7 +584,10 @@ class TranslationLayerInterface(DataLayerInterface, metaclass = ABCMeta):
# Halfway through a chunk, finish the chunk, then take more
if chunk_position != chunk_start:
chunk_size = min(chunk_position - chunk_start, scanner.chunk_size + scanner.overlap)
chunk_size = min(
chunk_position - chunk_start,
scanner.chunk_size + scanner.overlap,
)
output += [(return_name, chunk_position + conversion, chunk_size)]
chunk_start = chunk_position + chunk_size
chunk_position = chunk_start
@@ -517,10 +598,14 @@ class TranslationLayerInterface(DataLayerInterface, metaclass = ABCMeta):
yield output, chunk_position
output = []
chunk_position = chunk_start
# Take from chunk_position as far as far as the block can go,
# Take from chunk_position as far as the block can go,
# or as much left of a scanner chunk as we can
chunk_size = min(block_end - chunk_position,
scanner.chunk_size + scanner.overlap - (chunk_position - chunk_start))
chunk_size = min(
block_end - chunk_position,
scanner.chunk_size
+ scanner.overlap
- (chunk_position - chunk_start),
)
output += [(return_name, chunk_position + conversion, chunk_size)]
chunk_start = chunk_position + chunk_size
chunk_position = chunk_start
@@ -568,12 +653,20 @@ class LayerContainer(collections.abc.Mapping):
layer: the layer to add to the list of layers (based on layer.name)
"""
if layer.name in self._layers:
raise exceptions.LayerException(layer.name, f"Layer already exists: {layer.name}")
raise exceptions.LayerException(
layer.name, f"Layer already exists: {layer.name}"
)
if isinstance(layer, TranslationLayerInterface):
missing_list = [sublayer for sublayer in layer.dependencies if sublayer not in self._layers]
missing_list = [
sublayer
for sublayer in layer.dependencies
if sublayer not in self._layers
]
if missing_list:
raise exceptions.LayerException(
layer.name, f"Layer {layer.name} has unmet dependencies: {', '.join(missing_list)}")
layer.name,
f"Layer {layer.name} has unmet dependencies: {', '.join(missing_list)}",
)
self._layers[layer.name] = layer
def del_layer(self, name: str) -> None:
@@ -585,11 +678,12 @@ class LayerContainer(collections.abc.Mapping):
name: The name of the layer to delete
"""
for layer in self._layers:
depend_list = [superlayer for superlayer in self._layers if name in self._layers[layer].dependencies]
if depend_list:
if name in self._layers[layer].dependencies:
raise exceptions.LayerException(
self._layers[layer].name,
f"Layer {self._layers[layer].name} is depended upon: {', '.join(depend_list)}")
f"Layer {self._layers[layer].name} is depended upon by {layer}",
)
# Otherwise, wipe out the layer
self._layers[name].destroy()
del self._layers[name]
volatility3/framework/interfaces/objects.py
+95 -51
View File
@@ -6,6 +6,7 @@ interpreted values of data from a layer."""
import abc
import collections
import collections.abc
import contextlib
import logging
from typing import Any, Dict, List, Mapping, Optional
@@ -27,11 +28,13 @@ class ReadOnlyMapping(collections.abc.Mapping):
def __getattr__(self, attr: str) -> Any:
"""Returns the item as an attribute."""
if attr == '_dict':
if attr == "_dict":
return super().__getattribute__(attr)
if attr in self._dict:
return self._dict[attr]
raise AttributeError(f"Object has no attribute: {self.__class__.__name__}.{attr}")
raise AttributeError(
f"Object has no attribute: {self.__class__.__name__}.{attr}"
)
def __getitem__(self, name: str) -> Any:
"""Returns the item requested."""
@@ -60,13 +63,15 @@ class ObjectInformation(ReadOnlyMapping):
in a single place. These values are based on the :class:`ReadOnlyMapping` class, to prevent their modification.
"""
def __init__(self,
layer_name: str,
offset: int,
member_name: Optional[str] = None,
parent: Optional['ObjectInterface'] = None,
native_layer_name: Optional[str] = None,
size: Optional[int] = None):
def __init__(
self,
layer_name: str,
offset: int,
member_name: Optional[str] = None,
parent: Optional["ObjectInterface"] = None,
native_layer_name: Optional[str] = None,
size: Optional[int] = None,
):
"""Constructs a container for basic information about an object.
Args:
@@ -77,22 +82,29 @@ class ObjectInformation(ReadOnlyMapping):
native_layer_name: If this object references other objects (such as a pointer), what layer those objects live in
size: The size that the whole structure consumes in bytes
"""
super().__init__({
'layer_name': layer_name,
'offset': offset,
'member_name': member_name,
'parent': parent,
'native_layer_name': native_layer_name or layer_name,
'size': size
})
super().__init__(
{
"layer_name": layer_name,
"offset": offset,
"member_name": member_name,
"parent": parent,
"native_layer_name": native_layer_name or layer_name,
"size": size,
}
)
class ObjectInterface(metaclass = abc.ABCMeta):
class ObjectInterface(metaclass=abc.ABCMeta):
"""A base object required to be the ancestor of every object used in
volatility."""
def __init__(self, context: 'interfaces.context.ContextInterface', type_name: str, object_info: 'ObjectInformation',
**kwargs) -> None:
def __init__(
self,
context: "interfaces.context.ContextInterface",
type_name: str,
object_info: "ObjectInformation",
**kwargs,
) -> None:
"""Constructs an Object adhering to the ObjectInterface.
Args:
@@ -115,7 +127,7 @@ class ObjectInterface(metaclass = abc.ABCMeta):
mask = context.layers[object_info.layer_name].address_mask
normalized_offset = object_info.offset & mask
vol_info_dict = {'type_name': type_name, 'offset': normalized_offset}
vol_info_dict = {"type_name": type_name, "offset": normalized_offset}
self._vol = collections.ChainMap({}, vol_info_dict, object_info, kwargs)
self._context = context
@@ -142,13 +154,17 @@ class ObjectInterface(metaclass = abc.ABCMeta):
KeyError: If the table_name is not valid within the object's context
"""
if constants.BANG not in self.vol.type_name:
raise ValueError(f"Unable to determine table for symbol: {self.vol.type_name}")
table_name = self.vol.type_name[:self.vol.type_name.index(constants.BANG)]
raise ValueError(
f"Unable to determine table for symbol: {self.vol.type_name}"
)
table_name = self.vol.type_name[: self.vol.type_name.index(constants.BANG)]
if table_name not in self._context.symbol_space:
raise KeyError(f"Symbol table not found in context's symbol_space for symbol: {self.vol.type_name}")
raise KeyError(
f"Symbol table not found in context's symbol_space for symbol: {self.vol.type_name}"
)
return table_name
def cast(self, new_type_name: str, **additional) -> 'ObjectInterface':
def cast(self, new_type_name: str, **additional) -> "ObjectInterface":
"""Returns a new object at the offset and from the layer that the
current object inhabits.
@@ -162,13 +178,15 @@ class ObjectInterface(metaclass = abc.ABCMeta):
object_template = self._context.symbol_space.get_type(new_type_name)
object_template = object_template.clone()
object_template.update_vol(**additional)
object_info = ObjectInformation(layer_name = self.vol.layer_name,
offset = self.vol.offset,
member_name = self.vol.member_name,
parent = self.vol.parent,
native_layer_name = self.vol.native_layer_name,
size = object_template.size)
return object_template(context = self._context, object_info = object_info)
object_info = ObjectInformation(
layer_name=self.vol.layer_name,
offset=self.vol.offset,
member_name=self.vol.member_name,
parent=self.vol.parent,
native_layer_name=self.vol.native_layer_name,
size=object_template.size,
)
return object_template(context=self._context, object_info=object_info)
def has_member(self, member_name: str) -> bool:
"""Returns whether the object would contain a member called
@@ -187,11 +205,9 @@ class ObjectInterface(metaclass = abc.ABCMeta):
"""
if self.has_member(member_name):
# noinspection PyBroadException
try:
with contextlib.suppress(Exception):
_ = getattr(self, member_name)
return True
except Exception:
pass
return False
def has_valid_members(self, member_names: List[str]) -> bool:
@@ -202,7 +218,7 @@ class ObjectInterface(metaclass = abc.ABCMeta):
"""
return all([self.has_valid_member(member_name) for member_name in member_names])
class VolTemplateProxy(metaclass = abc.ABCMeta):
class VolTemplateProxy(metaclass=abc.ABCMeta):
"""A container for proxied methods that the ObjectTemplate of this
object will call. This is primarily to keep methods together for easy
organization/management, there is no significant need for it to be a
@@ -215,35 +231,52 @@ class ObjectInterface(metaclass = abc.ABCMeta):
to control how their templates respond without needing to write
new templates for each and every potential object type.
"""
_methods: List[str] = []
@classmethod
@abc.abstractmethod
def size(cls, template: 'Template') -> int:
def size(cls, template: "Template") -> int:
"""Returns the size of the template object."""
@classmethod
@abc.abstractmethod
def children(cls, template: 'Template') -> List['Template']:
def children(cls, template: "Template") -> List["Template"]:
"""Returns the children of the template."""
return []
@classmethod
@abc.abstractmethod
def replace_child(cls, template: 'Template', old_child: 'Template', new_child: 'Template') -> None:
def replace_child(
cls, template: "Template", old_child: "Template", new_child: "Template"
) -> None:
"""Substitutes the old_child for the new_child."""
raise KeyError(f"Template does not contain any children to replace: {template.vol.type_name}")
raise KeyError(
f"Template does not contain any children to replace: {template.vol.type_name}"
)
@classmethod
@abc.abstractmethod
def relative_child_offset(cls, template: 'Template', child: str) -> int:
def relative_child_offset(cls, template: "Template", child: str) -> int:
"""Returns the relative offset from the head of the parent data to
the child member."""
raise KeyError(f"Template does not contain any children: {template.vol.type_name}")
raise KeyError(
f"Template does not contain any children: {template.vol.type_name}"
)
@classmethod
@abc.abstractmethod
def has_member(cls, template: 'Template', member_name: str) -> bool:
def child_template(
cls, template: "Template", child: str
) -> "interfaces.objects.Template":
"""Returns the template of the child member from the parent."""
raise KeyError(
f"Template does not contain any children: {template.vol.type_name}"
)
@classmethod
@abc.abstractmethod
def has_member(cls, template: "Template", member_name: str) -> bool:
"""Returns whether the object would contain a member called
member_name."""
return False
@@ -277,7 +310,9 @@ class Template:
# Allow the updating of template arguments whilst still in template form
super().__init__()
empty_dict: Dict[str, Any] = {}
self._vol = collections.ChainMap(empty_dict, arguments, {'type_name': type_name})
self._vol = collections.ChainMap(
empty_dict, arguments, {"type_name": type_name}
)
@property
def vol(self) -> ReadOnlyMapping:
@@ -287,7 +322,7 @@ class Template:
return ReadOnlyMapping(self._vol)
@property
def children(self) -> List['Template']:
def children(self) -> List["Template"]:
"""The children of this template (such as member types, sub-types and
base-types where they are relevant).
@@ -306,7 +341,11 @@ class Template:
offset."""
@abc.abstractmethod
def replace_child(self, old_child: 'Template', new_child: 'Template') -> None:
def child_template(self, child: str) -> "interfaces.objects.Template":
"""Returns the `child` member template from its parent."""
@abc.abstractmethod
def replace_child(self, old_child: "Template", new_child: "Template") -> None:
"""Replaces `old_child` with `new_child` in the list of children."""
@abc.abstractmethod
@@ -314,7 +353,7 @@ class Template:
"""Returns whether the object would contain a member called
`member_name`"""
def clone(self) -> 'Template':
def clone(self) -> "Template":
"""Returns a copy of the original Template as constructed (without
`update_vol` additions having been made)"""
clone = self.__class__(**self._vol.parents.new_child())
@@ -328,11 +367,16 @@ class Template:
def __getattr__(self, attr: str) -> Any:
"""Exposes any other values stored in ._vol as attributes (for example,
enumeration choices)"""
if attr != '_vol':
if attr != "_vol":
if attr in self._vol:
return self._vol[attr]
raise AttributeError(f"{self.__class__.__name__} object has no attribute {attr}")
raise AttributeError(
f"{self.__class__.__name__} object has no attribute {attr}"
)
def __call__(self, context: 'interfaces.context.ContextInterface',
object_info: ObjectInformation) -> ObjectInterface:
def __call__(
self,
context: "interfaces.context.ContextInterface",
object_info: ObjectInformation,
) -> ObjectInterface:
"""Constructs the object."""
volatility3/framework/interfaces/plugins.py
+30 -10
View File
@@ -43,7 +43,7 @@ class FileHandlerInterface(io.RawIOBase):
return self._preferred_filename
@preferred_filename.setter
def preferred_filename(self, filename):
def preferred_filename(self, filename: str):
"""Sets the preferred filename"""
if self.closed:
raise IOError("FileHandler name cannot be changed once closed")
@@ -57,6 +57,18 @@ class FileHandlerInterface(io.RawIOBase):
def close(self):
"""Method that commits the file and fixes the final filename for use"""
@staticmethod
def sanitize_filename(filename: str) -> str:
"""Sanititizes the filename to ensure only a specific whitelist of characters is allowed through"""
allowed = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.- ()[]\{\}!$%^:#~?<>,|"
result = ""
for char in filename:
if char in allowed:
result += char
else:
result += "?"
return result
def __enter__(self):
return self
@@ -64,7 +76,9 @@ class FileHandlerInterface(io.RawIOBase):
if exc_type is None and exc_value is None and traceback is None:
self.close()
else:
vollog.warning(f"File {self._preferred_filename} could not be written: {str(exc_value)}")
vollog.warning(
f"File {self._preferred_filename} could not be written: {str(exc_value)}"
)
self.close()
@@ -82,9 +96,11 @@ class FileHandlerInterface(io.RawIOBase):
# The plugin runs and produces a TreeGrid output
class PluginInterface(interfaces.configuration.ConfigurableInterface,
interfaces.configuration.VersionableInterface,
metaclass = ABCMeta):
class PluginInterface(
interfaces.configuration.ConfigurableInterface,
interfaces.configuration.VersionableInterface,
metaclass=ABCMeta,
):
"""Class that defines the basic interface that all Plugins must maintain.
The constructor must only take a `context` and `config_path`, so
@@ -97,10 +113,12 @@ class PluginInterface(interfaces.configuration.ConfigurableInterface,
_required_framework_version: Tuple[int, int, int] = (0, 0, 0)
"""The _version variable is a quick way for plugins to define their current interface, it should follow SemVer rules"""
def __init__(self,
context: interfaces.context.ContextInterface,
config_path: str,
progress_callback: constants.ProgressCallback = None) -> None:
def __init__(
self,
context: interfaces.context.ContextInterface,
config_path: str,
progress_callback: constants.ProgressCallback = None,
) -> None:
"""
Args:
@@ -114,7 +132,9 @@ class PluginInterface(interfaces.configuration.ConfigurableInterface,
# the validation doesn't need to be repeated over and over again by externals
if self.unsatisfied(context, config_path):
vollog.warning("Plugin failed validation")
raise exceptions.PluginRequirementException("The plugin configuration failed to validate")
raise exceptions.PluginRequirementException(
"The plugin configuration failed to validate"
)
# Populate any optional defaults
for requirement in self.get_requirements():
if requirement.name not in self.config:
volatility3/framework/interfaces/renderers.py
+56 -24
View File
@@ -12,14 +12,26 @@ suitable output.
import datetime
from abc import abstractmethod, ABCMeta
from collections import abc
from typing import Any, Callable, ClassVar, Generator, List, NamedTuple, Optional, TypeVar, Type, Tuple, Union
from typing import (
Any,
Callable,
ClassVar,
Generator,
List,
NamedTuple,
Optional,
TypeVar,
Type,
Tuple,
Union,
)
Column = NamedTuple('Column', [('name', str), ('type', Any)])
Column = NamedTuple("Column", [("name", str), ("type", Any)])
RenderOption = Any
class Renderer(metaclass = ABCMeta):
class Renderer(metaclass=ABCMeta):
"""Class that defines the interface that all output renderers must
support."""
@@ -32,12 +44,12 @@ class Renderer(metaclass = ABCMeta):
"""Returns a list of rendering options."""
@abstractmethod
def render(self, grid: 'TreeGrid') -> None:
def render(self, grid: "TreeGrid") -> None:
"""Takes a grid object and renders it based on the object's
preferences."""
class ColumnSortKey(metaclass = ABCMeta):
class ColumnSortKey(metaclass=ABCMeta):
ascending: bool = True
@abstractmethod
@@ -46,14 +58,13 @@ class ColumnSortKey(metaclass = ABCMeta):
function."""
class TreeNode(abc.Sequence, metaclass = ABCMeta):
class TreeNode(abc.Sequence, metaclass=ABCMeta):
def __init__(self, path, treegrid, parent, values):
"""Initializes the TreeNode."""
@property
@abstractmethod
def values(self) -> List['BaseTypes']:
def values(self) -> List["BaseTypes"]:
"""Returns the list of values from the particular node, based on column
index."""
@@ -69,7 +80,7 @@ class TreeNode(abc.Sequence, metaclass = ABCMeta):
@property
@abstractmethod
def parent(self) -> Optional['TreeNode']:
def parent(self) -> Optional["TreeNode"]:
"""Returns the parent node of this node or None."""
@property
@@ -94,9 +105,12 @@ class BaseAbsentValue(object):
class Disassembly(object):
"""A class to indicate that the bytes provided should be disassembled
(based on the architecture)"""
possible_architectures = ['intel', 'intel64', 'arm', 'arm64']
def __init__(self, data: bytes, offset: int = 0, architecture: str = 'intel64') -> None:
possible_architectures = ["intel", "intel64", "arm", "arm64"]
def __init__(
self, data: bytes, offset: int = 0, architecture: str = "intel64"
) -> None:
self.data = data
self.architecture = None
if architecture in self.possible_architectures:
@@ -110,13 +124,20 @@ class Disassembly(object):
# contain the types that the validator will accept (which would not include the base)
_Type = TypeVar("_Type")
BaseTypes = Union[Type[int], Type[str], Type[float], Type[bytes], Type[datetime.datetime], Type[BaseAbsentValue],
Type[Disassembly]]
BaseTypes = Union[
Type[int],
Type[str],
Type[float],
Type[bytes],
Type[datetime.datetime],
Type[BaseAbsentValue],
Type[Disassembly],
]
ColumnsType = List[Tuple[str, BaseTypes]]
VisitorSignature = Callable[[TreeNode, _Type], _Type]
class TreeGrid(object, metaclass = ABCMeta):
class TreeGrid(object, metaclass=ABCMeta):
"""Class providing the interface for a TreeGrid (which contains TreeNodes)
The structure of a TreeGrid is designed to maintain the structure of the tree in a single object.
@@ -129,7 +150,14 @@ class TreeGrid(object, metaclass = ABCMeta):
and to create cycles.
"""
base_types: ClassVar[Tuple] = (int, str, float, bytes, datetime.datetime, Disassembly)
base_types: ClassVar[Tuple] = (
int,
str,
float,
bytes,
datetime.datetime,
Disassembly,
)
def __init__(self, columns: ColumnsType, generator: Generator) -> None:
"""Constructs a TreeGrid object using a specific set of columns.
@@ -149,10 +177,12 @@ class TreeGrid(object, metaclass = ABCMeta):
"""Method used to sanitize column names for TreeNodes."""
@abstractmethod
def populate(self,
function: VisitorSignature = None,
initial_accumulator: Any = None,
fail_on_errors: bool = True) -> Optional[Exception]:
def populate(
self,
function: VisitorSignature = None,
initial_accumulator: Any = None,
fail_on_errors: bool = True,
) -> Optional[Exception]:
"""Populates the tree by consuming the TreeGrid's construction
generator Func is called on every node, so can be used to create output
on demand.
@@ -196,11 +226,13 @@ class TreeGrid(object, metaclass = ABCMeta):
return node.path_depth
@abstractmethod
def visit(self,
node: Optional[TreeNode],
function: VisitorSignature,
initial_accumulator: _Type,
sort_key: ColumnSortKey = None) -> None:
def visit(
self,
node: Optional[TreeNode],
function: VisitorSignature,
initial_accumulator: _Type,
sort_key: ColumnSortKey = None,
) -> None:
"""Visits all the nodes in a tree, calling function on each one.
function should have the signature function(node, accumulator) and return new_accumulator
volatility3/framework/interfaces/symbols.py
+79 -41
View File
@@ -16,11 +16,13 @@ from volatility3.framework.interfaces.configuration import RequirementInterface
class SymbolInterface:
"""Contains information about a named location in a program's memory."""
def __init__(self,
name: str,
address: int,
type: Optional[objects.Template] = None,
constant_data: Optional[bytes] = None) -> None:
def __init__(
self,
name: str,
address: int,
type: Optional[objects.Template] = None,
constant_data: Optional[bytes] = None,
) -> None:
"""
Args:
@@ -31,7 +33,9 @@ class SymbolInterface:
"""
self._name = name
if constants.BANG in self._name:
raise ValueError(f"Symbol names cannot contain the symbol differentiator ({constants.BANG})")
raise ValueError(
f"Symbol names cannot contain the symbol differentiator ({constants.BANG})"
)
# Scope can be added at a later date
self._location = None
@@ -50,7 +54,7 @@ class SymbolInterface:
# Objects and ObjectTemplates should *always* get a type_name when they're constructed, so allow the IndexError
if self.type is None:
return None
return self.type.vol['type_name']
return self.type.vol["type_name"]
@property
def type(self) -> Optional[objects.Template]:
@@ -78,11 +82,13 @@ class BaseSymbolTableInterface:
Note: table_mapping is a rarely used feature (since symbol tables are typically self-contained)
"""
def __init__(self,
name: str,
native_types: 'NativeTableInterface',
table_mapping: Optional[Dict[str, str]] = None,
class_types: Optional[Mapping[str, Type[objects.ObjectInterface]]] = None) -> None:
def __init__(
self,
name: str,
native_types: "NativeTableInterface",
table_mapping: Optional[Dict[str, str]] = None,
class_types: Optional[Mapping[str, Type[objects.ObjectInterface]]] = None,
) -> None:
"""
Args:
@@ -110,44 +116,54 @@ class BaseSymbolTableInterface:
If the symbol isn't found, it raises a SymbolError exception
"""
raise NotImplementedError("Abstract property get_symbol not implemented by subclass.")
raise NotImplementedError(
"Abstract property get_symbol not implemented by subclass."
)
@property
def symbols(self) -> Iterable[str]:
"""Returns an iterator of the Symbol names."""
raise NotImplementedError("Abstract property symbols not implemented by subclass.")
raise NotImplementedError(
"Abstract property symbols not implemented by subclass."
)
# ## Required Type functions
@property
def types(self) -> Iterable[str]:
"""Returns an iterator of the Symbol type names."""
raise NotImplementedError("Abstract property types not implemented by subclass.")
raise NotImplementedError(
"Abstract property types not implemented by subclass."
)
def get_type(self, name: str) -> objects.Template:
"""Resolves a symbol name into an object template.
If the symbol isn't found it raises a SymbolError exception
"""
raise NotImplementedError("Abstract method get_type not implemented by subclass.")
raise NotImplementedError(
"Abstract method get_type not implemented by subclass."
)
# ## Required Symbol enumeration functions
@property
def enumerations(self) -> Iterable[Any]:
"""Returns an iterator of the Enumeration names."""
raise NotImplementedError("Abstract property enumerations not implemented by subclass.")
raise NotImplementedError(
"Abstract property enumerations not implemented by subclass."
)
# ## Native Type Handler
@property
def natives(self) -> 'NativeTableInterface':
def natives(self) -> "NativeTableInterface":
"""Returns None or a NativeTable for handling space specific native
types."""
return self._native_types
@natives.setter
def natives(self, value: 'NativeTableInterface') -> None:
def natives(self, value: "NativeTableInterface") -> None:
"""Checks the natives value and then applies it internally.
WARNING: This allows changing the underlying size of all the other types referenced in the SymbolTable
@@ -167,7 +183,9 @@ class BaseSymbolTableInterface:
"""
raise NotImplementedError("Abstract method set_type_class not implemented yet.")
def optional_set_type_class(self, name: str, clazz: Type[objects.ObjectInterface]) -> bool:
def optional_set_type_class(
self, name: str, clazz: Type[objects.ObjectInterface]
) -> bool:
"""Calls the set_type_class function but does not throw an exception.
Returns whether setting the type class was successful.
Args:
@@ -176,7 +194,7 @@ class BaseSymbolTableInterface:
"""
try:
self.set_type_class(name, clazz)
return True
except ValueError:
return False
@@ -206,8 +224,10 @@ class BaseSymbolTableInterface:
# This allows for searching with and without the table name (in case multiple tables contain
# the same symbol name and we've not specifically been told which one)
symbol = self.get_symbol(symbol_name)
if symbol.type_name is not None and (symbol.type_name == type_name or
(symbol.type_name.endswith(constants.BANG + type_name))):
if symbol.type_name is not None and (
symbol.type_name == type_name
or (symbol.type_name.endswith(constants.BANG + type_name))
):
yield symbol.name
def get_symbols_by_location(self, offset: int, size: int = 0) -> Iterable[str]:
@@ -216,11 +236,15 @@ class BaseSymbolTableInterface:
if size < 0:
raise ValueError("Size must be strictly non-negative")
if not self._sort_symbols:
self._sort_symbols = sorted([(self.get_symbol(sn).address, sn) for sn in self.symbols])
self._sort_symbols = sorted(
[(self.get_symbol(sn).address, sn) for sn in self.symbols]
)
sort_symbols = self._sort_symbols
result = bisect.bisect_left(sort_symbols, (offset, ""))
while result < len(sort_symbols) and \
(sort_symbols[result][0] >= offset and sort_symbols[result][0] <= offset + size):
while result < len(sort_symbols) and (
sort_symbols[result][0] >= offset
and sort_symbols[result][0] <= offset + size
):
yield sort_symbols[result][1]
result += 1
@@ -247,7 +271,9 @@ class SymbolSpaceInterface(collections.abc.Mapping):
"""Returns all symbols based on the type of the symbol."""
@abstractmethod
def get_symbols_by_location(self, offset: int, size: int = 0, table_name: Optional[str] = None) -> Iterable[str]:
def get_symbols_by_location(
self, offset: int, size: int = 0, table_name: Optional[str] = None
) -> Iterable[str]:
"""Returns all symbols that exist at a specific relative address."""
@abstractmethod
@@ -281,17 +307,21 @@ class SymbolSpaceInterface(collections.abc.Mapping):
"""Adds a symbol_list to the end of the space."""
class SymbolTableInterface(BaseSymbolTableInterface, configuration.ConfigurableInterface, ABC):
class SymbolTableInterface(
BaseSymbolTableInterface, configuration.ConfigurableInterface, ABC
):
"""Handles a table of symbols."""
# FIXME: native_types and table_mapping aren't recorded in the configuration
def __init__(self,
context: 'interfaces.context.ContextInterface',
config_path: str,
name: str,
native_types: 'NativeTableInterface',
table_mapping: Optional[Dict[str, str]] = None,
class_types: Optional[Mapping[str, Type[objects.ObjectInterface]]] = None) -> None:
def __init__(
self,
context: "interfaces.context.ContextInterface",
config_path: str,
name: str,
native_types: "NativeTableInterface",
table_mapping: Optional[Dict[str, str]] = None,
class_types: Optional[Mapping[str, Type[objects.ObjectInterface]]] = None,
) -> None:
"""Instantiates an SymbolTable based on an IntermediateSymbolFormat JSON file. This is validated against the
appropriate schema.
@@ -305,9 +335,11 @@ class SymbolTableInterface(BaseSymbolTableInterface, configuration.ConfigurableI
class_types: A dictionary of type names and classes that override StructType when they are instantiated
"""
configuration.ConfigurableInterface.__init__(self, context, config_path)
BaseSymbolTableInterface.__init__(self, name, native_types, table_mapping, class_types = class_types)
BaseSymbolTableInterface.__init__(
self, name, native_types, table_mapping, class_types=class_types
)
def build_configuration(self) -> 'configuration.HierarchicalDict':
def build_configuration(self) -> "configuration.HierarchicalDict":
config = super().build_configuration()
# Symbol Tables are constructable, and therefore require a class configuration variable
@@ -317,9 +349,13 @@ class SymbolTableInterface(BaseSymbolTableInterface, configuration.ConfigurableI
@classmethod
def get_requirements(cls) -> List[RequirementInterface]:
return super().get_requirements() + [
requirements.IntRequirement(name = 'symbol_mask', description = 'Address mask for symbols', optional = True,
default = 0),
]
requirements.IntRequirement(
name="symbol_mask",
description="Address mask for symbols",
optional=True,
default=0,
),
]
class NativeTableInterface(BaseSymbolTableInterface):
@@ -333,7 +369,9 @@ class NativeTableInterface(BaseSymbolTableInterface):
return []
def get_enumeration(self, name: str) -> objects.Template:
raise exceptions.SymbolError(name, self.name, "NativeTables never hold enumerations")
raise exceptions.SymbolError(
name, self.name, "NativeTables never hold enumerations"
)
@property
def enumerations(self) -> Iterable[str]:
volatility3/framework/layers/avml.py
+130 -36
View File
@@ -1,7 +1,12 @@
# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0
# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0
#
"""Functions that read AVML files.
The user of the file doesn't have to worry about the compression,
but random access is not allowed."""
import ctypes
import logging
import struct
from typing import Tuple, List, Optional
@@ -12,13 +17,56 @@ from volatility3.framework.layers import segmented
vollog = logging.getLogger(__name__)
try:
import snappy
# TODO: Find library for windows if needed
try:
# Linux
lib_snappy = ctypes.cdll.LoadLibrary("libsnappy.so.1")
except OSError:
lib_snappy = None
try:
if not lib_snappy:
# macOS
lib_snappy = ctypes.cdll.LoadLibrary("libsnappy.1.dylib")
except OSError:
lib_snappy = None
try:
if not lib_snappy:
# Windows 64
lib_snappy = ctypes.cdll.LoadLibrary("snappy64")
except OSError:
lib_snappy = None
if not lib_snappy:
# Windows 32
lib_snappy = ctypes.cdll.LoadLibrary("snappy32")
__snappy_uncompress = lib_snappy.snappy_uncompress
__snappy_uncompressed_length = lib_snappy.snappy_uncompressed_length
HAS_SNAPPY = True
except ImportError:
except (AttributeError, OSError):
HAS_SNAPPY = False
class SnappyException(exceptions.VolatilityException):
pass
def uncompress(s):
"""Uncompress a snappy compressed string."""
ulen = ctypes.c_int(0)
cresult = __snappy_uncompressed_length(s, len(s), ctypes.byref(ulen))
if cresult != 0:
raise SnappyException(f"Error in snappy_uncompressed_length: {cresult}")
ubuf = ctypes.create_string_buffer(ulen.value)
cresult = __snappy_uncompress(s, len(s), ubuf, ctypes.byref(ulen))
if cresult != 0:
raise SnappyException(f"Error in snappy_uncompress: {cresult}")
return ubuf.raw
class AVMLLayer(segmented.NonLinearlySegmentedLayer):
"""A Lime format TranslationLayer.
@@ -33,13 +81,20 @@ class AVMLLayer(segmented.NonLinearlySegmentedLayer):
@classmethod
def _check_header(cls, layer: interfaces.layers.DataLayerInterface):
header_structure = "<II"
magic, version = struct.unpack(header_structure,
layer.read(layer.minimum_address, struct.calcsize(header_structure)))
if magic not in [0x4c4d5641] or version != 2:
raise exceptions.LayerException("File not completely in AVML format")
magic, version = struct.unpack(
header_structure,
layer.read(layer.minimum_address, struct.calcsize(header_structure)),
)
if magic not in [0x4C4D5641] or version != 2:
raise exceptions.LayerException("File not in AVML format")
if not HAS_SNAPPY:
vollog.warning('AVML file detected, but snappy python library not installed')
raise exceptions.LayerException("AVML format dependencies not satisfied (snappy)")
vollog.warning(
"AVML file detected, but snappy library could not be found\n"
"Please install the snappy from your distribution or https://google.github.io/snappy/."
)
raise exceptions.LayerException(
"AVML format dependencies not satisfied (snappy)"
)
def _load_segments(self) -> None:
base_layer = self.context.layers[self._base_layer]
@@ -48,24 +103,38 @@ class AVMLLayer(segmented.NonLinearlySegmentedLayer):
avml_header_structure = "<IIQQQ"
avml_header_size = struct.calcsize(avml_header_structure)
avml_header_data = base_layer.read(offset, avml_header_size)
magic, version, start, end, padding = struct.unpack(avml_header_structure, avml_header_data)
magic, version, start, end, padding = struct.unpack(
avml_header_structure, avml_header_data
)
if magic not in [0x4c4d5641] or version != 2:
if magic not in [0x4C4D5641] or version != 2:
raise exceptions.LayerException("File not completely in AVML format")
chunk_data = base_layer.read(offset + avml_header_size,
min(end - start,
base_layer.maximum_address - (offset + avml_header_size)))
chunk_data = base_layer.read(
offset + avml_header_size,
min(
end - start,
base_layer.maximum_address - (offset + avml_header_size),
),
)
segments, consumed = self._read_snappy_frames(chunk_data, end - start)
# The returned segments are accurate the chunk_data that was passed in, but needs shifting
for (thing, mapped_offset, size, mapped_size, compressed) in segments:
self._segments.append((thing + start, offset + mapped_offset + avml_header_size, size, mapped_size))
for thing, mapped_offset, size, mapped_size, compressed in segments:
self._segments.append(
(
thing + start,
offset + mapped_offset + avml_header_size,
size,
mapped_size,
)
)
self._compressed[offset + mapped_offset + avml_header_size] = compressed
# TODO: Check whatever the remaining 8 bytes are
offset += avml_header_size + consumed + 8
def _read_snappy_frames(self, data: bytes, expected_length: int) -> Tuple[
List[Tuple[int, int, int, int, bool]], int]:
def _read_snappy_frames(
self, data: bytes, expected_length: int
) -> Tuple[List[Tuple[int, int, int, int, bool]], int]:
"""
Reads a framed-format snappy stream
@@ -80,41 +149,62 @@ class AVMLLayer(segmented.NonLinearlySegmentedLayer):
decompressed_len = 0
offset = 0
crc_len = 4
frame_header_struct = '<L'
frame_header_struct = "<L"
frame_header_len = struct.calcsize(frame_header_struct)
while decompressed_len <= expected_length:
if offset + frame_header_len < len(data):
frame_header = data[offset:offset + frame_header_len]
frame_header_val = struct.unpack('<L', frame_header)[0]
frame_type, frame_size = frame_header_val & 0xff, frame_header_val >> 8
if frame_type == 0xff:
if data[offset + frame_header_len:offset + frame_header_len + frame_size] != b'sNaPpY':
frame_header = data[offset : offset + frame_header_len]
frame_header_val = struct.unpack("<L", frame_header)[0]
frame_type, frame_size = frame_header_val & 0xFF, frame_header_val >> 8
if frame_type == 0xFF:
if (
data[
offset
+ frame_header_len : offset
+ frame_header_len
+ frame_size
]
!= b"sNaPpY"
):
raise ValueError(f"Snappy header missing at offset: {offset}")
elif frame_type in [0x00, 0x01]:
# CRC + (Un)compressed data
mapped_start = offset + frame_header_len
# frame_crc = data[mapped_start: mapped_start + crc_len]
frame_data = data[mapped_start + crc_len: mapped_start + frame_size]
frame_data = data[
mapped_start + crc_len : mapped_start + frame_size
]
if frame_type == 0x00:
# Compressed data
frame_data = snappy.decompress(frame_data)
frame_data = uncompress(frame_data)
# TODO: Verify CRC
segments.append((decompressed_len, mapped_start + crc_len, len(frame_data), frame_size - crc_len,
frame_type == 0x00))
segments.append(
(
decompressed_len,
mapped_start + crc_len,
len(frame_data),
frame_size - crc_len,
frame_type == 0x00,
)
)
decompressed_len += len(frame_data)
elif frame_type in range(0x2, 0x80):
# Unskippable
raise exceptions.LayerException(f"Unskippable chunk of type {frame_type} found: {offset}")
raise exceptions.LayerException(
f"Unskippable chunk of type {frame_type} found: {offset}"
)
offset += frame_header_len + frame_size
return segments, offset
def _decode_data(self, data: bytes, mapped_offset: int, offset: int, output_length: int) -> bytes:
def _decode_data(
self, data: bytes, mapped_offset: int, offset: int, output_length: int
) -> bytes:
start_offset, _, _, _ = self._find_segment(offset)
if self._compressed[mapped_offset]:
decoded_data = snappy.decompress(data)
decoded_data = uncompress(data)
else:
decoded_data = data
decoded_data = decoded_data[offset - start_offset:]
decoded_data = decoded_data[offset - start_offset :]
decoded_data = decoded_data[:output_length]
return decoded_data
@@ -123,14 +213,18 @@ class AVMLStacker(interfaces.automagic.StackerLayerInterface):
stack_order = 10
@classmethod
def stack(cls,
context: interfaces.context.ContextInterface,
layer_name: str,
progress_callback: constants.ProgressCallback = None) -> Optional[interfaces.layers.DataLayerInterface]:
def stack(
cls,
context: interfaces.context.ContextInterface,
layer_name: str,
progress_callback: constants.ProgressCallback = None,
) -> Optional[interfaces.layers.DataLayerInterface]:
try:
AVMLLayer._check_header(context.layers[layer_name])
except exceptions.LayerException:
return None
new_name = context.layers.free_layer_name("AVMLLayer")
context.config[interfaces.configuration.path_join(new_name, "base_layer")] = layer_name
context.config[interfaces.configuration.path_join(new_name, "base_layer")] = (
layer_name
)
return AVMLLayer(context, new_name, new_name)
volatility3/framework/layers/cloudstorage.py
+56
View File
@@ -0,0 +1,56 @@
# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0
# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0
#
import logging
import urllib.parse
from typing import Optional, Any, List
try:
import s3fs
HAS_S3FS = True
except ImportError:
HAS_S3FS = False
try:
import gcsfs
HAS_GCSFS = True
except ImportError:
HAS_GCSFS = False
from volatility3.framework.layers import resources
vollog = logging.getLogger(__file__)
if HAS_S3FS:
class S3FileSystemHandler(resources.VolatilityHandler):
@classmethod
def non_cached_schemes(cls) -> List[str]:
return ["s3"]
@staticmethod
def default_open(req: urllib.request.Request) -> Optional[Any]:
"""Handles the request if it's the s3 scheme."""
if req.type == "s3":
object_uri = "://".join(req.full_url.split("://")[1:])
return s3fs.S3FileSystem().open(object_uri)
return None
if HAS_GCSFS:
class GSFileSystemHandler(resources.VolatilityHandler):
@classmethod
def non_cached_schemes(cls) -> List[str]:
return ["gs"]
@staticmethod
def default_open(req: urllib.request.Request) -> Optional[Any]:
"""Handles the request if it's the gs scheme."""
if req.type == "gs":
object_uri = "://".join(req.full_url.split("://")[1:])
return gcsfs.GCSFileSystem().open(object_uri)
return None
volatility3/framework/layers/codecs/__init__.py
+4
View File
@@ -1,3 +1,7 @@
# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0
# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0
#
"""Codecs used for encoding or decoding data should live here
volatility3/framework/layers/crash.py
+110 -51
View File
@@ -1,6 +1,7 @@
# This file is Copyright 2021 Volatility Foundation and licensed under the Volatility Software License 1.0
# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0
#
import contextlib
import logging
import struct
from typing import Tuple, Optional
@@ -26,17 +27,18 @@ class WindowsCrashDump32Layer(segmented.SegmentedLayer):
provides = {"type": "physical"}
SIGNATURE = 0x45474150
VALIDDUMP = 0x504d5544
VALIDDUMP = 0x504D5544
crashdump_json = 'crash'
crashdump_json = "crash"
supported_dumptypes = [0x01, 0x05] # we need 0x5 for 32-bit bitmaps
dump_header_name = '_DUMP_HEADER'
dump_header_name = "_DUMP_HEADER"
_magic_struct = struct.Struct('<II')
_magic_struct = struct.Struct("<II")
headerpages = 1
def __init__(self, context: interfaces.context.ContextInterface, config_path: str, name: str) -> None:
def __init__(
self, context: interfaces.context.ContextInterface, config_path: str, name: str
) -> None:
# Construct these so we can use self.config
self._context = context
self._config_path = config_path
@@ -45,15 +47,18 @@ class WindowsCrashDump32Layer(segmented.SegmentedLayer):
self._base_layer = self.config["base_layer"]
# Create a custom SymbolSpace
self._crash_table_name = intermed.IntermediateSymbolTable.create(context, self._config_path, 'windows',
self.crashdump_json)
self._crash_table_name = intermed.IntermediateSymbolTable.create(
context, self._config_path, "windows", self.crashdump_json
)
# the _SUMMARY_DUMP is shared between 32- and 64-bit
self._crash_common_table_name = intermed.IntermediateSymbolTable.create(context,
self._config_path,
'windows',
'crash_common',
class_types = crash.class_types)
self._crash_common_table_name = intermed.IntermediateSymbolTable.create(
context,
self._config_path,
"windows",
"crash_common",
class_types=crash.class_types,
)
# Check Header
hdr_layer = self._context.layers[self._base_layer]
@@ -70,21 +75,30 @@ class WindowsCrashDump32Layer(segmented.SegmentedLayer):
# Verify that it is a supported format
if header.DumpType not in self.supported_dumptypes:
vollog.log(constants.LOGLEVEL_VVVV, f"unsupported dump format 0x{header.DumpType:x}")
raise WindowsCrashDumpFormatException(name, f"unsupported dump format 0x{header.DumpType:x}")
vollog.log(
constants.LOGLEVEL_VVVV,
f"unsupported dump format 0x{header.DumpType:x}",
)
raise WindowsCrashDumpFormatException(
name, f"unsupported dump format 0x{header.DumpType:x}"
)
# Then call the super, which will call load_segments (which needs the base_layer before it'll work)
super().__init__(context, config_path, name)
def get_header(self) -> interfaces.objects.ObjectInterface:
return self.context.object(self._crash_table_name + constants.BANG + self.dump_header_name,
offset = 0,
layer_name = self._base_layer)
return self.context.object(
self._crash_table_name + constants.BANG + self.dump_header_name,
offset=0,
layer_name=self._base_layer,
)
def get_summary_header(self) -> interfaces.objects.ObjectInterface:
return self.context.object(self._crash_common_table_name + constants.BANG + "_SUMMARY_DUMP",
offset = 0x1000 * self.headerpages,
layer_name = self._base_layer)
return self.context.object(
self._crash_common_table_name + constants.BANG + "_SUMMARY_DUMP",
offset=0x1000 * self.headerpages,
layer_name=self._base_layer,
)
def _load_segments(self) -> None:
"""Loads up the segments from the meta_layer."""
@@ -92,15 +106,25 @@ class WindowsCrashDump32Layer(segmented.SegmentedLayer):
segments = []
if self.dump_type == 0x1:
header = self.context.object(self._crash_table_name + constants.BANG + self.dump_header_name,
offset = 0,
layer_name = self._base_layer)
header = self.context.object(
self._crash_table_name + constants.BANG + self.dump_header_name,
offset=0,
layer_name=self._base_layer,
)
offset = self.headerpages
header.PhysicalMemoryBlockBuffer.Run.count = header.PhysicalMemoryBlockBuffer.NumberOfRuns
header.PhysicalMemoryBlockBuffer.Run.count = (
header.PhysicalMemoryBlockBuffer.NumberOfRuns
)
for run in header.PhysicalMemoryBlockBuffer.Run:
segments.append(
(run.BasePage * 0x1000, offset * 0x1000, run.PageCount * 0x1000, run.PageCount * 0x1000))
(
run.BasePage * 0x1000,
offset * 0x1000,
run.PageCount * 0x1000,
run.PageCount * 0x1000,
)
)
offset += run.PageCount
elif self.dump_type == 0x05:
@@ -117,7 +141,14 @@ class WindowsCrashDump32Layer(segmented.SegmentedLayer):
if first_bit is not None:
last_bit = ((outer_index - 1) * 32) + 31
segment_length = (last_bit - first_bit + 1) * 0x1000
segments.append((first_bit * 0x1000, first_offset, segment_length, segment_length))
segments.append(
(
first_bit * 0x1000,
first_offset,
segment_length,
segment_length,
)
)
first_bit = None
elif buffer_long[outer_index] == 0xFFFFFFFF:
if first_bit is None:
@@ -134,48 +165,74 @@ class WindowsCrashDump32Layer(segmented.SegmentedLayer):
offset = offset + 0x1000
else:
if first_bit is not None:
segment_length = ((bit_addr - 1) - first_bit + 1) * 0x1000
segments.append((first_bit * 0x1000, first_offset, segment_length, segment_length))
segment_length = (
(bit_addr - 1) - first_bit + 1
) * 0x1000
segments.append(
(
first_bit * 0x1000,
first_offset,
segment_length,
segment_length,
)
)
first_bit = None
last_bit_seen = (outer_index * 32) + 31
if first_bit is not None:
segment_length = (last_bit_seen - first_bit + 1) * 0x1000
segments.append((first_bit * 0x1000, first_offset, segment_length, segment_length))
segments.append(
(first_bit * 0x1000, first_offset, segment_length, segment_length)
)
else:
vollog.log(constants.LOGLEVEL_VVVV, f"unsupported dump format 0x{self.dump_type:x}")
raise WindowsCrashDumpFormatException(self.name, f"unsupported dump format 0x{self.dump_type:x}")
vollog.log(
constants.LOGLEVEL_VVVV, f"unsupported dump format 0x{self.dump_type:x}"
)
raise WindowsCrashDumpFormatException(
self.name, f"unsupported dump format 0x{self.dump_type:x}"
)
if len(segments) == 0:
raise WindowsCrashDumpFormatException(self.name, f"No Crash segments defined in {self._base_layer}")
raise WindowsCrashDumpFormatException(
self.name, f"No Crash segments defined in {self._base_layer}"
)
else:
# report the segments for debugging. this is valuable for dev/troubleshooting but
# not important enough for a dedicated plugin.
for idx, (start_position, mapped_offset, length, _) in enumerate(segments):
vollog.log(
constants.LOGLEVEL_VVVV,
"Segment {}: Position {:#x} Offset {:#x} Length {:#x}".format(idx, start_position, mapped_offset,
length))
"Segment {}: Position {:#x} Offset {:#x} Length {:#x}".format(
idx, start_position, mapped_offset, length
),
)
self._segments = segments
@classmethod
def check_header(cls, base_layer: interfaces.layers.DataLayerInterface, offset: int = 0) -> Tuple[int, int]:
def check_header(
cls, base_layer: interfaces.layers.DataLayerInterface, offset: int = 0
) -> Tuple[int, int]:
# Verify the Window's crash dump file magic
try:
header_data = base_layer.read(offset, cls._magic_struct.size)
except exceptions.InvalidAddressException:
raise WindowsCrashDumpFormatException(base_layer.name,
f"Crashdump header not found at offset {offset}")
raise WindowsCrashDumpFormatException(
base_layer.name, f"Crashdump header not found at offset {offset}"
)
(signature, validdump) = cls._magic_struct.unpack(header_data)
if signature != cls.SIGNATURE:
raise WindowsCrashDumpFormatException(
base_layer.name, f"Bad signature 0x{signature:x} at file offset 0x{offset:x}")
base_layer.name,
f"Bad signature 0x{signature:x} at file offset 0x{offset:x}",
)
if validdump != cls.VALIDDUMP:
raise WindowsCrashDumpFormatException(base_layer.name,
f"Invalid dump 0x{validdump:x} at file offset 0x{offset:x}")
raise WindowsCrashDumpFormatException(
base_layer.name,
f"Invalid dump 0x{validdump:x} at file offset 0x{offset:x}",
)
return signature, validdump
@@ -187,8 +244,8 @@ class WindowsCrashDump64Layer(WindowsCrashDump32Layer):
"""
VALIDDUMP = 0x34365544
crashdump_json = 'crash64'
dump_header_name = '_DUMP_HEADER64'
crashdump_json = "crash64"
dump_header_name = "_DUMP_HEADER64"
supported_dumptypes = [0x1, 0x05]
headerpages = 2
@@ -197,16 +254,18 @@ class WindowsCrashDumpStacker(interfaces.automagic.StackerLayerInterface):
stack_order = 11
@classmethod
def stack(cls,
context: interfaces.context.ContextInterface,
layer_name: str,
progress_callback: constants.ProgressCallback = None) -> Optional[interfaces.layers.DataLayerInterface]:
def stack(
cls,
context: interfaces.context.ContextInterface,
layer_name: str,
progress_callback: constants.ProgressCallback = None,
) -> Optional[interfaces.layers.DataLayerInterface]:
for layer in [WindowsCrashDump32Layer, WindowsCrashDump64Layer]:
try:
with contextlib.suppress(WindowsCrashDumpFormatException):
layer.check_header(context.layers[layer_name])
new_name = context.layers.free_layer_name(layer.__name__)
context.config[interfaces.configuration.path_join(new_name, "base_layer")] = layer_name
context.config[
interfaces.configuration.path_join(new_name, "base_layer")
] = layer_name
return layer(context, new_name, new_name)
except WindowsCrashDumpFormatException:
pass
return None
volatility3/framework/layers/elf.py
+64 -24
View File
@@ -18,50 +18,82 @@ class ElfFormatException(exceptions.LayerException):
class Elf64Layer(segmented.SegmentedLayer):
"""A layer that supports the Elf64 format as documented at: http://ftp.openwatcom.org/devel/docs/elf-64-gen.pdf"""
_header_struct = struct.Struct("<IBBB")
MAGIC = 0x464c457f # "\x7fELF"
MAGIC = 0x464C457F # "\x7fELF"
ELF_CLASS = 2
def __init__(self, context: interfaces.context.ContextInterface, config_path: str, name: str) -> None:
def __init__(
self, context: interfaces.context.ContextInterface, config_path: str, name: str
) -> None:
# Create a custom SymbolSpace
self._elf_table_name = intermed.IntermediateSymbolTable.create(context, config_path, 'linux', 'elf')
self._elf_table_name = intermed.IntermediateSymbolTable.create(
context, config_path, "linux", "elf"
)
super().__init__(context, config_path, name)
def _load_segments(self) -> None:
"""Load the segments from based on the PT_LOAD segments of the Elf64 format"""
ehdr = self.context.object(self._elf_table_name + constants.BANG + "Elf64_Ehdr",
layer_name = self._base_layer,
offset = 0)
ehdr = self.context.object(
self._elf_table_name + constants.BANG + "Elf64_Ehdr",
layer_name=self._base_layer,
offset=0,
)
segments = []
for pindex in range(ehdr.e_phnum):
phdr = self.context.object(self._elf_table_name + constants.BANG + "Elf64_Phdr",
layer_name = self._base_layer,
offset = ehdr.e_phoff + (pindex * ehdr.e_phentsize))
phdr = self.context.object(
self._elf_table_name + constants.BANG + "Elf64_Phdr",
layer_name=self._base_layer,
offset=ehdr.e_phoff + (pindex * ehdr.e_phentsize),
)
# We only want PT_TYPES with valid sizes
if phdr.p_type.lookup() == "PT_LOAD" and phdr.p_filesz == phdr.p_memsz and phdr.p_filesz > 0:
if (
phdr.p_type.lookup() == "PT_LOAD"
and phdr.p_filesz == phdr.p_memsz
and phdr.p_filesz > 0
):
# Cast these to ints to ensure the offsets don't need reconstructing
segments.append((int(phdr.p_paddr), int(phdr.p_offset), int(phdr.p_memsz), int(phdr.p_memsz)))
segments.append(
(
int(phdr.p_paddr),
int(phdr.p_offset),
int(phdr.p_memsz),
int(phdr.p_memsz),
)
)
if len(segments) == 0:
raise ElfFormatException(self.name, f"No ELF segments defined in {self._base_layer}")
raise ElfFormatException(
self.name, f"No ELF segments defined in {self._base_layer}"
)
self._segments = segments
@classmethod
def _check_header(cls, base_layer: interfaces.layers.DataLayerInterface, offset: int = 0) -> bool:
def _check_header(
cls, base_layer: interfaces.layers.DataLayerInterface, offset: int = 0
) -> bool:
try:
header_data = base_layer.read(offset, cls._header_struct.size)
except exceptions.InvalidAddressException:
raise ElfFormatException(base_layer.name,
f"Offset 0x{offset:0x} does not exist within the base layer")
(magic, elf_class, elf_data_encoding, elf_version) = cls._header_struct.unpack(header_data)
raise ElfFormatException(
base_layer.name,
f"Offset 0x{offset:0x} does not exist within the base layer",
)
(magic, elf_class, elf_data_encoding, elf_version) = cls._header_struct.unpack(
header_data
)
if magic != cls.MAGIC:
raise ElfFormatException(base_layer.name, f"Bad magic 0x{magic:x} at file offset 0x{offset:x}")
raise ElfFormatException(
base_layer.name, f"Bad magic 0x{magic:x} at file offset 0x{offset:x}"
)
if elf_class != cls.ELF_CLASS:
raise ElfFormatException(base_layer.name, f"ELF class is not 64-bit (2): {elf_class:d}")
raise ElfFormatException(
base_layer.name, f"ELF class is not 64-bit (2): {elf_class:d}"
)
# Virtualbox uses an ELF version of 0, which isn't to specification, but is ok to deal with
return True
@@ -70,10 +102,12 @@ class Elf64Stacker(interfaces.automagic.StackerLayerInterface):
stack_order = 10
@classmethod
def stack(cls,
context: interfaces.context.ContextInterface,
layer_name: str,
progress_callback: constants.ProgressCallback = None) -> Optional[interfaces.layers.DataLayerInterface]:
def stack(
cls,
context: interfaces.context.ContextInterface,
layer_name: str,
progress_callback: constants.ProgressCallback = None,
) -> Optional[interfaces.layers.DataLayerInterface]:
try:
if not Elf64Layer._check_header(context.layers[layer_name]):
return None
@@ -81,6 +115,12 @@ class Elf64Stacker(interfaces.automagic.StackerLayerInterface):
vollog.log(constants.LOGLEVEL_VVVV, f"Exception: {excp}")
return None
new_name = context.layers.free_layer_name("Elf64Layer")
context.config[interfaces.configuration.path_join(new_name, "base_layer")] = layer_name
context.config[interfaces.configuration.path_join(new_name, "base_layer")] = (
layer_name
)
return Elf64Layer(context, new_name, new_name)
try:
return Elf64Layer(context, new_name, new_name)
except ElfFormatException as excp:
vollog.log(constants.LOGLEVEL_VVVV, f"Exception: {excp}")
return None
volatility3/framework/layers/intel.py
+195 -77
View File
@@ -28,28 +28,44 @@ class Intel(linear.LinearlyMappedLayer):
# NOTE: _maxphyaddr is MAXPHYADDR as defined in the Intel specs *NOT* the maximum physical address
_maxphyaddr = 32
_maxvirtaddr = _maxphyaddr
_structure = [('page directory', 10, False), ('page table', 10, True)]
_direct_metadata = collections.ChainMap({'architecture': 'Intel32'}, {'mapped': True},
interfaces.layers.TranslationLayerInterface._direct_metadata)
_structure = [("page directory", 10, False), ("page table", 10, True)]
_direct_metadata = collections.ChainMap(
{"architecture": "Intel32"},
{"mapped": True},
interfaces.layers.TranslationLayerInterface._direct_metadata,
)
def __init__(self,
context: interfaces.context.ContextInterface,
config_path: str,
name: str,
metadata: Optional[Dict[str, Any]] = None) -> None:
super().__init__(context = context, config_path = config_path, name = name, metadata = metadata)
def __init__(
self,
context: interfaces.context.ContextInterface,
config_path: str,
name: str,
metadata: Optional[Dict[str, Any]] = None,
) -> None:
super().__init__(
context=context, config_path=config_path, name=name, metadata=metadata
)
self._base_layer = self.config["memory_layer"]
self._swap_layers: List[str] = []
self._page_map_offset = self.config["page_map_offset"]
# Assign constants
self._initial_position = min(self._maxvirtaddr, self._bits_per_register) - 1
self._initial_entry = self._mask(self._page_map_offset, self._initial_position, 0) | 0x1
self._initial_entry = (
self._mask(self._page_map_offset, self._initial_position, 0) | 0x1
)
self._entry_size = struct.calcsize(self._entry_format)
self._entry_number = self.page_size // self._entry_size
self._canonical_prefix = self._mask(
(1 << self._bits_per_register) - 1,
self._bits_per_register,
self._maxvirtaddr,
)
# These can vary depending on the type of space
self._index_shift = int(math.ceil(math.log2(struct.calcsize(self._entry_format))))
self._index_shift = int(
math.ceil(math.log2(struct.calcsize(self._entry_format)))
)
@classproperty
@functools.lru_cache()
@@ -86,7 +102,7 @@ class Intel(linear.LinearlyMappedLayer):
"""Returns the bits of a value between highbit and lowbit inclusive."""
high_mask = (1 << (high_bit + 1)) - 1
low_mask = (1 << low_bit) - 1
mask = (high_mask ^ low_mask)
mask = high_mask ^ low_mask
# print(high_bit, low_bit, bin(mask), bin(value))
return value & mask
@@ -95,6 +111,28 @@ class Intel(linear.LinearlyMappedLayer):
"""Returns whether a particular page is valid based on its entry."""
return bool(entry & 1)
@staticmethod
def _page_is_dirty(entry: int) -> bool:
"""Returns whether a particular page is dirty based on its entry."""
return bool(entry & (1 << 6))
def canonicalize(self, addr: int) -> int:
"""Canonicalizes an address by performing an appropiate sign extension on the higher addresses"""
if self._bits_per_register <= self._maxvirtaddr:
return addr & self.address_mask
elif addr < (1 << self._maxvirtaddr - 1):
return addr
return self._mask(addr, self._maxvirtaddr, 0) + self._canonical_prefix
def decanonicalize(self, addr: int) -> int:
"""Removes canonicalization to ensure an adress fits within the correct range if it has been canonicalized
This will produce an address outside the range if the canonicalization is incorrect
"""
if addr < (1 << self._maxvirtaddr - 1):
return addr
return addr ^ self._canonical_prefix
def _translate(self, offset: int) -> Tuple[int, int, str]:
"""Translates a specific offset based on paging tables.
@@ -106,9 +144,16 @@ class Intel(linear.LinearlyMappedLayer):
# Now we're done
if not self._page_is_valid(entry):
raise exceptions.PagedInvalidAddressException(self.name, offset, position + 1, entry,
f"Page Fault at entry {hex(entry)} in page entry")
page = self._mask(entry, self._maxphyaddr - 1, position + 1) | self._mask(offset, position, 0)
raise exceptions.PagedInvalidAddressException(
self.name,
offset,
position + 1,
entry,
f"Page Fault at entry {hex(entry)} in page entry",
)
page = self._mask(entry, self._maxphyaddr - 1, position + 1) | self._mask(
offset, position, 0
)
return page, 1 << (position + 1), self._base_layer
@@ -124,20 +169,30 @@ class Intel(linear.LinearlyMappedLayer):
entry = self._initial_entry
if self.minimum_address > offset > self.maximum_address:
raise exceptions.PagedInvalidAddressException(self.name, offset, position + 1, entry,
"Entry outside virtual address range: " + hex(entry))
raise exceptions.PagedInvalidAddressException(
self.name,
offset,
position + 1,
entry,
"Entry outside virtual address range: " + hex(entry),
)
# Run through the offset in various chunks
for (name, size, large_page) in self._structure:
for name, size, large_page in self._structure:
# Check we're valid
if not self._page_is_valid(entry):
raise exceptions.PagedInvalidAddressException(self.name, offset, position + 1, entry,
"Page Fault at entry " + hex(entry) + " in table " + name)
raise exceptions.PagedInvalidAddressException(
self.name,
offset,
position + 1,
entry,
"Page Fault at entry " + hex(entry) + " in table " + name,
)
# Check if we're a large page
if large_page and (entry & (1 << 7)):
# Mask off the PAT bit
if entry & (1 << 12):
entry -= (1 << 12)
entry -= 1 << 12
# We're a large page, the rest is finished below
# If we want to implement PSE-36, it would need to be done here
break
@@ -147,33 +202,51 @@ class Intel(linear.LinearlyMappedLayer):
index = self._mask(offset, start, position + 1) >> (position + 1)
# Grab the base address of the table we'll be getting the next entry from
base_address = self._mask(entry, self._maxphyaddr - 1, size + self._index_shift)
base_address = self._mask(
entry, self._maxphyaddr - 1, size + self._index_shift
)
table = self._get_valid_table(base_address)
if table is None:
raise exceptions.PagedInvalidAddressException(self.name, offset, position + 1, entry,
"Page Fault at entry " + hex(entry) + " in table " + name)
raise exceptions.PagedInvalidAddressException(
self.name,
offset,
position + 1,
entry,
"Page Fault at entry " + hex(entry) + " in table " + name,
)
# Read the data for the next entry
entry_data = table[(index << self._index_shift):(index << self._index_shift) + self._entry_size]
entry_data = table[
(index << self._index_shift) : (index << self._index_shift)
+ self._entry_size
]
if INTEL_TRANSLATION_DEBUGGING:
vollog.log(
constants.LOGLEVEL_VVVV, "Entry {} at index {} gives data {} as {}".format(
hex(entry), hex(index), hex(struct.unpack(self._entry_format, entry_data)[0]), name))
constants.LOGLEVEL_VVVV,
"Entry {} at index {} gives data {} as {}".format(
hex(entry),
hex(index),
hex(struct.unpack(self._entry_format, entry_data)[0]),
name,
),
)
# Read out the new entry from memory
entry, = struct.unpack(self._entry_format, entry_data)
(entry,) = struct.unpack(self._entry_format, entry_data)
return entry, position
@functools.lru_cache(1025)
def _get_valid_table(self, base_address: int) -> Optional[bytes]:
"""Extracts the table, validates it and returns it if it's valid."""
table = self._context.layers.read(self._base_layer, base_address, self.page_size)
table = self._context.layers.read(
self._base_layer, base_address, self.page_size
)
# If the table is entirely duplicates, then mark the whole table as bad
if (table == table[:self._entry_size] * self._entry_number):
if table == table[: self._entry_size] * self._entry_number:
return None
return table
@@ -182,27 +255,40 @@ class Intel(linear.LinearlyMappedLayer):
address."""
try:
# TODO: Consider reimplementing this, since calls to mapping can call is_valid
return all([
self._context.layers[layer].is_valid(mapped_offset)
for _, _, mapped_offset, _, layer in self.mapping(offset, length)
])
return all(
[
self._context.layers[layer].is_valid(mapped_offset)
for _, _, mapped_offset, _, layer in self.mapping(offset, length)
]
)
except exceptions.InvalidAddressException:
return False
def mapping(self,
offset: int,
length: int,
ignore_errors: bool = False) -> Iterable[Tuple[int, int, int, int, str]]:
def is_dirty(self, offset: int) -> bool:
"""Returns whether the page at offset is marked dirty"""
return self._page_is_dirty(self._translate_entry(offset)[0])
def mapping(
self, offset: int, length: int, ignore_errors: bool = False
) -> Iterable[Tuple[int, int, int, int, str]]:
"""Returns a sorted iterable of (offset, sublength, mapped_offset, mapped_length, layer)
mappings.
This allows translation layers to provide maps of contiguous
regions in one layer
"""
stashed_offset = stashed_mapped_offset = stashed_size = stashed_mapped_size = stashed_map_layer = None
for offset, size, mapped_offset, mapped_size, map_layer in self._mapping(offset, length, ignore_errors):
if stashed_offset is None or (stashed_offset + stashed_size != offset) or (
stashed_mapped_offset + stashed_mapped_size != mapped_offset) or (stashed_map_layer != map_layer):
stashed_offset = stashed_mapped_offset = stashed_size = stashed_mapped_size = (
stashed_map_layer
) = None
for offset, size, mapped_offset, mapped_size, map_layer in self._mapping(
offset, length, ignore_errors
):
if (
stashed_offset is None
or (stashed_offset + stashed_size != offset)
or (stashed_mapped_offset + stashed_mapped_size != mapped_offset)
or (stashed_map_layer != map_layer)
):
# The block isn't contiguous
if stashed_offset is not None:
yield stashed_offset, stashed_size, stashed_mapped_offset, stashed_mapped_size, stashed_map_layer
@@ -217,14 +303,18 @@ class Intel(linear.LinearlyMappedLayer):
stashed_size += size
stashed_mapped_size += mapped_size
# Yield whatever's left
if (stashed_offset is not None and stashed_mapped_offset is not None and stashed_size is not None
and stashed_mapped_size is not None and stashed_map_layer is not None):
if (
stashed_offset is not None
and stashed_mapped_offset is not None
and stashed_size is not None
and stashed_mapped_size is not None
and stashed_map_layer is not None
):
yield stashed_offset, stashed_size, stashed_mapped_offset, stashed_mapped_size, stashed_map_layer
def _mapping(self,
offset: int,
length: int,
ignore_errors: bool = False) -> Iterable[Tuple[int, int, int, int, str]]:
def _mapping(
self, offset: int, length: int, ignore_errors: bool = False
) -> Iterable[Tuple[int, int, int, int, str]]:
"""Returns a sorted iterable of (offset, sublength, mapped_offset, mapped_length, layer)
mappings.
@@ -235,20 +325,29 @@ class Intel(linear.LinearlyMappedLayer):
try:
mapped_offset, _, layer_name = self._translate(offset)
if not self._context.layers[layer_name].is_valid(mapped_offset):
raise exceptions.InvalidAddressException(layer_name = layer_name, invalid_address = mapped_offset)
raise exceptions.InvalidAddressException(
layer_name=layer_name, invalid_address=mapped_offset
)
except exceptions.InvalidAddressException:
if not ignore_errors:
raise
return
return None
yield offset, length, mapped_offset, length, layer_name
return
return None
while length > 0:
try:
chunk_offset, page_size, layer_name = self._translate(offset)
chunk_size = min(page_size - (chunk_offset % page_size), length)
if not self._context.layers[layer_name].is_valid(chunk_offset, chunk_size):
raise exceptions.InvalidAddressException(layer_name = layer_name, invalid_address = chunk_offset)
except (exceptions.PagedInvalidAddressException, exceptions.InvalidAddressException) as excp:
if not self._context.layers[layer_name].is_valid(
chunk_offset, chunk_size
):
raise exceptions.InvalidAddressException(
layer_name=layer_name, invalid_address=chunk_offset
)
except (
exceptions.PagedInvalidAddressException,
exceptions.InvalidAddressException,
) as excp:
if not ignore_errors:
raise
# We can jump more if we know where the page fault failed
@@ -256,7 +355,7 @@ class Intel(linear.LinearlyMappedLayer):
mask = (1 << excp.invalid_bits) - 1
else:
mask = (1 << self._page_size_in_bits) - 1
length_diff = (mask + 1 - (offset & mask))
length_diff = mask + 1 - (offset & mask)
length -= length_diff
offset += length_diff
else:
@@ -273,11 +372,13 @@ class Intel(linear.LinearlyMappedLayer):
@classmethod
def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]:
return [
requirements.TranslationLayerRequirement(name = 'memory_layer', optional = False),
requirements.LayerListRequirement(name = 'swap_layers', optional = True),
requirements.IntRequirement(name = 'page_map_offset', optional = False),
requirements.IntRequirement(name = 'kernel_virtual_offset', optional = True),
requirements.StringRequirement(name = 'kernel_banner', optional = True)
requirements.TranslationLayerRequirement(
name="memory_layer", optional=False
),
requirements.LayerListRequirement(name="swap_layers", optional=True),
requirements.IntRequirement(name="page_map_offset", optional=False),
requirements.IntRequirement(name="kernel_virtual_offset", optional=True),
requirements.StringRequirement(name="kernel_banner", optional=True),
]
@@ -289,25 +390,34 @@ class IntelPAE(Intel):
_bits_per_register = 32
_maxphyaddr = 40
_maxvirtaddr = 32
_structure = [('page directory pointer', 2, False), ('page directory', 9, True), ('page table', 9, True)]
_direct_metadata = collections.ChainMap({'pae': True}, Intel._direct_metadata)
_structure = [
("page directory pointer", 2, False),
("page directory", 9, True),
("page table", 9, True),
]
_direct_metadata = collections.ChainMap({"pae": True}, Intel._direct_metadata)
class Intel32e(Intel):
"""Class for handling 64-bit (32-bit extensions) for Intel
architectures."""
_direct_metadata = collections.ChainMap({'architecture': 'Intel64'}, Intel._direct_metadata)
_direct_metadata = collections.ChainMap(
{"architecture": "Intel64"}, Intel._direct_metadata
)
_entry_format = "<Q"
_bits_per_register = 64
_maxphyaddr = 52
_maxvirtaddr = 48
_structure = [('page map layer 4', 9, False), ('page directory pointer', 9, True), ('page directory', 9, True),
('page table', 9, True)]
_structure = [
("page map layer 4", 9, False),
("page directory pointer", 9, True),
("page directory", 9, True),
("page table", 9, True),
]
class WindowsMixin(Intel):
@staticmethod
def _page_is_valid(entry: int) -> bool:
"""Returns whether a particular page is valid based on its entry.
@@ -321,7 +431,9 @@ class WindowsMixin(Intel):
"""
return bool((entry & 1) or ((entry & 1 << 11) and not entry & 1 << 10))
def _translate_swap(self, layer: Intel, offset: int, bit_offset: int) -> Tuple[int, int, str]:
def _translate_swap(
self, layer: Intel, offset: int, bit_offset: int
) -> Tuple[int, int, str]:
try:
return super()._translate(offset)
except exceptions.PagedInvalidAddressException as excp:
@@ -331,19 +443,27 @@ class WindowsMixin(Intel):
unknown_bit = bool(entry & (1 << 7))
n = (entry >> 1) & 0xF
vbit = bool(entry & 1)
if (not tbit and not pbit and not vbit and unknown_bit) and ((entry >> bit_offset) != 0):
if (not tbit and not pbit and not vbit and unknown_bit) and (
(entry >> bit_offset) != 0
):
swap_offset = entry >> bit_offset << excp.invalid_bits
if layer.config.get('swap_layers', False):
if layer.config.get("swap_layers", False):
swap_layer_name = layer.config.get(
interfaces.configuration.path_join('swap_layers', 'swap_layers' + str(n)), None)
interfaces.configuration.path_join(
"swap_layers", "swap_layers" + str(n)
),
None,
)
if swap_layer_name:
return swap_offset, 1 << excp.invalid_bits, swap_layer_name
raise exceptions.SwappedInvalidAddressException(layer_name = excp.layer_name,
invalid_address = excp.invalid_address,
invalid_bits = excp.invalid_bits,
entry = excp.entry,
swap_offset = swap_offset)
raise exceptions.SwappedInvalidAddressException(
layer_name=excp.layer_name,
invalid_address=excp.invalid_address,
invalid_bits=excp.invalid_bits,
entry=excp.entry,
swap_offset=swap_offset,
)
raise
@@ -351,13 +471,11 @@ class WindowsMixin(Intel):
class WindowsIntel(WindowsMixin, Intel):
def _translate(self, offset):
return self._translate_swap(self, offset, self._page_size_in_bits)
class WindowsIntelPAE(WindowsMixin, IntelPAE):
def _translate(self, offset: int) -> Tuple[int, int, str]:
return self._translate_swap(self, offset, self._bits_per_register)
volatility3/framework/layers/leechcore.py
+22 -13
View File
@@ -1,3 +1,7 @@
# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0
# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0
#
import io
import logging
import urllib.parse
@@ -5,6 +9,7 @@ from typing import Optional, Any, List
try:
import leechcorepyc
HAS_LEECHCORE = True
except ImportError:
HAS_LEECHCORE = False
@@ -62,7 +67,7 @@ if HAS_LEECHCORE:
"""
return bool(self._handle)
def seek(self, offset, whence = io.SEEK_SET):
def seek(self, offset, whence=io.SEEK_SET):
if whence == io.SEEK_SET:
self._cursor = offset
elif whence == io.SEEK_CUR:
@@ -86,10 +91,14 @@ if HAS_LEECHCORE:
chunk_size = size
output = []
for entry in self.handle.memmap:
if entry['base'] + entry['size'] <= chunk_start or entry['base'] >= chunk_start + chunk_size:
if (
entry["base"] + entry["size"] <= chunk_start
or entry["base"] >= chunk_start + chunk_size
):
continue
output += [(max(entry['base'], chunk_start), min(entry['size'], chunk_size))]
output += [
(max(entry["base"], chunk_start), min(entry["size"], chunk_size))
]
chunk_start = output[-1][0] + output[-1][1]
chunk_size = max(0, size - chunk_start)
@@ -110,14 +119,16 @@ if HAS_LEECHCORE:
if len(data) > size:
data = data[:size]
else:
data = data + b'\x00' * (size - len(data))
data = data + b"\x00" * (size - len(data))
self._cursor += len(data)
if not len(data):
raise exceptions.InvalidAddressException('LeechCore layer read failure', self._cursor + len(data))
raise exceptions.InvalidAddressException(
"LeechCore layer read failure", self._cursor + len(data)
)
return data
def readline(self, __size: Optional[int] = ...) -> bytes:
data = b''
data = b""
while __size > self._chunk_size or __size < 0:
data += self.read(self._chunk_size)
index = data.find(b"\n")
@@ -155,20 +166,18 @@ if HAS_LEECHCORE:
def closed(self):
return self._handle
class LeechCoreHandler(resources.VolatilityHandler):
"""Handler for the invented `leechcore` scheme. This is an unofficial scheme and not registered with IANA
"""
"""Handler for the invented `leechcore` scheme. This is an unofficial scheme and not registered with IANA"""
@classmethod
def non_cached_schemes(cls) -> List[str]:
"""We need to turn caching *off* for a live filesystem"""
return ['leechcore']
return ["leechcore"]
@staticmethod
def default_open(req: urllib.request.Request) -> Optional[Any]:
"""Handles the request if it's the leechcore scheme."""
if req.type == 'leechcore':
device_uri = '://'.join(req.full_url.split('://')[1:])
if req.type == "leechcore":
device_uri = "://".join(req.full_url.split("://")[1:])
return LeechCoreFile(device_uri)
return None
volatility3/framework/layers/lime.py
+37 -17
View File
@@ -20,14 +20,16 @@ class LimeLayer(segmented.SegmentedLayer):
are large holes in the physical layer
"""
MAGIC = 0x4c694d45
MAGIC = 0x4C694D45
VERSION = 1
# Magic[4], Version[4], Start[8], End[8], Reserved[8]
# XXX move this to a custom SymbolSpace?
_header_struct = struct.Struct('<IIQQQ')
_header_struct = struct.Struct("<IIQQQ")
def __init__(self, context: interfaces.context.ContextInterface, config_path: str, name: str) -> None:
def __init__(
self, context: interfaces.context.ContextInterface, config_path: str, name: str
) -> None:
super().__init__(context, config_path, name)
# The base class loads the segments on initialization, but otherwise this must to get the right min/max addresses
@@ -45,31 +47,45 @@ class LimeLayer(segmented.SegmentedLayer):
if start < maxaddr or end < start:
raise LimeFormatException(
self.name, f"Bad start/end 0x{start:x}/0x{end:x} at file offset 0x{offset:x}")
self.name,
f"Bad start/end 0x{start:x}/0x{end:x} at file offset 0x{offset:x}",
)
segment_length = end - start + 1
segments.append((start, offset + header_size, segment_length, segment_length))
segments.append(
(start, offset + header_size, segment_length, segment_length)
)
maxaddr = end
offset = offset + header_size + segment_length
if len(segments) == 0:
raise LimeFormatException(self.name, f"No LiME segments defined in {self._base_layer}")
raise LimeFormatException(
self.name, f"No LiME segments defined in {self._base_layer}"
)
self._segments = segments
@classmethod
def _check_header(cls, base_layer: interfaces.layers.DataLayerInterface, offset: int = 0) -> Tuple[int, int]:
def _check_header(
cls, base_layer: interfaces.layers.DataLayerInterface, offset: int = 0
) -> Tuple[int, int]:
try:
header_data = base_layer.read(offset, cls._header_struct.size)
except exceptions.InvalidAddressException:
raise LimeFormatException(base_layer.name,
f"Offset 0x{offset:0x} does not exist within the base layer")
raise LimeFormatException(
base_layer.name,
f"Offset 0x{offset:0x} does not exist within the base layer",
)
(magic, version, start, end, reserved) = cls._header_struct.unpack(header_data)
if magic != cls.MAGIC:
raise LimeFormatException(base_layer.name, f"Bad magic 0x{magic:x} at file offset 0x{offset:x}")
raise LimeFormatException(
base_layer.name, f"Bad magic 0x{magic:x} at file offset 0x{offset:x}"
)
if version != cls.VERSION:
raise LimeFormatException(base_layer.name,
f"Unexpected version {version:d} at file offset 0x{offset:x}")
raise LimeFormatException(
base_layer.name,
f"Unexpected version {version:d} at file offset 0x{offset:x}",
)
return start, end
@@ -77,14 +93,18 @@ class LimeStacker(interfaces.automagic.StackerLayerInterface):
stack_order = 10
@classmethod
def stack(cls,
context: interfaces.context.ContextInterface,
layer_name: str,
progress_callback: constants.ProgressCallback = None) -> Optional[interfaces.layers.DataLayerInterface]:
def stack(
cls,
context: interfaces.context.ContextInterface,
layer_name: str,
progress_callback: constants.ProgressCallback = None,
) -> Optional[interfaces.layers.DataLayerInterface]:
try:
LimeLayer._check_header(context.layers[layer_name])
except LimeFormatException:
return None
new_name = context.layers.free_layer_name("LimeLayer")
context.config[interfaces.configuration.path_join(new_name, "base_layer")] = layer_name
context.config[interfaces.configuration.path_join(new_name, "base_layer")] = (
layer_name
)
return LimeLayer(context, new_name, new_name)
volatility3/framework/layers/linear.py
+41 -17
View File
@@ -1,3 +1,7 @@
# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0
# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0
#
import functools
from typing import List, Optional, Tuple, Iterable
@@ -10,41 +14,54 @@ class LinearlyMappedLayer(interfaces.layers.TranslationLayerInterface):
### Translation layer convenience function
def translate(self, offset: int, ignore_errors: bool = False) -> Tuple[Optional[int], Optional[str]]:
def translate(
self, offset: int, ignore_errors: bool = False
) -> Tuple[Optional[int], Optional[str]]:
mapping = list(self.mapping(offset, 0, ignore_errors))
if len(mapping) == 1:
original_offset, _, mapped_offset, _, layer = mapping[0]
if original_offset != offset:
raise exceptions.LayerException(self.name,
f"Layer {self.name} claims to map linearly but does not")
raise exceptions.LayerException(
self.name, f"Layer {self.name} claims to map linearly but does not"
)
else:
if ignore_errors:
# We should only hit this if we ignored errors, but check anyway
return None, None
raise exceptions.InvalidAddressException(self.name, offset,
f"Cannot translate {offset} in layer {self.name}")
raise exceptions.InvalidAddressException(
self.name, offset, f"Cannot translate {offset} in layer {self.name}"
)
return mapped_offset, layer
# ## Read/Write functions for mapped pages
# Redefine read here for speed reasons (so we don't call a processing method
@functools.lru_cache(maxsize = 512)
@functools.lru_cache(maxsize=512)
def read(self, offset: int, length: int, pad: bool = False) -> bytes:
"""Reads an offset for length bytes and returns 'bytes' (not 'str') of
length size."""
current_offset = offset
output: List[bytes] = []
for (offset, _, mapped_offset, mapped_length, layer) in self.mapping(offset, length, ignore_errors = pad):
for offset, _, mapped_offset, mapped_length, layer in self.mapping(
offset, length, ignore_errors=pad
):
if not pad and offset > current_offset:
raise exceptions.InvalidAddressException(
self.name, current_offset, f"Layer {self.name} cannot map offset: {current_offset}")
self.name,
current_offset,
f"Layer {self.name} cannot map offset: {current_offset}",
)
elif offset > current_offset:
output += [b"\x00" * (offset - current_offset)]
current_offset = offset
elif offset < current_offset:
raise exceptions.LayerException(self.name, "Mapping returned an overlapping element")
raise exceptions.LayerException(
self.name, "Mapping returned an overlapping element"
)
if mapped_length > 0:
output += [self._context.layers.read(layer, mapped_offset, mapped_length, pad)]
output += [
self._context.layers.read(layer, mapped_offset, mapped_length, pad)
]
current_offset += mapped_length
recovered_data = b"".join(output)
return recovered_data + b"\x00" * (length - len(recovered_data))
@@ -54,18 +71,25 @@ class LinearlyMappedLayer(interfaces.layers.TranslationLayerInterface):
underlying mapping."""
current_offset = offset
length = len(value)
for (offset, _, mapped_offset, length, layer) in self.mapping(offset, length):
for offset, _, mapped_offset, length, layer in self.mapping(offset, length):
if offset > current_offset:
raise exceptions.InvalidAddressException(
self.name, current_offset, f"Layer {self.name} cannot map offset: {current_offset}")
self.name,
current_offset,
f"Layer {self.name} cannot map offset: {current_offset}",
)
elif offset < current_offset:
raise exceptions.LayerException(self.name, "Mapping returned an overlapping element")
raise exceptions.LayerException(
self.name, "Mapping returned an overlapping element"
)
self._context.layers.write(layer, mapped_offset, value[:length])
value = value[length:]
current_offset += length
def _scan_iterator(self,
scanner: 'interfaces.layers.ScannerInterface',
sections: Iterable[Tuple[int, int]],
linear: bool = True) -> Iterable[interfaces.layers.IteratorValue]:
def _scan_iterator(
self,
scanner: "interfaces.layers.ScannerInterface",
sections: Iterable[Tuple[int, int]],
linear: bool = True,
) -> Iterable[interfaces.layers.IteratorValue]:
return super()._scan_iterator(scanner, sections, linear)
volatility3/framework/layers/msf.py
+108 -69
View File
@@ -21,15 +21,19 @@ class PdbMultiStreamFormat(linear.LinearlyMappedLayer):
"BIG_MSF_HDR": "Microsoft C/C++ MSF 7.00\r\n\x1a\x44\x53",
}
def __init__(self,
context: 'interfaces.context.ContextInterface',
config_path: str,
name: str,
metadata: Optional[Dict[str, Any]] = None) -> None:
def __init__(
self,
context: "interfaces.context.ContextInterface",
config_path: str,
name: str,
metadata: Optional[Dict[str, Any]] = None,
) -> None:
super().__init__(context, config_path, name, metadata)
self._base_layer = self.config["base_layer"]
self._pdb_symbol_table = intermed.IntermediateSymbolTable.create(context, self._config_path, 'windows', 'pdb')
self._pdb_symbol_table = intermed.IntermediateSymbolTable.create(
context, self._config_path, "windows", "pdb"
)
response = self._check_header()
if response is None:
raise PDBFormatException(name, "Could not find a suitable header")
@@ -43,59 +47,82 @@ class PdbMultiStreamFormat(linear.LinearlyMappedLayer):
def read_streams(self):
# Shortcut in case they've already been read
if self._streams:
return
return None
# Recover the root table, by recovering the root table index table...
module = self.context.module(self.pdb_symbol_table, self._base_layer, offset = 0)
module = self.context.module(self.pdb_symbol_table, self._base_layer, offset=0)
entry_size = module.get_type("unsigned long").size
root_table_num_pages = math.ceil(self._header.StreamInfo.StreamInfoSize / self._header.PageSize)
root_index_size = math.ceil((root_table_num_pages * entry_size) / self._header.PageSize)
root_index = module.object(object_type = "array",
offset = self._header.vol.size,
count = root_index_size,
subtype = module.get_type("unsigned long"))
root_index_layer_name = self.create_stream_from_pages("root_index", self._header.StreamInfo.StreamInfoSize,
[x for x in root_index])
root_table_num_pages = math.ceil(
self._header.StreamInfo.StreamInfoSize / self._header.PageSize
)
root_index_size = math.ceil(
(root_table_num_pages * entry_size) / self._header.PageSize
)
root_index = module.object(
object_type="array",
offset=self._header.vol.size,
count=root_index_size,
subtype=module.get_type("unsigned long"),
)
root_index_layer_name = self.create_stream_from_pages(
"root_index",
self._header.StreamInfo.StreamInfoSize,
[x for x in root_index],
)
module = self.context.module(self.pdb_symbol_table, root_index_layer_name, offset = 0)
root_pages = module.object(object_type = "array",
offset = 0,
count = root_table_num_pages,
subtype = module.get_type("unsigned long"))
root_layer_name = self.create_stream_from_pages("root", self._header.StreamInfo.StreamInfoSize,
[x for x in root_pages])
module = self.context.module(
self.pdb_symbol_table, root_index_layer_name, offset=0
)
root_pages = module.object(
object_type="array",
offset=0,
count=root_table_num_pages,
subtype=module.get_type("unsigned long"),
)
root_layer_name = self.create_stream_from_pages(
"root", self._header.StreamInfo.StreamInfoSize, [x for x in root_pages]
)
module = self.context.module(self.pdb_symbol_table, root_layer_name, offset = 0)
num_streams = module.object(object_type = "unsigned long", offset = 0)
stream_sizes = module.object(object_type = "array",
offset = entry_size,
count = num_streams,
subtype = module.get_type("unsigned long"))
module = self.context.module(self.pdb_symbol_table, root_layer_name, offset=0)
num_streams = module.object(object_type="unsigned long", offset=0)
stream_sizes = module.object(
object_type="array",
offset=entry_size,
count=num_streams,
subtype=module.get_type("unsigned long"),
)
current_offset = (num_streams + 1) * entry_size
for stream in range(num_streams):
list_size = math.ceil(stream_sizes[stream] / self.page_size)
if list_size == 0 or stream_sizes[stream] == 0xffffffff:
if list_size == 0 or stream_sizes[stream] == 0xFFFFFFFF:
self._streams[stream] = None
else:
stream_page_list = module.object(object_type = "array",
offset = current_offset,
count = list_size,
subtype = module.get_type("unsigned long"))
current_offset += (list_size * entry_size)
self._streams[stream] = self.create_stream_from_pages("stream" + str(stream), stream_sizes[stream],
[x for x in stream_page_list])
stream_page_list = module.object(
object_type="array",
offset=current_offset,
count=list_size,
subtype=module.get_type("unsigned long"),
)
current_offset += list_size * entry_size
self._streams[stream] = self.create_stream_from_pages(
"stream" + str(stream),
stream_sizes[stream],
[x for x in stream_page_list],
)
def create_stream_from_pages(self, stream_name: str, maximum_size: int, pages: List[int]) -> str:
def create_stream_from_pages(
self, stream_name: str, maximum_size: int, pages: List[int]
) -> str:
# Construct a root layer based on a number of pages
layer_name = self.name + "_" + stream_name
path_join = interfaces.configuration.path_join
config_path = path_join(self.config_path, stream_name)
self.context.config[path_join(config_path, 'base_layer')] = self.name
self.context.config[path_join(config_path, 'pages')] = pages
self.context.config[path_join(config_path, 'maximum_size')] = maximum_size
self.context.config[path_join(config_path, "base_layer")] = self.name
self.context.config[path_join(config_path, "pages")] = pages
self.context.config[path_join(config_path, "maximum_size")] = maximum_size
layer = PdbMSFStream(self.context, config_path, layer_name)
self.context.layers.add_layer(layer)
return layer_name
@@ -107,7 +134,10 @@ class PdbMultiStreamFormat(linear.LinearlyMappedLayer):
header_type = self.pdb_symbol_table + constants.BANG + header
current_header = self.context.object(header_type, self._base_layer, 0)
if utility.array_to_string(current_header.Magic) == self._headers[header]:
if not (current_header.PageSize < 0x100 or current_header.PageSize > (128 * 0x10000)):
if not (
current_header.PageSize < 0x100
or current_header.PageSize > (128 * 0x10000)
):
return header, current_header
return None
@@ -123,7 +153,9 @@ class PdbMultiStreamFormat(linear.LinearlyMappedLayer):
@classmethod
def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]:
return [requirements.TranslationLayerRequirement(name = 'base_layer', optional = False)]
return [
requirements.TranslationLayerRequirement(name="base_layer", optional=False)
]
@property
def maximum_address(self) -> int:
@@ -136,13 +168,12 @@ class PdbMultiStreamFormat(linear.LinearlyMappedLayer):
def is_valid(self, offset: int, length: int = 1) -> bool:
return self.context.layers[self._base_layer].is_valid(offset, length)
def mapping(self,
offset: int,
length: int,
ignore_errors: bool = False) -> Iterable[Tuple[int, int, int, int, str]]:
def mapping(
self, offset: int, length: int, ignore_errors: bool = False
) -> Iterable[Tuple[int, int, int, int, str]]:
yield offset, length, offset, length, self._base_layer
def get_stream(self, index) -> Optional['PdbMSFStream']:
def get_stream(self, index) -> Optional["PdbMSFStream"]:
self.read_streams()
if index not in self._streams:
raise PDBFormatException(self.name, "Stream not present")
@@ -154,12 +185,13 @@ class PdbMultiStreamFormat(linear.LinearlyMappedLayer):
class PdbMSFStream(linear.LinearlyMappedLayer):
def __init__(self,
context: 'interfaces.context.ContextInterface',
config_path: str,
name: str,
metadata: Optional[Dict[str, Any]] = None) -> None:
def __init__(
self,
context: "interfaces.context.ContextInterface",
config_path: str,
name: str,
metadata: Optional[Dict[str, Any]] = None,
) -> None:
super().__init__(context, config_path, name, metadata)
self._base_layer = self.config["base_layer"]
self._pages = self.config.get("pages", None)
@@ -180,28 +212,31 @@ class PdbMSFStream(linear.LinearlyMappedLayer):
@classmethod
def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]:
return [
requirements.ListRequirement(name = 'pages', element_type = int, min_elements = 1),
requirements.TranslationLayerRequirement(name = 'base_layer'),
requirements.IntRequirement(name = 'maximum_size')
requirements.ListRequirement(
name="pages", element_type=int, min_elements=1
),
requirements.TranslationLayerRequirement(name="base_layer"),
requirements.IntRequirement(name="maximum_size"),
]
def mapping(self,
offset: int,
length: int,
ignore_errors: bool = False) -> Iterable[Tuple[int, int, int, int, str]]:
def mapping(
self, offset: int, length: int, ignore_errors: bool = False
) -> Iterable[Tuple[int, int, int, int, str]]:
returned = 0
page_size = self._pdb_layer.page_size
while length > 0:
page = math.floor((offset + returned) / page_size)
page_position = ((offset + returned) % page_size)
page_position = (offset + returned) % page_size
chunk_size = min(page_size - page_position, length)
if page >= self._pages_len:
if not ignore_errors:
raise exceptions.InvalidAddressException(layer_name = self.name,
invalid_address = offset + returned)
raise exceptions.InvalidAddressException(
layer_name=self.name, invalid_address=offset + returned
)
else:
yield offset + returned, chunk_size, (self._pages[page] *
page_size) + page_position, chunk_size, self._base_layer
yield offset + returned, chunk_size, (
self._pages[page] * page_size
) + page_position, chunk_size, self._base_layer
returned += chunk_size
length -= chunk_size
@@ -218,13 +253,17 @@ class PdbMSFStream(linear.LinearlyMappedLayer):
@property
def maximum_address(self) -> int:
return self.config.get('maximum_size', len(self._pages) * self._pdb_layer.page_size)
return self.config.get(
"maximum_size", len(self._pages) * self._pdb_layer.page_size
)
@property
def _pdb_layer(self) -> PdbMultiStreamFormat:
if self._base_layer not in self._context.layers:
raise PDBFormatException(self._base_layer,
f"No PdbMultiStreamFormat layer found: {self._base_layer}")
raise PDBFormatException(
self._base_layer,
f"No PdbMultiStreamFormat layer found: {self._base_layer}",
)
result = self._context.layers[self._base_layer]
if isinstance(result, PdbMultiStreamFormat):
return result
volatility3/framework/layers/physical.py
+57 -34
View File
@@ -5,7 +5,7 @@ import logging
import threading
from typing import Any, Dict, IO, List, Optional, Union
from volatility3.framework import exceptions, interfaces, constants
from volatility3.framework import constants, exceptions, interfaces
from volatility3.framework.configuration import requirements
from volatility3.framework.layers import resources
@@ -16,13 +16,17 @@ class BufferDataLayer(interfaces.layers.DataLayerInterface):
"""A DataLayer class backed by a buffer in memory, designed for testing and
swift data access."""
def __init__(self,
context: interfaces.context.ContextInterface,
config_path: str,
name: str,
buffer: bytes,
metadata: Optional[Dict[str, Any]] = None) -> None:
super().__init__(context = context, config_path = config_path, name = name, metadata = metadata)
def __init__(
self,
context: interfaces.context.ContextInterface,
config_path: str,
name: str,
buffer: bytes,
metadata: Optional[Dict[str, Any]] = None,
) -> None:
super().__init__(
context=context, config_path=config_path, name=name, metadata=metadata
)
self._buffer = buffer
@property
@@ -37,8 +41,10 @@ class BufferDataLayer(interfaces.layers.DataLayerInterface):
def is_valid(self, offset: int, length: int = 1) -> bool:
"""Returns whether the offset is valid or not."""
return bool(self.minimum_address <= offset <= self.maximum_address
and self.minimum_address <= offset + length - 1 <= self.maximum_address)
return bool(
self.minimum_address <= offset <= self.maximum_address
and self.minimum_address <= offset + length - 1 <= self.maximum_address
)
def read(self, address: int, length: int, pad: bool = False) -> bytes:
"""Reads the data from the buffer."""
@@ -46,26 +52,30 @@ class BufferDataLayer(interfaces.layers.DataLayerInterface):
invalid_address = address
if self.minimum_address < address <= self.maximum_address:
invalid_address = self.maximum_address + 1
raise exceptions.InvalidAddressException(self.name, invalid_address,
"Offset outside of the buffer boundaries")
return self._buffer[address:address + length]
raise exceptions.InvalidAddressException(
self.name, invalid_address, "Offset outside of the buffer boundaries"
)
return self._buffer[address : address + length]
def write(self, address: int, data: bytes):
"""Writes the data from to the buffer."""
self._buffer = self._buffer[:address] + data + self._buffer[address + len(data):]
self._buffer = (
self._buffer[:address] + data + self._buffer[address + len(data) :]
)
@classmethod
def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]:
# No real requirements (only the buffer). Need to figure out if there's a better way of representing this
return [
requirements.BytesRequirement(name = 'buffer',
description = "The direct bytes to interact with",
optional = False)
requirements.BytesRequirement(
name="buffer",
description="The direct bytes to interact with",
optional=False,
)
]
class DummyLock:
def __enter__(self) -> None:
pass
@@ -76,12 +86,16 @@ class DummyLock:
class FileLayer(interfaces.layers.DataLayerInterface):
"""a DataLayer backed by a file on the filesystem."""
def __init__(self,
context: interfaces.context.ContextInterface,
config_path: str,
name: str,
metadata: Optional[Dict[str, Any]] = None) -> None:
super().__init__(context = context, config_path = config_path, name = name, metadata = metadata)
def __init__(
self,
context: interfaces.context.ContextInterface,
config_path: str,
name: str,
metadata: Optional[Dict[str, Any]] = None,
) -> None:
super().__init__(
context=context, config_path=config_path, name=name, metadata=metadata
)
self._write_warning = False
self._location = self.config["location"]
@@ -133,8 +147,10 @@ class FileLayer(interfaces.layers.DataLayerInterface):
"""Returns whether the offset is valid or not."""
if length <= 0:
raise ValueError("Length must be positive")
return bool(self.minimum_address <= offset <= self.maximum_address
and self.minimum_address <= offset + length - 1 <= self.maximum_address)
return bool(
self.minimum_address <= offset <= self.maximum_address
and self.minimum_address <= offset + length - 1 <= self.maximum_address
)
def read(self, offset: int, length: int, pad: bool = False) -> bytes:
"""Reads from the file at offset for length."""
@@ -142,8 +158,9 @@ class FileLayer(interfaces.layers.DataLayerInterface):
invalid_address = offset
if self.minimum_address < offset <= self.maximum_address:
invalid_address = self.maximum_address + 1
raise exceptions.InvalidAddressException(self.name, invalid_address,
"Offset outside of the buffer boundaries")
raise exceptions.InvalidAddressException(
self.name, invalid_address, "Offset outside of the buffer boundaries"
)
# TODO: implement locking for multi-threading
with self._lock:
@@ -152,10 +169,13 @@ class FileLayer(interfaces.layers.DataLayerInterface):
if len(data) < length:
if pad:
data += (b"\x00" * (length - len(data)))
data += b"\x00" * (length - len(data))
else:
raise exceptions.InvalidAddressException(
self.name, offset + len(data), "Could not read sufficient bytes from the " + self.name + " file")
self.name,
offset + len(data),
"Could not read sufficient bytes from the " + self.name + " file",
)
return data
def write(self, offset: int, data: bytes) -> None:
@@ -172,8 +192,11 @@ class FileLayer(interfaces.layers.DataLayerInterface):
invalid_address = offset
if self.minimum_address < offset <= self.maximum_address:
invalid_address = self.maximum_address + 1
raise exceptions.InvalidAddressException(self.name, invalid_address,
"Data segment outside of the " + self.name + " file boundaries")
raise exceptions.InvalidAddressException(
self.name,
invalid_address,
"Data segment outside of the " + self.name + " file boundaries",
)
with self._lock:
self._file.seek(offset)
self._file.write(data)
@@ -191,9 +214,9 @@ class FileLayer(interfaces.layers.DataLayerInterface):
"""Closes the file handle."""
self._file.close()
def __del__(self) -> None:
def __exit__(self, type, value, traceback) -> None:
self.destroy()
@classmethod
def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]:
return [requirements.StringRequirement(name = 'location', optional = False)]
return [requirements.StringRequirement(name="location", optional=False)]
volatility3/framework/layers/qemu.py
+250 -129
View File
@@ -26,7 +26,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer):
QEVM_SUBSECTION = 0x05
QEVM_VMDESCRIPTION = 0x06
QEVM_CONFIGURATION = 0x07
QEVM_SECTION_FOOTER = 0x7e
QEVM_SECTION_FOOTER = 0x7E
HASH_PTE_SIZE_64 = 16
SEGMENT_FLAG_COMPRESS = 0x02
@@ -56,57 +56,88 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer):
distro_re = r"(\w+[\d{1,2}\.]*)"
pci_hole_table = {re.compile(r"^pc-i440fx-([23456789]|\d\d+)\.\d$"): (0xe0000000, 0xc0000000, 0x100000000),
re.compile(r"^pc-i440fx-[01]\.\d$"): (0xe0000000, 0xe0000000, 0x100000000),
re.compile(r"^pc-q35-\d\.\d$"): (0xb0000000, 0x80000000, 0x100000000),
re.compile(r"^microvm$"): (0xc0000000, 0xc0000000, 0x100000000),
re.compile(r"^xen$"): (0xf0000000, 0xf0000000, 0x100000000),
re.compile(r"^pc-i440fx-" + distro_re + r"$"): (0xe0000000, 0xc0000000, 0x100000000),
re.compile(r"^pc-q35-" + distro_re + r"$"): (0xb0000000, 0x80000000, 0x100000000),
}
pci_hole_table = {
re.compile(r"^pc-i440fx-([23456789]|\d\d+)\.\d$"): (
0xE0000000,
0xC0000000,
0x100000000,
),
re.compile(r"^pc-i440fx-[01]\.\d$"): (0xE0000000, 0xE0000000, 0x100000000),
re.compile(r"^pc-q35-\d\.\d$"): (0xB0000000, 0x80000000, 0x100000000),
re.compile(r"^microvm$"): (0xC0000000, 0xC0000000, 0x100000000),
re.compile(r"^xen$"): (0xF0000000, 0xF0000000, 0x100000000),
re.compile(r"^pc-i440fx-" + distro_re + r"$"): (
0xE0000000,
0xC0000000,
0x100000000,
),
re.compile(r"^pc-q35-" + distro_re + r"$"): (
0xB0000000,
0x80000000,
0x100000000,
),
}
def __init__(self,
context: interfaces.context.ContextInterface,
config_path: str,
name: str,
metadata: Optional[Dict[str, Any]] = None) -> None:
self._qemu_table_name = intermed.IntermediateSymbolTable.create(context, config_path, 'generic', 'qemu')
def __init__(
self,
context: interfaces.context.ContextInterface,
config_path: str,
name: str,
metadata: Optional[Dict[str, Any]] = None,
) -> None:
self._qemu_table_name = intermed.IntermediateSymbolTable.create(
context, config_path, "generic", "qemu"
)
self._configuration = None
self._architecture = None
self._compressed: Set[int] = set()
self._current_segment_name = b''
self._current_segment_name = b""
self._pci_hole_start = 0
self._pci_hole_end = 0
self._pci_hole_minimum = 0
super().__init__(context = context, config_path = config_path, name = name, metadata = metadata)
super().__init__(
context=context, config_path=config_path, name=name, metadata=metadata
)
@classmethod
def _check_header(cls, base_layer: interfaces.layers.DataLayerInterface, name: str = ''):
def _check_header(
cls, base_layer: interfaces.layers.DataLayerInterface, name: str = ""
):
header = base_layer.read(0, 8)
if header[:4] != b'\x51\x45\x56\x4D':
raise exceptions.LayerException(name, 'No QEMU magic bytes')
if header[4:] != b'\x00\x00\x00\x03':
raise exceptions.LayerException(name, 'Unsupported QEMU version found')
if header[:4] != b"\x51\x45\x56\x4D":
raise exceptions.LayerException(name, "No QEMU magic bytes")
if header[4:] != b"\x00\x00\x00\x03":
raise exceptions.LayerException(name, "Unsupported QEMU version found")
vollog.debug("QEVM header found")
def _read_configuration(self, base_layer: interfaces.layers.DataLayerInterface, name: str) -> Any:
def _read_configuration(
self, base_layer: interfaces.layers.DataLayerInterface, name: str
) -> Any:
"""Reads the JSON configuration from the end of the file"""
chunk_size = 4096
data = b''
for i in range(base_layer.maximum_address, base_layer.minimum_address, -chunk_size):
if i != base_layer.maximum_address:
data = (base_layer.read(i, chunk_size) + data).rstrip(b'\x00')
if b'\x00' in data:
last_null_byte = data.rfind(b'\x00')
start_of_json = data.find(b'{', last_null_byte)
data = b""
for i in range(
base_layer.maximum_address + 1, base_layer.minimum_address, -chunk_size
):
# Since we're going backwards, we need to include one extra byte so the tail doesn't get chopped off
if i != base_layer.maximum_address + 1:
data = (base_layer.read(i, chunk_size) + data).rstrip(b"\x00")
if b"\x00" in data:
last_null_byte = data.rfind(b"\x00")
start_of_json = data.find(b"{", last_null_byte)
if start_of_json >= 0:
data = data[start_of_json:]
return json.loads(data)
# No JSON configuration found at the end of the file, return empty dict
return dict()
raise exceptions.LayerException(name, "Invalid JSON configuration at the end of the file")
raise exceptions.LayerException(
name, "Invalid JSON configuration at the end of the file"
)
def _get_ram_segments(self, index: int, page_size: int) -> Tuple[List[Tuple[int, int, int, int]], int]:
def _get_ram_segments(
self, index: int, page_size: int
) -> Tuple[List[Tuple[int, int, int, int]], int]:
"""Recovers the new index and any sections of memory from a ram section"""
done = None
segments = []
@@ -116,7 +147,7 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer):
while not done:
# Use struct.unpack here for performance improvements
addr = struct.unpack('>Q', base_layer.read(index, 8))[0]
addr = struct.unpack(">Q", base_layer.read(index, 8))[0]
# Flags are stored in the n least significant bits, where n equals the bit-length of pagesize
flags = addr & (page_size - 1)
@@ -129,43 +160,59 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer):
addr += self._pci_hole_end - self._pci_hole_start
if flags & self.SEGMENT_FLAG_MEM_SIZE:
namelen = self._context.object(self._qemu_table_name + constants.BANG + 'unsigned char',
offset = index,
layer_name = self._base_layer)
namelen = self._context.object(
self._qemu_table_name + constants.BANG + "unsigned char",
offset=index,
layer_name=self._base_layer,
)
while namelen != 0:
total_size = self._context.object(self._qemu_table_name + constants.BANG + 'unsigned long long',
offset = index + 1 + namelen,
layer_name = self._base_layer)
total_size = self._context.object(
self._qemu_table_name + constants.BANG + "unsigned long long",
offset=index + 1 + namelen,
layer_name=self._base_layer,
)
size_array[base_layer.read(index + 1, namelen)] = total_size
index += 1 + namelen + 8
namelen = self._context.object(self._qemu_table_name + constants.BANG + 'unsigned char',
offset = index,
layer_name = self._base_layer)
highest_possible_maximum = max([x[0] for x in self.pci_hole_table.values()]) + 1
if size_array.get(b'pc.ram', highest_possible_maximum) < self._pci_hole_minimum:
namelen = self._context.object(
self._qemu_table_name + constants.BANG + "unsigned char",
offset=index,
layer_name=self._base_layer,
)
highest_possible_maximum = (
max([x[0] for x in self.pci_hole_table.values()]) + 1
)
if (
size_array.get(b"pc.ram", highest_possible_maximum)
< self._pci_hole_minimum
):
# Turns off the pci_hole if it's not supposed to be there
vollog.debug(
f"QEVM turning off PCI hole due to small image size: 0x{size_array.get(b'pc.ram'):x} < 0x{self._pci_hole_minimum:x}")
f"QEVM turning off PCI hole due to small image size: 0x{size_array.get(b'pc.ram'):x} < 0x{self._pci_hole_minimum:x}"
)
self._pci_hole_start, self._pci_hole_end = 0, 0
if flags & (self.SEGMENT_FLAG_COMPRESS | self.SEGMENT_FLAG_PAGE):
if not (flags & self.SEGMENT_FLAG_CONTINUE):
namelen = self._context.object(self._qemu_table_name + constants.BANG + 'unsigned char',
offset = index,
layer_name = self._base_layer)
namelen = self._context.object(
self._qemu_table_name + constants.BANG + "unsigned char",
offset=index,
layer_name=self._base_layer,
)
self._current_segment_name = base_layer.read(index + 1, namelen)
index += 1 + namelen
if flags & self.SEGMENT_FLAG_COMPRESS:
if self._current_segment_name == b'pc.ram':
if self._current_segment_name == b"pc.ram":
segments.append((addr, index, page_size, 1))
self._compressed.add(addr)
index += 1
else:
if self._current_segment_name == b'pc.ram':
if self._current_segment_name == b"pc.ram":
segments.append((addr, index, page_size, page_size))
index += page_size
if flags & self.SEGMENT_FLAG_XBZRLE:
raise exceptions.LayerException(self.name, "XBZRLE compression not supported")
raise exceptions.LayerException(
self.name, "XBZRLE compression not supported"
)
if flags & self.SEGMENT_FLAG_EOS:
done = True
return segments, index
@@ -187,88 +234,136 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer):
if not self._architecture:
self._architecture = self._fallback_determine_architecture()
if self._architecture is None:
vollog.log(constants.LOGLEVEL_VV, f"QEVM architecture could not be determined")
vollog.log(
constants.LOGLEVEL_VV,
f"QEVM architecture could not be determined",
)
# Once all segments have been read, determine the PCI hole if any
for regex in self.pci_hole_table:
if regex.match(self._architecture):
self._pci_hole_minimum, self._pci_hole_start, self._pci_hole_end = self.pci_hole_table[regex]
vollog.log(constants.LOGLEVEL_VVVV, f"QEVM architecture detected as: {self._architecture}")
(
self._pci_hole_minimum,
self._pci_hole_start,
self._pci_hole_end,
) = self.pci_hole_table[regex]
vollog.log(
constants.LOGLEVEL_VVVV,
f"QEVM architecture detected as: {self._architecture}",
)
break
else:
vollog.log(constants.LOGLEVEL_VVVV, f"QEVM unknown architecture found: {self._architecture}")
vollog.log(
constants.LOGLEVEL_VVVV,
f"QEVM unknown architecture found: {self._architecture}",
)
arch_detected = True
section_byte = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned char',
offset = index,
layer_name = self._base_layer)
section_byte = self.context.object(
self._qemu_table_name + constants.BANG + "unsigned char",
offset=index,
layer_name=self._base_layer,
)
index += 1
if section_byte == self.QEVM_CONFIGURATION:
section_len = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned long',
offset = index,
layer_name = self._base_layer)
self._architecture = self.context.object(self._qemu_table_name + constants.BANG + 'string',
offset = index + 4, layer_name = self._base_layer,
max_length = section_len)
section_len = self.context.object(
self._qemu_table_name + constants.BANG + "unsigned long",
offset=index,
layer_name=self._base_layer,
)
self._architecture = self.context.object(
self._qemu_table_name + constants.BANG + "string",
offset=index + 4,
layer_name=self._base_layer,
max_length=section_len,
)
index += 4 + section_len
elif section_byte == self.QEVM_SECTION_START or section_byte == self.QEVM_SECTION_FULL:
section_id = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned long',
offset = index,
layer_name = self._base_layer)
elif (
section_byte == self.QEVM_SECTION_START
or section_byte == self.QEVM_SECTION_FULL
):
section_id = self.context.object(
self._qemu_table_name + constants.BANG + "unsigned long",
offset=index,
layer_name=self._base_layer,
)
current_section_id = section_id
index += 4
name_len = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned char',
offset = index,
layer_name = self._base_layer)
name_len = self.context.object(
self._qemu_table_name + constants.BANG + "unsigned char",
offset=index,
layer_name=self._base_layer,
)
index += 1
name = self.context.object(self._qemu_table_name + constants.BANG + 'string',
offset = index,
layer_name = self._base_layer,
max_length = name_len)
name = self.context.object(
self._qemu_table_name + constants.BANG + "string",
offset=index,
layer_name=self._base_layer,
max_length=name_len,
)
index += name_len
# instance_id = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned long',
# offset = index,
# layer_name = self._base_layer)
index += 4
version_id = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned long',
offset = index,
layer_name = self._base_layer)
version_id = self.context.object(
self._qemu_table_name + constants.BANG + "unsigned long",
offset=index,
layer_name=self._base_layer,
)
index += 4
# Store section info for handling QEVM_SECTION_PARTs later on
section_info[current_section_id] = {'name': name, 'version_id': version_id}
section_info[current_section_id] = {
"name": name,
"version_id": version_id,
}
# Read additional data
index = self.extract_data(index, name, version_id)
elif section_byte == self.QEVM_SECTION_PART or section_byte == self.QEVM_SECTION_END:
section_id = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned long',
offset = index,
layer_name = self._base_layer)
elif (
section_byte == self.QEVM_SECTION_PART
or section_byte == self.QEVM_SECTION_END
):
section_id = self.context.object(
self._qemu_table_name + constants.BANG + "unsigned long",
offset=index,
layer_name=self._base_layer,
)
current_section_id = section_id
index += 4
# Read additional data
index = self.extract_data(index, section_info[current_section_id]['name'],
section_info[current_section_id]['version_id'])
index = self.extract_data(
index,
section_info[current_section_id]["name"],
section_info[current_section_id]["version_id"],
)
elif section_byte == self.QEVM_SECTION_FOOTER:
section_id = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned long',
offset = index,
layer_name = self._base_layer)
section_id = self.context.object(
self._qemu_table_name + constants.BANG + "unsigned long",
offset=index,
layer_name=self._base_layer,
)
index += 4
if section_id != current_section_id:
raise exceptions.LayerException(
self._name, f'QEMU section footer mismatch: {current_section_id} and {section_id}')
self._name,
f"QEMU section footer mismatch: {current_section_id} and {section_id}",
)
elif section_byte == self.QEVM_EOF:
pass
else:
raise exceptions.LayerException(self._name, f'QEMU unknown section encountered: {section_byte}')
raise exceptions.LayerException(
self._name, f"QEMU unknown section encountered: {section_byte}"
)
def _fallback_determine_architecture(self) -> str:
architecture_pattern = rb'pc-(i440fx|q35)-(\d{1,2}\.\d{1,2}|\w+[\d{1,2}\.]*)'
architecture_pattern = rb"pc-(i440fx|q35)-(\d{1,2}\.\d{1,2}|\w+[\d{1,2}\.]*)"
default_suffix = "-2.0"
base_layer = self.context.layers[self._base_layer]
vollog.log(constants.LOGLEVEL_VVVV, "QEVM fallback architecture detection used")
res = scanners.RegExScanner(architecture_pattern)
for offset in base_layer.scan(context = self.context, scanner = res):
for offset in base_layer.scan(context=self.context, scanner=res):
line = base_layer.read(offset, 64)
regex_results = re.search(architecture_pattern, line)
architecture = regex_results.group().decode()
@@ -276,80 +371,102 @@ class QemuSuspendLayer(segmented.NonLinearlySegmentedLayer):
# If that does not work, look in configuration JSON for devices specific to a certain architecture
architecture = None
for device in self._configuration.get('devices', []):
device_name = device.get('vmsd_name', '').lower()
if 'i440fx' in device_name or 'piix' in device_name:
architecture = 'pc-i440fx' + default_suffix
for device in self._configuration.get("devices", []):
device_name = device.get("vmsd_name", "").lower()
if "i440fx" in device_name or "piix" in device_name:
architecture = "pc-i440fx" + default_suffix
break
elif 'ich9' in device_name:
architecture = 'pc-q35' + default_suffix
elif "ich9" in device_name:
architecture = "pc-q35" + default_suffix
break
if architecture:
vollog.log(constants.LOGLEVEL_VVV, f'Architecture version unknown, default used: {default_suffix}')
vollog.log(
constants.LOGLEVEL_VVV,
f"Architecture version unknown, default used: {default_suffix}",
)
return architecture
# Still haven't found architecture, switch to fallback-method
architecture_pattern = rb'Standard PC \((i440FX|Q35)'
architecture_pattern = rb"Standard PC \((i440FX|Q35)"
res = scanners.RegExScanner(architecture_pattern)
for offset in base_layer.scan(context = self.context, scanner = res):
for offset in base_layer.scan(context=self.context, scanner=res):
line = base_layer.read(offset, 64)
regex_results = re.search(architecture_pattern, line)
architecture = "pc-" + regex_results.groups()[0].decode().lower() + default_suffix
vollog.log(constants.LOGLEVEL_VVV, f'Architecture version unknown, default used: {default_suffix}')
architecture = (
"pc-" + regex_results.groups()[0].decode().lower() + default_suffix
)
vollog.log(
constants.LOGLEVEL_VVV,
f"Architecture version unknown, default used: {default_suffix}",
)
return architecture
vollog.warning("Could not determine QEMU target architecture!")
return None
def extract_data(self, index, name, version_id):
if name == 'ram':
if name == "ram":
if version_id != 4:
raise exceptions.LayerException(f"QEMU unknown RAM version_id {version_id}")
new_segments, index = self._get_ram_segments(index, self._configuration.get('page_size', 4096))
raise exceptions.LayerException(
f"QEMU unknown RAM version_id {version_id}"
)
new_segments, index = self._get_ram_segments(
index, self._configuration.get("page_size", 4096)
)
self._segments += new_segments
elif name == 'spapr/htab':
elif name == "spapr/htab":
if version_id != 1:
raise exceptions.LayerException(f"QEMU unknown HTAB version_id {version_id}")
header = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned long',
offset = index,
layer_name = self._base_layer)
raise exceptions.LayerException(
f"QEMU unknown HTAB version_id {version_id}"
)
header = self.context.object(
self._qemu_table_name + constants.BANG + "unsigned long",
offset=index,
layer_name=self._base_layer,
)
index += 4
if header == 0:
htab_index = -1
htab_n_valid = 0
htab_n_invalid = 0
while htab_index != 0 and htab_n_valid != 0 and htab_n_invalid != 0:
htab = self.context.object(self._qemu_table_name + constants.BANG + 'htab',
offset = index,
layer_name = self._base_layer)
htab = self.context.object(
self._qemu_table_name + constants.BANG + "htab",
offset=index,
layer_name=self._base_layer,
)
htab_index, htab_n_valid, htab_n_invalid = htab
index += 8 + (htab_n_valid * self.HASH_PTE_SIZE_64)
elif name == 'dirty-bitmap':
elif name == "dirty-bitmap":
index += 1
elif name == 'pbs-state':
section_len = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned long long',
offset = index,
layer_name = self._base_layer)
elif name == "pbs-state":
section_len = self.context.object(
self._qemu_table_name + constants.BANG + "unsigned long long",
offset=index,
layer_name=self._base_layer,
)
index += 8 + section_len
return index
def _decode_data(self, data: bytes, mapped_offset: int, offset: int, output_length: int) -> bytes:
def _decode_data(
self, data: bytes, mapped_offset: int, offset: int, output_length: int
) -> bytes:
"""Takes the full segment from the base_layer that the data occurs in, checks whether it's compressed
(by locating it in the segment list and verifying if that address is compressed), then reading/expanding the
data, and finally cutting it to the right size. Offset may be the address requested rather than the location
of the starting data. It is the responsibility of the layer to turn the provided data chunk into the right
portion of data necessary.
"""
page_size = self._configuration.get('page_size', 4096)
page_size = self._configuration.get("page_size", 4096)
# start_offset equals the highest multiple of pagesize <= offset
# (We assume that page_size is a power of 2)
start_offset = offset ^ (offset & (page_size - 1))
if start_offset in self._compressed:
data = (data * page_size)
result = data[offset - start_offset:output_length + offset - start_offset]
data = data * page_size
result = data[offset - start_offset : output_length + offset - start_offset]
return result
@functools.lru_cache(maxsize = 512)
@functools.lru_cache(maxsize=512)
def read(self, offset: int, length: int, pad: bool = False) -> bytes:
return super().read(offset, length, pad)
@@ -358,16 +475,20 @@ class QemuStacker(interfaces.automagic.StackerLayerInterface):
stack_order = 10
@classmethod
def stack(cls,
context: interfaces.context.ContextInterface,
layer_name: str,
progress_callback: constants.ProgressCallback = None) -> Optional[interfaces.layers.DataLayerInterface]:
def stack(
cls,
context: interfaces.context.ContextInterface,
layer_name: str,
progress_callback: constants.ProgressCallback = None,
) -> Optional[interfaces.layers.DataLayerInterface]:
try:
QemuSuspendLayer._check_header(context.layers[layer_name])
except exceptions.LayerException:
return None
new_name = context.layers.free_layer_name("QemuSuspendLayer")
context.config[interfaces.configuration.path_join(new_name, "base_layer")] = layer_name
context.config[interfaces.configuration.path_join(new_name, "base_layer")] = (
layer_name
)
layer = QemuSuspendLayer(context, new_name, new_name)
cls.stacker_slow_warning()
return layer
volatility3/framework/layers/registry.py
+143 -79
View File
@@ -1,13 +1,16 @@
# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0
# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0
#
import contextlib
import logging
from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union
from volatility3.framework import constants, exceptions, interfaces, objects
from volatility3.framework.configuration import requirements
from volatility3.framework.configuration.requirements import IntRequirement, TranslationLayerRequirement
from volatility3.framework.configuration.requirements import (
IntRequirement,
TranslationLayerRequirement,
)
from volatility3.framework.exceptions import InvalidAddressException
from volatility3.framework.layers import linear
from volatility3.framework.symbols import intermed
@@ -25,35 +28,49 @@ class RegistryInvalidIndex(exceptions.LayerException):
class RegistryHive(linear.LinearlyMappedLayer):
def __init__(self,
context: interfaces.context.ContextInterface,
config_path: str,
name: str,
metadata: Optional[Dict[str, Any]] = None) -> None:
super().__init__(context = context, config_path = config_path, name = name, metadata = metadata)
def __init__(
self,
context: interfaces.context.ContextInterface,
config_path: str,
name: str,
metadata: Optional[Dict[str, Any]] = None,
) -> None:
super().__init__(
context=context, config_path=config_path, name=name, metadata=metadata
)
self._base_layer = self.config["base_layer"]
self._hive_offset = self.config["hive_offset"]
self._table_name = self.config["nt_symbols"]
self._page_size = 1 << 12
self._reg_table_name = intermed.IntermediateSymbolTable.create(context, self._config_path, 'windows',
'registry')
self._reg_table_name = intermed.IntermediateSymbolTable.create(
context, self._config_path, "windows", "registry"
)
cmhive = self.context.object(self._table_name + constants.BANG + "_CMHIVE", self._base_layer, self._hive_offset)
cmhive = self.context.object(
self._table_name + constants.BANG + "_CMHIVE",
self._base_layer,
self._hive_offset,
)
self._cmhive_name = cmhive.get_name()
self.hive = cmhive.Hive
# TODO: Check the checksum
if self.hive.Signature != 0xbee0bee0:
if self.hive.Signature != 0xBEE0BEE0:
raise RegistryFormatException(
self.name, f"Registry hive at {self._hive_offset} does not have a valid signature")
self.name,
f"Registry hive at {self._hive_offset} does not have a valid signature",
)
# Win10 17063 introduced the Registry process to map most hives. Check
# if it exists and update RegistryHive._base_layer
for proc in pslist.PsList.list_processes(self.context, self.config['base_layer'], self.config['nt_symbols']):
proc_name = proc.ImageFileName.cast("string", max_length = proc.ImageFileName.vol.count, errors = 'replace')
for proc in pslist.PsList.list_processes(
self.context, self.config["base_layer"], self.config["nt_symbols"]
):
proc_name = proc.ImageFileName.cast(
"string", max_length=proc.ImageFileName.vol.count, errors="replace"
)
if proc_name == "Registry" and proc.InheritedFromUniqueProcessId == 4:
proc_layer_name = proc.add_process_layer()
self._base_layer = proc_layer_name
@@ -66,16 +83,23 @@ class RegistryHive(linear.LinearlyMappedLayer):
self._hive_maxaddr_non_volatile = self.hive.Storage[0].Length
self._hive_maxaddr_volatile = self.hive.Storage[1].Length
self._maxaddr = 0x80000000 | self._hive_maxaddr_volatile
vollog.log(constants.LOGLEVEL_VVVV, f"Setting hive {self.name} max address to {hex(self._maxaddr)}")
vollog.log(
constants.LOGLEVEL_VVVV,
f"Setting hive {self.name} max address to {hex(self._maxaddr)}",
)
except exceptions.InvalidAddressException:
self._hive_maxaddr_non_volatile = 0x7fffffff
self._hive_maxaddr_volatile = 0x7fffffff
self._hive_maxaddr_non_volatile = 0x7FFFFFFF
self._hive_maxaddr_volatile = 0x7FFFFFFF
self._maxaddr = 0x80000000 | self._hive_maxaddr_volatile
vollog.log(constants.LOGLEVEL_VVVV,
f"Exception when setting hive {self.name} max address, using {hex(self._maxaddr)}")
vollog.log(
constants.LOGLEVEL_VVVV,
f"Exception when setting hive {self.name} max address, using {hex(self._maxaddr)}",
)
def _get_hive_maxaddr(self, volatile):
return self._hive_maxaddr_volatile if volatile else self._hive_maxaddr_non_volatile
return (
self._hive_maxaddr_volatile if volatile else self._hive_maxaddr_non_volatile
)
def get_name(self) -> str:
return self._cmhive_name or "[NONAME]"
@@ -92,55 +116,73 @@ class RegistryHive(linear.LinearlyMappedLayer):
@property
def root_cell_offset(self) -> int:
"""Returns the offset for the root cell in this hive."""
try:
if self._base_block.Signature.cast("string", max_length = 4, encoding = "latin-1") == 'regf':
with contextlib.suppress(InvalidAddressException):
if (
self._base_block.Signature.cast(
"string", max_length=4, encoding="latin-1"
)
== "regf"
):
return self._base_block.RootCell
except InvalidAddressException:
pass
return 0x20
def get_cell(self, cell_offset: int) -> 'objects.StructType':
def get_cell(self, cell_offset: int) -> "objects.StructType":
"""Returns the appropriate Cell value for a cell offset."""
# This would be an _HCELL containing CELL_DATA, but to save time we skip the size of the HCELL
cell = self._context.object(object_type = self._table_name + constants.BANG + "_CELL_DATA",
offset = cell_offset + 4,
layer_name = self.name)
cell = self._context.object(
object_type=self._table_name + constants.BANG + "_CELL_DATA",
offset=cell_offset + 4,
layer_name=self.name,
)
return cell
def get_node(self, cell_offset: int) -> 'objects.StructType':
def get_node(self, cell_offset: int) -> "objects.StructType":
"""Returns the appropriate Node, interpreted from the Cell based on its
Signature."""
cell = self.get_cell(cell_offset)
signature = cell.cast('string', max_length = 2, encoding = 'latin-1')
if signature == 'nk':
signature = cell.cast("string", max_length=2, encoding="latin-1")
if signature == "nk":
return cell.u.KeyNode
elif signature == 'sk':
elif signature == "sk":
return cell.u.KeySecurity
elif signature == 'vk':
elif signature == "vk":
return cell.u.KeyValue
elif signature == 'db':
elif signature == "db":
# Big Data
return cell.u.ValueData
elif signature == 'lf' or signature == 'lh' or signature == 'ri':
elif signature == "lf" or signature == "lh" or signature == "ri":
# Fast Leaf, Hash Leaf, Index Root
return cell.u.KeyIndex
else:
# It doesn't matter that we use KeyNode, we're just after the first two bytes
vollog.debug("Unknown Signature {} (0x{:x}) at offset {}".format(signature, cell.u.KeyNode.Signature,
cell_offset))
vollog.debug(
"Unknown Signature {} (0x{:x}) at offset {}".format(
signature, cell.u.KeyNode.Signature, cell_offset
)
)
return cell
def get_key(self, key: str, return_list: bool = False) -> Union[List[objects.StructType], objects.StructType]:
def get_key(
self, key: str, return_list: bool = False
) -> Union[List[objects.StructType], objects.StructType]:
"""Gets a specific registry key by key path.
return_list specifies whether the return result will be a single
node (default) or a list of nodes from root to the current node
(if return_list is true).
"""
node_key = [self.get_node(self.root_cell_offset)]
root_node = self.get_node(self.root_cell_offset)
if not root_node.vol.type_name.endswith(constants.BANG + "_CM_KEY_NODE"):
raise RegistryFormatException(
self.name,
"Encountered {} instead of _CM_KEY_NODE".format(
root_node.vol.type_name
),
)
node_key = [root_node]
if key.endswith("\\"):
key = key[:-1]
key_array = key.split('\\')
key_array = key.split("\\")
found_key: List[str] = []
while key_array and node_key:
subkeys = node_key[-1].get_subkeys()
@@ -154,14 +196,18 @@ class RegistryHive(linear.LinearlyMappedLayer):
else:
node_key = []
if not node_key:
raise KeyError("Key {} not found under {}".format(key_array[0], '\\'.join(found_key)))
raise KeyError(
"Key {} not found under {}".format(key_array[0], "\\".join(found_key))
)
if return_list:
return node_key
return node_key[-1]
def visit_nodes(self,
visitor: Callable[[objects.StructType], None],
node: Optional[objects.StructType] = None) -> None:
def visit_nodes(
self,
visitor: Callable[[objects.StructType], None],
node: Optional[objects.StructType] = None,
) -> None:
"""Applies a callable (visitor) to all nodes within the registry tree
from a given node."""
if not node:
@@ -174,22 +220,28 @@ class RegistryHive(linear.LinearlyMappedLayer):
def _mask(value: int, high_bit: int, low_bit: int) -> int:
"""Returns the bits of a value between highbit and lowbit inclusive."""
high_mask = (2 ** (high_bit + 1)) - 1
low_mask = (2 ** low_bit) - 1
mask = (high_mask ^ low_mask)
low_mask = (2**low_bit) - 1
mask = high_mask ^ low_mask
# print(high_bit, low_bit, bin(mask), bin(value))
return value & mask
@classmethod
def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]:
return [
IntRequirement(name = 'hive_offset',
description = 'Offset within the base layer at which the hive lives',
default = 0,
optional = False),
requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"),
TranslationLayerRequirement(name = 'base_layer',
description = 'Layer in which the registry hive lives',
optional = False)
IntRequirement(
name="hive_offset",
description="Offset within the base layer at which the hive lives",
default=0,
optional=False,
),
requirements.SymbolTableRequirement(
name="nt_symbols", description="Windows kernel symbols"
),
TranslationLayerRequirement(
name="base_layer",
description="Layer in which the registry hive lives",
optional=False,
),
]
def _translate(self, offset: int) -> int:
@@ -198,15 +250,20 @@ class RegistryHive(linear.LinearlyMappedLayer):
# Ignore the volatile bit when determining maxaddr validity
volatile = self._mask(offset, 31, 31) >> 31
if offset & 0x7fffffff > self._get_hive_maxaddr(volatile):
vollog.log(constants.LOGLEVEL_VVV,
"Layer {} couldn't translate offset {}, greater than {} in {} store of {}".format(
self.name,
hex(offset & 0x7fffffff),
hex(self._get_hive_maxaddr(volatile)),
"volative" if volatile else "non-volatile",
self.get_name()))
raise RegistryInvalidIndex(self.name, "Mapping request for value greater than maxaddr")
if offset & 0x7FFFFFFF > self._get_hive_maxaddr(volatile):
vollog.log(
constants.LOGLEVEL_VVV,
"Layer {} couldn't translate offset {}, greater than {} in {} store of {}".format(
self.name,
hex(offset & 0x7FFFFFFF),
hex(self._get_hive_maxaddr(volatile)),
"volative" if volatile else "non-volatile",
self.get_name(),
),
)
raise RegistryInvalidIndex(
self.name, "Mapping request for value greater than maxaddr"
)
storage = self.hive.Storage[volatile]
dir_index = self._mask(offset, 30, 21) >> 21
@@ -217,11 +274,9 @@ class RegistryHive(linear.LinearlyMappedLayer):
entry = table.Table[table_index]
return entry.get_block_offset() + suboffset
def mapping(self,
offset: int,
length: int,
ignore_errors: bool = False) -> Iterable[Tuple[int, int, int, int, str]]:
def mapping(
self, offset: int, length: int, ignore_errors: bool = False
) -> Iterable[Tuple[int, int, int, int, str]]:
if length < 0:
raise ValueError("Mapping length of RegistryHive must be positive or zero")
@@ -236,7 +291,15 @@ class RegistryHive(linear.LinearlyMappedLayer):
chunk_size = min(chunk_size, remaining_length, self._page_size)
try:
translated_offset = self._translate(current_offset)
response.append((current_offset, chunk_size, translated_offset, chunk_size, self._base_layer))
response.append(
(
current_offset,
chunk_size,
translated_offset,
chunk_size,
self._base_layer,
)
)
except exceptions.LayerException:
if not ignore_errors:
raise
@@ -248,18 +311,19 @@ class RegistryHive(linear.LinearlyMappedLayer):
@property
def dependencies(self) -> List[str]:
"""Returns a list of layer names that this layer translates onto."""
return [self.config['base_layer']]
return [self.config["base_layer"]]
def is_valid(self, offset: int, length: int = 1) -> bool:
"""Returns a boolean based on whether the offset is valid or not."""
try:
with contextlib.suppress(exceptions.InvalidAddressException):
# Pass this to the lower layers for now
return all([
self.context.layers[layer].is_valid(offset, length)
for (_, _, offset, length, layer) in self.mapping(offset, length)
])
except exceptions.InvalidAddressException:
return False
return all(
[
self.context.layers[layer].is_valid(offset, length)
for (_, _, offset, length, layer) in self.mapping(offset, length)
]
)
return False
@property
def minimum_address(self) -> int:
volatility3/framework/layers/resources.py
+90 -57
View File
@@ -31,6 +31,7 @@ try:
# Import so that the handler is found by the framework.class_subclasses callc
import smb.SMBHandler # lgtm [py/unused-import]
except ImportError:
# If we fail to import this, it means that SMB handling won't be available
pass
vollog = logging.getLogger(__name__)
@@ -62,10 +63,12 @@ class ResourceAccessor(object):
list_handlers = True
def __init__(self,
progress_callback: Optional[constants.ProgressCallback] = None,
context: Optional[ssl.SSLContext] = None,
enable_cache: bool = True) -> None:
def __init__(
self,
progress_callback: Optional[constants.ProgressCallback] = None,
context: Optional[ssl.SSLContext] = None,
enable_cache: bool = True,
) -> None:
"""Creates a resource accessor.
Note: context is an SSL context, not a volatility context
@@ -75,20 +78,24 @@ class ResourceAccessor(object):
self._handlers = list(framework.class_subclasses(urllib.request.BaseHandler))
self._enable_cache = enable_cache
if self.list_handlers:
vollog.log(constants.LOGLEVEL_VVV,
f"Available URL handlers: {', '.join([x.__name__ for x in self._handlers])}")
vollog.log(
constants.LOGLEVEL_VVV,
f"Available URL handlers: {', '.join([x.__name__ for x in self._handlers])}",
)
self.__class__.list_handlers = False
def uses_cache(self, url: str) -> bool:
"""Determines whether a URLs contents should be cached"""
parsed_url = urllib.parse.urlparse(url)
return self._enable_cache and parsed_url.scheme not in self._non_cached_schemes()
return (
self._enable_cache and parsed_url.scheme not in self._non_cached_schemes()
)
@staticmethod
def _non_cached_schemes() -> List[str]:
"""Returns the list of schemes not to be cached"""
result = ['file']
result = ["file"]
for clazz in framework.class_subclasses(VolatilityHandler):
result += clazz.non_cached_schemes()
return result
@@ -102,34 +109,44 @@ class ResourceAccessor(object):
urllib.request.install_opener(urllib.request.build_opener(*self._handlers))
# Python bug 46654
if sys.platform == 'win32':
if sys.platform == "win32":
# We only need to worry about UNC paths on windows, on linux they'd be smb:// and need pysmb or similar
parsed_url = urllib.parse.urlparse(url, scheme = 'file')
parsed_url = urllib.parse.urlparse(url, scheme="file")
# Only worry about file scheme URLs, make sure that there's either a host or
# the unparsing left an extra slash at the start (which will get lost with urlunparse)
if parsed_url.scheme == 'file' and (parsed_url.netloc or parsed_url.path.startswith('//')):
if parsed_url.scheme == "file" and (
parsed_url.netloc or parsed_url.path.startswith("//")
):
# Change the netloc to '/' and then prepend the netloc to the path
# Urlunparse will remove extra initial slashes from path, hence setting netloc
new_url = urllib.parse.urlunparse((parsed_url.scheme, '/',
'/' + parsed_url.netloc + parsed_url.path, parsed_url.params,
parsed_url.query, parsed_url.fragment))
vollog.log(constants.LOGLEVEL_VVVV, f'UNC path detected, converted path {url} to {new_url}')
new_url = urllib.parse.urlunparse(
(
parsed_url.scheme,
"/",
"/" + parsed_url.netloc + parsed_url.path,
parsed_url.params,
parsed_url.query,
parsed_url.fragment,
)
)
vollog.log(
constants.LOGLEVEL_VVVV,
f"UNC path detected, converted path {url} to {new_url}",
)
url = new_url
try:
fp = urllib.request.urlopen(url, context = self._context)
fp = urllib.request.urlopen(url, context=self._context)
except error.URLError as excp:
if excp.args:
# TODO: As of python3.7 this can be removed
unverified_retrieval = (hasattr(ssl, "SSLCertVerificationError") and isinstance(
excp.args[0], ssl.SSLCertVerificationError)) or (isinstance(excp.args[0], ssl.SSLError) and
excp.args[0].reason == "CERTIFICATE_VERIFY_FAILED")
if unverified_retrieval:
vollog.warning("SSL certificate verification failed: attempting UNVERIFIED retrieval")
if isinstance(excp.args[0], ssl.SSLCertVerificationError):
vollog.warning(
"SSL certificate verification failed: attempting UNVERIFIED retrieval"
)
non_verifying_ctx = ssl.SSLContext()
non_verifying_ctx.check_hostname = False
non_verifying_ctx.verify_mode = ssl.CERT_NONE
fp = urllib.request.urlopen(url, context = non_verifying_ctx)
fp = urllib.request.urlopen(url, context=non_verifying_ctx)
else:
raise excp
else:
@@ -143,40 +160,43 @@ class ResourceAccessor(object):
if not self.uses_cache(url):
# ZipExtFiles (files in zips) cannot seek, so must be cached in order to use and/or decompress
curfile = urllib.request.urlopen(url, context = self._context)
curfile = urllib.request.urlopen(url, context=self._context)
else:
# TODO: find a way to check if we already have this file (look at http headers?)
block_size = 1028 * 8
temp_filename = os.path.join(
constants.CACHE_PATH,
"data_" + hashlib.sha512(bytes(url, 'raw_unicode_escape')).hexdigest() + ".cache")
"data_"
+ hashlib.sha512(bytes(url, "raw_unicode_escape")).hexdigest()
+ ".cache",
)
if not os.path.exists(temp_filename):
vollog.debug(f"Caching file at: {temp_filename}")
try:
content_length = fp.info().get('Content-Length', -1)
content_length = fp.info().get("Content-Length", -1)
except AttributeError:
# If our fp doesn't have an info member, carry on gracefully
content_length = -1
cache_file = open(temp_filename, "wb")
count = 0
block = fp.read(block_size)
while block:
count += len(block)
if self._progress_callback:
self._progress_callback(count * 100 / max(count, int(content_length)),
f"Reading file {url}")
cache_file.write(block)
with open(temp_filename, "wb") as cache_file:
count = 0
block = fp.read(block_size)
cache_file.close()
while block:
count += len(block)
if self._progress_callback:
self._progress_callback(
count * 100 / max(count, int(content_length)),
f"Reading file {url}",
)
cache_file.write(block)
block = fp.read(block_size)
else:
vollog.debug(f"Using already cached file at: {temp_filename}")
# Re-open the cache with a different mode
# Since we don't want people thinking they're able to save to the cache file,
# open it in read mode only and allow breakages to happen if they wanted to write
curfile = open(temp_filename, mode = "rb")
curfile = open(temp_filename, mode="rb")
# Determine whether the file is a particular type of file, and if so, open it as such
IMPORTED_MAGIC = False
@@ -184,23 +204,29 @@ class ResourceAccessor(object):
stop = False
while not stop:
detected = None
try:
with contextlib.suppress(AttributeError, IOError):
# Detect the content
detected = magic.detect_from_fobj(curfile)
IMPORTED_MAGIC = True
# This is because python-magic and file provide a magic module
# Only file's python has magic.detect_from_fobj
except (AttributeError, IOError):
pass
if detected:
if detected.mime_type == 'application/x-xz':
curfile = cascadeCloseFile(lzma.LZMAFile(curfile, mode), curfile)
elif detected.mime_type == 'application/x-bzip2':
if detected.mime_type == "application/x-xz":
curfile = cascadeCloseFile(
lzma.LZMAFile(curfile, mode), curfile
)
elif detected.mime_type == "application/x-bzip2":
curfile = cascadeCloseFile(bz2.BZ2File(curfile, mode), curfile)
elif detected.mime_type == 'application/x-gzip':
curfile = cascadeCloseFile(gzip.GzipFile(fileobj = curfile, mode = mode), curfile)
if detected.mime_type in ['application/x-xz', 'application/x-bzip2', 'application/x-gzip']:
elif detected.mime_type == "application/x-gzip":
curfile = cascadeCloseFile(
gzip.GzipFile(fileobj=curfile, mode=mode), curfile
)
if detected.mime_type in [
"application/x-xz",
"application/x-bzip2",
"application/x-gzip",
]:
# Read and rewind to ensure we're inside any compressed file layers
curfile.read(1)
curfile.seek(0)
@@ -223,7 +249,9 @@ class ResourceAccessor(object):
elif extension == "bz2":
curfile = cascadeCloseFile(bz2.BZ2File(curfile, mode), curfile)
elif extension == "gz":
curfile = cascadeCloseFile(gzip.GzipFile(fileobj = curfile, mode = mode), curfile)
curfile = cascadeCloseFile(
gzip.GzipFile(fileobj=curfile, mode=mode), curfile
)
else:
stop = True
@@ -234,7 +262,6 @@ class ResourceAccessor(object):
class VolatilityHandler(urllib.request.BaseHandler):
@classmethod
def non_cached_schemes(cls) -> List[str]:
return []
@@ -252,21 +279,27 @@ class JarHandler(VolatilityHandler):
@classmethod
def non_cached_schemes(cls) -> List[str]:
return ['jar']
return ["jar"]
@staticmethod
def default_open(req: urllib.request.Request) -> Optional[Any]:
"""Handles the request if it's the jar scheme."""
if req.type == 'jar':
subscheme, remainder = req.full_url.split(":")[1], ":".join(req.full_url.split(":")[2:])
if subscheme != 'file':
vollog.log(constants.LOGLEVEL_VVV, f"Unsupported jar subscheme {subscheme}")
if req.type == "jar":
subscheme, remainder = req.full_url.split(":")[1], ":".join(
req.full_url.split(":")[2:]
)
if subscheme != "file":
vollog.log(
constants.LOGLEVEL_VVV, f"Unsupported jar subscheme {subscheme}"
)
return None
zipsplit = remainder.split("!")
if len(zipsplit) != 2:
vollog.log(constants.LOGLEVEL_VVV,
f"Path did not contain exactly one fragment indicator: {remainder}")
vollog.log(
constants.LOGLEVEL_VVV,
f"Path did not contain exactly one fragment indicator: {remainder}",
)
return None
zippath, filepath = zipsplit
@@ -277,6 +310,6 @@ class JarHandler(VolatilityHandler):
class OfflineHandler(VolatilityHandler):
@staticmethod
def default_open(req: urllib.request.Request) -> Optional[Any]:
if constants.OFFLINE and req.type in ['http', 'https']:
if constants.OFFLINE and req.type in ["http", "https"]:
raise exceptions.OfflineException(req.full_url)
return None
volatility3/framework/layers/scanners/__init__.py
+11 -6
View File
@@ -35,6 +35,7 @@ class RegExScanner(layers.ScannerInterface):
The default flags include DOTALL, since the searches are through binary data and the newline character should
have no specific significance in such searches"""
thread_safe = True
_required_framework_version = (2, 0, 0)
@@ -80,7 +81,7 @@ class MultiStringScanner(layers.ScannerInterface):
def _process_trie(self, trie: Optional[Dict[int, Optional[Dict]]]) -> bytes:
if trie is None or len(trie) == 1 and -1 in trie:
# We've reached the end of this path, return the empty byte string
return b''
return b""
choices = []
suffixes = []
@@ -101,16 +102,16 @@ class MultiStringScanner(layers.ScannerInterface):
if len(suffixes) == 1:
choices.append(suffixes[0])
elif len(suffixes) > 1:
choices.append(b'[' + b''.join(suffixes) + b']')
choices.append(b"[" + b"".join(suffixes) + b"]")
if len(choices) == 0:
# If there's none, return the empty byte string
response = b''
response = b""
elif len(choices) == 1:
# If there's only one return it
response = choices[0]
else:
response = b'(?:' + b'|'.join(choices) + b')'
response = b"(?:" + b"|".join(choices) + b")"
if finished:
# We finished one string, so everything after this is optional
@@ -118,7 +119,9 @@ class MultiStringScanner(layers.ScannerInterface):
return response
def __call__(self, data: bytes, data_offset: int) -> Generator[Tuple[int, bytes], None, None]:
def __call__(
self, data: bytes, data_offset: int
) -> Generator[Tuple[int, bytes], None, None]:
"""Runs through the data looking for the needles."""
for offset, pattern in self.search(data):
if offset < self.chunk_size:
@@ -128,6 +131,8 @@ class MultiStringScanner(layers.ScannerInterface):
if not isinstance(haystack, bytes):
raise TypeError("Search haystack must be a byte string")
if not self._regex:
raise ValueError("MultiRegexp cannot be used with an empty set of search strings")
raise ValueError(
"MultiRegexp cannot be used with an empty set of search strings"
)
for match in re.finditer(self._regex, haystack):
yield match.start(0), match.group()
volatility3/framework/layers/scanners/multiregexp.py
+5 -3
View File
@@ -11,7 +11,7 @@ class MultiRegexp(object):
def __init__(self) -> None:
self._pattern_strings: List[bytes] = []
self._regex = re.compile(b'')
self._regex = re.compile(b"")
def add_pattern(self, pattern: bytes) -> None:
self._pattern_strings.append(pattern)
@@ -19,12 +19,14 @@ class MultiRegexp(object):
def preprocess(self) -> None:
if not self._pattern_strings:
raise ValueError("No strings to compile into a regular expression")
self._regex = re.compile(b'|'.join(map(re.escape, self._pattern_strings)))
self._regex = re.compile(b"|".join(map(re.escape, self._pattern_strings)))
def search(self, haystack: bytes) -> Generator[Tuple[int, bytes], None, None]:
if not isinstance(haystack, bytes):
raise TypeError("Search haystack must be a byte string")
if not self._regex.pattern:
raise ValueError("MultiRegexp cannot be used with an empty set of search strings")
raise ValueError(
"MultiRegexp cannot be used with an empty set of search strings"
)
for match in re.finditer(self._regex, haystack):
yield (match.start(0), match.group())
volatility3/framework/layers/segmented.py
+54 -26
View File
@@ -10,19 +10,25 @@ from volatility3.framework.configuration import requirements
from volatility3.framework.layers import linear
class NonLinearlySegmentedLayer(interfaces.layers.TranslationLayerInterface, metaclass = ABCMeta):
class NonLinearlySegmentedLayer(
interfaces.layers.TranslationLayerInterface, metaclass=ABCMeta
):
"""A class to handle a single run-based layer-to-layer mapping.
In the documentation "mapped address" or "mapped offset" refers to
an offset once it has been mapped to the underlying layer
"""
def __init__(self,
context: interfaces.context.ContextInterface,
config_path: str,
name: str,
metadata: Optional[Dict[str, Any]] = None) -> None:
super().__init__(context = context, config_path = config_path, name = name, metadata = metadata)
def __init__(
self,
context: interfaces.context.ContextInterface,
config_path: str,
name: str,
metadata: Optional[Dict[str, Any]] = None,
) -> None:
super().__init__(
context=context, config_path=config_path, name=name, metadata=metadata
)
self._base_layer = self.config["base_layer"]
self._segments: List[Tuple[int, int, int, int]] = []
@@ -45,11 +51,17 @@ class NonLinearlySegmentedLayer(interfaces.layers.TranslationLayerInterface, met
try:
base_layer = self._context.layers[self._base_layer]
return all(
[base_layer.is_valid(mapped_offset) for _i, _i, mapped_offset, _i, _s in self.mapping(offset, length)])
[
base_layer.is_valid(mapped_offset)
for _i, _i, mapped_offset, _i, _s in self.mapping(offset, length)
]
)
except exceptions.InvalidAddressException:
return False
def _find_segment(self, offset: int, next: bool = False) -> Tuple[int, int, int, int]:
def _find_segment(
self, offset: int, next: bool = False
) -> Tuple[int, int, int, int]:
"""Finds the segment containing a given offset.
Returns the segment tuple (offset, mapped_offset, length, mapped_length)
@@ -59,7 +71,10 @@ class NonLinearlySegmentedLayer(interfaces.layers.TranslationLayerInterface, met
self._load_segments()
# Find rightmost value less than or equal to x
i = bisect_right(self._segments, (offset, self.context.layers[self._base_layer].maximum_address))
i = bisect_right(
self._segments,
(offset, self.context.layers[self._base_layer].maximum_address),
)
if i and not next:
segment = self._segments[i - 1]
if segment[0] <= offset < segment[0] + segment[2]:
@@ -67,16 +82,17 @@ class NonLinearlySegmentedLayer(interfaces.layers.TranslationLayerInterface, met
if next:
if i < len(self._segments):
return self._segments[i]
raise exceptions.InvalidAddressException(self.name, offset, f"Invalid address at {offset:0x}")
raise exceptions.InvalidAddressException(
self.name, offset, f"Invalid address at {offset:0x}"
)
# Determines whether larger segments are in use and the offsets within them should be tracked linearly
# When no decoding of the data occurs, this should be set to true
_track_offset = False
def mapping(self,
offset: int,
length: int,
ignore_errors: bool = False) -> Iterable[Tuple[int, int, int, int, str]]:
def mapping(
self, offset: int, length: int, ignore_errors: bool = False
) -> Iterable[Tuple[int, int, int, int, str]]:
"""Returns a sorted iterable of (offset, length, mapped_offset, mapped_length, layer)
mappings."""
done = False
@@ -84,7 +100,9 @@ class NonLinearlySegmentedLayer(interfaces.layers.TranslationLayerInterface, met
while not done:
try:
# Search for the appropriate segment that contains the current_offset
logical_offset, mapped_offset, size, mapped_size = self._find_segment(current_offset)
logical_offset, mapped_offset, size, mapped_size = self._find_segment(
current_offset
)
# If it starts before the current_offset, bring the lower edge up to the right place
if current_offset > logical_offset:
difference = current_offset - logical_offset
@@ -98,14 +116,19 @@ class NonLinearlySegmentedLayer(interfaces.layers.TranslationLayerInterface, met
raise
try:
# Find the next valid segment after our current_offset
logical_offset, mapped_offset, size, mapped_size = self._find_segment(current_offset, next = True)
(
logical_offset,
mapped_offset,
size,
mapped_size,
) = self._find_segment(current_offset, next=True)
# We know that the logical_offset must be greater than current_offset so skip to that value
current_offset = logical_offset
# If it starts too late then we're done
if logical_offset > offset + length:
return
return None
except exceptions.InvalidAddressException:
return
return None
# Crop it to the amount we need left
chunk_size = min(size, length + offset - logical_offset)
yield logical_offset, chunk_size, mapped_offset, mapped_size, self._base_layer
@@ -140,16 +163,21 @@ class NonLinearlySegmentedLayer(interfaces.layers.TranslationLayerInterface, met
@classmethod
def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]:
return [requirements.TranslationLayerRequirement(name = 'base_layer', optional = False)]
return [
requirements.TranslationLayerRequirement(name="base_layer", optional=False)
]
class SegmentedLayer(NonLinearlySegmentedLayer, linear.LinearlyMappedLayer, metaclass = ABCMeta):
class SegmentedLayer(
NonLinearlySegmentedLayer, linear.LinearlyMappedLayer, metaclass=ABCMeta
):
_track_offset = True
def mapping(self,
offset: int,
length: int,
ignore_errors: bool = False) -> Iterable[Tuple[int, int, int, int, str]]:
def mapping(
self, offset: int, length: int, ignore_errors: bool = False
) -> Iterable[Tuple[int, int, int, int, str]]:
# Linear mappings must return the same length of segment as that requested
for offset, length, mapped_offset, mapped_length, layer in super().mapping(offset, length, ignore_errors):
for offset, length, mapped_offset, mapped_length, layer in super().mapping(
offset, length, ignore_errors
):
yield offset, length, mapped_offset, length, layer
volatility3/framework/layers/vmware.py
+131 -63
View File
@@ -1,14 +1,15 @@
# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0
# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0
#
import contextlib
import logging
import struct
import os
from typing import Any, Dict, List, Optional
from volatility3.framework import interfaces, constants, exceptions
from volatility3.framework import constants, exceptions, interfaces
from volatility3.framework.configuration import requirements
from volatility3.framework.layers import physical, segmented, resources
from volatility3.framework.layers import physical, resources, segmented
from volatility3.framework.symbols import native
vollog = logging.getLogger(__name__)
@@ -22,18 +23,23 @@ class VmwareLayer(segmented.SegmentedLayer):
header_structure = "<4sII"
group_structure = "64sQQ"
def __init__(self,
context: interfaces.context.ContextInterface,
config_path: str,
name: str,
metadata: Optional[Dict[str, Any]] = None) -> None:
def __init__(
self,
context: interfaces.context.ContextInterface,
config_path: str,
name: str,
metadata: Optional[Dict[str, Any]] = None,
) -> None:
# Construct these so we can use self.config
self._context = context
self._config_path = config_path
self._page_size = 0x1000
self._base_layer, self._meta_layer = self.config["base_layer"], self.config["meta_layer"]
self._base_layer, self._meta_layer = (
self.config["base_layer"],
self.config["meta_layer"],
)
# Then call the super, which will call load_segments (which needs the base_layer before it'll work)
super().__init__(context, config_path = config_path, name = name, metadata = metadata)
super().__init__(context, config_path=config_path, name=name, metadata=metadata)
def _load_segments(self) -> None:
"""Loads up the segments from the meta_layer."""
@@ -46,22 +52,33 @@ class VmwareLayer(segmented.SegmentedLayer):
def _read_header(self) -> None:
"""Checks the vmware header to make sure it's valid."""
if "vmware" not in self._context.symbol_space:
self._context.symbol_space.append(native.NativeTable("vmware", native.std_ctypes))
self._context.symbol_space.append(
native.NativeTable("vmware", native.std_ctypes)
)
meta_layer = self.context.layers.get(self._meta_layer, None)
header_size = struct.calcsize(self.header_structure)
data = meta_layer.read(0, header_size)
magic, unknown, groupCount = struct.unpack(self.header_structure, data)
if magic not in [b"\xD0\xBE\xD2\xBE", b"\xD1\xBA\xD1\xBA", b"\xD2\xBE\xD2\xBE", b"\xD3\xBE\xD3\xBE"]:
raise VmwareFormatException(self.name, f"Wrong magic bytes for Vmware layer: {repr(magic)}")
if magic not in [
b"\xD0\xBE\xD2\xBE",
b"\xD1\xBA\xD1\xBA",
b"\xD2\xBE\xD2\xBE",
b"\xD3\xBE\xD3\xBE",
]:
raise VmwareFormatException(
self.name, f"Wrong magic bytes for Vmware layer: {repr(magic)}"
)
version = magic[0] & 0xf
version = magic[0] & 0xF
group_size = struct.calcsize(self.group_structure)
groups = {}
for group in range(groupCount):
name, tag_location, _unknown = struct.unpack(
self.group_structure, meta_layer.read(header_size + (group * group_size), group_size))
self.group_structure,
meta_layer.read(header_size + (group * group_size), group_size),
)
name = name.rstrip(b"\x00")
groups[name] = tag_location
memory = groups[b"memory"]
@@ -75,47 +92,74 @@ class VmwareLayer(segmented.SegmentedLayer):
name_len = ord(meta_layer.read(offset + 1, 1))
tags_read = (flags == 0) and (name_len == 0)
if not tags_read:
name = self._context.object("vmware!string",
layer_name = self._meta_layer,
offset = offset + 2,
max_length = name_len)
name = self._context.object(
"vmware!string",
layer_name=self._meta_layer,
offset=offset + 2,
max_length=name_len,
)
indices_len = (flags >> 6) & 3
indices = []
for index in range(indices_len):
indices.append(
self._context.object("vmware!unsigned int",
offset = offset + name_len + 2 + (index * index_len),
layer_name = self._meta_layer))
data_len = flags & 0x3f
if data_len in [62, 63]: # Handle special data sizes that indicate a longer data stream
self._context.object(
"vmware!unsigned int",
offset=offset + name_len + 2 + (index * index_len),
layer_name=self._meta_layer,
)
)
data_len = flags & 0x3F
if data_len in [
62,
63,
]: # Handle special data sizes that indicate a longer data stream
data_len = 4 if version == 0 else 8
# Read the size of the data
data_size = self._context.object(self._choose_type(data_len),
layer_name = self._meta_layer,
offset = offset + 2 + name_len + (indices_len * index_len))
data_size = self._context.object(
self._choose_type(data_len),
layer_name=self._meta_layer,
offset=offset + 2 + name_len + (indices_len * index_len),
)
# Skip two bytes of padding (as it seems?)
# Read the actual data
data = self._context.object("vmware!bytes",
layer_name = self._meta_layer,
offset = offset + 2 + name_len + (indices_len * index_len) +
2 * data_len + 2,
length = data_size)
offset += 2 + name_len + (indices_len * index_len) + 2 * data_len + 2 + data_size
data = self._context.object(
"vmware!bytes",
layer_name=self._meta_layer,
offset=offset
+ 2
+ name_len
+ (indices_len * index_len)
+ 2 * data_len
+ 2,
length=data_size,
)
offset += (
2
+ name_len
+ (indices_len * index_len)
+ 2 * data_len
+ 2
+ data_size
)
else: # Handle regular cases
data = self._context.object(self._choose_type(data_len),
layer_name = self._meta_layer,
offset = offset + 2 + name_len + (indices_len * index_len))
data = self._context.object(
self._choose_type(data_len),
layer_name=self._meta_layer,
offset=offset + 2 + name_len + (indices_len * index_len),
)
offset += 2 + name_len + (indices_len * index_len) + data_len
tags[(name, tuple(indices))] = (flags, data)
if tags[("regionsCount", ())][1] == 0:
raise VmwareFormatException(self.name, "VMware VMEM is not split into regions")
raise VmwareFormatException(
self.name, "VMware VMEM is not split into regions"
)
for region in range(tags[("regionsCount", ())][1]):
offset = tags[("regionPPN", (region, ))][1] * self._page_size
mapped_offset = tags[("regionPageNum", (region, ))][1] * self._page_size
length = tags[("regionSize", (region, ))][1] * self._page_size
offset = tags[("regionPPN", (region,))][1] * self._page_size
mapped_offset = tags[("regionPageNum", (region,))][1] * self._page_size
length = tags[("regionSize", (region,))][1] * self._page_size
self._segments.append((offset, mapped_offset, length, length))
@property
@@ -127,8 +171,8 @@ class VmwareLayer(segmented.SegmentedLayer):
"""This vmware translation layer always requires a separate metadata
layer."""
return [
requirements.TranslationLayerRequirement(name = 'base_layer', optional = False),
requirements.TranslationLayerRequirement(name = 'meta_layer', optional = False)
requirements.TranslationLayerRequirement(name="base_layer", optional=False),
requirements.TranslationLayerRequirement(name="meta_layer", optional=False),
]
@@ -136,10 +180,12 @@ class VmwareStacker(interfaces.automagic.StackerLayerInterface):
stack_order = 20
@classmethod
def stack(cls,
context: interfaces.context.ContextInterface,
layer_name: str,
progress_callback: constants.ProgressCallback = None) -> Optional[interfaces.layers.DataLayerInterface]:
def stack(
cls,
context: interfaces.context.ContextInterface,
layer_name: str,
progress_callback: constants.ProgressCallback = None,
) -> Optional[interfaces.layers.DataLayerInterface]:
"""Attempt to stack this based on the starting information."""
memlayer = context.layers[layer_name]
if not isinstance(memlayer, physical.FileLayer):
@@ -149,35 +195,57 @@ class VmwareStacker(interfaces.automagic.StackerLayerInterface):
vmss = location[:-5] + ".vmss"
vmsn = location[:-5] + ".vmsn"
current_layer_name = context.layers.free_layer_name("VmwareMetaLayer")
current_config_path = interfaces.configuration.path_join("automagic", "layer_stacker", "stack",
current_layer_name)
current_config_path = interfaces.configuration.path_join(
"automagic", "layer_stacker", "stack", current_layer_name
)
vmss_success = False
try:
_ = resources.ResourceAccessor().open(vmss).read(10)
context.config[interfaces.configuration.path_join(current_config_path, "location")] = vmss
context.layers.add_layer(physical.FileLayer(context, current_config_path, current_layer_name))
with contextlib.suppress(IOError):
with resources.ResourceAccessor().open(vmss) as fp:
_ = fp.read(10)
context.config[
interfaces.configuration.path_join(current_config_path, "location")
] = vmss
context.layers.add_layer(
physical.FileLayer(context, current_config_path, current_layer_name)
)
vmss_success = True
except IOError:
pass
vmsn_success = False
if not vmss_success:
try:
with contextlib.suppress(IOError):
_ = resources.ResourceAccessor().open(vmsn).read(10)
context.config[interfaces.configuration.path_join(current_config_path, "location")] = vmsn
context.layers.add_layer(physical.FileLayer(context, current_config_path, current_layer_name))
context.config[
interfaces.configuration.path_join(
current_config_path, "location"
)
] = vmsn
context.layers.add_layer(
physical.FileLayer(
context, current_config_path, current_layer_name
)
)
vmsn_success = True
except IOError:
pass
vollog.log(constants.LOGLEVEL_VVVV, f"Metadata found: VMSS ({vmss_success}) or VMSN ({vmsn_success})")
vollog.log(
constants.LOGLEVEL_VVVV,
f"Metadata found: VMSS ({vmss_success}) or VMSN ({vmsn_success})",
)
if not vmss_success and not vmsn_success:
vmem_file_basename = os.path.basename(location)
example_vmss_file_basename = os.path.basename(vmss)
vollog.warning(
f"No metadata file found alongside VMEM file. A VMSS or VMSN file may be required to correctly process a VMEM file. These should be placed in the same directory with the same file name, e.g. {vmem_file_basename} and {example_vmss_file_basename}.",
)
return None
new_layer_name = context.layers.free_layer_name("VmwareLayer")
context.config[interfaces.configuration.path_join(current_config_path, "base_layer")] = layer_name
context.config[interfaces.configuration.path_join(current_config_path, "meta_layer")] = current_layer_name
context.config[
interfaces.configuration.path_join(current_config_path, "base_layer")
] = layer_name
context.config[
interfaces.configuration.path_join(current_config_path, "meta_layer")
] = current_layer_name
new_layer = VmwareLayer(context, current_config_path, new_layer_name)
return new_layer
return None
volatility3/framework/layers/xen.py
+180
View File
@@ -0,0 +1,180 @@
import logging
import struct
from typing import Optional
from volatility3.framework import constants, interfaces, exceptions
from volatility3.framework.layers import elf
from volatility3.framework.symbols import intermed
vollog = logging.getLogger(__name__)
class XenCoreDumpLayer(elf.Elf64Layer):
"""A layer that supports the Xen Dump-Core format as documented at: https://xenbits.xen.org/docs/4.6-testing/misc/dump-core-format.txt"""
_header_struct = struct.Struct("<IBBB")
MAGIC = 0x464C457F # "\x7fELF"
ELF_CLASS = 2
def __init__(
self, context: interfaces.context.ContextInterface, config_path: str, name: str
) -> None:
# Create a custom SymbolSpace
self._elf_table_name = intermed.IntermediateSymbolTable.create(
context, config_path, "linux", "elf"
)
self._xen_table_name = intermed.IntermediateSymbolTable.create(
context, config_path, "linux", "xen"
)
self._segment_headers = {}
super().__init__(context, config_path, name)
def _extract_result_array(
self, varname: str, segment_index: int
) -> interfaces.objects.ObjectInterface:
hdr = self._segment_headers[segment_index]
result = self.context.object(
self._xen_table_name + constants.BANG + varname,
layer_name=self._base_layer,
offset=hdr.sh_offset,
size=hdr.sh_size,
)
result.entries.count = hdr.sh_size // result.entries.vol.subtype.size
return result
def _load_segments(self) -> None:
"""Load the segments from based on the PT_LOAD segments of the Elf64 format"""
ehdr = self.context.object(
self._elf_table_name + constants.BANG + "Elf64_Ehdr",
layer_name=self._base_layer,
offset=0,
)
segments = []
self._segment_headers = []
for sindex in range(ehdr.e_shnum):
shdr = self.context.object(
self._elf_table_name + constants.BANG + "Elf64_Shdr",
layer_name=self._base_layer,
offset=ehdr.e_shoff + (sindex * ehdr.e_shentsize),
)
self._segment_headers.append(shdr)
if sindex == ehdr.e_shstrndx:
segment_names = self.context.layers[self._base_layer].read(
shdr.sh_offset, shdr.sh_size
)
segment_names = segment_names.split(b"\x00")
if not segment_names:
raise elf.ElfFormatException("No segment names, not a Xen Core Dump")
try:
p2m_data = self._extract_result_array(
"xen_p2m", segment_names.index(b".xen_p2m")
)
except ValueError:
p2m_data = None
try:
pfn_data = self._extract_result_array(
"xen_pfn", segment_names.index(b".xen_pfn")
)
except ValueError:
pfn_data = None
pages_hdr = self._segment_headers[segment_names.index(b".xen_pages")]
page_size = 0x1000
if pfn_data and not p2m_data:
for entry_index in range(len(pfn_data.entries)):
entry = pfn_data.entries[entry_index]
# TODO: Don't hardcode the maximum value here
if entry and entry != 0xFFFFFFFF:
segments.append(
(
entry * page_size,
pages_hdr.sh_offset + (entry_index * page_size),
page_size,
page_size,
)
)
elif p2m_data and not pfn_data:
for entry_index in range(len(p2m_data.entries)):
entry = p2m_data.entries[entry_index]
# TODO: Don't hardcode the maximum value here
if entry.pfn != 0xFFFFFFFF:
segments.append(
(
entry.pfn * page_size,
pages_hdr.sh_offset + (entry_index * page_size),
page_size,
page_size,
)
)
elif p2m_data and pfn_data:
raise elf.ElfFormatException(
self.name, f"Both P2M and PFN in Xen Core Dump"
)
else:
raise elf.ElfFormatException(
self.name, f"Neither P2M nor PFN in Xen Core Dump"
)
if len(segments) == 0:
raise elf.ElfFormatException(
self.name, f"No ELF segments defined in {self._base_layer}"
)
self._segments = segments
@classmethod
def _check_header(
cls, base_layer: interfaces.layers.DataLayerInterface, offset: int = 0
) -> bool:
try:
header_data = base_layer.read(offset, cls._header_struct.size)
except exceptions.InvalidAddressException:
raise elf.ElfFormatException(
base_layer.name,
f"Offset 0x{offset:0x} does not exist within the base layer",
)
(magic, elf_class, elf_data_encoding, elf_version) = cls._header_struct.unpack(
header_data
)
if magic != cls.MAGIC:
raise elf.ElfFormatException(
base_layer.name, f"Bad magic 0x{magic:x} at file offset 0x{offset:x}"
)
if elf_class != cls.ELF_CLASS:
raise elf.ElfFormatException(
base_layer.name, f"ELF class is not 64-bit (2): {elf_class:d}"
)
# Virtualbox uses an ELF version of 0, which isn't to specification, but is ok to deal with
return True
class XenCoreDumpStacker(elf.Elf64Stacker):
stack_order = 10
@classmethod
def stack(
cls,
context: interfaces.context.ContextInterface,
layer_name: str,
progress_callback: constants.ProgressCallback = None,
) -> Optional[interfaces.layers.DataLayerInterface]:
try:
if not XenCoreDumpLayer._check_header(context.layers[layer_name]):
return None
except elf.ElfFormatException as excp:
vollog.log(constants.LOGLEVEL_VVVV, f"Exception: {excp}")
return None
new_name = context.layers.free_layer_name("XenCoreDumpLayer")
context.config[interfaces.configuration.path_join(new_name, "base_layer")] = (
layer_name
)
return XenCoreDumpLayer(context, new_name, new_name)
volatility3/framework/objects/__init__.py
+448 -246
View File
File diff suppressed because it is too large Load Diff
volatility3/framework/objects/templates.py
+48 -15
View File
@@ -22,13 +22,22 @@ class ObjectTemplate(interfaces.objects.Template):
* etc
"""
def __init__(self, object_class: Type[interfaces.objects.ObjectInterface], type_name: str, **arguments) -> None:
arguments['object_class'] = object_class
super().__init__(type_name = type_name, **arguments)
def __init__(
self,
object_class: Type[interfaces.objects.ObjectInterface],
type_name: str,
**arguments,
) -> None:
arguments["object_class"] = object_class
super().__init__(type_name=type_name, **arguments)
proxy_cls = self.vol.object_class.VolTemplateProxy
for method_name in proxy_cls._methods:
setattr(self, method_name, functools.partial(getattr(proxy_cls, method_name), self))
setattr(
self,
method_name,
functools.partial(getattr(proxy_cls, method_name), self),
)
@property
def size(self) -> int:
@@ -48,28 +57,45 @@ class ObjectTemplate(interfaces.objects.Template):
plateProxy`)"""
return self.vol.object_class.VolTemplateProxy.relative_child_offset(self, child)
def replace_child(self, old_child: interfaces.objects.Template, new_child: interfaces.objects.Template) -> None:
def child_template(self, child: str) -> interfaces.objects.Template:
"""Returns the template of a child of the templated object (see
:class:`~volatility3.framework.interfaces.objects.ObjectInterface.VolTem
plateProxy`)"""
return self.vol.object_class.VolTemplateProxy.child_template(self, child)
def replace_child(
self,
old_child: interfaces.objects.Template,
new_child: interfaces.objects.Template,
) -> None:
"""Replaces `old_child` for `new_child` in the templated object's child
list (see :class:`~volatility3.framework.interfaces.objects.ObjectInterf
ace.VolTemplateProxy`)"""
return self.vol.object_class.VolTemplateProxy.replace_child(self, old_child, new_child)
return self.vol.object_class.VolTemplateProxy.replace_child(
self, old_child, new_child
)
def has_member(self, member_name: str) -> bool:
"""Returns whether the object would contain a member called
member_name."""
return self.vol.object_class.VolTemplateProxy.has_member(self, member_name)
def __call__(self, context: interfaces.context.ContextInterface,
object_info: interfaces.objects.ObjectInformation) -> interfaces.objects.ObjectInterface:
def __call__(
self,
context: interfaces.context.ContextInterface,
object_info: interfaces.objects.ObjectInformation,
) -> interfaces.objects.ObjectInterface:
"""Constructs the object.
Returns: an object adhereing to the :class:`~volatility3.framework.interfaces.objects.ObjectInterface`
Returns: an object adhering to the :class:`~volatility3.framework.interfaces.objects.ObjectInterface`
"""
arguments: Dict[str, Any] = {}
for arg in self.vol:
if arg != 'object_class':
if arg != "object_class":
arguments[arg] = self.vol[arg]
return self.vol.object_class(context = context, object_info = object_info, **arguments)
return self.vol.object_class(
context=context, object_info=object_info, **arguments
)
class ReferenceTemplate(interfaces.objects.Template):
@@ -93,14 +119,21 @@ class ReferenceTemplate(interfaces.objects.Template):
table_name = type_name[0]
symbol_name = type_name[-1]
raise exceptions.SymbolError(
symbol_name, table_name,
f"Template contains no information about its structure: {self.vol.type_name}")
symbol_name,
table_name,
f"Template contains no information about its structure: {self.vol.type_name}",
)
size: ClassVar[Any] = property(_unresolved)
replace_child: ClassVar[Any] = _unresolved
relative_child_offset: ClassVar[Any] = _unresolved
child_template: ClassVar[Any] = _unresolved
has_member: ClassVar[Any] = _unresolved
def __call__(self, context: interfaces.context.ContextInterface, object_info: interfaces.objects.ObjectInformation):
def __call__(
self,
context: interfaces.context.ContextInterface,
object_info: interfaces.objects.ObjectInformation,
):
template = context.symbol_space.get_type(self.vol.type_name)
return template(context = context, object_info = object_info)
return template(context=context, object_info=object_info)
volatility3/framework/objects/utility.py
+21 -13
View File
@@ -7,9 +7,9 @@ from typing import Optional, Union
from volatility3.framework import interfaces, objects, constants
def array_to_string(array: 'objects.Array',
count: Optional[int] = None,
errors: str = 'replace') -> interfaces.objects.ObjectInterface:
def array_to_string(
array: "objects.Array", count: Optional[int] = None, errors: str = "replace"
) -> interfaces.objects.ObjectInterface:
"""Takes a volatility Array of characters and returns a string."""
# TODO: Consider checking the Array's target is a native char
if count is None:
@@ -17,28 +17,36 @@ def array_to_string(array: 'objects.Array',
if not isinstance(array, objects.Array):
raise TypeError("Array_to_string takes an Array of char")
return array.cast("string", max_length = count, errors = errors)
return array.cast("string", max_length=count, errors=errors)
def pointer_to_string(pointer: 'objects.Pointer', count: int, errors: str = 'replace'):
def pointer_to_string(pointer: "objects.Pointer", count: int, errors: str = "replace"):
"""Takes a volatility Pointer to characters and returns a string."""
if not isinstance(pointer, objects.Pointer):
raise TypeError("pointer_to_string takes a Pointer")
if count < 1:
raise ValueError("pointer_to_string requires a positive count")
char = pointer.dereference()
return char.cast("string", max_length = count, errors = errors)
return char.cast("string", max_length=count, errors=errors)
def array_of_pointers(array: interfaces.objects.ObjectInterface, count: int,
subtype: Union[str, interfaces.objects.Template],
context: interfaces.context.ContextInterface) -> interfaces.objects.ObjectInterface:
def array_of_pointers(
array: interfaces.objects.ObjectInterface,
count: int,
subtype: Union[str, interfaces.objects.Template],
context: interfaces.context.ContextInterface,
) -> interfaces.objects.ObjectInterface:
"""Takes an object, and recasts it as an array of pointers to subtype."""
symbol_table = array.vol.type_name.split(constants.BANG)[0]
if isinstance(subtype, str) and context is not None:
subtype = context.symbol_space.get_type(subtype)
if not isinstance(subtype, interfaces.objects.Template) or subtype is None:
raise TypeError("Subtype must be a valid template (or string name of an object template)")
subtype_pointer = context.symbol_space.get_type(symbol_table + constants.BANG + "pointer")
subtype_pointer.update_vol(subtype = subtype)
return array.cast("array", count = count, subtype = subtype_pointer)
raise TypeError(
"Subtype must be a valid template (or string name of an object template)"
)
# We have to clone the pointer class, or we'll be defining the pointer subtype for all future pointers
subtype_pointer = context.symbol_space.get_type(
symbol_table + constants.BANG + "pointer"
).clone()
subtype_pointer.update_vol(subtype=subtype)
return array.cast("array", count=count, subtype=subtype_pointer)
volatility3/framework/plugins/__init__.py
+22 -9
View File
@@ -15,11 +15,14 @@ from volatility3.framework import interfaces, automagic, exceptions, constants
vollog = logging.getLogger(__name__)
def construct_plugin(context: interfaces.context.ContextInterface,
automagics: List[interfaces.automagic.AutomagicInterface],
plugin: Type[interfaces.plugins.PluginInterface], base_config_path: str,
progress_callback: constants.ProgressCallback,
open_method: Type[interfaces.plugins.FileHandlerInterface]) -> interfaces.plugins.PluginInterface:
def construct_plugin(
context: interfaces.context.ContextInterface,
automagics: List[interfaces.automagic.AutomagicInterface],
plugin: Type[interfaces.plugins.PluginInterface],
base_config_path: str,
progress_callback: constants.ProgressCallback,
open_method: Type[interfaces.plugins.FileHandlerInterface],
) -> interfaces.plugins.PluginInterface:
"""Constructs a plugin object based on the parameters.
Clever magic figures out how to fulfill each requirement that might not be fulfilled
@@ -35,9 +38,17 @@ def construct_plugin(context: interfaces.context.ContextInterface,
Returns:
The constructed plugin object
"""
errors = automagic.run(automagics, context, plugin, base_config_path, progress_callback = progress_callback)
errors = automagic.run(
automagics,
context,
plugin,
base_config_path,
progress_callback=progress_callback,
)
# Plugins always get their configuration stored under their plugin name
plugin_config_path = interfaces.configuration.path_join(base_config_path, plugin.__name__)
plugin_config_path = interfaces.configuration.path_join(
base_config_path, plugin.__name__
)
# Check all the requirements and/or go back to the automagic step
unsatisfied = plugin.unsatisfied(context, plugin_config_path)
@@ -45,10 +56,12 @@ def construct_plugin(context: interfaces.context.ContextInterface,
for error in errors:
error_string = [x for x in error.format_exception_only()][-1]
vollog.warning(f"Automagic exception occurred: {error_string[:-1]}")
vollog.log(constants.LOGLEVEL_V, "".join(error.format(chain = True)))
vollog.log(constants.LOGLEVEL_V, "".join(error.format(chain=True)))
raise exceptions.UnsatisfiedException(unsatisfied)
constructed = plugin(context, plugin_config_path, progress_callback = progress_callback)
constructed = plugin(
context, plugin_config_path, progress_callback=progress_callback
)
if open_method:
constructed.set_open_method(open_method)
return constructed
volatility3/framework/plugins/banners.py
+27 -12
View File
@@ -19,32 +19,47 @@ class Banners(interfaces.plugins.PluginInterface):
@classmethod
def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]:
return [requirements.TranslationLayerRequirement(name = 'primary', description = 'Memory layer to scan')]
return [
requirements.TranslationLayerRequirement(
name="primary", description="Memory layer to scan"
)
]
def _generator(self):
layer = self.context.layers[self.config['primary']]
layer = self.context.layers[self.config["primary"]]
if isinstance(layer, layers.intel.Intel):
layer = self.context.layers[layer.config['memory_layer']]
layer = self.context.layers[layer.config["memory_layer"]]
for offset, banner in self.locate_banners(self.context, layer.name):
yield 0, (offset, banner)
@classmethod
def locate_banners(cls, context: interfaces.context.ContextInterface, layer_name: str):
def locate_banners(
cls, context: interfaces.context.ContextInterface, layer_name: str
):
"""Identifies banners from a memory image"""
layer = context.layers[layer_name]
for offset in layer.scan(
context = context,
scanner = scanners.RegExScanner(rb"(Linux version|Darwin Kernel Version) [0-9]+\.[0-9]+\.[0-9]+")):
data = layer.read(offset, 0xfff)
data_index = data.find(b'\x00')
context=context,
scanner=scanners.RegExScanner(
rb"(Linux version|Darwin Kernel Version) [0-9]+\.[0-9]+\.[0-9]+"
),
):
data = layer.read(offset, 0xFFF)
data_index = data.find(b"\x00")
if data_index > 0:
data = data[:data_index].strip()
failed = [
char for char in data
if char not in b' #()+,;/-.0123456789:@ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz~'
char
for char in data
if char
not in b" #()+,;/-.0123456789:@ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz~"
]
if not failed:
yield format_hints.Hex(offset), str(data, encoding = 'latin-1', errors = '?')
yield format_hints.Hex(offset), str(
data, encoding="latin-1", errors="?"
)
def run(self):
return renderers.TreeGrid([("Offset", format_hints.Hex), ("Banner", str)], self._generator())
return renderers.TreeGrid(
[("Offset", format_hints.Hex), ("Banner", str)], self._generator()
)
volatility3/framework/plugins/configwriter.py
+21 -10
View File
@@ -22,25 +22,36 @@ class ConfigWriter(plugins.PluginInterface):
@classmethod
def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]:
return [
requirements.TranslationLayerRequirement(name = 'primary',
description = 'Memory layer for the kernel',
architectures = ["Intel32", "Intel64"]),
requirements.BooleanRequirement(name = 'extra',
description = 'Outputs whole configuration tree',
default = False,
optional = True)
requirements.TranslationLayerRequirement(
name="primary",
description="Memory layer for the kernel",
architectures=["Intel32", "Intel64"],
),
requirements.BooleanRequirement(
name="extra",
description="Outputs whole configuration tree",
default=False,
optional=True,
),
]
def _generator(self):
filename = "config.json"
config = dict(self.build_configuration())
if self.config.get('extra', False):
vollog.debug("Outputting additional information, this will NOT work with the -c option")
if self.config.get("extra", False):
vollog.debug(
"Outputting additional information, this will NOT work with the -c option"
)
config = dict(self.context.config)
filename = "config.extra"
try:
with self.open(filename) as file_data:
file_data.write(bytes(json.dumps(config, sort_keys = True, indent = 2), 'raw_unicode_escape'))
file_data.write(
bytes(
json.dumps(config, sort_keys=True, indent=2),
"raw_unicode_escape",
)
)
except Exception as excp:
vollog.warning(f"Unable to JSON encode configuration: {excp}")
volatility3/framework/plugins/frameworkinfo.py
+13 -9
View File
@@ -1,3 +1,7 @@
# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0
# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0
#
from typing import List
from volatility3 import framework
@@ -16,19 +20,19 @@ class FrameworkInfo(plugins.PluginInterface):
def _generator(self):
categories = {
'Automagic': interfaces.automagic.AutomagicInterface,
'Requirement': interfaces.configuration.RequirementInterface,
'Layer': interfaces.layers.DataLayerInterface,
'LayerStacker': interfaces.automagic.StackerLayerInterface,
'Object': interfaces.objects.ObjectInterface,
'Plugin': interfaces.plugins.PluginInterface,
'Renderer': interfaces.renderers.Renderer
"Automagic": interfaces.automagic.AutomagicInterface,
"Requirement": interfaces.configuration.RequirementInterface,
"Layer": interfaces.layers.DataLayerInterface,
"LayerStacker": interfaces.automagic.StackerLayerInterface,
"Object": interfaces.objects.ObjectInterface,
"Plugin": interfaces.plugins.PluginInterface,
"Renderer": interfaces.renderers.Renderer,
}
for category, module_interface in categories.items():
yield (0, (category, ))
yield (0, (category,))
for clazz in framework.class_subclasses(module_interface):
yield (1, (clazz.__name__, ))
yield (1, (clazz.__name__,))
def run(self):
return renderers.TreeGrid([("Data", str)], self._generator())
volatility3/framework/plugins/isfinfo.py
+132 -59
View File
@@ -1,17 +1,16 @@
# This file is Copyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0
# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0
#
import base64
import json
import logging
import os
import pathlib
import zipfile
from typing import List, Type, Any, Generator
from typing import Generator, List
from volatility3 import schemas, symbols
from volatility3.framework import interfaces, renderers, constants
from volatility3.framework.automagic import mac, linux, symbol_cache
from volatility3.framework import constants, interfaces, renderers
from volatility3.framework.automagic import symbol_cache
from volatility3.framework.configuration import requirements
from volatility3.framework.interfaces import plugins
from volatility3.framework.layers import resources
@@ -23,108 +22,182 @@ class IsfInfo(plugins.PluginInterface):
"""Determines information about the currently available ISF files, or a specific one"""
_required_framework_version = (2, 0, 0)
_version = (1, 0, 0)
_version = (2, 0, 0)
@classmethod
def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]:
return [
requirements.ListRequirement(name = 'filter',
description = 'String that must be present in the file URI to display the ISF',
optional = True,
default = []),
requirements.URIRequirement(name = 'isf',
description = "Specific ISF file to process",
default = None,
optional = True),
requirements.BooleanRequirement(name = 'validate',
description = 'Validate against schema if possible',
default = False,
optional = True)
requirements.ListRequirement(
name="filter",
description="String that must be present in the file URI to display the ISF",
optional=True,
default=[],
),
requirements.URIRequirement(
name="isf",
description="Specific ISF file to process",
default=None,
optional=True,
),
requirements.BooleanRequirement(
name="validate",
description="Validate against schema if possible",
default=False,
optional=True,
),
requirements.VersionRequirement(
name="SQLiteCache",
component=symbol_cache.SqliteCache,
version=(1, 0, 0),
),
requirements.BooleanRequirement(
name="live",
description="Traverse all files, rather than use the cache",
default=False,
optional=True,
),
]
@classmethod
def list_all_isf_files(cls) -> Generator[str, None, None]:
"""Lists all the ISF files that can be found"""
for symbol_path in symbols.__path__:
for root, dirs, files in os.walk(symbol_path, followlinks = True):
for root, dirs, files in os.walk(symbol_path, followlinks=True):
for filename in files:
base_name = os.path.join(root, filename)
if filename.endswith('zip'):
with zipfile.ZipFile(base_name, 'r') as zfile:
if filename.endswith("zip"):
with zipfile.ZipFile(base_name, "r") as zfile:
for name in zfile.namelist():
for extension in constants.ISF_EXTENSIONS:
# By ending with an extension (and therefore, not /), we should not return any directories
if name.endswith(extension):
yield "jar:file:" + str(pathlib.Path(base_name)) + "!" + name
yield "jar:file:" + str(
pathlib.Path(base_name)
) + "!" + name
else:
for extension in constants.ISF_EXTENSIONS:
if filename.endswith(extension):
yield pathlib.Path(base_name).as_uri()
def _get_banner(self, clazz: Type[symbol_cache.SymbolBannerCache], data: Any) -> str:
"""Gets a banner from an ISF file"""
banner_symbol = data.get('symbols', {}).get(clazz.symbol_name, {}).get('constant_data',
renderers.NotAvailableValue())
if not isinstance(banner_symbol, interfaces.renderers.BaseAbsentValue):
banner_symbol = str(base64.b64decode(banner_symbol), encoding = 'latin-1')
return banner_symbol
def _generator(self):
if self.config.get('isf', None) is not None:
file_list = [self.config['isf']]
if self.config.get("isf", None) is not None:
file_list = [self.config["isf"]]
else:
file_list = list(self.list_all_isf_files())
# Filter the files
filtered_list = []
if not len(self.config['filter']):
if not len(self.config["filter"]):
filtered_list = file_list
else:
for isf_file in file_list:
for filter_item in self.config['filter']:
for filter_item in self.config["filter"]:
if filter_item in isf_file:
filtered_list.append(isf_file)
try:
import jsonschema
if not self.config['validate']:
if not self.config["validate"]:
raise ImportError # Act as if we couldn't import if validation is turned off
def check_valid(data):
return "True" if schemas.validate(data, True) else "False"
except ImportError:
def check_valid(data):
return "Unknown"
# Process the filtered list
for entry in filtered_list:
num_types = num_enums = num_bases = num_symbols = 0
windows_info = linux_banner = mac_banner = renderers.NotAvailableValue()
valid = "Unknown"
with resources.ResourceAccessor().open(url = entry) as fp:
try:
data = json.load(fp)
num_symbols = len(data.get('symbols', []))
num_types = len(data.get('user_types', []))
num_enums = len(data.get('enums', []))
num_bases = len(data.get('base_types', []))
if self.config["live"]:
# Process the filtered list
for entry in filtered_list:
num_types = num_enums = num_bases = num_symbols = 0
valid = "Unknown"
with resources.ResourceAccessor().open(url=entry) as fp:
try:
data = json.load(fp)
num_symbols = len(data.get("symbols", []))
num_types = len(data.get("user_types", []))
num_enums = len(data.get("enums", []))
num_bases = len(data.get("base_types", []))
linux_banner = self._get_banner(linux.LinuxBannerCache, data)
mac_banner = self._get_banner(mac.MacBannerCache, data)
if not linux_banner and not mac_banner:
windows_info = os.path.splitext(os.path.basename(entry))[0]
valid = check_valid(data)
except (UnicodeDecodeError, json.decoder.JSONDecodeError):
vollog.warning(f"Invalid ISF: {entry}")
yield (0, (entry, valid, num_bases, num_types, num_symbols, num_enums, windows_info, linux_banner,
mac_banner))
identifiers_path = os.path.join(
constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME
)
identifier_cache = symbol_cache.SqliteCache(identifiers_path)
identifier = identifier_cache.get_identifier(location=entry)
if identifier:
identifier = identifier.decode("utf-8", errors="replace")
else:
identifier = renderers.NotAvailableValue()
valid = check_valid(data)
except (UnicodeDecodeError, json.decoder.JSONDecodeError):
vollog.warning(f"Invalid ISF: {entry}")
yield (
0,
(
entry,
valid,
num_bases,
num_types,
num_symbols,
num_enums,
identifier,
),
)
else:
identifiers_path = os.path.join(
constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME
)
cache = symbol_cache.SqliteCache(identifiers_path)
valid = "Unknown"
for identifier, location in cache.get_identifier_dictionary().items():
(
num_bases,
num_types,
num_enums,
num_symbols,
) = cache.get_location_statistics(location)
if identifier:
json_hash = cache.get_hash(location)
if json_hash and json_hash in schemas.cached_validations:
valid = "True (cached)"
if self.config["validate"]:
# Even if we're not live, if we've been explicitly asked to validate, then do-so
with resources.ResourceAccessor().open(url=location) as fp:
try:
data = json.load(fp)
valid = check_valid(data)
except (UnicodeDecodeError, json.decoder.JSONDecodeError):
vollog.warning(f"Invalid ISF: {location}")
yield (
0,
(
location,
valid,
num_bases,
num_types,
num_symbols,
num_enums,
str(identifier),
),
)
# Try to open the file, load it as JSON, read the data from it
def run(self):
return renderers.TreeGrid([("URI", str), ("Valid", str),
("Number of base_types", int), ("Number of types", int), ("Number of symbols", int),
("Number of enums", int), ("Windows info", str), ("Linux banner", str),
("Mac banner", str)], self._generator())
return renderers.TreeGrid(
[
("URI", str),
("Valid", str),
("Number of base_types", int),
("Number of types", int),
("Number of symbols", int),
("Number of enums", int),
("Identifying information", str),
],
self._generator(),
)
volatility3/framework/plugins/layerwriter.py
+58 -42
View File
@@ -23,32 +23,40 @@ class LayerWriter(plugins.PluginInterface):
@classmethod
def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]:
return [
requirements.TranslationLayerRequirement(name = 'primary', description = 'Memory layer for the kernel'),
requirements.IntRequirement(name = 'block_size',
description = "Size of blocks to copy over",
default = cls.default_block_size,
optional = True),
requirements.BooleanRequirement(name = 'list',
description = 'List available layers',
default = False,
optional = True),
requirements.TranslationLayerRequirement(
name="primary", description="Memory layer for the kernel"
),
requirements.IntRequirement(
name="block_size",
description="Size of blocks to copy over",
default=cls.default_block_size,
optional=True,
),
requirements.BooleanRequirement(
name="list",
description="List available layers",
default=False,
optional=True,
),
requirements.ListRequirement(
name = 'layers',
element_type = str,
description = 'Names of layers to write (defaults to the highest non-mapped layer)',
default = None,
optional = True)
name="layers",
element_type=str,
description="Names of layers to write (defaults to the highest non-mapped layer)",
default=None,
optional=True,
),
]
@classmethod
def write_layer(
cls,
context: interfaces.context.ContextInterface,
layer_name: str,
preferred_name: str,
open_method: Type[plugins.FileHandlerInterface],
chunk_size: Optional[int] = None,
progress_callback: Optional[constants.ProgressCallback] = None) -> Optional[plugins.FileHandlerInterface]:
cls,
context: interfaces.context.ContextInterface,
layer_name: str,
preferred_name: str,
open_method: Type[plugins.FileHandlerInterface],
chunk_size: Optional[int] = None,
progress_callback: Optional[constants.ProgressCallback] = None,
) -> Optional[plugins.FileHandlerInterface]:
"""Produces a FileHandler from the named layer in the provided context or None on failure
Args:
@@ -70,42 +78,48 @@ class LayerWriter(plugins.PluginInterface):
file_handle = open_method(preferred_name)
for i in range(0, layer.maximum_address, chunk_size):
current_chunk_size = min(chunk_size, layer.maximum_address - i)
data = layer.read(i, current_chunk_size, pad = True)
data = layer.read(i, current_chunk_size, pad=True)
file_handle.write(data)
if progress_callback:
progress_callback((i / layer.maximum_address) * 100, f'Writing layer {layer_name}')
progress_callback(
(i / layer.maximum_address) * 100, f"Writing layer {layer_name}"
)
return file_handle
def _generator(self):
if self.config['list']:
if self.config["list"]:
for name in self.context.layers:
yield 0, (name, )
yield 0, (name,)
else:
# Choose the most recently added layer that isn't virtual
if not self.config['layers']:
self.config['layers'] = []
if not self.config["layers"]:
self.config["layers"] = []
for name in self.context.layers:
if not self.context.layers[name].metadata.get('mapped', False):
self.config['layers'] = [name]
if not self.context.layers[name].metadata.get("mapped", False):
self.config["layers"] = [name]
for name in self.config['layers']:
for name in self.config["layers"]:
# Check the layer exists and validate the output file
if name not in self.context.layers:
yield 0, (f'Layer Name {name} does not exist', )
yield 0, (f"Layer Name {name} does not exist",)
else:
output_name = self.config.get('output', ".".join([name, "raw"]))
output_name = self.config.get("output", ".".join([name, "raw"]))
try:
file_handle = self.write_layer(self.context,
name,
output_name,
self.open,
self.config.get('block_size', self.default_block_size),
progress_callback = self._progress_callback)
file_handle = self.write_layer(
self.context,
name,
output_name,
self.open,
self.config.get("block_size", self.default_block_size),
progress_callback=self._progress_callback,
)
file_handle.close()
except IOError as excp:
yield 0, (f"Layer cannot be written to {self.config['output_name']}: {excp}", )
yield 0, (
f"Layer cannot be written to {self.config['output_name']}: {excp}",
)
yield 0, (f'Layer has been written to {output_name}', )
yield 0, (f"Layer has been written to {output_name}",)
def _generate_layers(self):
"""List layer names from this run"""
@@ -113,6 +127,8 @@ class LayerWriter(plugins.PluginInterface):
yield (0, (name, self.context.layers[name].__class__.__name__))
def run(self):
if self.config['list']:
return renderers.TreeGrid([("Layer name", str), ('Layer type', str)], self._generate_layers())
if self.config["list"]:
return renderers.TreeGrid(
[("Layer name", str), ("Layer type", str)], self._generate_layers()
)
return renderers.TreeGrid([("Status", str)], self._generator())
volatility3/framework/plugins/linux/bash.py
+68 -34
View File
@@ -26,18 +26,27 @@ class Bash(plugins.PluginInterface, timeliner.TimeLinerInterface):
@classmethod
def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]:
return [
requirements.ModuleRequirement(name = 'kernel', description = 'Linux kernel',
architectures = ["Intel32", "Intel64"]),
requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)),
requirements.ListRequirement(name = 'pid',
element_type = int,
description = "Process IDs to include (all other processes are excluded)",
optional = True)
requirements.ModuleRequirement(
name="kernel",
description="Linux kernel",
architectures=["Intel32", "Intel64"],
),
requirements.PluginRequirement(
name="pslist", plugin=pslist.PsList, version=(2, 0, 0)
),
requirements.ListRequirement(
name="pid",
element_type=int,
description="Process IDs to include (all other processes are excluded)",
optional=True,
),
]
def _generator(self, tasks):
vmlinux = self.context.modules[self.config["kernel"]]
is_32bit = not symbols.symbol_table_is_64bit(self.context, vmlinux.symbol_table_name)
is_32bit = not symbols.symbol_table_is_64bit(
self.context, vmlinux.symbol_table_name
)
if is_32bit:
pack_format = "I"
bash_json_file = "bash32"
@@ -45,10 +54,13 @@ class Bash(plugins.PluginInterface, timeliner.TimeLinerInterface):
pack_format = "Q"
bash_json_file = "bash64"
bash_table_name = BashIntermedSymbols.create(self.context, self.config_path, "linux", bash_json_file)
bash_table_name = BashIntermedSymbols.create(
self.context, self.config_path, "linux", bash_json_file
)
ts_offset = self.context.symbol_space.get_type(bash_table_name + constants.BANG +
"hist_entry").relative_child_offset("timestamp")
ts_offset = self.context.symbol_space.get_type(
bash_table_name + constants.BANG + "hist_entry"
).relative_child_offset("timestamp")
for task in tasks:
task_name = utility.array_to_string(task.comm)
@@ -63,45 +75,67 @@ class Bash(plugins.PluginInterface, timeliner.TimeLinerInterface):
bang_addrs = []
# get task memory sections to be used by scanners
task_memory_sections = [
section for section in task.get_process_memory_sections(heap_only=True)
]
# find '#' values on the heap
for address in proc_layer.scan(self.context,
scanners.BytesScanner(b"#"),
sections = task.get_process_memory_sections(heap_only = True)):
for address in proc_layer.scan(
self.context,
scanners.BytesScanner(b"#"),
sections=task_memory_sections,
):
bang_addrs.append(struct.pack(pack_format, address))
history_entries = []
if bang_addrs:
for address, _ in proc_layer.scan(self.context,
scanners.MultiStringScanner(bang_addrs),
sections = task.get_process_memory_sections(heap_only = True)):
hist = self.context.object(bash_table_name + constants.BANG + "hist_entry",
offset = address - ts_offset,
layer_name = proc_layer_name)
for address, _ in proc_layer.scan(
self.context,
scanners.MultiStringScanner(bang_addrs),
sections=task_memory_sections,
):
hist = self.context.object(
bash_table_name + constants.BANG + "hist_entry",
offset=address - ts_offset,
layer_name=proc_layer_name,
)
if hist.is_valid():
history_entries.append(hist)
for hist in sorted(history_entries, key = lambda x: x.get_time_as_integer()):
yield (0, (task.pid, task_name, hist.get_time_object(), hist.get_command()))
for hist in sorted(history_entries, key=lambda x: x.get_time_as_integer()):
yield (
0,
(task.pid, task_name, hist.get_time_object(), hist.get_command()),
)
def run(self):
filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None))
filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None))
return renderers.TreeGrid([("PID", int), ("Process", str), ("CommandTime", datetime.datetime),
("Command", str)],
self._generator(
pslist.PsList.list_tasks(self.context,
self.config['kernel'],
filter_func = filter_func)))
return renderers.TreeGrid(
[
("PID", int),
("Process", str),
("CommandTime", datetime.datetime),
("Command", str),
],
self._generator(
pslist.PsList.list_tasks(
self.context, self.config["kernel"], filter_func=filter_func
)
),
)
def generate_timeline(self):
filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None))
filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None))
for row in self._generator(
pslist.PsList.list_tasks(self.context,
self.config['kernel'],
filter_func = filter_func)):
pslist.PsList.list_tasks(
self.context, self.config["kernel"], filter_func=filter_func
)
):
_depth, row_data = row
description = f"{row_data[0]} ({row_data[1]}): \"{row_data[3]}\""
description = f'{row_data[0]} ({row_data[1]}): "{row_data[3]}"'
yield (description, timeliner.TimeLinerType.CREATED, row_data[2])

Some files were not shown because too many files have changed in this diff Show More