From a5fe38339a038852cdff47acb0d4942e98fdaefd Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 24 Aug 2022 21:15:50 +0100 Subject: [PATCH 1/4] Core: Allow for deprecation of constants gracefully --- volatility3/framework/__init__.py | 2 +- volatility3/framework/automagic/linux.py | 4 ++- volatility3/framework/automagic/mac.py | 4 ++- .../framework/automagic/symbol_cache.py | 3 +- .../framework/automagic/symbol_finder.py | 4 ++- volatility3/framework/constants/__init__.py | 29 ++++++++++++++----- volatility3/framework/plugins/isfinfo.py | 6 ++-- .../framework/symbols/windows/pdbutil.py | 3 +- 8 files changed, 40 insertions(+), 15 deletions(-) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 176eb2242..9b11143b2 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -7,7 +7,7 @@ import glob import sys import zipfile -required_python_version = (3, 6, 0) +required_python_version = (3, 7, 0) if (sys.version_info.major != required_python_version[0] or sys.version_info.minor < required_python_version[1] or (sys.version_info.minor == required_python_version[1] and sys.version_info.micro < required_python_version[2])): raise RuntimeError( diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 2c152996d..9bb2dae9b 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -3,6 +3,7 @@ # import logging +import os from typing import Optional, Tuple, Type from volatility3.framework import constants, interfaces @@ -40,7 +41,8 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): if isinstance(layer, intel.Intel): return None - linux_banners = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH).get_identifier_dictionary( + identifiers_path = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME) + linux_banners = symbol_cache.SqliteCache(identifiers_path).get_identifier_dictionary( operating_system = 'linux') # If we have no banners, don't bother scanning if not linux_banners: diff --git a/volatility3/framework/automagic/mac.py b/volatility3/framework/automagic/mac.py index 246462878..9bb3ad5f0 100644 --- a/volatility3/framework/automagic/mac.py +++ b/volatility3/framework/automagic/mac.py @@ -3,6 +3,7 @@ # import logging +import os import struct from typing import Optional @@ -42,7 +43,8 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): if isinstance(layer, intel.Intel): return None - mac_banners = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH).get_identifier_dictionary( + identifiers_path = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME) + mac_banners = symbol_cache.SqliteCache(identifiers_path).get_identifier_dictionary( operating_system = 'mac') # If we have no banners, don't bother scanning if not mac_banners: diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 558bfb2f1..d69009721 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -388,7 +388,8 @@ class SymbolCacheMagic(interfaces.automagic.AutomagicInterface): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self._cache = SqliteCache(constants.IDENTIFIERS_PATH) + identifiers_path = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME) + self._cache = SqliteCache(identifiers_path) def __call__(self, context, config_path, configurable, progress_callback = None): """Runs the automagic over the configurable.""" diff --git a/volatility3/framework/automagic/symbol_finder.py b/volatility3/framework/automagic/symbol_finder.py index a9221a7cc..7a197dffc 100644 --- a/volatility3/framework/automagic/symbol_finder.py +++ b/volatility3/framework/automagic/symbol_finder.py @@ -3,6 +3,7 @@ # import logging +import os from typing import Any, Callable, Iterable, List, Optional, Tuple from volatility3.framework import constants, interfaces, layers @@ -40,7 +41,8 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface): """Creates a cached copy of the results, but only it's been requested.""" if not self._banners: - cache = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH) + identifiers_path = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME) + cache = symbol_cache.SqliteCache(identifiers_path) self._banners = cache.get_identifier_dictionary(operating_system = self.operating_system) return self._banners diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 3b499adea..1f646416b 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -9,6 +9,7 @@ volatility This includes default scanning block sizes, etc. import enum import os.path import sys +import warnings from typing import Callable, Optional import volatility3.framework.constants.linux @@ -67,13 +68,7 @@ if sys.platform == 'win32': CACHE_PATH = os.path.join(os.environ.get("APPDATA", os.path.expanduser("~")), "volatility3") os.makedirs(CACHE_PATH, exist_ok = True) -LINUX_BANNERS_PATH = os.path.join(CACHE_PATH, "linux_banners.cache") -"""Default location to record information about available linux banners""" - -MAC_BANNERS_PATH = os.path.join(CACHE_PATH, "mac_banners.cache") -"""Default location to record information about available mac banners""" - -IDENTIFIERS_PATH = os.path.join(CACHE_PATH, "identifiers.cache") +IDENTIFIERS_FILENAME = "identifier.cache" """Default location to record information about available identifiers""" CACHE_SQLITE_SCEMA_VERSION = 1 @@ -107,3 +102,23 @@ OFFLINE = False REMOTE_ISF_URL = None # 'http://localhost:8000/banners.json' """Remote URL to query for a list of ISF addresses""" + +### +# DEPRECATED VALUES +### + +_deprecated_LINUX_BANNERS_FILENAME = os.path.join(CACHE_PATH, 'linux_banners.cache') +"""This value is deprecated and is no longer used within volatility""" + +_deprecated_MAC_BANNERS_PATH = os.path.join(CACHE_PATH, 'mac_banners.cache') +"""This value is deprecated and is no longer used within volatility""" + +_deprecated_IDENTIFIERS_PATH = os.path.join(CACHE_PATH, IDENTIFIERS_FILENAME) +"""This value is deprecated in favour of CACHE_PATH joined to IDENTIFIER_FILENAME""" + + +def __getattr__(name): + deprecated_tag = '_deprecated_' + if name in [x[len(deprecated_tag):] for x in globals() if x.startswith(deprecated_tag)]: + warnings.warn(f"{name} is deprecated", FutureWarning) + return globals()[f"{deprecated_tag}{name}"] diff --git a/volatility3/framework/plugins/isfinfo.py b/volatility3/framework/plugins/isfinfo.py index 6b13f10b6..efffa9b87 100644 --- a/volatility3/framework/plugins/isfinfo.py +++ b/volatility3/framework/plugins/isfinfo.py @@ -109,7 +109,8 @@ class IsfInfo(plugins.PluginInterface): num_enums = len(data.get('enums', [])) num_bases = len(data.get('base_types', [])) - identifier_cache = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH) + identifiers_path = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME) + identifier_cache = symbol_cache.SqliteCache(identifiers_path) identifier = identifier_cache.get_identifier(location = entry) if identifier: identifier = identifier.decode('utf-8', errors = 'replace') @@ -120,7 +121,8 @@ class IsfInfo(plugins.PluginInterface): vollog.warning(f"Invalid ISF: {entry}") yield (0, (entry, valid, num_bases, num_types, num_symbols, num_enums, identifier)) else: - cache = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH) + identifiers_path = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME) + cache = symbol_cache.SqliteCache(identifiers_path) valid = 'Unknown' for identifier, location in cache.get_identifier_dictionary().items(): num_bases, num_types, num_enums, num_symbols = cache.get_location_statistics(location) diff --git a/volatility3/framework/symbols/windows/pdbutil.py b/volatility3/framework/symbols/windows/pdbutil.py index 430ad6a30..079b0e826 100644 --- a/volatility3/framework/symbols/windows/pdbutil.py +++ b/volatility3/framework/symbols/windows/pdbutil.py @@ -80,7 +80,8 @@ class PDBUtility(interfaces.configuration.VersionableInterface): vollog.debug(f"Required version of SQLiteCache not found") return None - value = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH).find_location( + identifiers_path = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME) + value = symbol_cache.SqliteCache(identifiers_path).find_location( symbol_cache.WindowsIdentifier.generate(pdb_name.strip('\x00'), guid.upper(), age), 'windows') if value: From a337ec732a6feaf70032c405a48f1f3ceae39ae5 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 24 Aug 2022 21:20:56 +0100 Subject: [PATCH 2/4] Test: Update build tests to new minimum python version --- .github/workflows/test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index cf70b66cd..2d3729981 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -7,10 +7,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.6 + - name: Set up Python 3.7 uses: actions/setup-python@v2 with: - python-version: '3.6' + python-version: '3.7' - name: Install dependencies run: | From d7301d653fca9c1195f83642c7133514c1f6a9a7 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 25 Aug 2022 10:55:58 +0100 Subject: [PATCH 3/4] Core: Additional updates with the bump to python 3.7.0 Kindly pointed out by @digitalisx --- README.md | 2 +- setup.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 348121e44..502e26f10 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ more details. ## Requirements -Volatility 3 requires Python 3.6.0 or later. To install the most minimal set of dependencies (some plugins will not work) use a command such as: +Volatility 3 requires Python 3.7.0 or later. To install the most minimal set of dependencies (some plugins will not work) use a command such as: ```shell pip3 install -r requirements-minimal.txt diff --git a/setup.py b/setup.py index f6bb687f2..bce21ca66 100644 --- a/setup.py +++ b/setup.py @@ -9,9 +9,10 @@ from volatility3.framework import constants with open("README.md", "r", encoding = "utf-8") as fh: long_description = fh.read() + def get_install_requires(): requirements = [] - with open("requirements-minimal.txt", "r", encoding="utf-8") as fh: + with open("requirements-minimal.txt", "r", encoding = "utf-8") as fh: for line in fh.readlines(): stripped_line = line.strip() if stripped_line == "" or stripped_line.startswith("#"): @@ -19,6 +20,7 @@ def get_install_requires(): requirements.append(stripped_line) return requirements + setuptools.setup(name = "volatility3", description = "Memory forensics framework", version = constants.PACKAGE_VERSION, @@ -34,7 +36,7 @@ setuptools.setup(name = "volatility3", "Documentation": "https://volatility3.readthedocs.io/", "Source Code": "https://github.com/volatilityfoundation/volatility3", }, - python_requires = '>=3.6.0', + python_requires = '>=3.7.0', include_package_data = True, exclude_package_data = { '': ['development', 'development.*'], From 7529c7b246734ae02b51bb80dc22bbeddb819078 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 21 Sep 2022 21:15:40 +0100 Subject: [PATCH 4/4] Core: Revert volatility 3.7 bump and associated features --- .github/workflows/test.yaml | 4 ++-- README.md | 2 +- setup.py | 6 ++---- volatility3/framework/__init__.py | 2 +- volatility3/framework/constants/__init__.py | 21 --------------------- 5 files changed, 6 insertions(+), 29 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 2d3729981..cf70b66cd 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -7,10 +7,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.7 + - name: Set up Python 3.6 uses: actions/setup-python@v2 with: - python-version: '3.7' + python-version: '3.6' - name: Install dependencies run: | diff --git a/README.md b/README.md index 502e26f10..348121e44 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ more details. ## Requirements -Volatility 3 requires Python 3.7.0 or later. To install the most minimal set of dependencies (some plugins will not work) use a command such as: +Volatility 3 requires Python 3.6.0 or later. To install the most minimal set of dependencies (some plugins will not work) use a command such as: ```shell pip3 install -r requirements-minimal.txt diff --git a/setup.py b/setup.py index bce21ca66..f6bb687f2 100644 --- a/setup.py +++ b/setup.py @@ -9,10 +9,9 @@ from volatility3.framework import constants with open("README.md", "r", encoding = "utf-8") as fh: long_description = fh.read() - def get_install_requires(): requirements = [] - with open("requirements-minimal.txt", "r", encoding = "utf-8") as fh: + with open("requirements-minimal.txt", "r", encoding="utf-8") as fh: for line in fh.readlines(): stripped_line = line.strip() if stripped_line == "" or stripped_line.startswith("#"): @@ -20,7 +19,6 @@ def get_install_requires(): requirements.append(stripped_line) return requirements - setuptools.setup(name = "volatility3", description = "Memory forensics framework", version = constants.PACKAGE_VERSION, @@ -36,7 +34,7 @@ setuptools.setup(name = "volatility3", "Documentation": "https://volatility3.readthedocs.io/", "Source Code": "https://github.com/volatilityfoundation/volatility3", }, - python_requires = '>=3.7.0', + python_requires = '>=3.6.0', include_package_data = True, exclude_package_data = { '': ['development', 'development.*'], diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 9b11143b2..176eb2242 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -7,7 +7,7 @@ import glob import sys import zipfile -required_python_version = (3, 7, 0) +required_python_version = (3, 6, 0) if (sys.version_info.major != required_python_version[0] or sys.version_info.minor < required_python_version[1] or (sys.version_info.minor == required_python_version[1] and sys.version_info.micro < required_python_version[2])): raise RuntimeError( diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 1f646416b..d6fb96e1c 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -9,7 +9,6 @@ volatility This includes default scanning block sizes, etc. import enum import os.path import sys -import warnings from typing import Callable, Optional import volatility3.framework.constants.linux @@ -102,23 +101,3 @@ OFFLINE = False REMOTE_ISF_URL = None # 'http://localhost:8000/banners.json' """Remote URL to query for a list of ISF addresses""" - -### -# DEPRECATED VALUES -### - -_deprecated_LINUX_BANNERS_FILENAME = os.path.join(CACHE_PATH, 'linux_banners.cache') -"""This value is deprecated and is no longer used within volatility""" - -_deprecated_MAC_BANNERS_PATH = os.path.join(CACHE_PATH, 'mac_banners.cache') -"""This value is deprecated and is no longer used within volatility""" - -_deprecated_IDENTIFIERS_PATH = os.path.join(CACHE_PATH, IDENTIFIERS_FILENAME) -"""This value is deprecated in favour of CACHE_PATH joined to IDENTIFIER_FILENAME""" - - -def __getattr__(name): - deprecated_tag = '_deprecated_' - if name in [x[len(deprecated_tag):] for x in globals() if x.startswith(deprecated_tag)]: - warnings.warn(f"{name} is deprecated", FutureWarning) - return globals()[f"{deprecated_tag}{name}"]