From 48e29ae530ecfb65fa3311a7547c16ea77f7f5cd Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 29 May 2022 23:44:29 +0100 Subject: [PATCH 1/6] CLI: Add support for a configuration options file --- volatility3/cli/__init__.py | 58 ++++++++++++++++++++++------ volatility3/cli/volshell/__init__.py | 51 +++++++++++++++++------- 2 files changed, 83 insertions(+), 26 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 8851e2b18..62be115d3 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -19,7 +19,7 @@ import os import sys import tempfile import traceback -from typing import Any, Dict, Type, Union +from typing import Any, Dict, List, Tuple, Type, Union from urllib import parse, request import volatility3.plugins @@ -92,6 +92,9 @@ class CommandLine: renderers = dict([(x.name.lower(), x) for x in framework.class_subclasses(text_renderer.CLIRenderer)]) + # Load up system defaults + delayed_logs, default_config = self.load_system_defaults('vol.json') + parser = volargparse.HelpfulArgParser(add_help = False, prog = self.CLI_NAME, description = "An open-source memory forensics framework") @@ -174,6 +177,8 @@ class CommandLine: default = False, action = 'store_true') + parser.set_defaults(**default_config) + # We have to filter out help, otherwise parse_known_args will trigger the help message before having # processed the plugin choice or had the plugin subparser added. known_args = [arg for arg in sys.argv if arg != '--help' and arg != '-h'] @@ -184,17 +189,7 @@ class CommandLine: banner_output = sys.stderr banner_output.write(f"Volatility 3 Framework {constants.PACKAGE_VERSION}\n") - if partial_args.plugin_dirs: - volatility3.plugins.__path__ = [os.path.abspath(p) - for p in partial_args.plugin_dirs.split(";")] + constants.PLUGINS_PATH - - if partial_args.symbol_dirs: - volatility3.symbols.__path__ = [os.path.abspath(p) - for p in partial_args.symbol_dirs.split(";")] + constants.SYMBOL_BASEPATHS - - if partial_args.cache_path: - constants.CACHE_PATH = partial_args.cache_path - + ### Start up logging if partial_args.log: file_logger = logging.FileHandler(partial_args.log) file_logger.setLevel(1) @@ -210,6 +205,21 @@ class CommandLine: else: console.setLevel(10 - (partial_args.verbosity - 2)) + for level, msg in delayed_logs: + vollog.log(level, msg) + + ### Alter constants if necessary + if partial_args.plugin_dirs: + volatility3.plugins.__path__ = [os.path.abspath(p) + for p in partial_args.plugin_dirs.split(";")] + constants.PLUGINS_PATH + + if partial_args.symbol_dirs: + volatility3.symbols.__path__ = [os.path.abspath(p) + for p in partial_args.symbol_dirs.split(";")] + constants.SYMBOL_BASEPATHS + + if partial_args.cache_path: + constants.CACHE_PATH = partial_args.cache_path + vollog.info(f"Volatility plugins path: {volatility3.plugins.__path__}") vollog.info(f"Volatility symbols path: {volatility3.symbols.__path__}") @@ -366,6 +376,30 @@ class CommandLine: raise ValueError(f"File does not exist: {filename}") return parse.urlunparse(single_location) + def load_system_defaults(self, filename: str) -> Tuple[List[Tuple[int, str]], Dict[str, Any]]: + """Modify the main configuration based on the default configuration override""" + # Build the config path + default_config_path = os.path.join(os.path.expanduser("~"), ".config", "volatility3", filename) + if sys.platform == 'win32': + default_config_path = os.path.join(os.environ.get("APPDATA", os.path.expanduser("~")), "volatility3", + filename) + + delayed_logs = [] + + # Process it if the files exist + if os.path.exists(default_config_path): + result = json.load(open(default_config_path, 'rb')) + if not isinstance(result, dict): + delayed_logs.append((logging.INFO, + f'Default configuration file {default_config_path} does not contain a dictionary')) + else: + delayed_logs.append( + (logging.INFO, f"Loading default configuration options from {default_config_path}")) + delayed_logs.append((logging.DEBUG, + f"Loaded configuration: {json.dumps(result, indent = 2, sort_keys = True)}")) + return delayed_logs, result + return delayed_logs, {} + def process_exceptions(self, excp): """Provide useful feedback if an exception occurs during a run of a plugin.""" # Ensure there's nothing in the cache diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 769e958fd..647be8a1a 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -45,6 +45,9 @@ class VolShell(cli.CommandLine): framework.require_interface_version(2, 0, 0) + # Load up system defaults + delayed_logs, default_config = self.load_system_defaults('volshell.json') + parser = argparse.ArgumentParser(prog = self.CLI_NAME, description = "A tool for interactivate forensic analysis of memory images") parser.add_argument("-c", @@ -68,13 +71,16 @@ class VolShell(cli.CommandLine): default = "", type = str) parser.add_argument("-v", "--verbosity", help = "Increase output verbosity", default = 0, action = "count") + parser.add_argument("--log", + help = "Log output to a file as well as the console", + default = None, + type = str) parser.add_argument("-o", "--output-dir", help = "Directory in which to output any generated files", default = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')), type = str) parser.add_argument("-q", "--quiet", help = "Remove progress feedback", default = False, action = 'store_true') - parser.add_argument("--log", help = "Log output to a file as well as the console", default = None, type = str) parser.add_argument("-f", "--file", metavar = 'FILE', @@ -97,6 +103,10 @@ class VolShell(cli.CommandLine): help = f"Change the default path ({constants.CACHE_PATH}) used to store the cache", default = constants.CACHE_PATH, type = str) + parser.add_argument("--offline", + help = "Do not search online for additional JSON files", + default = False, + action = 'store_true') # Volshell specific flags os_specific = parser.add_mutually_exclusive_group(required = False) @@ -108,10 +118,33 @@ class VolShell(cli.CommandLine): os_specific.add_argument("-l", "--linux", default = False, action = "store_true", help = "Run a Linux volshell") os_specific.add_argument("-m", "--mac", default = False, action = "store_true", help = "Run a Mac volshell") + parser.set_defaults(**default_config) + # We have to filter out help, otherwise parse_known_args will trigger the help message before having # processed the plugin choice or had the plugin subparser added. known_args = [arg for arg in sys.argv if arg != '--help' and arg != '-h'] partial_args, _ = parser.parse_known_args(known_args) + + ### Start up logging + if partial_args.log: + file_logger = logging.FileHandler(partial_args.log) + file_logger.setLevel(1) + file_formatter = logging.Formatter(datefmt = '%y-%m-%d %H:%M:%S', + fmt = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s') + file_logger.setFormatter(file_formatter) + rootlog.addHandler(file_logger) + vollog.info("Logging started") + if partial_args.verbosity < 3: + if partial_args.verbosity < 1: + sys.tracebacklimit = None + console.setLevel(30 - (partial_args.verbosity * 10)) + else: + console.setLevel(10 - (partial_args.verbosity - 2)) + + for level, msg in delayed_logs: + vollog.log(level, msg) + + ### Alter constants if necessary if partial_args.plugin_dirs: volatility3.plugins.__path__ = [os.path.abspath(p) for p in partial_args.plugin_dirs.split(";")] + constants.PLUGINS_PATH @@ -126,23 +159,13 @@ class VolShell(cli.CommandLine): vollog.info(f"Volatility plugins path: {volatility3.plugins.__path__}") vollog.info(f"Volatility symbols path: {volatility3.symbols.__path__}") - if partial_args.log: - file_logger = logging.FileHandler(partial_args.log) - file_logger.setLevel(0) - file_formatter = logging.Formatter(datefmt = '%y-%m-%d %H:%M:%S', - fmt = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s') - file_logger.setFormatter(file_formatter) - vollog.addHandler(file_logger) - vollog.info("Logging started") - - if partial_args.verbosity < 3: - console.setLevel(30 - (partial_args.verbosity * 10)) - else: - console.setLevel(10 - (partial_args.verbosity - 2)) if partial_args.clear_cache: framework.clear_cache() + if partial_args.offline: + constants.OFFLINE = partial_args.offline + # Do the initialization ctx = contexts.Context() # Construct a blank context failures = framework.import_files(volatility3.plugins, From 9ea1a960cdece1c6897735adc58f592f73c44e86 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 30 May 2022 01:34:19 +0100 Subject: [PATCH 2/6] CLI: Synchonize volshell code a little better courtesy of @digitalisx --- volatility3/cli/volshell/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 647be8a1a..e2e7bc394 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -15,12 +15,14 @@ from volatility3.cli.volshell import generic, linux, mac, windows from volatility3.framework import automagic, constants, contexts, exceptions, interfaces, plugins # Make sure we log everything + +rootlog = logging.getLogger() vollog = logging.getLogger() vollog.setLevel(0) -# Trim the console down by default console = logging.StreamHandler() console.setLevel(logging.WARNING) formatter = logging.Formatter('%(levelname)-8s %(name)-12s: %(message)s') +# Trim the console down by default console.setFormatter(formatter) vollog.addHandler(console) From 2e4c257e15e83b33fb40cd9369c90dac2c1786e9 Mon Sep 17 00:00:00 2001 From: ikelos Date: Mon, 30 May 2022 01:53:07 +0100 Subject: [PATCH 3/6] Update volatility3/cli/__init__.py Co-authored-by: Donghyun Kim --- volatility3/cli/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 62be115d3..6881311bf 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -388,7 +388,8 @@ class CommandLine: # Process it if the files exist if os.path.exists(default_config_path): - result = json.load(open(default_config_path, 'rb')) + with open(default_config_path, 'rb') as config_json: + result = json.load(config_json) if not isinstance(result, dict): delayed_logs.append((logging.INFO, f'Default configuration file {default_config_path} does not contain a dictionary')) From 9127509db9a8bcfffc971e7b959d429ffcecea6e Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 8 Feb 2024 11:13:14 +0000 Subject: [PATCH 4/6] CLI: Reblack after updating PR --- volatility3/cli/__init__.py | 45 ++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 0e7ba41ee..9393c0805 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -106,7 +106,7 @@ class CommandLine: ) # Load up system defaults - delayed_logs, default_config = self.load_system_defaults('vol.json') + delayed_logs, default_config = self.load_system_defaults("vol.json") parser = volargparse.HelpfulArgParser( add_help=False, @@ -280,7 +280,7 @@ class CommandLine: if partial_args.cache_path: constants.CACHE_PATH = partial_args.cache_path - + vollog.info(f"Volatility plugins path: {volatility3.plugins.__path__}") vollog.info(f"Volatility symbols path: {volatility3.symbols.__path__}") @@ -473,28 +473,47 @@ class CommandLine: ) return requirements.URIRequirement.location_from_file(filename) - def load_system_defaults(self, filename: str) -> Tuple[List[Tuple[int, str]], Dict[str, Any]]: + def load_system_defaults( + self, filename: str + ) -> Tuple[List[Tuple[int, str]], Dict[str, Any]]: """Modify the main configuration based on the default configuration override""" # Build the config path - default_config_path = os.path.join(os.path.expanduser("~"), ".config", "volatility3", filename) - if sys.platform == 'win32': - default_config_path = os.path.join(os.environ.get("APPDATA", os.path.expanduser("~")), "volatility3", - filename) + default_config_path = os.path.join( + os.path.expanduser("~"), ".config", "volatility3", filename + ) + if sys.platform == "win32": + default_config_path = os.path.join( + os.environ.get("APPDATA", os.path.expanduser("~")), + "volatility3", + filename, + ) delayed_logs = [] # Process it if the files exist if os.path.exists(default_config_path): - with open(default_config_path, 'rb') as config_json: + with open(default_config_path, "rb") as config_json: result = json.load(config_json) if not isinstance(result, dict): - delayed_logs.append((logging.INFO, - f'Default configuration file {default_config_path} does not contain a dictionary')) + delayed_logs.append( + ( + logging.INFO, + f"Default configuration file {default_config_path} does not contain a dictionary", + ) + ) else: delayed_logs.append( - (logging.INFO, f"Loading default configuration options from {default_config_path}")) - delayed_logs.append((logging.DEBUG, - f"Loaded configuration: {json.dumps(result, indent = 2, sort_keys = True)}")) + ( + logging.INFO, + f"Loading default configuration options from {default_config_path}", + ) + ) + delayed_logs.append( + ( + logging.DEBUG, + f"Loaded configuration: {json.dumps(result, indent = 2, sort_keys = True)}", + ) + ) return delayed_logs, result return delayed_logs, {} From a66a4461906e7661b957559634559a56ee6356f6 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 8 Feb 2024 11:16:35 +0000 Subject: [PATCH 5/6] CLI: Reblack after updating PR - take 2 --- volatility3/cli/volshell/__init__.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 40f42f3af..9e74acfec 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -56,7 +56,7 @@ class VolShell(cli.CommandLine): framework.require_interface_version(2, 0, 0) # Load up system defaults - delayed_logs, default_config = self.load_system_defaults('volshell.json') + delayed_logs, default_config = self.load_system_defaults("volshell.json") parser = argparse.ArgumentParser( prog=self.CLI_NAME, @@ -151,10 +151,12 @@ class VolShell(cli.CommandLine): default=constants.CACHE_PATH, type=str, ) - parser.add_argument("--offline", - help = "Do not search online for additional JSON files", - default = False, - action = 'store_true') + parser.add_argument( + "--offline", + help="Do not search online for additional JSON files", + default=False, + action="store_true", + ) # Volshell specific flags os_specific = parser.add_mutually_exclusive_group(required=False) From 7b42ef0a9f23ecb6b4fff2425c0ac33c85b30d04 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 8 Feb 2024 11:59:50 +0000 Subject: [PATCH 6/6] Documentation: Add in more information about overriding configuration values --- doc/source/vol-cli.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/doc/source/vol-cli.rst b/doc/source/vol-cli.rst index cc6f7fe6a..7b91e815d 100644 --- a/doc/source/vol-cli.rst +++ b/doc/source/vol-cli.rst @@ -143,3 +143,18 @@ Options `hivescan` would match `windows.registry.hivescan.HiveScan`, but `pslist` is ambiguous because it could match `windows.pslist` or `linux.pslist`. + +Overriding options +------------------ + +The default values for the command line interface are defined by constants within the code, +but can be overridden by creating a JSON file (`%APPDATA%/volatility3/vol.json` for Windows +systems, or `~/.config/volatility3/vol.json` or `volshell.json` for all others). + +The format of this file is a JSON dictionary, containing the options above and their value. +It should be noted that the ordering is (`<` means is overridden by): + +`in-built default value < config file value < command line parameter` + +It should also be noted that boolean flags (such as `offline`) that are overridden as true will +not be unset by not specifying the command line flag.