From 50767ffb8d809b6b2c7fab1d599c95965422a5d7 Mon Sep 17 00:00:00 2001 From: kyrre Date: Fri, 27 Jun 2025 16:33:28 +0200 Subject: [PATCH 01/22] add arrow and parquet renderers --- pyproject.toml | 2 + volatility3/cli/text_renderer.py | 141 ++++++++++++++++++++++++++++++- 2 files changed, 142 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b88ac7752..0acbfcbbe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,8 @@ dev = [ "types-jsonschema>=4.23.0,<5", ] +arrow = ["pyarrow>=17.0.0"] + test = [ "volatility3[dev]", "pytest>=8.3.3,<9", diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index 1e437a0be..69d2d6d31 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -9,7 +9,7 @@ import random import string import sys from functools import wraps -from typing import Any, Callable, Dict, List, Optional, Set, Tuple, TypeVar, Union +from typing import Any, Callable, Dict, List, Optional, Set, Tuple, TypeVar, Union, TextIO from volatility3.cli import text_filter from volatility3.framework import exceptions, interfaces, renderers @@ -25,6 +25,15 @@ except ImportError: vollog.debug("Disassembly library capstone not found") +try: + ARROW_PRESENT = True + import pyarrow as pa + import pyarrow.parquet as pq +except ImportError: + ARROW_PRESENT = False + vollog.debug("Arrow/Parquet libraries not found") + + def hex_bytes_as_text(value: bytes, width: int = 16) -> str: """Renders HexBytes as text. @@ -624,3 +633,133 @@ class JsonLinesRenderer(JsonRenderer): for line in result: outfd.write(json.dumps(line, sort_keys=True)) outfd.write("\n") + +class ArrowRenderer(CLIRenderer): + def __init__( + self, options: List[interfaces.renderers.RenderOption] | None = None + ) -> None: + super().__init__(options) + + if not ARROW_PRESENT: + raise RuntimeError("Arrow output format requires the pyarrow package") + + _to_arrow_type = { + renderers.Disassembly: pa.utf8, + bool: pa.bool_, + int: pa.int64, + float: pa.float64, + str: pa.utf8, + datetime.datetime: lambda: pa.timestamp("ms"), + format_hints.Bin: pa.int64, + format_hints.Hex: pa.int64, + format_hints.MultiTypeData: pa.utf8, + format_hints.HexBytes: pa.binary, + } + + name = "arrow" + structured_output = True + + def get_render_options(self) -> List[interfaces.renderers.RenderOption]: + return [] + + def to_arrow_schema(self, grid: interfaces.renderers.TreeGrid) -> pa.Schema: + fields = [] + for column in grid.columns: + arrow_type = self._to_arrow_type[column.type] + fields.append(pa.field(column.name, arrow_type())) + return pa.schema(fields) + + def output_result(self, schema, outfd: TextIO, result): + """Outputs the JSON data to a file in a particular format""" + + t = pa.Table.from_pylist(result, schema=schema) + self.write_data(t, outfd) + + def write_data(self, t: pa.Table, outfd: TextIO) -> None: + buf = pa.BufferOutputStream() + + writer = pa.ipc.new_stream(buf, t.schema) + writer.write_table(t) + writer.close() + + # Get the buffer bytes and write to output + buf_bytes = buf.getvalue().to_pybytes() + outfd.buffer.write(buf_bytes) + + def render(self, grid: interfaces.renderers.TreeGrid): + outfd = sys.stdout + final_output: Tuple[ + Dict[str, List[interfaces.renderers.TreeNode]], + List[interfaces.renderers.TreeNode], + ] = ({}, []) + + ignore_columns = self.ignored_columns(grid) + + def visitor( + node: interfaces.renderers.TreeNode, + accumulator: Tuple[Dict[str, Dict[str, Any]], List[Dict[str, Any]]], + ) -> Tuple[Dict[str, Dict[str, Any]], List[Dict[str, Any]]]: + # Nodes always have a path value, giving them a path_depth of at least 1, we use max just in case + acc_map, final_tree = accumulator + node_dict: Dict[str, Any] = {"__children": []} + line = [] + for column_index, column in enumerate(grid.columns): + if column in ignore_columns: + continue + + data = list(node.values)[column_index] + + if isinstance(data, interfaces.renderers.BaseAbsentValue): + data = None + + if isinstance(data, renderers.Disassembly): + data = display_disassembly(data) + + node_dict[column.name] = data + line.append(data) + + if self.filter and self.filter.filter(line): + return accumulator + + if node.parent: + acc_map[node.parent.path]["__children"].append(node_dict) + else: + final_tree.append(node_dict) + acc_map[node.path] = node_dict + + return (acc_map, final_tree) + + if not grid.populated: + grid.populate(visitor, final_output) + else: + grid.visit(node=None, function=visitor, initial_accumulator=final_output) + + schema = self.to_arrow_schema(grid) + self.output_result(schema, outfd, final_output[1]) + + +class ParquetRenderer(ArrowRenderer): + name = "parquet" + structured_output = True + + def get_render_options(self) -> List[interfaces.renderers.RenderOption]: + return [] + + def write_data(self, t: pa.Table, outfd: TextIO) -> None: + """ + Writes a table to stdout using the Parquet format. + + Args: + t: The Arrow table to write + outfd: The output file descriptor + + Returns: + Nothing + """ + # Write DataFrame to a temporary file-like object + buf = pa.BufferOutputStream() + pq.write_table(t, buf, compression="snappy") + + # Get the buffer as a bytes object + buf_bytes = buf.getvalue().to_pybytes() + outfd.buffer.write(buf_bytes) \ No newline at end of file From a0b00766f8df6cd4351e850ee9424a1c18786aa9 Mon Sep 17 00:00:00 2001 From: kyrre Date: Fri, 27 Jun 2025 16:58:35 +0200 Subject: [PATCH 02/22] formatting with ruff --- volatility3/cli/text_renderer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index 69d2d6d31..13276520d 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -748,11 +748,11 @@ class ParquetRenderer(ArrowRenderer): def write_data(self, t: pa.Table, outfd: TextIO) -> None: """ Writes a table to stdout using the Parquet format. - + Args: t: The Arrow table to write outfd: The output file descriptor - + Returns: Nothing """ @@ -762,4 +762,4 @@ class ParquetRenderer(ArrowRenderer): # Get the buffer as a bytes object buf_bytes = buf.getvalue().to_pybytes() - outfd.buffer.write(buf_bytes) \ No newline at end of file + outfd.buffer.write(buf_bytes) From 1c7856212f6c05feec3aa4d5527b7b00f4bbc456 Mon Sep 17 00:00:00 2001 From: kyrre Date: Fri, 27 Jun 2025 17:01:08 +0200 Subject: [PATCH 03/22] linting --- volatility3/cli/text_renderer.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index 13276520d..1bdca3803 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -9,7 +9,18 @@ import random import string import sys from functools import wraps -from typing import Any, Callable, Dict, List, Optional, Set, Tuple, TypeVar, Union, TextIO +from typing import ( + Any, + Callable, + Dict, + List, + Optional, + Set, + Tuple, + TypeVar, + Union, + TextIO, +) from volatility3.cli import text_filter from volatility3.framework import exceptions, interfaces, renderers @@ -634,6 +645,7 @@ class JsonLinesRenderer(JsonRenderer): outfd.write(json.dumps(line, sort_keys=True)) outfd.write("\n") + class ArrowRenderer(CLIRenderer): def __init__( self, options: List[interfaces.renderers.RenderOption] | None = None From 03895969875f682031466cb84793b6dc9d74f617 Mon Sep 17 00:00:00 2001 From: Kyrre-Wahl-Kongsgard Date: Sat, 28 Jun 2025 11:46:13 +0200 Subject: [PATCH 04/22] Fix: Avoid import-time errors if pyarrow is missing - Use string type annotations for pa.* types in Arrow/Parquet renderers. - Ensure pyarrow dependency is checked only in __init__. --- volatility3/cli/text_renderer.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index 1bdca3803..d2334a9f9 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -655,18 +655,18 @@ class ArrowRenderer(CLIRenderer): if not ARROW_PRESENT: raise RuntimeError("Arrow output format requires the pyarrow package") - _to_arrow_type = { - renderers.Disassembly: pa.utf8, - bool: pa.bool_, - int: pa.int64, - float: pa.float64, - str: pa.utf8, - datetime.datetime: lambda: pa.timestamp("ms"), - format_hints.Bin: pa.int64, - format_hints.Hex: pa.int64, - format_hints.MultiTypeData: pa.utf8, - format_hints.HexBytes: pa.binary, - } + self._to_arrow_type = { + renderers.Disassembly: pa.utf8, + bool: pa.bool_, + int: pa.int64, + float: pa.float64, + str: pa.utf8, + datetime.datetime: lambda: pa.timestamp("ms"), + format_hints.Bin: pa.int64, + format_hints.Hex: pa.int64, + format_hints.MultiTypeData: pa.utf8, + format_hints.HexBytes: pa.binary, + } name = "arrow" structured_output = True @@ -674,20 +674,20 @@ class ArrowRenderer(CLIRenderer): def get_render_options(self) -> List[interfaces.renderers.RenderOption]: return [] - def to_arrow_schema(self, grid: interfaces.renderers.TreeGrid) -> pa.Schema: + def to_arrow_schema(self, grid: interfaces.renderers.TreeGrid) -> "pa.Schema": fields = [] for column in grid.columns: arrow_type = self._to_arrow_type[column.type] fields.append(pa.field(column.name, arrow_type())) return pa.schema(fields) - def output_result(self, schema, outfd: TextIO, result): + def output_result(self, schema: "pa.Schema", outfd: TextIO, result): """Outputs the JSON data to a file in a particular format""" t = pa.Table.from_pylist(result, schema=schema) self.write_data(t, outfd) - def write_data(self, t: pa.Table, outfd: TextIO) -> None: + def write_data(self, t: "pa.Table", outfd: TextIO) -> None: buf = pa.BufferOutputStream() writer = pa.ipc.new_stream(buf, t.schema) @@ -757,7 +757,7 @@ class ParquetRenderer(ArrowRenderer): def get_render_options(self) -> List[interfaces.renderers.RenderOption]: return [] - def write_data(self, t: pa.Table, outfd: TextIO) -> None: + def write_data(self, t: "pa.Table", outfd: TextIO) -> None: """ Writes a table to stdout using the Parquet format. From 78d56534c0b9b17fff2e9e62c06f53fb1edc7015 Mon Sep 17 00:00:00 2001 From: Kyrre-Wahl-Kongsgard Date: Sat, 28 Jun 2025 12:16:36 +0200 Subject: [PATCH 05/22] fix: handle layerdata to pyarrow conversion --- volatility3/cli/text_renderer.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index d2334a9f9..52a499ffe 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -666,6 +666,8 @@ class ArrowRenderer(CLIRenderer): format_hints.Hex: pa.int64, format_hints.MultiTypeData: pa.utf8, format_hints.HexBytes: pa.binary, + renderers.LayerData: pa.binary, + bytes: pa.binary, } name = "arrow" @@ -727,6 +729,9 @@ class ArrowRenderer(CLIRenderer): if isinstance(data, renderers.Disassembly): data = display_disassembly(data) + if isinstance(data, renderers.LayerData): + data = LayerDataRenderer().render_bytes(data)[0] + node_dict[column.name] = data line.append(data) From d3a6b030b7604f6289f59dc7fa3e2882eb69225a Mon Sep 17 00:00:00 2001 From: Kyrre-Wahl-Kongsgard Date: Mon, 14 Jul 2025 13:38:35 +0200 Subject: [PATCH 06/22] moved arrow/parquet parsers to it's own file and reverted changes --- volatility3/cli/text_renderer.py | 166 +----------------- .../framework/plugins/parquet_renderer.py | 161 +++++++++++++++++ 2 files changed, 165 insertions(+), 162 deletions(-) create mode 100644 volatility3/framework/plugins/parquet_renderer.py diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index 52a499ffe..89526c90c 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -9,18 +9,7 @@ import random import string import sys from functools import wraps -from typing import ( - Any, - Callable, - Dict, - List, - Optional, - Set, - Tuple, - TypeVar, - Union, - TextIO, -) +from typing import Any, Callable, Dict, List, Optional, Set, Tuple, TypeVar, Union from volatility3.cli import text_filter from volatility3.framework import exceptions, interfaces, renderers @@ -36,15 +25,6 @@ except ImportError: vollog.debug("Disassembly library capstone not found") -try: - ARROW_PRESENT = True - import pyarrow as pa - import pyarrow.parquet as pq -except ImportError: - ARROW_PRESENT = False - vollog.debug("Arrow/Parquet libraries not found") - - def hex_bytes_as_text(value: bytes, width: int = 16) -> str: """Renders HexBytes as text. @@ -298,7 +278,6 @@ class CLIRenderer(interfaces.renderers.Renderer): class QuickTextRenderer(CLIRenderer): - name = "quick" def get_render_options(self): @@ -368,7 +347,6 @@ class NoneRenderer(CLIRenderer): class CSVRenderer(CLIRenderer): - name = "csv" structured_output = True @@ -486,9 +464,9 @@ class PrettyTextRenderer(CLIRenderer): accumulator.append((node.path_depth, line)) return accumulator - final_output: List[Tuple[int, Dict[interfaces.renderers.Column, list[str]]]] = ( - [] - ) + final_output: List[ + Tuple[int, Dict[interfaces.renderers.Column, list[str]]] + ] = [] if not grid.populated: grid.populate(visitor, final_output) else: @@ -644,139 +622,3 @@ class JsonLinesRenderer(JsonRenderer): for line in result: outfd.write(json.dumps(line, sort_keys=True)) outfd.write("\n") - - -class ArrowRenderer(CLIRenderer): - def __init__( - self, options: List[interfaces.renderers.RenderOption] | None = None - ) -> None: - super().__init__(options) - - if not ARROW_PRESENT: - raise RuntimeError("Arrow output format requires the pyarrow package") - - self._to_arrow_type = { - renderers.Disassembly: pa.utf8, - bool: pa.bool_, - int: pa.int64, - float: pa.float64, - str: pa.utf8, - datetime.datetime: lambda: pa.timestamp("ms"), - format_hints.Bin: pa.int64, - format_hints.Hex: pa.int64, - format_hints.MultiTypeData: pa.utf8, - format_hints.HexBytes: pa.binary, - renderers.LayerData: pa.binary, - bytes: pa.binary, - } - - name = "arrow" - structured_output = True - - def get_render_options(self) -> List[interfaces.renderers.RenderOption]: - return [] - - def to_arrow_schema(self, grid: interfaces.renderers.TreeGrid) -> "pa.Schema": - fields = [] - for column in grid.columns: - arrow_type = self._to_arrow_type[column.type] - fields.append(pa.field(column.name, arrow_type())) - return pa.schema(fields) - - def output_result(self, schema: "pa.Schema", outfd: TextIO, result): - """Outputs the JSON data to a file in a particular format""" - - t = pa.Table.from_pylist(result, schema=schema) - self.write_data(t, outfd) - - def write_data(self, t: "pa.Table", outfd: TextIO) -> None: - buf = pa.BufferOutputStream() - - writer = pa.ipc.new_stream(buf, t.schema) - writer.write_table(t) - writer.close() - - # Get the buffer bytes and write to output - buf_bytes = buf.getvalue().to_pybytes() - outfd.buffer.write(buf_bytes) - - def render(self, grid: interfaces.renderers.TreeGrid): - outfd = sys.stdout - final_output: Tuple[ - Dict[str, List[interfaces.renderers.TreeNode]], - List[interfaces.renderers.TreeNode], - ] = ({}, []) - - ignore_columns = self.ignored_columns(grid) - - def visitor( - node: interfaces.renderers.TreeNode, - accumulator: Tuple[Dict[str, Dict[str, Any]], List[Dict[str, Any]]], - ) -> Tuple[Dict[str, Dict[str, Any]], List[Dict[str, Any]]]: - # Nodes always have a path value, giving them a path_depth of at least 1, we use max just in case - acc_map, final_tree = accumulator - node_dict: Dict[str, Any] = {"__children": []} - line = [] - for column_index, column in enumerate(grid.columns): - if column in ignore_columns: - continue - - data = list(node.values)[column_index] - - if isinstance(data, interfaces.renderers.BaseAbsentValue): - data = None - - if isinstance(data, renderers.Disassembly): - data = display_disassembly(data) - - if isinstance(data, renderers.LayerData): - data = LayerDataRenderer().render_bytes(data)[0] - - node_dict[column.name] = data - line.append(data) - - if self.filter and self.filter.filter(line): - return accumulator - - if node.parent: - acc_map[node.parent.path]["__children"].append(node_dict) - else: - final_tree.append(node_dict) - acc_map[node.path] = node_dict - - return (acc_map, final_tree) - - if not grid.populated: - grid.populate(visitor, final_output) - else: - grid.visit(node=None, function=visitor, initial_accumulator=final_output) - - schema = self.to_arrow_schema(grid) - self.output_result(schema, outfd, final_output[1]) - - -class ParquetRenderer(ArrowRenderer): - name = "parquet" - structured_output = True - - def get_render_options(self) -> List[interfaces.renderers.RenderOption]: - return [] - - def write_data(self, t: "pa.Table", outfd: TextIO) -> None: - """ - Writes a table to stdout using the Parquet format. - - Args: - t: The Arrow table to write - outfd: The output file descriptor - - Returns: - Nothing - """ - # Write DataFrame to a temporary file-like object - buf = pa.BufferOutputStream() - pq.write_table(t, buf, compression="snappy") - - # Get the buffer as a bytes object - buf_bytes = buf.getvalue().to_pybytes() - outfd.buffer.write(buf_bytes) diff --git a/volatility3/framework/plugins/parquet_renderer.py b/volatility3/framework/plugins/parquet_renderer.py new file mode 100644 index 000000000..806dcd801 --- /dev/null +++ b/volatility3/framework/plugins/parquet_renderer.py @@ -0,0 +1,161 @@ +# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# +import datetime +import logging +import sys +from typing import ( + Any, + Dict, + List, + Tuple, + TextIO, +) +from volatility3.framework import interfaces, renderers +from volatility3.framework.renderers import format_hints +from volatility3.cli.text_renderer import CLIRenderer, display_disassembly, LayerDataRenderer + +vollog = logging.getLogger(__name__) + +try: + ARROW_PRESENT = True + import pyarrow as pa + import pyarrow.parquet as pq +except ImportError: + ARROW_PRESENT = False + vollog.debug("Arrow/Parquet libraries not found") + +class ArrowRenderer(CLIRenderer): + def __init__( + self, options: List[interfaces.renderers.RenderOption] | None = None + ) -> None: + super().__init__(options) + + if not ARROW_PRESENT: + raise RuntimeError("Arrow output format requires the pyarrow package") + + self._to_arrow_type = { + renderers.Disassembly: pa.utf8, + bool: pa.bool_, + int: pa.int64, + float: pa.float64, + str: pa.utf8, + datetime.datetime: lambda: pa.timestamp("ms"), + format_hints.Bin: pa.int64, + format_hints.Hex: pa.int64, + format_hints.MultiTypeData: pa.utf8, + format_hints.HexBytes: pa.binary, + renderers.LayerData: pa.binary, + bytes: pa.binary, + } + + name = "arrow" + structured_output = True + + def get_render_options(self) -> List[interfaces.renderers.RenderOption]: + return [] + + def to_arrow_schema(self, grid: interfaces.renderers.TreeGrid) -> "pa.Schema": + fields = [] + for column in grid.columns: + arrow_type = self._to_arrow_type[column.type] + fields.append(pa.field(column.name, arrow_type())) + return pa.schema(fields) + + def output_result(self, schema: "pa.Schema", outfd: TextIO, result): + """Outputs the JSON data to a file in a particular format""" + + t = pa.Table.from_pylist(result, schema=schema) + self.write_data(t, outfd) + + def write_data(self, t: "pa.Table", outfd: TextIO) -> None: + buf = pa.BufferOutputStream() + + writer = pa.ipc.new_stream(buf, t.schema) + writer.write_table(t) + writer.close() + + # Get the buffer bytes and write to output + buf_bytes = buf.getvalue().to_pybytes() + outfd.buffer.write(buf_bytes) + + def render(self, grid: interfaces.renderers.TreeGrid): + outfd = sys.stdout + final_output: Tuple[ + Dict[str, List[interfaces.renderers.TreeNode]], + List[interfaces.renderers.TreeNode], + ] = ({}, []) + + ignore_columns = self.ignored_columns(grid) + + def visitor( + node: interfaces.renderers.TreeNode, + accumulator: Tuple[Dict[str, Dict[str, Any]], List[Dict[str, Any]]], + ) -> Tuple[Dict[str, Dict[str, Any]], List[Dict[str, Any]]]: + # Nodes always have a path value, giving them a path_depth of at least 1, we use max just in case + acc_map, final_tree = accumulator + node_dict: Dict[str, Any] = {"__children": []} + line = [] + for column_index, column in enumerate(grid.columns): + if column in ignore_columns: + continue + + data = list(node.values)[column_index] + + if isinstance(data, interfaces.renderers.BaseAbsentValue): + data = None + + if isinstance(data, renderers.Disassembly): + data = display_disassembly(data) + + if isinstance(data, renderers.LayerData): + data = LayerDataRenderer().render_bytes(data)[0] + + node_dict[column.name] = data + line.append(data) + + if self.filter and self.filter.filter(line): + return accumulator + + if node.parent: + acc_map[node.parent.path]["__children"].append(node_dict) + else: + final_tree.append(node_dict) + acc_map[node.path] = node_dict + + return (acc_map, final_tree) + + if not grid.populated: + grid.populate(visitor, final_output) + else: + grid.visit(node=None, function=visitor, initial_accumulator=final_output) + + schema = self.to_arrow_schema(grid) + self.output_result(schema, outfd, final_output[1]) + + +class ParquetRenderer(ArrowRenderer): + name = "parquet" + structured_output = True + + def get_render_options(self) -> List[interfaces.renderers.RenderOption]: + return [] + + def write_table(self, table: "pa.Table", outfd: TextIO) -> None: + """ + Writes a table to stdout using the Parquet format. + + Args: + t: The Arrow table to write + outfd: The output file descriptor + + Returns: + Nothing + """ + # Write DataFrame to a temporary file-like object + buf = pa.BufferOutputStream() + pq.write_table(table, buf, compression="snappy") + + # Get the buffer as a bytes object + buf_bytes = buf.getvalue().to_pybytes() + outfd.buffer.write(buf_bytes) From 8479fa1c136c3f71c7fb8602184219326fca1fae Mon Sep 17 00:00:00 2001 From: Kyrre-Wahl-Kongsgard Date: Mon, 14 Jul 2025 14:07:44 +0200 Subject: [PATCH 07/22] moved renderer parsing to after the plugins are imported --- volatility3/cli/__init__.py | 45 ++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index cc97c4fcb..4c379a15e 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -106,13 +106,6 @@ class CommandLine: volatility3.framework.require_interface_version(2, 0, 0) - renderers = dict( - [ - (x.name.lower(), x) - for x in framework.class_subclasses(text_renderer.CLIRenderer) - ] - ) - # Load up system defaults delayed_logs, default_config = self.load_system_defaults("vol.json") @@ -193,14 +186,6 @@ class CommandLine: default=False, action="store_true", ) - parser.add_argument( - "-r", - "--renderer", - metavar="RENDERER", - help=f"Determines how to render the output ({', '.join(list(renderers))})", - default="quick", - choices=list(renderers), - ) parser.add_argument( "-f", "--file", @@ -270,11 +255,6 @@ class CommandLine: known_args = [arg for arg in sys.argv if arg != "--help" and arg != "-h"] partial_args, _ = parser.parse_known_args(known_args) - banner_output = sys.stdout - if renderers[partial_args.renderer].structured_output: - banner_output = sys.stderr - banner_output.write(f"Volatility 3 Framework {constants.PACKAGE_VERSION}\n") - ### Start up logging if partial_args.log: file_logger = logging.FileHandler(partial_args.log) @@ -346,6 +326,24 @@ class CommandLine: plugin_list = framework.list_plugins() + # Discover renderers after plugin directories are loaded + # This allows custom renderers to be found in plugin directories + renderers = dict( + [ + (x.name.lower(), x) + for x in framework.class_subclasses(text_renderer.CLIRenderer) + ] + ) + + parser.add_argument( + "-r", + "--renderer", + metavar="RENDERER", + help=f"Determines how to render the output ({', '.join(list(renderers))})", + default="quick", + choices=list(renderers), + ) + seen_automagics = set() chosen_configurables_list = {} for amagic in automagics: @@ -392,6 +390,13 @@ class CommandLine: # before all the plugins have been added argcomplete.autocomplete(parser) args = parser.parse_args() + + # Display banner - redirect to stderr if using structured output + banner_output = sys.stdout + if renderers[args.renderer].structured_output: + banner_output = sys.stderr + banner_output.write(f"Volatility 3 Framework {constants.PACKAGE_VERSION}\n") + if args.plugin is None: parser.error( f"Please select a plugin to run (see '{self.CLI_NAME} --help' for options" From 066076b4d83ff47b766f250bef379076ab8b8369 Mon Sep 17 00:00:00 2001 From: Kyrre-Wahl-Kongsgard Date: Mon, 14 Jul 2025 14:10:31 +0200 Subject: [PATCH 08/22] black formatting --- volatility3/cli/text_renderer.py | 6 +++--- volatility3/framework/plugins/parquet_renderer.py | 7 ++++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index 89526c90c..d55201371 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -464,9 +464,9 @@ class PrettyTextRenderer(CLIRenderer): accumulator.append((node.path_depth, line)) return accumulator - final_output: List[ - Tuple[int, Dict[interfaces.renderers.Column, list[str]]] - ] = [] + final_output: List[Tuple[int, Dict[interfaces.renderers.Column, list[str]]]] = ( + [] + ) if not grid.populated: grid.populate(visitor, final_output) else: diff --git a/volatility3/framework/plugins/parquet_renderer.py b/volatility3/framework/plugins/parquet_renderer.py index 806dcd801..adc3a97e1 100644 --- a/volatility3/framework/plugins/parquet_renderer.py +++ b/volatility3/framework/plugins/parquet_renderer.py @@ -13,7 +13,11 @@ from typing import ( ) from volatility3.framework import interfaces, renderers from volatility3.framework.renderers import format_hints -from volatility3.cli.text_renderer import CLIRenderer, display_disassembly, LayerDataRenderer +from volatility3.cli.text_renderer import ( + CLIRenderer, + display_disassembly, + LayerDataRenderer, +) vollog = logging.getLogger(__name__) @@ -25,6 +29,7 @@ except ImportError: ARROW_PRESENT = False vollog.debug("Arrow/Parquet libraries not found") + class ArrowRenderer(CLIRenderer): def __init__( self, options: List[interfaces.renderers.RenderOption] | None = None From 0f32bec2638e69ac543f27f0ce2b0993cc9d75c6 Mon Sep 17 00:00:00 2001 From: Kyrre-Wahl-Kongsgard Date: Mon, 14 Jul 2025 14:20:37 +0200 Subject: [PATCH 09/22] restore orginal text_renderer --- volatility3/cli/text_renderer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index d55201371..1e437a0be 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -278,6 +278,7 @@ class CLIRenderer(interfaces.renderers.Renderer): class QuickTextRenderer(CLIRenderer): + name = "quick" def get_render_options(self): @@ -347,6 +348,7 @@ class NoneRenderer(CLIRenderer): class CSVRenderer(CLIRenderer): + name = "csv" structured_output = True From 72422d0db9ab255a03d50f3830681011c8b3191c Mon Sep 17 00:00:00 2001 From: Kyrre-Wahl-Kongsgard Date: Mon, 21 Jul 2025 13:38:54 +0200 Subject: [PATCH 10/22] fix bug: hadn't properly renamed the write_data method name --- volatility3/framework/plugins/parquet_renderer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/parquet_renderer.py b/volatility3/framework/plugins/parquet_renderer.py index adc3a97e1..6564dbaeb 100644 --- a/volatility3/framework/plugins/parquet_renderer.py +++ b/volatility3/framework/plugins/parquet_renderer.py @@ -71,9 +71,9 @@ class ArrowRenderer(CLIRenderer): """Outputs the JSON data to a file in a particular format""" t = pa.Table.from_pylist(result, schema=schema) - self.write_data(t, outfd) + self.write_table(t, outfd) - def write_data(self, t: "pa.Table", outfd: TextIO) -> None: + def write_table(self, t: "pa.Table", outfd: TextIO) -> None: buf = pa.BufferOutputStream() writer = pa.ipc.new_stream(buf, t.schema) From e7c1126b05996ecb3fd718ba559422d68c852023 Mon Sep 17 00:00:00 2001 From: Kyrre-Wahl-Kongsgard Date: Mon, 21 Jul 2025 23:46:14 +0200 Subject: [PATCH 11/22] moved the arrow/parquet renderers to a subdir for renderers --- .../plugins/renderers/parquet_renderer.py | 173 ++++++++++++++++++ 1 file changed, 173 insertions(+) create mode 100644 volatility3/framework/plugins/renderers/parquet_renderer.py diff --git a/volatility3/framework/plugins/renderers/parquet_renderer.py b/volatility3/framework/plugins/renderers/parquet_renderer.py new file mode 100644 index 000000000..c9ef95ee7 --- /dev/null +++ b/volatility3/framework/plugins/renderers/parquet_renderer.py @@ -0,0 +1,173 @@ +# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# +import datetime +import logging +import sys +from typing import ( + Any, + Dict, + List, + Tuple, + TextIO, +) +from volatility3.framework import interfaces, renderers +from volatility3.framework.renderers import format_hints +from volatility3.cli.text_renderer import ( + CLIRenderer, + display_disassembly, + LayerDataRenderer, +) + +vollog = logging.getLogger(__name__) + +ARROW_PRESENT = False +try: + import pyarrow as pa + import pyarrow.parquet as pq + + ARROW_PRESENT = True +except ImportError: + vollog.debug("Arrow/Parquet libraries not found") + + +class ArrowRenderer(CLIRenderer): + """Renderer that outputs Arrow IPC format data.""" + + name = "arrow" + structured_output = True + _version = (1, 0, 0) + + def __init__( + self, options: List[interfaces.renderers.RenderOption] | None = None + ) -> None: + super().__init__(options) + + if not ARROW_PRESENT: + raise RuntimeError("Arrow output format requires the pyarrow package") + + self._to_arrow_type = { + renderers.Disassembly: pa.utf8, + bool: pa.bool_, + int: pa.int64, + float: pa.float64, + str: pa.utf8, + datetime.datetime: lambda: pa.timestamp("ms"), + format_hints.Bin: pa.int64, + format_hints.Hex: pa.int64, + format_hints.MultiTypeData: pa.utf8, + format_hints.HexBytes: pa.binary, + renderers.LayerData: pa.binary, + bytes: pa.binary, + } + + def get_render_options(self) -> List[interfaces.renderers.RenderOption]: + return [] + + def to_arrow_schema(self, grid: interfaces.renderers.TreeGrid) -> "pa.Schema": + fields = [] + for column in grid.columns: + arrow_type = self._to_arrow_type[column.type] + fields.append(pa.field(column.name, arrow_type())) + return pa.schema(fields) + + def output_result(self, schema: "pa.Schema", outfd: TextIO, result): + """Outputs the JSON data to a file in a particular format""" + + t = pa.Table.from_pylist(result, schema=schema) + self.write_table(t, outfd) + + def write_table(self, t: "pa.Table", outfd: TextIO) -> None: + buf = pa.BufferOutputStream() + + writer = pa.ipc.new_stream(buf, t.schema) + writer.write_table(t) + writer.close() + + # Get the buffer bytes and write to output + buf_bytes = buf.getvalue().to_pybytes() + outfd.buffer.write(buf_bytes) + + def render(self, grid: interfaces.renderers.TreeGrid): + outfd = sys.stdout + final_output: Tuple[ + Dict[str, List[interfaces.renderers.TreeNode]], + List[interfaces.renderers.TreeNode], + ] = ({}, []) + + ignore_columns = self.ignored_columns(grid) + + def visitor( + node: interfaces.renderers.TreeNode, + accumulator: Tuple[Dict[str, Dict[str, Any]], List[Dict[str, Any]]], + ) -> Tuple[Dict[str, Dict[str, Any]], List[Dict[str, Any]]]: + # Nodes always have a path value, giving them a path_depth of at least 1, we use max just in case + acc_map, final_tree = accumulator + node_dict: Dict[str, Any] = {"__children": []} + line = [] + for column_index, column in enumerate(grid.columns): + if column in ignore_columns: + continue + + data = list(node.values)[column_index] + + if isinstance(data, interfaces.renderers.BaseAbsentValue): + data = None + + if isinstance(data, renderers.Disassembly): + data = display_disassembly(data) + + if isinstance(data, renderers.LayerData): + data = LayerDataRenderer().render_bytes(data)[0] + + node_dict[column.name] = data + line.append(data) + + if self.filter and self.filter.filter(line): + return accumulator + + if node.parent: + acc_map[node.parent.path]["__children"].append(node_dict) + else: + final_tree.append(node_dict) + acc_map[node.path] = node_dict + + return (acc_map, final_tree) + + if not grid.populated: + grid.populate(visitor, final_output) + else: + grid.visit(node=None, function=visitor, initial_accumulator=final_output) + + schema = self.to_arrow_schema(grid) + self.output_result(schema, outfd, final_output[1]) + + +class ParquetRenderer(ArrowRenderer): + """Renderer that outputs Parquet format data.""" + + name = "parquet" + structured_output = True + _version = (1, 0, 0) + + def get_render_options(self) -> List[interfaces.renderers.RenderOption]: + return [] + + def write_table(self, table: "pa.Table", outfd: TextIO) -> None: + """ + Writes a table to stdout using the Parquet format. + + Args: + t: The Arrow table to write + outfd: The output file descriptor + + Returns: + Nothing + """ + # Write DataFrame to a temporary file-like object + buf = pa.BufferOutputStream() + pq.write_table(table, buf, compression="snappy") + + # Get the buffer as a bytes object + buf_bytes = buf.getvalue().to_pybytes() + outfd.buffer.write(buf_bytes) From 6437148dc8d5d976e8f36a7b3e18780833de0716 Mon Sep 17 00:00:00 2001 From: Kyrre-Wahl-Kongsgard Date: Mon, 21 Jul 2025 23:47:24 +0200 Subject: [PATCH 12/22] remove old parquet renderer --- .../framework/plugins/parquet_renderer.py | 166 ------------------ 1 file changed, 166 deletions(-) delete mode 100644 volatility3/framework/plugins/parquet_renderer.py diff --git a/volatility3/framework/plugins/parquet_renderer.py b/volatility3/framework/plugins/parquet_renderer.py deleted file mode 100644 index 6564dbaeb..000000000 --- a/volatility3/framework/plugins/parquet_renderer.py +++ /dev/null @@ -1,166 +0,0 @@ -# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 -# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 -# -import datetime -import logging -import sys -from typing import ( - Any, - Dict, - List, - Tuple, - TextIO, -) -from volatility3.framework import interfaces, renderers -from volatility3.framework.renderers import format_hints -from volatility3.cli.text_renderer import ( - CLIRenderer, - display_disassembly, - LayerDataRenderer, -) - -vollog = logging.getLogger(__name__) - -try: - ARROW_PRESENT = True - import pyarrow as pa - import pyarrow.parquet as pq -except ImportError: - ARROW_PRESENT = False - vollog.debug("Arrow/Parquet libraries not found") - - -class ArrowRenderer(CLIRenderer): - def __init__( - self, options: List[interfaces.renderers.RenderOption] | None = None - ) -> None: - super().__init__(options) - - if not ARROW_PRESENT: - raise RuntimeError("Arrow output format requires the pyarrow package") - - self._to_arrow_type = { - renderers.Disassembly: pa.utf8, - bool: pa.bool_, - int: pa.int64, - float: pa.float64, - str: pa.utf8, - datetime.datetime: lambda: pa.timestamp("ms"), - format_hints.Bin: pa.int64, - format_hints.Hex: pa.int64, - format_hints.MultiTypeData: pa.utf8, - format_hints.HexBytes: pa.binary, - renderers.LayerData: pa.binary, - bytes: pa.binary, - } - - name = "arrow" - structured_output = True - - def get_render_options(self) -> List[interfaces.renderers.RenderOption]: - return [] - - def to_arrow_schema(self, grid: interfaces.renderers.TreeGrid) -> "pa.Schema": - fields = [] - for column in grid.columns: - arrow_type = self._to_arrow_type[column.type] - fields.append(pa.field(column.name, arrow_type())) - return pa.schema(fields) - - def output_result(self, schema: "pa.Schema", outfd: TextIO, result): - """Outputs the JSON data to a file in a particular format""" - - t = pa.Table.from_pylist(result, schema=schema) - self.write_table(t, outfd) - - def write_table(self, t: "pa.Table", outfd: TextIO) -> None: - buf = pa.BufferOutputStream() - - writer = pa.ipc.new_stream(buf, t.schema) - writer.write_table(t) - writer.close() - - # Get the buffer bytes and write to output - buf_bytes = buf.getvalue().to_pybytes() - outfd.buffer.write(buf_bytes) - - def render(self, grid: interfaces.renderers.TreeGrid): - outfd = sys.stdout - final_output: Tuple[ - Dict[str, List[interfaces.renderers.TreeNode]], - List[interfaces.renderers.TreeNode], - ] = ({}, []) - - ignore_columns = self.ignored_columns(grid) - - def visitor( - node: interfaces.renderers.TreeNode, - accumulator: Tuple[Dict[str, Dict[str, Any]], List[Dict[str, Any]]], - ) -> Tuple[Dict[str, Dict[str, Any]], List[Dict[str, Any]]]: - # Nodes always have a path value, giving them a path_depth of at least 1, we use max just in case - acc_map, final_tree = accumulator - node_dict: Dict[str, Any] = {"__children": []} - line = [] - for column_index, column in enumerate(grid.columns): - if column in ignore_columns: - continue - - data = list(node.values)[column_index] - - if isinstance(data, interfaces.renderers.BaseAbsentValue): - data = None - - if isinstance(data, renderers.Disassembly): - data = display_disassembly(data) - - if isinstance(data, renderers.LayerData): - data = LayerDataRenderer().render_bytes(data)[0] - - node_dict[column.name] = data - line.append(data) - - if self.filter and self.filter.filter(line): - return accumulator - - if node.parent: - acc_map[node.parent.path]["__children"].append(node_dict) - else: - final_tree.append(node_dict) - acc_map[node.path] = node_dict - - return (acc_map, final_tree) - - if not grid.populated: - grid.populate(visitor, final_output) - else: - grid.visit(node=None, function=visitor, initial_accumulator=final_output) - - schema = self.to_arrow_schema(grid) - self.output_result(schema, outfd, final_output[1]) - - -class ParquetRenderer(ArrowRenderer): - name = "parquet" - structured_output = True - - def get_render_options(self) -> List[interfaces.renderers.RenderOption]: - return [] - - def write_table(self, table: "pa.Table", outfd: TextIO) -> None: - """ - Writes a table to stdout using the Parquet format. - - Args: - t: The Arrow table to write - outfd: The output file descriptor - - Returns: - Nothing - """ - # Write DataFrame to a temporary file-like object - buf = pa.BufferOutputStream() - pq.write_table(table, buf, compression="snappy") - - # Get the buffer as a bytes object - buf_bytes = buf.getvalue().to_pybytes() - outfd.buffer.write(buf_bytes) From 11b312383c5bce78b9bb144a47e9ad47a6954f29 Mon Sep 17 00:00:00 2001 From: Kyrre-Wahl-Kongsgard Date: Tue, 22 Jul 2025 10:09:55 +0200 Subject: [PATCH 13/22] fix: update data type mappings for format_hints.Bin and format_hints.Hex to pa.uint64 --- volatility3/framework/plugins/renderers/parquet_renderer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/plugins/renderers/parquet_renderer.py b/volatility3/framework/plugins/renderers/parquet_renderer.py index c9ef95ee7..3d8f55ecb 100644 --- a/volatility3/framework/plugins/renderers/parquet_renderer.py +++ b/volatility3/framework/plugins/renderers/parquet_renderer.py @@ -53,8 +53,8 @@ class ArrowRenderer(CLIRenderer): float: pa.float64, str: pa.utf8, datetime.datetime: lambda: pa.timestamp("ms"), - format_hints.Bin: pa.int64, - format_hints.Hex: pa.int64, + format_hints.Bin: pa.uint64, + format_hints.Hex: pa.uint64, format_hints.MultiTypeData: pa.utf8, format_hints.HexBytes: pa.binary, renderers.LayerData: pa.binary, From 533b305afbb3a85a92e9f8466279cac38da4a1ed Mon Sep 17 00:00:00 2001 From: Kyrre-Wahl-Kongsgard Date: Sat, 2 Aug 2025 18:07:05 +0200 Subject: [PATCH 14/22] Added tests for the Parquet and Arrow renderers --- test/renderers/__init__.py | 0 test/renderers/test_parquet_renderers.py | 108 +++++++++++++++++++++++ 2 files changed, 108 insertions(+) create mode 100644 test/renderers/__init__.py create mode 100644 test/renderers/test_parquet_renderers.py diff --git a/test/renderers/__init__.py b/test/renderers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test/renderers/test_parquet_renderers.py b/test/renderers/test_parquet_renderers.py new file mode 100644 index 000000000..b45a602aa --- /dev/null +++ b/test/renderers/test_parquet_renderers.py @@ -0,0 +1,108 @@ +import io +import pytest +from abc import ABC, abstractmethod +from test import test_volatility + +HAS_PYARROW = False +try: + import pyarrow as pa + import pyarrow.parquet as pq + import pyarrow.compute as pc + HAS_PYARROW = True +except ImportError: + pass + + +@pytest.mark.skipif(not HAS_PYARROW, reason="pyarrow not installed") +class TestArrowRendererBase(ABC): + """Base class for testing Arrow-based renderers. + + Re-implements Windows and Linux plugin tests using PyArrow operations + instead of text-based assertions. + """ + + renderer_format = None # Override in subclasses + + @abstractmethod + def _get_table_from_output(self, output_bytes) -> "pa.Table": + """Parse output bytes into Arrow table. Override in subclasses.""" + + def test_windows_generic_pslist(self, volatility, python, image): + rc, out, _err = test_volatility.runvol_plugin( + "windows.pslist.PsList", + image, + volatility, + python, + globalargs=("-r", self.renderer_format), + ) + assert rc == 0 + + table = self._get_table_from_output(out) + assert table.num_rows > 10 + + assert table.filter(pc.match_substring(pc.utf8_lower(table.column('ImageFileName')), "system")).num_rows > 0 + assert table.filter(pc.match_substring(pc.utf8_lower(table.column('ImageFileName')), "csrss.exe")).num_rows > 0 + assert table.filter(pc.match_substring(pc.utf8_lower(table.column('ImageFileName')), "svchost.exe")).num_rows > 0 + assert table.filter(pc.greater(table.column('PID'), 0)).num_rows == table.num_rows + + def test_linux_generic_pslist(self, volatility, python, image): + rc, out, _err = test_volatility.runvol_plugin( + "linux.pslist.PsList", + image, + volatility, + python, + globalargs=("-r", self.renderer_format), + ) + assert rc == 0 + + table = self._get_table_from_output(out) + assert table.num_rows > 10 + + init_rows = table.filter(pc.match_substring(pc.utf8_lower(table.column('COMM')), "init")) + systemd_rows = table.filter(pc.match_substring(pc.utf8_lower(table.column('COMM')), "systemd")) + assert (init_rows.num_rows > 0) or (systemd_rows.num_rows > 0) + + assert table.filter(pc.match_substring(pc.utf8_lower(table.column('COMM')), "watchdog")).num_rows > 0 + assert table.filter(pc.greater(table.column('PID'), 0)).num_rows == table.num_rows + + def test_windows_generic_handles(self, volatility, python, image): + rc, out, _err = test_volatility.runvol_plugin( + "windows.handles.Handles", + image, + volatility, + python, + globalargs=("-r", self.renderer_format), + pluginargs=("--pid", "4"), + ) + assert rc == 0 + + table = self._get_table_from_output(out) + assert table.num_rows > 500 + assert table.filter(pc.match_substring(pc.utf8_lower(table.column('Name')), "machine\\system")).num_rows > 0 + + def test_linux_generic_lsof(self, volatility, python, image): + rc, out, _err = test_volatility.runvol_plugin( + "linux.lsof.Lsof", + image, + volatility, + python, + globalargs=("-r", self.renderer_format), + ) + assert rc == 0 + + table = self._get_table_from_output(out) + assert table.num_rows > 35 + +class TestParquetRenderer(TestArrowRendererBase): + renderer_format = "parquet" + + def _get_table_from_output(self, output_bytes): + return pq.read_table(io.BytesIO(output_bytes)) + + +class TestArrowRenderer(TestArrowRendererBase): + renderer_format = "arrow" + + def _get_table_from_output(self, output_bytes): + return pa.ipc.open_stream(io.BytesIO(output_bytes)).read_all() + From eb7daa95a0e19c9022d78361420646d60b706107 Mon Sep 17 00:00:00 2001 From: Kyrre-Wahl-Kongsgard Date: Wed, 6 Aug 2025 15:38:09 +0200 Subject: [PATCH 15/22] flatten tree output --- .../plugins/renderers/parquet_renderer.py | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/volatility3/framework/plugins/renderers/parquet_renderer.py b/volatility3/framework/plugins/renderers/parquet_renderer.py index 3d8f55ecb..b596349e4 100644 --- a/volatility3/framework/plugins/renderers/parquet_renderer.py +++ b/volatility3/framework/plugins/renderers/parquet_renderer.py @@ -61,6 +61,11 @@ class ArrowRenderer(CLIRenderer): bytes: pa.binary, } + # indicates if the output from the plugin is nested, e.g., pstree + # which would then need to be flattened + self._is_tree_result = False + self._node_id_counter = 0 + def get_render_options(self) -> List[interfaces.renderers.RenderOption]: return [] @@ -69,11 +74,58 @@ class ArrowRenderer(CLIRenderer): for column in grid.columns: arrow_type = self._to_arrow_type[column.type] fields.append(pa.field(column.name, arrow_type())) + + # if the output is nested, e.g., windows.pstree + if self._is_tree_result: + fields.append(pa.field("_vol_id", pa.uint64())) + fields.append(pa.field("_vol_parent_id", pa.uint64())) + return pa.schema(fields) + + def _flatten_tree_structure(self, nested: list[dict]) -> list[dict]: + """ + Flattens a list of nested dicts using the `__children` key. + + Each node gets a `_vol_id` and a `_vol_parent_id` to preserve + the original tree structure in a flat format suitable for tabular output. + + Args: + nested: A list of dicts with optional `__children` lists (tree nodes). + + Returns: + A flat list of dicts with `_vol_id` and `_vol_parent_id`. + """ + rows = [] + self._node_id_counter = 0 + + def _process_node(node: dict, parent_id: int | None): + current_id = self._node_id_counter + self._node_id_counter += 1 + + entry = {k: v for k, v in node.items() if k != "__children"} + entry["_vol_id"] = current_id + entry["_vol_parent_id"] = parent_id + rows.append(entry) + + for child in node.get("__children", []): + _process_node(child, current_id) + + for root in nested: + _process_node(root, None) + + return rows + + + + def output_result(self, schema: "pa.Schema", outfd: TextIO, result): """Outputs the JSON data to a file in a particular format""" + + if self._is_tree_result: + result = self._flatten_tree_structure(result) + t = pa.Table.from_pylist(result, schema=schema) self.write_table(t, outfd) @@ -128,6 +180,7 @@ class ArrowRenderer(CLIRenderer): if node.parent: acc_map[node.parent.path]["__children"].append(node_dict) + self._is_tree_result = True else: final_tree.append(node_dict) acc_map[node.path] = node_dict From 3c830e6e67ab5e2e1743165ef21fc63e13bb3448 Mon Sep 17 00:00:00 2001 From: Kyrre-Wahl-Kongsgard Date: Wed, 6 Aug 2025 15:43:46 +0200 Subject: [PATCH 16/22] black & ruff --- volatility3/framework/plugins/renderers/parquet_renderer.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/volatility3/framework/plugins/renderers/parquet_renderer.py b/volatility3/framework/plugins/renderers/parquet_renderer.py index b596349e4..7593fa4ef 100644 --- a/volatility3/framework/plugins/renderers/parquet_renderer.py +++ b/volatility3/framework/plugins/renderers/parquet_renderer.py @@ -82,7 +82,6 @@ class ArrowRenderer(CLIRenderer): return pa.schema(fields) - def _flatten_tree_structure(self, nested: list[dict]) -> list[dict]: """ Flattens a list of nested dicts using the `__children` key. @@ -116,13 +115,9 @@ class ArrowRenderer(CLIRenderer): return rows - - - def output_result(self, schema: "pa.Schema", outfd: TextIO, result): """Outputs the JSON data to a file in a particular format""" - if self._is_tree_result: result = self._flatten_tree_structure(result) From d21a81d8c328af72a7bb85edb89eaadf7ec1ff64 Mon Sep 17 00:00:00 2001 From: Kyrre-Wahl-Kongsgard Date: Fri, 5 Sep 2025 06:55:00 +0200 Subject: [PATCH 17/22] fix: update type hints to support Python 3.8 --- .../framework/plugins/renderers/parquet_renderer.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/plugins/renderers/parquet_renderer.py b/volatility3/framework/plugins/renderers/parquet_renderer.py index 7593fa4ef..7bc4b82ab 100644 --- a/volatility3/framework/plugins/renderers/parquet_renderer.py +++ b/volatility3/framework/plugins/renderers/parquet_renderer.py @@ -8,6 +8,7 @@ from typing import ( Any, Dict, List, + Optional, Tuple, TextIO, ) @@ -39,7 +40,7 @@ class ArrowRenderer(CLIRenderer): _version = (1, 0, 0) def __init__( - self, options: List[interfaces.renderers.RenderOption] | None = None + self, options: Optional[List[interfaces.renderers.RenderOption]] = None ) -> None: super().__init__(options) @@ -82,7 +83,7 @@ class ArrowRenderer(CLIRenderer): return pa.schema(fields) - def _flatten_tree_structure(self, nested: list[dict]) -> list[dict]: + def _flatten_tree_structure(self, nested: List[Dict]) -> List[Dict]: """ Flattens a list of nested dicts using the `__children` key. @@ -98,7 +99,7 @@ class ArrowRenderer(CLIRenderer): rows = [] self._node_id_counter = 0 - def _process_node(node: dict, parent_id: int | None): + def _process_node(node: Dict, parent_id: Optional[int]): current_id = self._node_id_counter self._node_id_counter += 1 From 86fe4485f0abcb4c10a2f07ae369dfbaa02d1f4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kyrre=20Wahl=20Kongsg=C3=A5rd?= Date: Mon, 15 Sep 2025 09:31:20 +0200 Subject: [PATCH 18/22] Update volatility3/framework/plugins/renderers/parquet_renderer.py Co-authored-by: ikelos --- volatility3/framework/plugins/renderers/parquet_renderer.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/renderers/parquet_renderer.py b/volatility3/framework/plugins/renderers/parquet_renderer.py index 7bc4b82ab..ff2e07aea 100644 --- a/volatility3/framework/plugins/renderers/parquet_renderer.py +++ b/volatility3/framework/plugins/renderers/parquet_renderer.py @@ -14,11 +14,7 @@ from typing import ( ) from volatility3.framework import interfaces, renderers from volatility3.framework.renderers import format_hints -from volatility3.cli.text_renderer import ( - CLIRenderer, - display_disassembly, - LayerDataRenderer, -) +from volatility3.cli import text_renderer vollog = logging.getLogger(__name__) From 4c7a9b517f45a50e58f5321ad041871ece98386a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kyrre=20Wahl=20Kongsg=C3=A5rd?= Date: Mon, 15 Sep 2025 09:31:28 +0200 Subject: [PATCH 19/22] Update volatility3/framework/plugins/renderers/parquet_renderer.py Co-authored-by: ikelos --- volatility3/framework/plugins/renderers/parquet_renderer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/renderers/parquet_renderer.py b/volatility3/framework/plugins/renderers/parquet_renderer.py index ff2e07aea..197944b85 100644 --- a/volatility3/framework/plugins/renderers/parquet_renderer.py +++ b/volatility3/framework/plugins/renderers/parquet_renderer.py @@ -159,7 +159,7 @@ class ArrowRenderer(CLIRenderer): data = None if isinstance(data, renderers.Disassembly): - data = display_disassembly(data) + data = text_renderer.display_disassembly(data) if isinstance(data, renderers.LayerData): data = LayerDataRenderer().render_bytes(data)[0] From 4af48ae3b0c10911c8da526154da02fff2560373 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kyrre=20Wahl=20Kongsg=C3=A5rd?= Date: Mon, 15 Sep 2025 09:31:34 +0200 Subject: [PATCH 20/22] Update volatility3/framework/plugins/renderers/parquet_renderer.py Co-authored-by: ikelos --- volatility3/framework/plugins/renderers/parquet_renderer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/renderers/parquet_renderer.py b/volatility3/framework/plugins/renderers/parquet_renderer.py index 197944b85..4e8a4c043 100644 --- a/volatility3/framework/plugins/renderers/parquet_renderer.py +++ b/volatility3/framework/plugins/renderers/parquet_renderer.py @@ -28,7 +28,7 @@ except ImportError: vollog.debug("Arrow/Parquet libraries not found") -class ArrowRenderer(CLIRenderer): +class ArrowRenderer(text_renderer.CLIRenderer): """Renderer that outputs Arrow IPC format data.""" name = "arrow" From 8090d72149f7b64c2512ef0370d4dfcd61bbb442 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kyrre=20Wahl=20Kongsg=C3=A5rd?= Date: Mon, 15 Sep 2025 09:32:10 +0200 Subject: [PATCH 21/22] Update volatility3/framework/plugins/renderers/parquet_renderer.py Co-authored-by: ikelos --- volatility3/framework/plugins/renderers/parquet_renderer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/renderers/parquet_renderer.py b/volatility3/framework/plugins/renderers/parquet_renderer.py index 4e8a4c043..f4eaf9c9b 100644 --- a/volatility3/framework/plugins/renderers/parquet_renderer.py +++ b/volatility3/framework/plugins/renderers/parquet_renderer.py @@ -162,7 +162,7 @@ class ArrowRenderer(text_renderer.CLIRenderer): data = text_renderer.display_disassembly(data) if isinstance(data, renderers.LayerData): - data = LayerDataRenderer().render_bytes(data)[0] + data = text_renderer.LayerDataRenderer().render_bytes(data)[0] node_dict[column.name] = data line.append(data) From 81f2e7376ad93a94930de6b11ea99e646229dc22 Mon Sep 17 00:00:00 2001 From: ikelos Date: Mon, 15 Sep 2025 08:58:04 +0100 Subject: [PATCH 22/22] Update test/renderers/test_parquet_renderers.py --- test/renderers/test_parquet_renderers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/renderers/test_parquet_renderers.py b/test/renderers/test_parquet_renderers.py index b45a602aa..aa5474636 100644 --- a/test/renderers/test_parquet_renderers.py +++ b/test/renderers/test_parquet_renderers.py @@ -10,6 +10,7 @@ try: import pyarrow.compute as pc HAS_PYARROW = True except ImportError: + # The user doesn't have pyarrow installed, but HAS_PYARROW will be false so just continue pass