diff --git a/pyproject.toml b/pyproject.toml index b88ac7752..0acbfcbbe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,8 @@ dev = [ "types-jsonschema>=4.23.0,<5", ] +arrow = ["pyarrow>=17.0.0"] + test = [ "volatility3[dev]", "pytest>=8.3.3,<9", diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index 1e437a0be..69d2d6d31 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -9,7 +9,7 @@ import random import string import sys from functools import wraps -from typing import Any, Callable, Dict, List, Optional, Set, Tuple, TypeVar, Union +from typing import Any, Callable, Dict, List, Optional, Set, Tuple, TypeVar, Union, TextIO from volatility3.cli import text_filter from volatility3.framework import exceptions, interfaces, renderers @@ -25,6 +25,15 @@ except ImportError: vollog.debug("Disassembly library capstone not found") +try: + ARROW_PRESENT = True + import pyarrow as pa + import pyarrow.parquet as pq +except ImportError: + ARROW_PRESENT = False + vollog.debug("Arrow/Parquet libraries not found") + + def hex_bytes_as_text(value: bytes, width: int = 16) -> str: """Renders HexBytes as text. @@ -624,3 +633,133 @@ class JsonLinesRenderer(JsonRenderer): for line in result: outfd.write(json.dumps(line, sort_keys=True)) outfd.write("\n") + +class ArrowRenderer(CLIRenderer): + def __init__( + self, options: List[interfaces.renderers.RenderOption] | None = None + ) -> None: + super().__init__(options) + + if not ARROW_PRESENT: + raise RuntimeError("Arrow output format requires the pyarrow package") + + _to_arrow_type = { + renderers.Disassembly: pa.utf8, + bool: pa.bool_, + int: pa.int64, + float: pa.float64, + str: pa.utf8, + datetime.datetime: lambda: pa.timestamp("ms"), + format_hints.Bin: pa.int64, + format_hints.Hex: pa.int64, + format_hints.MultiTypeData: pa.utf8, + format_hints.HexBytes: pa.binary, + } + + name = "arrow" + structured_output = True + + def get_render_options(self) -> List[interfaces.renderers.RenderOption]: + return [] + + def to_arrow_schema(self, grid: interfaces.renderers.TreeGrid) -> pa.Schema: + fields = [] + for column in grid.columns: + arrow_type = self._to_arrow_type[column.type] + fields.append(pa.field(column.name, arrow_type())) + return pa.schema(fields) + + def output_result(self, schema, outfd: TextIO, result): + """Outputs the JSON data to a file in a particular format""" + + t = pa.Table.from_pylist(result, schema=schema) + self.write_data(t, outfd) + + def write_data(self, t: pa.Table, outfd: TextIO) -> None: + buf = pa.BufferOutputStream() + + writer = pa.ipc.new_stream(buf, t.schema) + writer.write_table(t) + writer.close() + + # Get the buffer bytes and write to output + buf_bytes = buf.getvalue().to_pybytes() + outfd.buffer.write(buf_bytes) + + def render(self, grid: interfaces.renderers.TreeGrid): + outfd = sys.stdout + final_output: Tuple[ + Dict[str, List[interfaces.renderers.TreeNode]], + List[interfaces.renderers.TreeNode], + ] = ({}, []) + + ignore_columns = self.ignored_columns(grid) + + def visitor( + node: interfaces.renderers.TreeNode, + accumulator: Tuple[Dict[str, Dict[str, Any]], List[Dict[str, Any]]], + ) -> Tuple[Dict[str, Dict[str, Any]], List[Dict[str, Any]]]: + # Nodes always have a path value, giving them a path_depth of at least 1, we use max just in case + acc_map, final_tree = accumulator + node_dict: Dict[str, Any] = {"__children": []} + line = [] + for column_index, column in enumerate(grid.columns): + if column in ignore_columns: + continue + + data = list(node.values)[column_index] + + if isinstance(data, interfaces.renderers.BaseAbsentValue): + data = None + + if isinstance(data, renderers.Disassembly): + data = display_disassembly(data) + + node_dict[column.name] = data + line.append(data) + + if self.filter and self.filter.filter(line): + return accumulator + + if node.parent: + acc_map[node.parent.path]["__children"].append(node_dict) + else: + final_tree.append(node_dict) + acc_map[node.path] = node_dict + + return (acc_map, final_tree) + + if not grid.populated: + grid.populate(visitor, final_output) + else: + grid.visit(node=None, function=visitor, initial_accumulator=final_output) + + schema = self.to_arrow_schema(grid) + self.output_result(schema, outfd, final_output[1]) + + +class ParquetRenderer(ArrowRenderer): + name = "parquet" + structured_output = True + + def get_render_options(self) -> List[interfaces.renderers.RenderOption]: + return [] + + def write_data(self, t: pa.Table, outfd: TextIO) -> None: + """ + Writes a table to stdout using the Parquet format. + + Args: + t: The Arrow table to write + outfd: The output file descriptor + + Returns: + Nothing + """ + # Write DataFrame to a temporary file-like object + buf = pa.BufferOutputStream() + pq.write_table(t, buf, compression="snappy") + + # Get the buffer as a bytes object + buf_bytes = buf.getvalue().to_pybytes() + outfd.buffer.write(buf_bytes) \ No newline at end of file