From f0153817c5bcbb72093c1db70e79023405db1144 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 24 Mar 2025 00:58:28 +0000 Subject: [PATCH 01/10] Add in slots to object model --- volatility3/framework/interfaces/objects.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/volatility3/framework/interfaces/objects.py b/volatility3/framework/interfaces/objects.py index 62c31481b..7317469f1 100644 --- a/volatility3/framework/interfaces/objects.py +++ b/volatility3/framework/interfaces/objects.py @@ -23,6 +23,8 @@ class ReadOnlyMapping(collections.abc.Mapping): modified, making an immutable mapping. """ + __slots__ = ("_dict",) + def __init__(self, dictionary: Mapping[str, Any]) -> None: self._dict = dictionary @@ -63,6 +65,8 @@ class ObjectInformation(ReadOnlyMapping): in a single place. These values are based on the :class:`ReadOnlyMapping` class, to prevent their modification. """ + __slots__ = () + def __init__( self, layer_name: str, @@ -98,6 +102,8 @@ class ObjectInterface(metaclass=abc.ABCMeta): """A base object required to be the ancestor of every object used in volatility.""" + __slots__ = () + def __init__( self, context: "interfaces.context.ContextInterface", @@ -305,6 +311,8 @@ class Template: constructed at resolution time and then cached. """ + __slots__ = "_vol" + def __init__(self, type_name: str, **arguments) -> None: """Stores the keyword arguments for later object creation.""" # Allow the updating of template arguments whilst still in template form From 3153cd7e30dd7444eb6e35be9f4ed35bc4febc8f Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 24 Mar 2025 19:21:51 +0000 Subject: [PATCH 02/10] Remove the chainmap and multiple dictionaries to reduce memory consumption --- volatility3/framework/interfaces/objects.py | 12 ++++++------ volatility3/framework/renderers/__init__.py | 7 +++++-- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/interfaces/objects.py b/volatility3/framework/interfaces/objects.py index 7317469f1..8419f9d4d 100644 --- a/volatility3/framework/interfaces/objects.py +++ b/volatility3/framework/interfaces/objects.py @@ -133,8 +133,10 @@ class ObjectInterface(metaclass=abc.ABCMeta): mask = context.layers[object_info.layer_name].address_mask normalized_offset = object_info.offset & mask + self._vol = kwargs vol_info_dict = {"type_name": type_name, "offset": normalized_offset} - self._vol = collections.ChainMap({}, vol_info_dict, object_info, kwargs) + self._vol.update(object_info) + self._vol.update(vol_info_dict) self._context = context def __getattr__(self, attr: str) -> Any: @@ -317,10 +319,8 @@ class Template: """Stores the keyword arguments for later object creation.""" # Allow the updating of template arguments whilst still in template form super().__init__() - empty_dict: Dict[str, Any] = {} - self._vol = collections.ChainMap( - empty_dict, arguments, {"type_name": type_name} - ) + self._vol = {"type_name": type_name} + self._vol.update(arguments) @property def vol(self) -> ReadOnlyMapping: @@ -364,7 +364,7 @@ class Template: def clone(self) -> "Template": """Returns a copy of the original Template as constructed (without `update_vol` additions having been made)""" - clone = self.__class__(**self._vol.parents.new_child()) + clone = self.__class__(**self._vol) return clone def update_vol(self, **new_arguments) -> None: diff --git a/volatility3/framework/renderers/__init__.py b/volatility3/framework/renderers/__init__.py index 093edf8cc..cc3129e87 100644 --- a/volatility3/framework/renderers/__init__.py +++ b/volatility3/framework/renderers/__init__.py @@ -61,7 +61,7 @@ class TreeNode(interfaces.renderers.TreeNode): self._treegrid = treegrid self._parent = parent self._path = path - self._validate_values(values) + validated_values = self._validate_values(values) self._values = treegrid.RowStructure(*values) # type: ignore def __repr__(self) -> str: @@ -73,9 +73,12 @@ class TreeNode(interfaces.renderers.TreeNode): def __len__(self) -> int: return len(self._treegrid.children(self)) - def _validate_values(self, values: List[interfaces.renderers.BaseTypes]) -> None: + def _validate_values( + self, values: List[interfaces.renderers.BaseTypes] + ) -> List[interfaces.renderers.BaseTypes]: """A function for raising exceptions if a given set of values is invalid according to the column properties.""" + new_values = () if not ( isinstance(values, collections.abc.Sequence) and len(values) == len(self._treegrid.columns) From 8610c681aba380b7f77ddcce9ed22e716e10b7f5 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 24 Mar 2025 22:33:25 +0000 Subject: [PATCH 03/10] Restore the chainmap, since we need it for cloning --- volatility3/framework/interfaces/objects.py | 50 ++++++++++++++++----- volatility3/framework/renderers/__init__.py | 7 +-- 2 files changed, 41 insertions(+), 16 deletions(-) diff --git a/volatility3/framework/interfaces/objects.py b/volatility3/framework/interfaces/objects.py index 8419f9d4d..995a5f29b 100644 --- a/volatility3/framework/interfaces/objects.py +++ b/volatility3/framework/interfaces/objects.py @@ -54,7 +54,7 @@ class ReadOnlyMapping(collections.abc.Mapping): return dict(self) == dict(other) -class ObjectInformation(ReadOnlyMapping): +class ObjectInformation(collections.abc.Mapping): """Contains common information useful/pertinent only to an individual object (like an instance) @@ -65,7 +65,14 @@ class ObjectInformation(ReadOnlyMapping): in a single place. These values are based on the :class:`ReadOnlyMapping` class, to prevent their modification. """ - __slots__ = () + __slots__ = ( + "layer_name", + "offset", + "member_name", + "parent", + "native_layer_name", + "size", + ) def __init__( self, @@ -86,17 +93,36 @@ class ObjectInformation(ReadOnlyMapping): native_layer_name: If this object references other objects (such as a pointer), what layer those objects live in size: The size that the whole structure consumes in bytes """ - super().__init__( - { - "layer_name": layer_name, - "offset": offset, - "member_name": member_name, - "parent": parent, - "native_layer_name": native_layer_name or layer_name, - "size": size, - } + self.layer_name = layer_name + self.offset = offset + self.member_name = member_name + self.parent = parent + self.native_layer_name = native_layer_name or layer_name + self.size = size + + def __getattr__(self, attr: str) -> Any: + """Returns the item as an attribute.""" + if attr in self.__slots__: + return getattr(self, attr) + raise AttributeError( + f"Object has no attribute: {self.__class__.__name__}.{attr}" ) + def __getitem__(self, name: str) -> Any: + """Returns the item requested.""" + return getattr(self, name) + + def __iter__(self): + """Returns an iterator of the dictionary items.""" + return self.__slots__.__iter__() + + def __len__(self) -> int: + """Returns the length of the internal dictionary.""" + return len(self.__slots__) + + def __eq__(self, other): + return dict(self) == dict(other) + class ObjectInterface(metaclass=abc.ABCMeta): """A base object required to be the ancestor of every object used in @@ -137,6 +163,7 @@ class ObjectInterface(metaclass=abc.ABCMeta): vol_info_dict = {"type_name": type_name, "offset": normalized_offset} self._vol.update(object_info) self._vol.update(vol_info_dict) + self._vol = collections.ChainMap({}, self._vol) self._context = context def __getattr__(self, attr: str) -> Any: @@ -321,6 +348,7 @@ class Template: super().__init__() self._vol = {"type_name": type_name} self._vol.update(arguments) + self._vol = collections.ChainMap({}, self._vol) @property def vol(self) -> ReadOnlyMapping: diff --git a/volatility3/framework/renderers/__init__.py b/volatility3/framework/renderers/__init__.py index cc3129e87..093edf8cc 100644 --- a/volatility3/framework/renderers/__init__.py +++ b/volatility3/framework/renderers/__init__.py @@ -61,7 +61,7 @@ class TreeNode(interfaces.renderers.TreeNode): self._treegrid = treegrid self._parent = parent self._path = path - validated_values = self._validate_values(values) + self._validate_values(values) self._values = treegrid.RowStructure(*values) # type: ignore def __repr__(self) -> str: @@ -73,12 +73,9 @@ class TreeNode(interfaces.renderers.TreeNode): def __len__(self) -> int: return len(self._treegrid.children(self)) - def _validate_values( - self, values: List[interfaces.renderers.BaseTypes] - ) -> List[interfaces.renderers.BaseTypes]: + def _validate_values(self, values: List[interfaces.renderers.BaseTypes]) -> None: """A function for raising exceptions if a given set of values is invalid according to the column properties.""" - new_values = () if not ( isinstance(values, collections.abc.Sequence) and len(values) == len(self._treegrid.columns) From 0ea5d795fdfbe3f562d6d04fde5a6192de29003b Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 24 Mar 2025 22:36:12 +0000 Subject: [PATCH 04/10] Fix ruff issue --- volatility3/framework/interfaces/objects.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/interfaces/objects.py b/volatility3/framework/interfaces/objects.py index 995a5f29b..93c0ac16f 100644 --- a/volatility3/framework/interfaces/objects.py +++ b/volatility3/framework/interfaces/objects.py @@ -8,7 +8,7 @@ import collections import collections.abc import contextlib import logging -from typing import Any, Dict, List, Mapping, Optional +from typing import Any, List, Mapping, Optional from volatility3.framework import constants, interfaces From 62d1d818b3f751534baae6b2fbc0a063e43d183c Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 24 Mar 2025 22:40:14 +0000 Subject: [PATCH 05/10] Restore use of ChainMap as well --- volatility3/framework/interfaces/objects.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/interfaces/objects.py b/volatility3/framework/interfaces/objects.py index 93c0ac16f..e1d36abb4 100644 --- a/volatility3/framework/interfaces/objects.py +++ b/volatility3/framework/interfaces/objects.py @@ -346,9 +346,8 @@ class Template: """Stores the keyword arguments for later object creation.""" # Allow the updating of template arguments whilst still in template form super().__init__() - self._vol = {"type_name": type_name} - self._vol.update(arguments) - self._vol = collections.ChainMap({}, self._vol) + vol = {"type_name": type_name}.update(arguments) + self._vol = collections.ChainMap({}, vol) @property def vol(self) -> ReadOnlyMapping: @@ -392,7 +391,7 @@ class Template: def clone(self) -> "Template": """Returns a copy of the original Template as constructed (without `update_vol` additions having been made)""" - clone = self.__class__(**self._vol) + clone = self.__class__(**self._vol.parents.new_child()) return clone def update_vol(self, **new_arguments) -> None: From 292ed6aeb46ff9bbbfbd0a8f73460bebf21f32b7 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 24 Mar 2025 22:41:52 +0000 Subject: [PATCH 06/10] Try to avoid variables changing types --- volatility3/framework/interfaces/objects.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/interfaces/objects.py b/volatility3/framework/interfaces/objects.py index e1d36abb4..d6ad47bec 100644 --- a/volatility3/framework/interfaces/objects.py +++ b/volatility3/framework/interfaces/objects.py @@ -159,11 +159,11 @@ class ObjectInterface(metaclass=abc.ABCMeta): mask = context.layers[object_info.layer_name].address_mask normalized_offset = object_info.offset & mask - self._vol = kwargs + vol = kwargs vol_info_dict = {"type_name": type_name, "offset": normalized_offset} - self._vol.update(object_info) - self._vol.update(vol_info_dict) - self._vol = collections.ChainMap({}, self._vol) + vol.update(object_info) + vol.update(vol_info_dict) + self._vol = collections.ChainMap({}, vol) self._context = context def __getattr__(self, attr: str) -> Any: From 200746cbe6264e8e84b9b4ade0f9116b0848c45d Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 24 Mar 2025 22:46:36 +0000 Subject: [PATCH 07/10] Fix silly usage of update --- volatility3/framework/interfaces/objects.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/interfaces/objects.py b/volatility3/framework/interfaces/objects.py index d6ad47bec..7f4667b4e 100644 --- a/volatility3/framework/interfaces/objects.py +++ b/volatility3/framework/interfaces/objects.py @@ -346,7 +346,8 @@ class Template: """Stores the keyword arguments for later object creation.""" # Allow the updating of template arguments whilst still in template form super().__init__() - vol = {"type_name": type_name}.update(arguments) + vol = {"type_name": type_name} + vol.update(arguments) self._vol = collections.ChainMap({}, vol) @property From acfedd6d9cbbdca0a8058cec5cefc303455f591e Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 24 Mar 2025 23:23:22 +0000 Subject: [PATCH 08/10] Sets slots to none has no effect on memory as long as __dict__ isn't instanciated --- volatility3/framework/interfaces/objects.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/volatility3/framework/interfaces/objects.py b/volatility3/framework/interfaces/objects.py index 7f4667b4e..e77a5893c 100644 --- a/volatility3/framework/interfaces/objects.py +++ b/volatility3/framework/interfaces/objects.py @@ -128,8 +128,6 @@ class ObjectInterface(metaclass=abc.ABCMeta): """A base object required to be the ancestor of every object used in volatility.""" - __slots__ = () - def __init__( self, context: "interfaces.context.ContextInterface", From a013170a2d2c22d48ae0f90354640b19117b5cd2 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 24 Mar 2025 23:58:45 +0000 Subject: [PATCH 09/10] Slotting has little effect, so don't change so much --- volatility3/framework/interfaces/objects.py | 54 +++++---------------- 1 file changed, 11 insertions(+), 43 deletions(-) diff --git a/volatility3/framework/interfaces/objects.py b/volatility3/framework/interfaces/objects.py index e77a5893c..b7ea616c7 100644 --- a/volatility3/framework/interfaces/objects.py +++ b/volatility3/framework/interfaces/objects.py @@ -8,7 +8,7 @@ import collections import collections.abc import contextlib import logging -from typing import Any, List, Mapping, Optional +from typing import Any, Dict, List, Mapping, Optional from volatility3.framework import constants, interfaces @@ -23,8 +23,6 @@ class ReadOnlyMapping(collections.abc.Mapping): modified, making an immutable mapping. """ - __slots__ = ("_dict",) - def __init__(self, dictionary: Mapping[str, Any]) -> None: self._dict = dictionary @@ -54,7 +52,7 @@ class ReadOnlyMapping(collections.abc.Mapping): return dict(self) == dict(other) -class ObjectInformation(collections.abc.Mapping): +class ObjectInformation(ReadOnlyMapping): """Contains common information useful/pertinent only to an individual object (like an instance) @@ -65,15 +63,6 @@ class ObjectInformation(collections.abc.Mapping): in a single place. These values are based on the :class:`ReadOnlyMapping` class, to prevent their modification. """ - __slots__ = ( - "layer_name", - "offset", - "member_name", - "parent", - "native_layer_name", - "size", - ) - def __init__( self, layer_name: str, @@ -93,36 +82,17 @@ class ObjectInformation(collections.abc.Mapping): native_layer_name: If this object references other objects (such as a pointer), what layer those objects live in size: The size that the whole structure consumes in bytes """ - self.layer_name = layer_name - self.offset = offset - self.member_name = member_name - self.parent = parent - self.native_layer_name = native_layer_name or layer_name - self.size = size - - def __getattr__(self, attr: str) -> Any: - """Returns the item as an attribute.""" - if attr in self.__slots__: - return getattr(self, attr) - raise AttributeError( - f"Object has no attribute: {self.__class__.__name__}.{attr}" + super().__init__( + { + "layer_name": layer_name, + "offset": offset, + "member_name": member_name, + "parent": parent, + "native_layer_name": native_layer_name or layer_name, + "size": size, + } ) - def __getitem__(self, name: str) -> Any: - """Returns the item requested.""" - return getattr(self, name) - - def __iter__(self): - """Returns an iterator of the dictionary items.""" - return self.__slots__.__iter__() - - def __len__(self) -> int: - """Returns the length of the internal dictionary.""" - return len(self.__slots__) - - def __eq__(self, other): - return dict(self) == dict(other) - class ObjectInterface(metaclass=abc.ABCMeta): """A base object required to be the ancestor of every object used in @@ -338,8 +308,6 @@ class Template: constructed at resolution time and then cached. """ - __slots__ = "_vol" - def __init__(self, type_name: str, **arguments) -> None: """Stores the keyword arguments for later object creation.""" # Allow the updating of template arguments whilst still in template form From de11e87f28661f8b30d7d2c38ea8480461a0536c Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Tue, 25 Mar 2025 00:08:07 +0000 Subject: [PATCH 10/10] Fix ruff error (again) --- volatility3/framework/interfaces/objects.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/interfaces/objects.py b/volatility3/framework/interfaces/objects.py index b7ea616c7..1bca7a045 100644 --- a/volatility3/framework/interfaces/objects.py +++ b/volatility3/framework/interfaces/objects.py @@ -8,7 +8,7 @@ import collections import collections.abc import contextlib import logging -from typing import Any, Dict, List, Mapping, Optional +from typing import Any, List, Mapping, Optional from volatility3.framework import constants, interfaces