From ee12b81f4adf12e3eed32a7402a8ad38e917c822 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 10 Dec 2017 14:57:26 +0000 Subject: [PATCH] Add in type-annotations for renderers and start on symbols. --- volatility/framework/interfaces/renderers.py | 15 ++-- volatility/framework/interfaces/symbols.py | 14 ++-- volatility/framework/renderers/__init__.py | 50 +++++++----- volatility/framework/renderers/text.py | 9 ++- volatility/framework/symbols/__init__.py | 83 +++++++++++--------- 5 files changed, 97 insertions(+), 74 deletions(-) diff --git a/volatility/framework/interfaces/renderers.py b/volatility/framework/interfaces/renderers.py index c5f7454f4..b3c40e54b 100644 --- a/volatility/framework/interfaces/renderers.py +++ b/volatility/framework/interfaces/renderers.py @@ -43,7 +43,7 @@ class TreeNode(collections.Sequence, metaclass = ABCMeta): @property @abstractmethod - def values(self) -> typing.Any: + def values(self) -> typing.Iterable['SimpleTypes']: """Returns the list of values from the particular node, based on column.index""" @property @@ -73,9 +73,10 @@ class TreeNode(collections.Sequence, metaclass = ABCMeta): """ +_Type = typing.TypeVar("_Type") ColumnsType = typing.List[typing.Tuple[str, typing.Type]] -SimpleTypes = typing.Union[int, str, float, bytes] -_T = typing.TypeVar("_T") +SimpleTypes = typing.Union[typing.Type[int], typing.Type[str], typing.Type[float], typing.Type[bytes]] +VisitorSignature = typing.Callable[[TreeNode, _Type], _Type] class TreeGrid(object, metaclass = ABCMeta): @@ -91,7 +92,7 @@ class TreeGrid(object, metaclass = ABCMeta): and to create cycles. """ - simple_types = {int, str, float, bytes} # type: typing.ClassVar[typing.Set[typing.Type]] + simple_types = (int, str, float, bytes) # type: typing.ClassVar[typing.Tuple] def __init__(self, columns: ColumnsType, generator: typing.Generator) -> None: """Constructs a TreeGrid object using a specific set of columns @@ -106,7 +107,7 @@ class TreeGrid(object, metaclass = ABCMeta): @abstractmethod def populate(self, - func: typing.Callable[[typing.Tuple[SimpleTypes]], TreeNode] = None, + func: VisitorSignature = None, initial_accumulator: typing.Any = None) \ -> typing.Generator[typing.Tuple[SimpleTypes, ...], None, None]: """Generator that returns the next available Node @@ -155,8 +156,8 @@ class TreeGrid(object, metaclass = ABCMeta): @abstractmethod def visit(self, node: TreeNode, - function: typing.Callable[[TreeNode, _T], _T], - initial_accumulator: _T = None, + function: VisitorSignature, + initial_accumulator: _Type = None, sort_key: ColumnSortKey = None) -> None: """Visits all the nodes in a tree, calling function on each one. diff --git a/volatility/framework/interfaces/symbols.py b/volatility/framework/interfaces/symbols.py index 7e5ad5991..263607732 100644 --- a/volatility/framework/interfaces/symbols.py +++ b/volatility/framework/interfaces/symbols.py @@ -68,11 +68,11 @@ class SymbolSpaceInterface(collections.abc.Mapping): """Returns an unused table name to ensure no collision occurs when inserting a symbol table""" @abstractmethod - def get_symbols_by_type(self, type_name: str) -> typing.List[Symbol]: + def get_symbols_by_type(self, type_name: str) -> typing.Iterable[str]: """Returns all symbols based on the type of the symbol""" @abstractmethod - def get_symbols_by_location(self, address: int, table_name: typing.Optional[str] = None) -> typing.List[Symbol]: + def get_symbols_by_location(self, address: int, table_name: typing.Optional[str] = None) -> typing.Iterable[str]: """Returns all symbols that exist at a specific relative address""" @abstractmethod @@ -195,7 +195,7 @@ class BaseSymbolTableInterface(validity.ValidityRoutines): """Resolves a symbol name into a symbol and then resolves the symbol's type""" return self.get_type(self.get_symbol(name).type_name) - def get_symbols_by_type(self, type_name: str) -> typing.Generator[str, None, None]: + def get_symbols_by_type(self, type_name: str) -> typing.Iterable[str]: """Returns the name of all symbols in this table that have type matching type_name""" for symbol_name in self.symbols: # This allows for searching with and without the table name (in case multiple tables contain @@ -204,7 +204,7 @@ class BaseSymbolTableInterface(validity.ValidityRoutines): if symbol.type_name == type_name or (symbol.type_name.endswith(constants.BANG + type_name)): yield symbol.name - def get_symbols_by_location(self, offset: int) -> typing.Generator[str, None, None]: + def get_symbols_by_location(self, offset: int) -> typing.Iterable[str]: """Returns the name of all symbols in this table that live at a particular offset""" sort_symbols = sorted([(self.get_symbol(sn).address, sn) for sn in self.symbols]) result = bisect.bisect_left(sort_symbols, (offset, "")) @@ -239,12 +239,12 @@ class NativeTableInterface(BaseSymbolTableInterface): raise exceptions.SymbolError("NativeTables never hold symbols") @property - def symbols(self) -> typing.List[str]: + def symbols(self) -> typing.Iterable[str]: return [] - def get_enumeration(self, name: str): + def get_enumeration(self, name: str) -> typing.Dict[str, typing.Any]: raise exceptions.SymbolError("NativeTables never hold enumerations") @property - def enumerations(self) -> typing.List[str]: + def enumerations(self) -> typing.Iterable[str]: return [] diff --git a/volatility/framework/renderers/__init__.py b/volatility/framework/renderers/__init__.py index a5e17ef71..c6ca3ac1b 100644 --- a/volatility/framework/renderers/__init__.py +++ b/volatility/framework/renderers/__init__.py @@ -11,25 +11,29 @@ from volatility.framework import interfaces class TreeNode(interfaces.renderers.TreeNode): """Class representing a particular node in a tree grid""" - def __init__(self, path, treegrid, parent, values): + def __init__(self, + path: str, + treegrid: 'TreeGrid', + parent: typing.Optional['TreeNode'], + values: typing.List[interfaces.renderers.SimpleTypes]) -> None: if not isinstance(treegrid, TreeGrid): raise TypeError("Treegrid must be an instance of TreeGrid") self._treegrid = treegrid self._parent = parent self._path = path self._validate_values(values) - self._values = treegrid.RowStructure(*values) + self._values = treegrid.RowStructure(*values) # type: ignore - def __repr__(self): + def __repr__(self) -> str: return "".format(self.path, self._values) - def __getitem__(self, item): + def __getitem__(self, item: typing.Union[int, slice]) -> typing.Any: return self._treegrid.children(self).__getitem__(item) - def __len__(self): + def __len__(self) -> int: return len(self._treegrid.children(self)) - def _validate_values(self, values): + def _validate_values(self, values: typing.List[interfaces.renderers.SimpleTypes]) -> None: """A function for raising exceptions if a given set of values is invalid according to the column properties.""" if not (isinstance(values, collections.Sequence) and len(values) == len(self._treegrid.columns)): raise TypeError( @@ -45,12 +49,12 @@ class TreeNode(interfaces.renderers.TreeNode): column.type)) @property - def values(self): + def values(self) -> typing.Iterable[interfaces.renderers.SimpleTypes]: """Returns the list of values from the particular node, based on column.index""" return self._values @property - def path(self): + def path(self) -> str: """Returns a path identifying string This should be seen as opaque by external classes, @@ -59,16 +63,16 @@ class TreeNode(interfaces.renderers.TreeNode): return self._path @property - def parent(self): + def parent(self) -> typing.Optional['TreeNode']: """Returns the parent node of this node or None""" return self._parent @property - def path_depth(self): + def path_depth(self) -> int: """Return the path depth of the current node""" return len(self.path.split(TreeGrid.path_sep)) - def path_changed(self, path, added = False): + def path_changed(self, path: str, added: bool = False) -> None: """Updates the path based on the addition or removal of a node higher up in the tree This should only be called by the containing TreeGrid and expects to only be called for affected nodes. @@ -96,7 +100,9 @@ class TreeGrid(interfaces.renderers.TreeGrid): path_sep = "|" - def __init__(self, columns, generator): + def __init__(self, + columns: typing.List[typing.Tuple[str, interfaces.renderers.SimpleTypes]], + generator: typing.Optional[typing.Iterable[typing.Tuple[int, typing.Tuple]]]) -> None: """Constructs a TreeGrid object using a specific set of columns The TreeGrid itself is a root element, that can have children but no values. @@ -108,14 +114,12 @@ class TreeGrid(interfaces.renderers.TreeGrid): """ self._populated = False self._row_count = 0 - self._children = [] - converted_columns = [] + self._children = [] # type: typing.List[TreeNode] + converted_columns = [] # type: typing.List[interfaces.renderers.Column] if len(columns) < 1: raise ValueError("Columns must be a list containing at least one column") for (name, column_type) in columns: - is_simple_type = False - for stype in self.simple_types: - is_simple_type = is_simple_type or issubclass(column_type, stype) + is_simple_type = issubclass(column_type, self.simple_types) if not is_simple_type: raise TypeError( "Column {}'s type is not a simple type: {}".format(name, column_type.__class__.__name__)) @@ -130,14 +134,17 @@ class TreeGrid(interfaces.renderers.TreeGrid): self._generator = generator @staticmethod - def _sanitize(text): + def _sanitize(text: str) -> str: output = "" for letter in text.lower(): if letter != ' ': output += (letter if letter in 'abcdefghiljklmnopqrstuvwxyz_' else '_') return output - def populate(self, func = None, initial_accumulator = None): + def populate(self, + func: interfaces.renderers.VisitorSignature = None, + initial_accumulator: typing.Any = None) \ + -> typing.Generator[typing.Tuple[interfaces.renderers.SimpleTypes, ...], None, None]: """Populates the tree by consuming the TreeGrid's construction generator Func is called on every node, so can be used to create output on demand @@ -145,11 +152,11 @@ class TreeGrid(interfaces.renderers.TreeGrid): """ accumulator = initial_accumulator if func is None: - def func(_x, _y): + def func(_x: interfaces.renderers.TreeNode, _y: typing.Any) -> typing.Any: return None if not self.populated: - prev_nodes = [] + prev_nodes = [] # type: typing.List[TreeNode] for (level, item) in self._generator: parent_index = min(len(prev_nodes), level) parent = prev_nodes[parent_index - 1] if parent_index > 0 else None @@ -158,6 +165,7 @@ class TreeGrid(interfaces.renderers.TreeGrid): accumulator = func(treenode, accumulator) self._row_count += 1 self._populated = True + return None @property def populated(self): diff --git a/volatility/framework/renderers/text.py b/volatility/framework/renderers/text.py index 3c70c98d0..b4dd6e299 100644 --- a/volatility/framework/renderers/text.py +++ b/volatility/framework/renderers/text.py @@ -4,7 +4,7 @@ from volatility.framework import interfaces from volatility.framework.renderers import format_hints -def hex_bytes_as_text(value): +def hex_bytes_as_text(value: bytes) -> str: """Renders HexBytes as text""" if not isinstance(value, bytes): raise TypeError("hex_bytes_as_text takes bytes not: {}".format(type(value))) @@ -31,19 +31,20 @@ class QuickTextRenderer(interfaces.renderers.Renderer): bytes: lambda x: x.decode("utf-8"), 'default': lambda x: "{}".format(x)} - def __init__(self, options = None): + def __init__(self, options = None) -> None: super().__init__(options) def get_render_options(self): pass - def render(self, grid): + def render(self, grid: interfaces.renderers.TreeGrid) -> None: # TODO: Docstrings # TODO: Improve text output outfd = sys.stdout for column in grid.columns: - outfd.write("\t{}".format(column.name)) + # Ignore the type because namedtuples don't realize they have accessible attributes + outfd.write("\t{}".format(column.name)) # type: ignore outfd.write("\n") def visitor(node, accumulator): diff --git a/volatility/framework/symbols/__init__.py b/volatility/framework/symbols/__init__.py index 249ab182c..530022ce1 100644 --- a/volatility/framework/symbols/__init__.py +++ b/volatility/framework/symbols/__init__.py @@ -2,6 +2,7 @@ import collections import collections.abc import enum import logging +import typing from volatility.framework import constants, exceptions, interfaces, objects, validity from volatility.framework.symbols import native, windows, linux @@ -15,6 +16,12 @@ class SymbolType(enum.Enum): ENUM = 3 +SymbolSpaceReturnType = typing.TypeVar("SymbolSpaceReturnType", + interfaces.objects.Template, + interfaces.symbols.Symbol, + typing.Dict[str, typing.Any]) + + class SymbolSpace(interfaces.symbols.SymbolSpaceInterface, validity.ValidityRoutines): """Handles an ordered collection of SymbolTables @@ -22,13 +29,13 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface, validity.ValidityRout proceed down through the ranks if a namespace isn't specified. """ - def __init__(self): + def __init__(self) -> None: super().__init__() - self._dict = collections.OrderedDict() + self._dict = collections.OrderedDict() # type: typing.Dict[str, interfaces.symbols.BaseSymbolTableInterface] # Permanently cache all resolved symbols - self._resolved = {} + self._resolved = {} # type: typing.Dict[str, interfaces.objects.Template] - def free_table_name(self, prefix = "layer"): + def free_table_name(self, prefix: str = "layer") -> str: """Returns an unused table name to ensure no collision occurs when inserting a symbol table""" self._check_type(prefix, str) @@ -39,39 +46,39 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface, validity.ValidityRout ### Symbol functions - def get_symbols_by_type(self, type_name): + def get_symbols_by_type(self, type_name: str) -> typing.Iterable[str]: """Returns all symbols based on the type of the symbol""" for table in self._dict.keys(): for symbol_name in self._dict[table].get_symbols_by_type(type_name): yield table + constants.BANG + symbol_name - def get_symbols_by_location(self, address, table_name = None): + def get_symbols_by_location(self, offset: int, table_name: str = None) -> typing.Iterable[str]: """Returns all symbols that exist at a specific relative address""" - table_list = self._dict.values() + table_list = self._dict.values() # type: typing.Iterable[interfaces.symbols.BaseSymbolTableInterface] if table_name is not None: if table_name in self._dict: table_list = [self._dict[table_name]] else: table_list = [] for table in table_list: - for symbol_name in self._dict[table].get_symbols_by_location(address = address): - yield table + constants.BANG + symbol_name + for symbol_name in table.get_symbols_by_location(offset = offset): + yield table.name + constants.BANG + symbol_name ### Space functions - def __len__(self): + def __len__(self) -> int: """Returns the number of tables within the space""" return len(self._dict) - def __getitem__(self, i): + def __getitem__(self, i: str) -> typing.Any: """Returns a specific table from the space""" return self._dict[i] - def __iter__(self): + def __iter__(self) -> typing.Iterator[str]: """Iterates through all available tables in the symbol space""" return iter(self._dict) - def append(self, value): + def append(self, value: interfaces.symbols.BaseSymbolTableInterface) -> None: """Adds a symbol_list to the end of the space""" if not isinstance(value, interfaces.symbols.BaseSymbolTableInterface): raise TypeError(value) @@ -79,7 +86,7 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface, validity.ValidityRout self.remove(value.name) self._dict[value.name] = value - def remove(self, key): + def remove(self, key: str) -> None: """Removes a named symbol_list from the space""" # Reset the resolved list, since we're removing some symbols self._resolved = {} @@ -98,11 +105,11 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface, validity.ValidityRout indicate this failure in resolution. """ - def __init__(self, type_name = None, **kwargs): + def __init__(self, type_name: str = None, **kwargs) -> None: vollog.debug("Unresolved reference: {}".format(type_name)) super().__init__(type_name = type_name, **kwargs) - def _weak_resolve(self, resolve_type, name): + def _weak_resolve(self, resolve_type: SymbolType, name: str) -> SymbolSpaceReturnType: """Takes a symbol name and resolves it with ReferentialTemplates""" if resolve_type == SymbolType.TYPE: get_function = 'get_type' @@ -120,13 +127,10 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface, validity.ValidityRout try: return getattr(self._dict[table_name], get_function)(component_name) except KeyError as e: - vollog.debug('Type {} references missing Type/Symbol/Enum: {}'.format(name, e)) - return self._UnresolvedTemplate(name) - except exceptions.SymbolError: - return self._UnresolvedTemplate(name) + raise exceptions.SymbolError('Type {} references missing Type/Symbol/Enum: {}'.format(name, e)) raise exceptions.SymbolError("Malformed name: {}".format(name)) - def get_type(self, type_name): + def get_type(self, type_name: str) -> interfaces.objects.Template: """Takes a symbol name and resolves it This method ensures that all referenced templates (including self-referential templates) @@ -134,7 +138,7 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface, validity.ValidityRout """ # Traverse down any resolutions if type_name not in self._resolved: - self._resolved[type_name] = self._weak_resolve(SymbolType.TYPE, type_name) + self._resolved[type_name] = self._weak_resolve(SymbolType.TYPE, type_name) # type: ignore traverse_list = [type_name] replacements = set() # Whole Symbols that still need traversing @@ -149,8 +153,11 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface, validity.ValidityRout # to the "symbols that still need traversing" list if child.vol.type_name not in self._resolved: traverse_list.append(child.vol.type_name) - self._resolved[child.vol.type_name] = self._weak_resolve(SymbolType.TYPE, - child.vol.type_name) + try: + self._resolved[child.vol.type_name] = self._weak_resolve(SymbolType.TYPE, + child.vol.type_name) + except exceptions.SymbolError: + self._resolved[child.vol.type_name] = self._UnresolvedTemplate(child.vol.type_name) # Stash the replacement replacements.add((traverser, child)) elif child.children: @@ -161,18 +168,23 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface, validity.ValidityRout raise exceptions.SymbolError("Unresolvable symbol requested: {}".format(type_name)) return self._resolved[type_name] - def get_symbol(self, symbol_name): + def get_symbol(self, symbol_name: str) -> interfaces.symbols.Symbol: """Look-up a symbol name across all the contained symbol spaces""" - return self._weak_resolve(SymbolType.SYMBOL, symbol_name) + retval = self._weak_resolve(SymbolType.SYMBOL, symbol_name) + if not isinstance(retval, interfaces.symbols.Symbol): + raise exceptions.SymbolError("Unresolvable Symbol: {}".format(symbol_name)) + return retval - def get_enumeration(self, enum_name): + def get_enumeration(self, enum_name: str) -> typing.Dict[str, typing.Any]: """Look-up a set of enumeration choices from a specific symbol table""" - return self._weak_resolve(SymbolType.ENUM, enum_name) + retval = self._weak_resolve(SymbolType.ENUM, enum_name) + if not isinstance(retval, dict): + raise exceptions.SymbolError("Unresolvable Enumeration: {}".format(enum_name)) + return retval - def _membership(self, member_type, name): + def _membership(self, member_type: SymbolType, name: str) -> bool: """Test for membership of a component within a table""" - table = [] name_array = name.split(constants.BANG) if len(name_array) == 2: table_name = name_array[0] @@ -180,8 +192,9 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface, validity.ValidityRout else: return False - if table_name in self: - table = self[table_name] + if table_name not in self: + return False + table = self[table_name] if member_type == SymbolType.TYPE: return component_name in table.types @@ -191,11 +204,11 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface, validity.ValidityRout return component_name in table.enumerations return False - def has_type(self, name): + def has_type(self, name: str) -> bool: return self._membership(SymbolType.TYPE, name) - def has_symbol(self, name): + def has_symbol(self, name: str) -> bool: return self._membership(SymbolType.SYMBOL, name) - def has_enumeration(self, name): + def has_enumeration(self, name: str) -> bool: return self._membership(SymbolType.ENUM, name)