Add in type-annotations for renderers and start on symbols.

volatility/framework/interfaces/renderers.py

@@ -43,7 +43,7 @@ class TreeNode(collections.Sequence, metaclass = ABCMeta):
 
     @property
     @abstractmethod
-    def values(self) -> typing.Any:
+    def values(self) -> typing.Iterable['SimpleTypes']:
         """Returns the list of values from the particular node, based on column.index"""
 
     @property
@@ -73,9 +73,10 @@ class TreeNode(collections.Sequence, metaclass = ABCMeta):
         """
 
 
+_Type = typing.TypeVar("_Type")
 ColumnsType = typing.List[typing.Tuple[str, typing.Type]]
-SimpleTypes = typing.Union[int, str, float, bytes]
-_T = typing.TypeVar("_T")
+SimpleTypes = typing.Union[typing.Type[int], typing.Type[str], typing.Type[float], typing.Type[bytes]]
+VisitorSignature = typing.Callable[[TreeNode, _Type], _Type]
 
 
 class TreeGrid(object, metaclass = ABCMeta):
@@ -91,7 +92,7 @@ class TreeGrid(object, metaclass = ABCMeta):
     and to create cycles.
     """
 
-    simple_types = {int, str, float, bytes}  # type: typing.ClassVar[typing.Set[typing.Type]]
+    simple_types = (int, str, float, bytes)  # type: typing.ClassVar[typing.Tuple]
 
     def __init__(self, columns: ColumnsType, generator: typing.Generator) -> None:
         """Constructs a TreeGrid object using a specific set of columns
@@ -106,7 +107,7 @@ class TreeGrid(object, metaclass = ABCMeta):
 
     @abstractmethod
     def populate(self,
-                 func: typing.Callable[[typing.Tuple[SimpleTypes]], TreeNode] = None,
+                 func: VisitorSignature = None,
                  initial_accumulator: typing.Any = None) \
             -> typing.Generator[typing.Tuple[SimpleTypes, ...], None, None]:
         """Generator that returns the next available Node
@@ -155,8 +156,8 @@ class TreeGrid(object, metaclass = ABCMeta):
     @abstractmethod
     def visit(self,
               node: TreeNode,
-              function: typing.Callable[[TreeNode, _T], _T],
-              initial_accumulator: _T = None,
+              function: VisitorSignature,
+              initial_accumulator: _Type = None,
               sort_key: ColumnSortKey = None) -> None:
         """Visits all the nodes in a tree, calling function on each one.
 

volatility/framework/interfaces/symbols.py

@@ -68,11 +68,11 @@ class SymbolSpaceInterface(collections.abc.Mapping):
         """Returns an unused table name to ensure no collision occurs when inserting a symbol table"""
 
     @abstractmethod
-    def get_symbols_by_type(self, type_name: str) -> typing.List[Symbol]:
+    def get_symbols_by_type(self, type_name: str) -> typing.Iterable[str]:
         """Returns all symbols based on the type of the symbol"""
 
     @abstractmethod
-    def get_symbols_by_location(self, address: int, table_name: typing.Optional[str] = None) -> typing.List[Symbol]:
+    def get_symbols_by_location(self, address: int, table_name: typing.Optional[str] = None) -> typing.Iterable[str]:
         """Returns all symbols that exist at a specific relative address"""
 
     @abstractmethod
@@ -195,7 +195,7 @@ class BaseSymbolTableInterface(validity.ValidityRoutines):
         """Resolves a symbol name into a symbol and then resolves the symbol's type"""
         return self.get_type(self.get_symbol(name).type_name)
 
-    def get_symbols_by_type(self, type_name: str) -> typing.Generator[str, None, None]:
+    def get_symbols_by_type(self, type_name: str) -> typing.Iterable[str]:
         """Returns the name of all symbols in this table that have type matching type_name"""
         for symbol_name in self.symbols:
             # This allows for searching with and without the table name (in case multiple tables contain
@@ -204,7 +204,7 @@ class BaseSymbolTableInterface(validity.ValidityRoutines):
             if symbol.type_name == type_name or (symbol.type_name.endswith(constants.BANG + type_name)):
                 yield symbol.name
 
-    def get_symbols_by_location(self, offset: int) -> typing.Generator[str, None, None]:
+    def get_symbols_by_location(self, offset: int) -> typing.Iterable[str]:
         """Returns the name of all symbols in this table that live at a particular offset"""
         sort_symbols = sorted([(self.get_symbol(sn).address, sn) for sn in self.symbols])
         result = bisect.bisect_left(sort_symbols, (offset, ""))
@@ -239,12 +239,12 @@ class NativeTableInterface(BaseSymbolTableInterface):
         raise exceptions.SymbolError("NativeTables never hold symbols")
 
     @property
-    def symbols(self) -> typing.List[str]:
+    def symbols(self) -> typing.Iterable[str]:
         return []
 
-    def get_enumeration(self, name: str):
+    def get_enumeration(self, name: str) -> typing.Dict[str, typing.Any]:
         raise exceptions.SymbolError("NativeTables never hold enumerations")
 
     @property
-    def enumerations(self) -> typing.List[str]:
+    def enumerations(self) -> typing.Iterable[str]:
         return []

volatility/framework/renderers/__init__.py

@@ -11,25 +11,29 @@ from volatility.framework import interfaces
 class TreeNode(interfaces.renderers.TreeNode):
     """Class representing a particular node in a tree grid"""
 
-    def __init__(self, path, treegrid, parent, values):
+    def __init__(self,
+                 path: str,
+                 treegrid: 'TreeGrid',
+                 parent: typing.Optional['TreeNode'],
+                 values: typing.List[interfaces.renderers.SimpleTypes]) -> None:
         if not isinstance(treegrid, TreeGrid):
             raise TypeError("Treegrid must be an instance of TreeGrid")
         self._treegrid = treegrid
         self._parent = parent
         self._path = path
         self._validate_values(values)
-        self._values = treegrid.RowStructure(*values)
+        self._values = treegrid.RowStructure(*values)  # type: ignore
 
-    def __repr__(self):
+    def __repr__(self) -> str:
         return "<TreeNode [{}] - {}>".format(self.path, self._values)
 
-    def __getitem__(self, item):
+    def __getitem__(self, item: typing.Union[int, slice]) -> typing.Any:
         return self._treegrid.children(self).__getitem__(item)
 
-    def __len__(self):
+    def __len__(self) -> int:
         return len(self._treegrid.children(self))
 
-    def _validate_values(self, values):
+    def _validate_values(self, values: typing.List[interfaces.renderers.SimpleTypes]) -> None:
         """A function for raising exceptions if a given set of values is invalid according to the column properties."""
         if not (isinstance(values, collections.Sequence) and len(values) == len(self._treegrid.columns)):
             raise TypeError(
@@ -45,12 +49,12 @@ class TreeNode(interfaces.renderers.TreeNode):
                         column.type))
 
     @property
-    def values(self):
+    def values(self) -> typing.Iterable[interfaces.renderers.SimpleTypes]:
         """Returns the list of values from the particular node, based on column.index"""
         return self._values
 
     @property
-    def path(self):
+    def path(self) -> str:
         """Returns a path identifying string
 
         This should be seen as opaque by external classes,
@@ -59,16 +63,16 @@ class TreeNode(interfaces.renderers.TreeNode):
         return self._path
 
     @property
-    def parent(self):
+    def parent(self) -> typing.Optional['TreeNode']:
         """Returns the parent node of this node or None"""
         return self._parent
 
     @property
-    def path_depth(self):
+    def path_depth(self) -> int:
         """Return the path depth of the current node"""
         return len(self.path.split(TreeGrid.path_sep))
 
-    def path_changed(self, path, added = False):
+    def path_changed(self, path: str, added: bool = False) -> None:
         """Updates the path based on the addition or removal of a node higher up in the tree
 
            This should only be called by the containing TreeGrid and expects to only be called for affected nodes.
@@ -96,7 +100,9 @@ class TreeGrid(interfaces.renderers.TreeGrid):
 
     path_sep = "|"
 
-    def __init__(self, columns, generator):
+    def __init__(self,
+                 columns: typing.List[typing.Tuple[str, interfaces.renderers.SimpleTypes]],
+                 generator: typing.Optional[typing.Iterable[typing.Tuple[int, typing.Tuple]]]) -> None:
         """Constructs a TreeGrid object using a specific set of columns
 
         The TreeGrid itself is a root element, that can have children but no values.
@@ -108,14 +114,12 @@ class TreeGrid(interfaces.renderers.TreeGrid):
         """
         self._populated = False
         self._row_count = 0
-        self._children = []
-        converted_columns = []
+        self._children = []  # type: typing.List[TreeNode]
+        converted_columns = []  # type: typing.List[interfaces.renderers.Column]
         if len(columns) < 1:
             raise ValueError("Columns must be a list containing at least one column")
         for (name, column_type) in columns:
-            is_simple_type = False
-            for stype in self.simple_types:
-                is_simple_type = is_simple_type or issubclass(column_type, stype)
+            is_simple_type = issubclass(column_type, self.simple_types)
             if not is_simple_type:
                 raise TypeError(
                     "Column {}'s type is not a simple type: {}".format(name, column_type.__class__.__name__))
@@ -130,14 +134,17 @@ class TreeGrid(interfaces.renderers.TreeGrid):
         self._generator = generator
 
     @staticmethod
-    def _sanitize(text):
+    def _sanitize(text: str) -> str:
         output = ""
         for letter in text.lower():
             if letter != ' ':
                 output += (letter if letter in 'abcdefghiljklmnopqrstuvwxyz_' else '_')
         return output
 
-    def populate(self, func = None, initial_accumulator = None):
+    def populate(self,
+                 func: interfaces.renderers.VisitorSignature = None,
+                 initial_accumulator: typing.Any = None) \
+            -> typing.Generator[typing.Tuple[interfaces.renderers.SimpleTypes, ...], None, None]:
         """Populates the tree by consuming the TreeGrid's construction generator
            Func is called on every node, so can be used to create output on demand
 
@@ -145,11 +152,11 @@ class TreeGrid(interfaces.renderers.TreeGrid):
         """
         accumulator = initial_accumulator
         if func is None:
-            def func(_x, _y):
+            def func(_x: interfaces.renderers.TreeNode, _y: typing.Any) -> typing.Any:
                 return None
 
         if not self.populated:
-            prev_nodes = []
+            prev_nodes = []  # type: typing.List[TreeNode]
             for (level, item) in self._generator:
                 parent_index = min(len(prev_nodes), level)
                 parent = prev_nodes[parent_index - 1] if parent_index > 0 else None
@@ -158,6 +165,7 @@ class TreeGrid(interfaces.renderers.TreeGrid):
                 accumulator = func(treenode, accumulator)
                 self._row_count += 1
         self._populated = True
+        return None
 
     @property
     def populated(self):

volatility/framework/renderers/text.py

@@ -4,7 +4,7 @@ from volatility.framework import interfaces
 from volatility.framework.renderers import format_hints
 
 
-def hex_bytes_as_text(value):
+def hex_bytes_as_text(value: bytes) -> str:
     """Renders HexBytes as text"""
     if not isinstance(value, bytes):
         raise TypeError("hex_bytes_as_text takes bytes not: {}".format(type(value)))
@@ -31,19 +31,20 @@ class QuickTextRenderer(interfaces.renderers.Renderer):
                       bytes: lambda x: x.decode("utf-8"),
                       'default': lambda x: "{}".format(x)}
 
-    def __init__(self, options = None):
+    def __init__(self, options = None) -> None:
         super().__init__(options)
 
     def get_render_options(self):
         pass
 
-    def render(self, grid):
+    def render(self, grid: interfaces.renderers.TreeGrid) -> None:
         # TODO: Docstrings
         # TODO: Improve text output
         outfd = sys.stdout
 
         for column in grid.columns:
-            outfd.write("\t{}".format(column.name))
+            # Ignore the type because namedtuples don't realize they have accessible attributes
+            outfd.write("\t{}".format(column.name))  # type: ignore
         outfd.write("\n")
 
         def visitor(node, accumulator):

volatility/framework/symbols/__init__.py

@@ -2,6 +2,7 @@ import collections
 import collections.abc
 import enum
 import logging
+import typing
 
 from volatility.framework import constants, exceptions, interfaces, objects, validity
 from volatility.framework.symbols import native, windows, linux
@@ -15,6 +16,12 @@ class SymbolType(enum.Enum):
     ENUM = 3
 
 
+SymbolSpaceReturnType = typing.TypeVar("SymbolSpaceReturnType",
+                                       interfaces.objects.Template,
+                                       interfaces.symbols.Symbol,
+                                       typing.Dict[str, typing.Any])
+
+
 class SymbolSpace(interfaces.symbols.SymbolSpaceInterface, validity.ValidityRoutines):
     """Handles an ordered collection of SymbolTables
 
@@ -22,13 +29,13 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface, validity.ValidityRout
        proceed down through the ranks if a namespace isn't specified.
     """
 
-    def __init__(self):
+    def __init__(self) -> None:
         super().__init__()
-        self._dict = collections.OrderedDict()
+        self._dict = collections.OrderedDict()  # type: typing.Dict[str, interfaces.symbols.BaseSymbolTableInterface]
         # Permanently cache all resolved symbols
-        self._resolved = {}
+        self._resolved = {}  # type: typing.Dict[str, interfaces.objects.Template]
 
-    def free_table_name(self, prefix = "layer"):
+    def free_table_name(self, prefix: str = "layer") -> str:
         """Returns an unused table name to ensure no collision occurs when inserting a symbol table"""
         self._check_type(prefix, str)
 
@@ -39,39 +46,39 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface, validity.ValidityRout
 
     ### Symbol functions
 
-    def get_symbols_by_type(self, type_name):
+    def get_symbols_by_type(self, type_name: str) -> typing.Iterable[str]:
         """Returns all symbols based on the type of the symbol"""
         for table in self._dict.keys():
             for symbol_name in self._dict[table].get_symbols_by_type(type_name):
                 yield table + constants.BANG + symbol_name
 
-    def get_symbols_by_location(self, address, table_name = None):
+    def get_symbols_by_location(self, offset: int, table_name: str = None) -> typing.Iterable[str]:
         """Returns all symbols that exist at a specific relative address"""
-        table_list = self._dict.values()
+        table_list = self._dict.values()  # type: typing.Iterable[interfaces.symbols.BaseSymbolTableInterface]
         if table_name is not None:
             if table_name in self._dict:
                 table_list = [self._dict[table_name]]
             else:
                 table_list = []
         for table in table_list:
-            for symbol_name in self._dict[table].get_symbols_by_location(address = address):
-                yield table + constants.BANG + symbol_name
+            for symbol_name in table.get_symbols_by_location(offset = offset):
+                yield table.name + constants.BANG + symbol_name
 
     ### Space functions
 
-    def __len__(self):
+    def __len__(self) -> int:
         """Returns the number of tables within the space"""
         return len(self._dict)
 
-    def __getitem__(self, i):
+    def __getitem__(self, i: str) -> typing.Any:
         """Returns a specific table from the space"""
         return self._dict[i]
 
-    def __iter__(self):
+    def __iter__(self) -> typing.Iterator[str]:
         """Iterates through all available tables in the symbol space"""
         return iter(self._dict)
 
-    def append(self, value):
+    def append(self, value: interfaces.symbols.BaseSymbolTableInterface) -> None:
         """Adds a symbol_list to the end of the space"""
         if not isinstance(value, interfaces.symbols.BaseSymbolTableInterface):
             raise TypeError(value)
@@ -79,7 +86,7 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface, validity.ValidityRout
             self.remove(value.name)
         self._dict[value.name] = value
 
-    def remove(self, key):
+    def remove(self, key: str) -> None:
         """Removes a named symbol_list from the space"""
         # Reset the resolved list, since we're removing some symbols
         self._resolved = {}
@@ -98,11 +105,11 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface, validity.ValidityRout
            indicate this failure in resolution.
         """
 
-        def __init__(self, type_name = None, **kwargs):
+        def __init__(self, type_name: str = None, **kwargs) -> None:
             vollog.debug("Unresolved reference: {}".format(type_name))
             super().__init__(type_name = type_name, **kwargs)
 
-    def _weak_resolve(self, resolve_type, name):
+    def _weak_resolve(self, resolve_type: SymbolType, name: str) -> SymbolSpaceReturnType:
         """Takes a symbol name and resolves it with ReferentialTemplates"""
         if resolve_type == SymbolType.TYPE:
             get_function = 'get_type'
@@ -120,13 +127,10 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface, validity.ValidityRout
             try:
                 return getattr(self._dict[table_name], get_function)(component_name)
             except KeyError as e:
-                vollog.debug('Type {} references missing Type/Symbol/Enum: {}'.format(name, e))
-                return self._UnresolvedTemplate(name)
-            except exceptions.SymbolError:
-                return self._UnresolvedTemplate(name)
+                raise exceptions.SymbolError('Type {} references missing Type/Symbol/Enum: {}'.format(name, e))
         raise exceptions.SymbolError("Malformed name: {}".format(name))
 
-    def get_type(self, type_name):
+    def get_type(self, type_name: str) -> interfaces.objects.Template:
         """Takes a symbol name and resolves it
 
            This method ensures that all referenced templates (including self-referential templates)
@@ -134,7 +138,7 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface, validity.ValidityRout
         """
         # Traverse down any resolutions
         if type_name not in self._resolved:
-            self._resolved[type_name] = self._weak_resolve(SymbolType.TYPE, type_name)
+            self._resolved[type_name] = self._weak_resolve(SymbolType.TYPE, type_name)  # type: ignore
             traverse_list = [type_name]
             replacements = set()
             # Whole Symbols that still need traversing
@@ -149,8 +153,11 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface, validity.ValidityRout
                             # to the "symbols that still need traversing" list
                             if child.vol.type_name not in self._resolved:
                                 traverse_list.append(child.vol.type_name)
-                                self._resolved[child.vol.type_name] = self._weak_resolve(SymbolType.TYPE,
-                                                                                         child.vol.type_name)
+                                try:
+                                    self._resolved[child.vol.type_name] = self._weak_resolve(SymbolType.TYPE,
+                                                                                             child.vol.type_name)
+                                except exceptions.SymbolError:
+                                    self._resolved[child.vol.type_name] = self._UnresolvedTemplate(child.vol.type_name)
                             # Stash the replacement
                             replacements.add((traverser, child))
                         elif child.children:
@@ -161,18 +168,23 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface, validity.ValidityRout
             raise exceptions.SymbolError("Unresolvable symbol requested: {}".format(type_name))
         return self._resolved[type_name]
 
-    def get_symbol(self, symbol_name):
+    def get_symbol(self, symbol_name: str) -> interfaces.symbols.Symbol:
         """Look-up a symbol name across all the contained symbol spaces"""
-        return self._weak_resolve(SymbolType.SYMBOL, symbol_name)
+        retval = self._weak_resolve(SymbolType.SYMBOL, symbol_name)
+        if not isinstance(retval, interfaces.symbols.Symbol):
+            raise exceptions.SymbolError("Unresolvable Symbol: {}".format(symbol_name))
+        return retval
 
-    def get_enumeration(self, enum_name):
+    def get_enumeration(self, enum_name: str) -> typing.Dict[str, typing.Any]:
         """Look-up a set of enumeration choices from a specific symbol table"""
-        return self._weak_resolve(SymbolType.ENUM, enum_name)
+        retval = self._weak_resolve(SymbolType.ENUM, enum_name)
+        if not isinstance(retval, dict):
+            raise exceptions.SymbolError("Unresolvable Enumeration: {}".format(enum_name))
+        return retval
 
-    def _membership(self, member_type, name):
+    def _membership(self, member_type: SymbolType, name: str) -> bool:
         """Test for membership of a component within a table"""
 
-        table = []
         name_array = name.split(constants.BANG)
         if len(name_array) == 2:
             table_name = name_array[0]
@@ -180,8 +192,9 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface, validity.ValidityRout
         else:
             return False
 
-        if table_name in self:
-            table = self[table_name]
+        if table_name not in self:
+            return False
+        table = self[table_name]
 
         if member_type == SymbolType.TYPE:
             return component_name in table.types
@@ -191,11 +204,11 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface, validity.ValidityRout
             return component_name in table.enumerations
         return False
 
-    def has_type(self, name):
+    def has_type(self, name: str) -> bool:
         return self._membership(SymbolType.TYPE, name)
 
-    def has_symbol(self, name):
+    def has_symbol(self, name: str) -> bool:
         return self._membership(SymbolType.SYMBOL, name)
 
-    def has_enumeration(self, name):
+    def has_enumeration(self, name: str) -> bool:
         return self._membership(SymbolType.ENUM, name)