Initial implementation of the Intermediate Format, breaks the vtypes format and needs a lot of tidying.

volatility/cli/__init__.py

@@ -59,9 +59,10 @@ class CommandLine(object):
         ctx.config["ui.single_location"] = "file:///run/media/mike/disk/memory/private/jon-fres.dmp"
         ctx.config["plugins.pslist.offset"] = 0x01bcc830
 
-        ctx.config["plugins.pslist.ntkrnlmp.class"] = "volatility.framework.symbols.windows.WindowsKernelVTypeSymbols"
-        ctx.config["plugins.pslist.ntkrnlmp.vtype_pymodule"] = "volatility.framework.symbols.windows.xp_sp2_x86_vtypes"
-        ctx.config["plugins.pslist.ntkrnlmp.vtype_variable"] = "ntkrnlmp_types"
+        ctx.config[
+            "plugins.pslist.ntkrnlmp.class"] = "volatility.framework.symbols.windows.WindowsKernelIntermedSymbols"
+        ctx.config[
+            "plugins.pslist.ntkrnlmp.idd_filepath"] = "file:///home/mike/workspace/volatility3/aux/ntoskrnl.pdb.json"
 
         ###
         # BACK TO THE FRAMEWORK

volatility/framework/symbols/__init__.py

@@ -105,7 +105,7 @@ class SymbolSpace(collections.abc.Mapping):
             return getattr(self._dict[table_name], get_function)(component_name)
         elif name in self.natives.types:
             return getattr(self.natives, get_function)(name)
-        raise exceptions.SymbolError("Malformed symbol name")
+        raise exceptions.SymbolError("Malformed symbol name: " + repr(name))
 
     def get_type(self, type_name):
         """Takes a symbol name and resolves it

volatility/framework/symbols/intermed.py

@@ -0,0 +1,106 @@
+import copy
+import json
+import logging
+import urllib.parse
+
+from volatility.framework import constants, exceptions, interfaces, objects
+
+vollog = logging.getLogger(__name__)
+
+
+class IntermediateSymbolTable(interfaces.symbols.SymbolTableInterface):
+    """Class for storing intermediate debugging data as objects and classes"""
+
+    def __init__(self, name, idd_filepath, native_types = None):
+        super().__init__(name, native_types)
+        url = urllib.parse.urlparse(idd_filepath)
+        if url.scheme != 'file':
+            raise NotImplementedError("The {0} scheme is not yet implement for the Intermediate Symbol Format.")
+        with open(url.path, "r") as fp:
+            self._json = json.load(fp)
+        self._validate_json()
+        self._overrides = {}
+
+    def _validate_json(self):
+        if (not 'user_types' in self._json or
+                not 'base_types' in self._json or
+                not 'metadata' in self._json or
+                not 'symbols' in self._json or
+                not 'enums' in self._json):
+            raise exceptions.SymbolSpaceError("Malformed JSON file provided")
+
+    def get_type_class(self, name):
+        return self._overrides.get(name, objects.Struct)
+
+    def set_type_class(self, name, clazz):
+        if name not in self.types:
+            raise ValueError("Symbol type " + name + " not in " + self.name + " SymbolTable")
+        self._overrides[name] = clazz
+
+    def del_type_class(self, name):
+        if name in self._overrides:
+            del self._overrides[name]
+
+    @property
+    def types(self):
+        """Returns an iterator of the symbol names"""
+        return self._json.get('user_types', {})
+
+    def _interdict_to_template(self, dictionary):
+        """Converts an intermediate format dict into an object template"""
+        if not dictionary:
+            raise exceptions.SymbolSpaceError("Invalid intermediate dictionary: " + repr(dictionary))
+
+        type_name = dictionary['kind']
+        if type_name == 'base':
+            type_name = dictionary['name']
+
+        if type_name in self.natives.types:
+            # The symbol is a native type
+            native_template = self.natives.get_type(type_name)
+
+            # Add specific additional parameters, etc
+            update = {}
+            if type_name == 'array':
+                update['count'] = dictionary['count']
+                update['target'] = self._interdict_to_template(dictionary['subtype'])
+            elif type_name == 'pointer':
+                update["target"] = self._interdict_to_template(dictionary['subtype'])
+            elif type_name == 'enum':
+                update = self._lookup_enum(dictionary['name'])
+            elif type_name == 'bitfield':
+                update = {'start_bit': dictionary['bit_position'], 'end_bit': dictionary['bit_length']}
+                update['target'] = self._interdict_to_template(dictionary['type'])
+            native_template.update_vol(**update)  # pylint: disable=W0142
+            return native_template
+
+        # Otherwise
+        if dictionary['kind'] not in ['struct', 'union']:
+            raise exceptions.SymbolSpaceError("Unknown Intermediate format: " + repr(dictionary))
+
+        return objects.templates.ReferenceTemplate(type_name = self.name + constants.BANG + dictionary['name'])
+
+    def _lookup_enum(self, name):
+        """Looks up an enumeration and returns a dictionary of __init__ parameters for an Enum"""
+        lookup = self._json['enums'].get(name, None)
+        if not lookup:
+            raise exceptions.SymbolSpaceError("Unknown enumeration found: " + repr(name))
+        result = {"choices": copy.deepcopy(lookup['constants']),
+                  "target": self.natives.get_type(lookup['base'])}
+        return result
+
+    def get_type(self, type_name):
+        """Resolves an individual symbol"""
+        if type_name not in self._json['user_types']:
+            raise exceptions.SymbolError("Unknown symbol:" + repr(type_name))
+        curdict = self._json['user_types'][type_name]
+        members = {}
+        for member_name in curdict['fields']:
+            interdict = curdict['fields'][member_name]
+            member = (interdict['offset'], self._interdict_to_template(interdict['type']))
+            members[member_name] = member
+        object_class = self.get_type_class(type_name)
+        return objects.templates.ObjectTemplate(type_name = self.name + constants.BANG + type_name,
+                                                object_class = object_class,
+                                                size = curdict['length'],
+                                                members = members)

volatility/framework/symbols/native.py

@@ -20,7 +20,7 @@ class NativeTable(interfaces.symbols.NativeTableInterface):
             self._overrides[native_type] = native_class
         # Create this once early, because it may get used a lot
         self._types = set(self._native_dictionary).union(
-            {'Enumeration', 'array', 'BitField', 'void', 'pointer', 'String', 'Bytes'})
+            {'enum', 'array', 'bitfield', 'void', 'pointer', 'string', 'bytes', 'function'})
 
     def get_type_class(self, name):
         ntype, fmt = native_types.get(name, (objects.Integer, ''))
@@ -39,21 +39,21 @@ class NativeTable(interfaces.symbols.NativeTableInterface):
         # NOTE: These need updating whenever the object init signatures change
         additional = {}
         obj = None
-        if type_name == 'void':
+        if type_name == 'void' or type_name == 'function':
             obj = objects.Void
         elif type_name == 'array':
             obj = objects.Array
             additional = {"count": 0, "target": self.get_type('void')}
-        elif type_name == 'Enumeration':
+        elif type_name == 'enum':
             obj = objects.Enumeration
             additional = {"target": self.get_type('void'), "choices": {}}
-        elif type_name == 'BitField':
+        elif type_name == 'bitfield':
             obj = objects.BitField
             additional = {"start_bit": 0, "end_bit": 0}
-        elif type_name == 'String':
+        elif type_name == 'string':
             obj = objects.String
             additional = {"max_length": 0}
-        elif type_name == 'Bytes':
+        elif type_name == 'bytes':
             obj = objects.Bytes
             additional = {"length": 0}
         if obj is not None:

volatility/framework/symbols/windows/__init__.py

@@ -1,5 +1,5 @@
 from volatility.framework.configuration import requirements
-from volatility.framework.symbols import vtypes
+from volatility.framework.symbols import vtypes, intermed
 from volatility.framework.symbols.windows import extensions
 
 __author__ = 'mike'
@@ -21,3 +21,18 @@ class WindowsKernelVTypeSymbols(vtypes.VTypeSymbolTable):
         return [requirements.StringRequirement("vtype_pymodule", description = "Python module containing the vtypes"),
                 requirements.StringRequirement("vtype_variable",
                                                description = "Python vtypes variable within the module")]
+
+
+class WindowsKernelIntermedSymbols(intermed.IntermediateSymbolTable):
+    provides = {"type": "interface"}
+
+    def __init__(self, context, config_path, name, idd_filepath):
+        super().__init__(name = name, idd_filepath = idd_filepath, native_types = context.symbol_space.natives)
+
+        # Set-up windows specific types
+        self.set_type_class('_ETHREAD', extensions._ETHREAD)
+        self.set_type_class('_LIST_ENTRY', extensions._LIST_ENTRY)
+
+    @classmethod
+    def get_requirements(cls):
+        return [requirements.StringRequirement("idd_filepath", description = "JSON file containnig the symbols")]

volatility/plugins/windows/pslist.py

@@ -34,7 +34,7 @@ class PsList(plugins.PluginInterface):
     def _generator(self, eproc):
         for proc in eproc.ActiveProcessLinks:
             yield (0, (proc.UniqueProcessId, proc.InheritedFromUniqueProcessId,
-                       proc.ImageFileName.cast("String", max_length = proc.ImageFileName.vol.count,
+                       proc.ImageFileName.cast("string", max_length = proc.ImageFileName.vol.count,
                                                errors = 'replace')))
 
     def run(self):