From a72314a2b2c64a22cc4383cfa56048cfc153cef2 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Mon, 8 Jul 2019 00:31:07 +0100 Subject: [PATCH] Add in omap mapping and symbol support. --- volatility/framework/layers/msf.py | 6 +- .../framework/symbols/windows/mspdb.json | 416 ++++++++++++++++++ volatility/framework/symbols/windows/mspdb.py | 241 +++++++--- 3 files changed, 603 insertions(+), 60 deletions(-) diff --git a/volatility/framework/layers/msf.py b/volatility/framework/layers/msf.py index 80cb06ef5..c89117b41 100644 --- a/volatility/framework/layers/msf.py +++ b/volatility/framework/layers/msf.py @@ -8,7 +8,7 @@ from volatility.framework.symbols import intermed class PdbMSF(interfaces.layers.TranslationLayerInterface): - headers = { + _headers = { "MSF_HDR": "Microsoft C/C++ program database 2.00\r\n\x1a\x4a\x47", "BIG_MSF_HDR": "Microsoft C/C++ MSF 7.00\r\n\x1a\x44\x53", } @@ -93,10 +93,10 @@ class PdbMSF(interfaces.layers.TranslationLayerInterface): def _check_header(self) -> Optional[Tuple[str, interfaces.objects.ObjectInterface]]: """Verifies the header of the PDB file and returns the version of the file""" - for header in self.headers: + for header in self._headers: header_type = self.pdb_symbol_table + constants.BANG + header current_header = self.context.object(header_type, self._base_layer, 0) - if utility.array_to_string(current_header.Magic) == self.headers[header]: + if utility.array_to_string(current_header.Magic) == self._headers[header]: if not (current_header.PageSize < 0x100 or current_header.PageSize > (128 * 0x10000)): return header, current_header return None diff --git a/volatility/framework/symbols/windows/mspdb.json b/volatility/framework/symbols/windows/mspdb.json index ec3cc8920..0c21d4c1a 100644 --- a/volatility/framework/symbols/windows/mspdb.json +++ b/volatility/framework/symbols/windows/mspdb.json @@ -417,6 +417,360 @@ "kind": "struct", "size": 56 }, + "DBI_HEADER": { + "fields": { + "magic": { + "offset": 0, + "type": { + "count": 4, + "kind": "array", + "subtype": { + "kind": "base", + "name": "unsigned char" + } + } + }, + "version": { + "offset": 4, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "age": { + "offset": 8, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "gssymStream": { + "offset": 12, + "type": { + "kind": "base", + "name": "unsigned short" + } + }, + "vers": { + "offset": 14, + "type": { + "kind": "base", + "name": "unsigned short" + } + }, + "pssymStream": { + "offset": 16, + "type": { + "kind": "base", + "name": "unsigned short" + } + }, + "pdbver": { + "offset": 18, + "type": { + "kind": "base", + "name": "unsigned short" + } + }, + "symrecStream": { + "offset": 20, + "type": { + "kind": "base", + "name": "unsigned short" + } + }, + "pdbver2": { + "offset": 22, + "type": { + "kind": "base", + "name": "unsigned short" + } + }, + "module_size": { + "offset": 24, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "secconSize": { + "offset": 28, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "secmapSize": { + "offset": 32, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "filinfSize": { + "offset": 36, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "tsmapSize": { + "offset": 40, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "mfcIndex": { + "offset": 44, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "dbghdrSize": { + "offset": 48, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "ecinfoSize": { + "offset": 52, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "flags": { + "offset": 56, + "type": { + "kind": "base", + "name": "unsigned short" + } + }, + "reserved": { + "offset": 60, + "type": { + "kind": "base", + "name": "unsigned long" + } + } + }, + "kind": "struct", + "size": 64 + }, + "DBI_DBG_HEADER": { + "fields": { + "snFPO": { + "offset": 0, + "type": { + "kind": "base", + "name": "short" + } + }, + "snException": { + "offset": 2, + "type": { + "kind": "base", + "name": "short" + } + }, + "snFixup": { + "offset": 4, + "type": { + "kind": "base", + "name": "short" + } + }, + "snOmapToSrc": { + "offset": 6, + "type": { + "kind": "base", + "name": "short" + } + }, + "snOmapFromSrc": { + "offset": 8, + "type": { + "kind": "base", + "name": "short" + } + }, + "snSectionHdr": { + "offset": 10, + "type": { + "kind": "base", + "name": "short" + } + }, + "snTokenRidMap": { + "offset": 12, + "type": { + "kind": "base", + "name": "short" + } + }, + "snXdata": { + "offset": 14, + "type": { + "kind": "base", + "name": "short" + } + }, + "snPdata": { + "offset": 16, + "type": { + "kind": "base", + "name": "short" + } + }, + "snNewFPO": { + "offset": 18, + "type": { + "kind": "base", + "name": "short" + } + }, + "snSectionHdrOrig": { + "offset": 20, + "type": { + "kind": "base", + "name": "short" + } + } + }, + "kind": "struct", + "size": 22 + }, + "GLOBAL_SYMBOL": { + "fields": { + "length": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned short" + } + }, + "leaf_type": { + "offset": 2, + "type": { + "kind": "base", + "name": "unsigned short" + } + }, + "symtype": { + "offset": 4, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "offset": { + "offset": 8, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "segment": { + "offset": 12, + "type": { + "kind": "base", + "name": "unsigned short" + } + }, + "name": { + "offset": 14, + "type": { + "kind": "base", + "name": "string" + } + } + }, + "kind": "struct", + "size": 14 + }, + "IMAGE_SECTION_HEADER": { + "fields": { + "Characteristics": { + "offset": 36, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "Misc": { + "offset": 8, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "Name": { + "offset": 0, + "type": { + "count": 8, + "kind": "array", + "subtype": { + "kind": "base", + "name": "unsigned char" + } + } + }, + "NumberOfLinenumbers": { + "offset": 34, + "type": { + "kind": "base", + "name": "unsigned short" + } + }, + "NumberOfRelocations": { + "offset": 32, + "type": { + "kind": "base", + "name": "unsigned short" + } + }, + "PointerToLinenumbers": { + "offset": 28, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "PointerToRawData": { + "offset": 20, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "PointerToRelocations": { + "offset": 24, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "SizeOfRawData": { + "offset": 16, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "VirtualAddress": { + "offset": 12, + "type": { + "kind": "base", + "name": "unsigned long" + } + } + }, + "kind": "struct", + "size": 40 + }, "LF_ARRAY": { "fields": { "element_type": { @@ -903,9 +1257,65 @@ }, "kind": "struct", "size": 12 + }, + "OMAP_RECORD": { + "fields": { + "source_address": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "target_address": { + "offset": 4, + "type": { + "kind": "base", + "name": "unsigned long" + } + } + }, + "kind": "struct", + "size": 8 } }, "enums": { + "MACHINE_TYPE": { + "base": "unsigned short", + "constants": { + "IMAGE_FILE_MACHINE_UNKNOWN": 0, + "IMAGE_FILE_MACHINE_I386": 332, + "IMAGE_FILE_MACHINE_R3000": 354, + "IMAGE_FILE_MACHINE_R4000": 358, + "IMAGE_FILE_MACHINE_R10000": 360, + "IMAGE_FILE_MACHINE_WCEMIPSV2": 361, + "IMAGE_FILE_MACHINE_ALPHA": 388, + "IMAGE_FILE_MACHINE_SH3": 418, + "IMAGE_FILE_MACHINE_SH3DSP": 419, + "IMAGE_FILE_MACHINE_SH3E": 420, + "IMAGE_FILE_MACHINE_SH4": 422, + "IMAGE_FILE_MACHINE_SH5": 424, + "IMAGE_FILE_MACHINE_ARM": 448, + "IMAGE_FILE_MACHINE_THUMB": 450, + "IMAGE_FILE_MACHINE_ARMNT": 452, + "IMAGE_FILE_MACHINE_AM33": 467, + "IMAGE_FILE_MACHINE_POWERPC": 496, + "IMAGE_FILE_MACHINE_POWERPCFP": 497, + "IMAGE_FILE_MACHINE_IA64": 512, + "IMAGE_FILE_MACHINE_MIPS16": 614, + "IMAGE_FILE_MACHINE_ALPHA64": 644, + "IMAGE_FILE_MACHINE_AXP64": 644, + "IMAGE_FILE_MACHINE_MIPSFPU": 870, + "IMAGE_FILE_MACHINE_MIPSFPU16": 1126, + "IMAGE_FILE_MACHINE_TRICORE": 1312, + "IMAGE_FILE_MACHINE_CEF": 3311, + "IMAGE_FILE_MACHINE_EBC": 3772, + "IMAGE_FILE_MACHINE_AMD64": 34404, + "IMAGE_FILE_MACHINE_M32R": 36929, + "IMAGE_FILE_MACHINE_CEE": 49390 + }, + "size": 2 + }, "LEAF_TYPE": { "base": "unsigned short", "constants": { @@ -1096,6 +1506,12 @@ "signed": false, "size": 2 }, + "short": { + "endian": "little", + "kind": "int", + "signed": true, + "size": 2 + }, "long": { "endian": "little", "kind": "int", diff --git a/volatility/framework/symbols/windows/mspdb.py b/volatility/framework/symbols/windows/mspdb.py index 3beb2c136..63406596d 100644 --- a/volatility/framework/symbols/windows/mspdb.py +++ b/volatility/framework/symbols/windows/mspdb.py @@ -1,12 +1,16 @@ import argparse import json +import logging import os -from typing import Tuple, Dict, Any, Optional +from bisect import bisect +from typing import Tuple, Dict, Any, Optional, Union, List from urllib import request -from volatility.framework import contexts, interfaces +from volatility.framework import contexts, interfaces, constants from volatility.framework.layers import physical, msf +vollog = logging.getLogger(__name__) + primatives = { 0x03: ("void", { "endian": "little", @@ -229,10 +233,14 @@ class PdbReader: def __init__(self, context: interfaces.context.ContextInterface, location: str): self._layer_name, self._context = self.load_pdb_layer(context, location) + self._dbiheader = None self.types = [] self.bases = {} self.user_types = {} self.enumerations = {} + self.symbols = {} + self.omap_mapping = [] + self.sections = [] @property def context(self): @@ -271,7 +279,16 @@ class PdbReader: return msf_layer_name, new_context - def read_tpi_stream(self): + def reset(self): + self.bases = {} + self.user_types = {} + self.enumerations = {} + self.symbols = {} + self.sections = [] + self.omap_mapping = [] + + def read_tpi_stream(self) -> None: + print("Reading TPI") tpi_layer = self._context.layers.get(self._layer_name + "_stream2", None) if not tpi_layer: raise ValueError("No TPI stream available") @@ -289,10 +306,6 @@ class PdbReader: # Reset the state self.types = [] - self.bases = {} - self.user_types = {} - self.enumerations = {} - type_references = {} offset = header.header_size @@ -318,6 +331,72 @@ class PdbReader: if tpi_layer.maximum_address - offset != 0: raise ValueError("Type values did not fill the TPI stream correctly") + self.process_types(type_references) + + def read_dbi_stream(self) -> None: + print("Reading DBI") + dbi_layer = self._context.layers.get(self._layer_name + "_stream3", None) + if not dbi_layer: + raise ValueError("No DBI stream available") + module = self._context.module(module_name = dbi_layer.pdb_symbol_table, layer_name = dbi_layer.name, offset = 0) + self._dbiheader = module.object(type_name = "DBI_HEADER", offset = 0) + + # Skip past sections we don't care about to get to the DBG header + dbg_hdr_offset = (self._dbiheader.vol.size + self._dbiheader.module_size + self._dbiheader.secconSize + + self._dbiheader.secmapSize + self._dbiheader.filinfSize + self._dbiheader.tsmapSize + + self._dbiheader.ecinfoSize) + self._dbidbgheader = module.object(type_name = "DBI_DBG_HEADER", offset = dbg_hdr_offset) + + self.sections = [] + self.omap_mapping = [] + + if self._dbidbgheader.snSectionHdrOrig != -1: + section_orig_layer_name = self._layer_name + "_stream" + str(self._dbidbgheader.snSectionHdrOrig) + consumed, length = 0, self.context.layers[section_orig_layer_name].maximum_address + while consumed < length: + section = self.context.object( + dbi_layer.pdb_symbol_table + constants.BANG + "IMAGE_SECTION_HEADER", + offset = consumed, + layer_name = section_orig_layer_name) + self.sections.append(section) + consumed += section.vol.size + + if self._dbidbgheader.snOmapFromSrc != -1: + omap_layer_name = self._layer_name + "_stream" + str(self._dbidbgheader.snOmapFromSrc) + length = self.context.layers[omap_layer_name].maximum_address + data = self.context.layers[omap_layer_name].read(0, length) + # For speed we don't use the framework to read this (usually sizeable) data + for i in range(0, length, 8): + self.omap_mapping.append((int.from_bytes(data[i:i + 4], byteorder = 'little'), + int.from_bytes(data[i + 4:i + 8], byteorder = 'little'))) + elif self._dbidbgheader.snSectionHdr != -1: + section_layer_name = self._layer_name + "_stream" + str(self._dbidbgheader.snSectionHdr) + consumed, length = 0, self.context.layers[section_layer_name].maximum_address + while consumed < length: + section = self.context.object( + dbi_layer.pdb_symbol_table + constants.BANG + "IMAGE_SECTION_HEADER", + offset = consumed, + layer_name = section_layer_name) + self.sections.append(section) + consumed += section.vol.size + + def omap_lookup(self, address): + """Looks up an address using the omap mapping""" + pos = bisect(self.omap_mapping, (address, 0)) + if self.omap_mapping[pos][0] != address: + pos = pos - 1 + + if not self.omap_mapping[pos][1]: + return 0 + return self.omap_mapping[pos][1] + (address - self.omap_mapping[pos][0]) + + def process_types(self, type_references: Dict[str, int]) -> None: + """Reads the TPI and symbol streams to populate the reader's variables""" + + self.bases = {} + self.user_types = {} + self.enumerations = {} + for index in range(len(self.types)): leaf_type, name, value = self.types[index] if leaf_type in [ @@ -350,36 +429,62 @@ class PdbReader: # Re-run through for ForwardSizeReferences self.user_types = self.replace_forward_references(self.user_types, type_references) - with open("file.out", "w") as f: - json.dump(self.get_json(), f, indent = 2, sort_keys = True) + def read_symbol_stream(self): + """Reads in the symbol stream""" + self.symbols = {} - return header + if not self._dbiheader: + self.read_dbi_stream() + + print("Reading Symbols") + + symrec_layer = self._context.layers.get(self._layer_name + "_stream" + str(self._dbiheader.symrecStream), None) + if not symrec_layer: + raise ValueError("No SymRec stream available") + module = self._context.module( + module_name = symrec_layer.pdb_symbol_table, layer_name = symrec_layer.name, offset = 0) + + offset = 0 + max_address = symrec_layer.maximum_address + + while offset < max_address: + sym = module.object(type_name = "GLOBAL_SYMBOL", offset = offset) + leaf_type = module.object(type_name = "unsigned short", offset = sym.leaf_type.vol.offset) + name = None + if leaf_type == 0x110e: + # v3 symbol (c-string) + name = self.parse_string(sym.name, False, sym.length - sym.vol.size - 4) + address = self.sections[sym.segment - 1].VirtualAddress + sym.offset + elif leaf_type == 0x1009: + # v2 symbol (pascal-string) + name = self.parse_string(sym.name, True, sym.length - sym.vol.size - 4) + address = self.sections[sym.segment - 1].VirtualAddress + sym.offset + else: + vollog.warning("Only v2 and v3 symbols are supported") + if name: + if self.omap_mapping: + address = self.omap_lookup(address) + self.symbols[name] = {"address": address} + offset += sym.length + 2 # Add on length itself + + def read_necessary_streams(self): + if not self.user_types: + self.read_tpi_stream() + if not self.symbols: + self.read_symbol_stream() def get_json(self): - return {"user_types": self.user_types, "enums": self.enumerations, "base_types": self.bases} + """Returns the intermediate format JSON data from this pdb file""" + self.read_necessary_streams() - def replace_forward_references(self, types, type_references): - """Finds all ForwardArrayCounts and calculates them one ForwardReferences have been resolved""" - if isinstance(types, dict): - for k, v in types.items(): - types[k] = self.replace_forward_references(v, type_references) - elif isinstance(types, list): - new_types = [] - for v in types: - new_types.append(self.replace_forward_references(v, type_references)) - types = new_types - elif isinstance(types, ForwardArrayCount): - element_type = types.element_type - # If we're a forward array count, we need to do the calculation now after all the types have been processed - if element_type > 0x1000: - _, name, _ = self.types[types.element_type - 0x1000] - # If there's no name, the original size is probably fine - if name: - element_type = type_references[name] + 0x1000 - return types.size // self.get_size_from_index(element_type) - return types + return { + "user_types": self.user_types, + "enums": self.enumerations, + "base_types": self.bases, + "symbols": self.symbols + } - def get_type_from_index(self, index: int) -> Dict[str, Any]: + def get_type_from_index(self, index: int) -> Union[List[Dict[str, Any]], Dict[str, Any]]: """Takes a type index and returns appropriate dictionary""" if index < 0x1000: base_name, base = primatives[index & 0xff] @@ -424,6 +529,7 @@ class PdbReader: return result def get_size_from_index(self, index: int) -> int: + """Returns the size of the structure based on the type index provided""" if index < 0x1000: if (index & 0xf00): _, base = indirections[index & 0xf00] @@ -511,7 +617,8 @@ class PdbReader: consumed = length elif leaf_type in [leaf_type.LF_ARGLIST, leaf_type.LF_ENUM]: enum = module.object(type_name = "LF_ENUM", offset = offset) - name = self.parse_string(leaf_type, enum.name, size = length - enum.vol.size - consumed) + name = self.parse_string( + enum.name, leaf_type < leaf_type.LF_ST_MAX, size = length - enum.vol.size - consumed) enum.name = name result = leaf_type, name, enum consumed = length @@ -524,7 +631,8 @@ class PdbReader: consumed += enum.vol.size + len(name) + 1 + excess elif leaf_type in [leaf_type.LF_UNION]: union = module.object(type_name = "LF_UNION", offset = offset) - name = self.parse_string(leaf_type, union.name, size = length - union.vol.size - consumed) + name = self.parse_string( + union.name, leaf_type < leaf_type.LF_ST_MAX, size = length - union.vol.size - consumed) result = leaf_type, name, union consumed = length else: @@ -532,6 +640,13 @@ class PdbReader: return result, consumed + def consume_padding(self, layer_name: str, offset: int) -> int: + """Returns the amount of padding used between fields""" + val = self.context.layers[layer_name].read(offset, 1) + if not (int(val[0]) & 0xf0): + return 0 + return (int(val[0]) & 0x0f) + def determine_extended_value(self, leaf_type: interfaces.objects.ObjectInterface, value: interfaces.objects.ObjectInterface, module: interfaces.context.ModuleInterface, length: int) -> Tuple[str, interfaces.objects.ObjectInterface, int]: @@ -559,29 +674,11 @@ class PdbReader: excess = value.vol.data_format.length # Updated the consume/offset counters name = module.object(type_name = "string", offset = value.vol.offset + value.vol.data_format.length) - name = self.parse_string(leaf_type, name, size = length) + name = self.parse_string(name, leaf_type < leaf_type.LF_ST_MAX, size = length) return name, value, excess - def consume_padding(self, layer_name: str, offset: int) -> int: - """Returns the amount of padding used between fields""" - val = self.context.layers[layer_name].read(offset, 1) - if not (int(val[0]) & 0xf0): - return 0 - return (int(val[0]) & 0x0f) - - def parse_string(self, - leaf_type: interfaces.objects.ObjectInterface, - structure: interfaces.objects.ObjectInterface, - size: int = 0) -> str: - """Consumes either a c-string or a pascal string depending on the leaf_type""" - if leaf_type > leaf_type.LF_ST_MAX: - name = structure.cast("string", max_length = size, encoding = "latin-1") - else: - name = structure.cast("pascal_string") - name = name.string.cast("string", max_length = name.length, encoding = "latin-1") - return name - def convert_fields(self, fields: int): + """Converts a field list into a list of fields""" result = {} _, _, fields_struct = self.types[fields] for field in fields_struct: @@ -589,6 +686,37 @@ class PdbReader: result[name] = {"offset": member.offset, "type": self.get_type_from_index(member.field_type)} return result + def replace_forward_references(self, types, type_references): + """Finds all ForwardArrayCounts and calculates them one ForwardReferences have been resolved""" + if isinstance(types, dict): + for k, v in types.items(): + types[k] = self.replace_forward_references(v, type_references) + elif isinstance(types, list): + new_types = [] + for v in types: + new_types.append(self.replace_forward_references(v, type_references)) + types = new_types + elif isinstance(types, ForwardArrayCount): + element_type = types.element_type + # If we're a forward array count, we need to do the calculation now after all the types have been processed + if element_type > 0x1000: + _, name, _ = self.types[types.element_type - 0x1000] + # If there's no name, the original size is probably fine + if name: + element_type = type_references[name] + 0x1000 + return types.size // self.get_size_from_index(element_type) + return types + + def parse_string(self, structure: interfaces.objects.ObjectInterface, parse_as_pascal: bool = False, + size: int = 0) -> str: + """Consumes either a c-string or a pascal string depending on the leaf_type""" + if not parse_as_pascal: + name = structure.cast("string", max_length = size, encoding = "latin-1") + else: + name = structure.cast("pascal_string") + name = name.string.cast("string", max_length = name.length, encoding = "latin-1") + return name + if __name__ == '__main__': @@ -603,6 +731,5 @@ if __name__ == '__main__': reader = PdbReader(ctx, location) - ### TESTING - # x = ctx.object('pdb1!BIG_MSF_HDR', reader.pdb_layer_name, 0) - header = reader.read_tpi_stream() + with open("file.out", "w") as f: + json.dump(reader.get_json(), f, indent = 2, sort_keys = True)