diff --git a/development/dtbfinder.py b/development/dtbfinder.py deleted file mode 100644 index 7a267a92c..000000000 --- a/development/dtbfinder.py +++ /dev/null @@ -1,122 +0,0 @@ -import statistics -import sys - -# TODO: Rather nasty hack, when volatility's actually installed this would be unnecessary -sys.path += ".." - -from volatility.framework import contexts -from volatility.framework import interfaces -from volatility.framework import layers -from volatility.framework.automagic import windows - -if __name__ == '__main__': - import argparse - - parser = argparse.ArgumentParser() - parser.add_argument("filenames", metavar = "FILE", nargs = "+", action = "store", help = "FILE to read for testing") - parser.add_argument("--32bit", action = "store_false", dest = "bit32", help = "Disable 32-bit scanning") - parser.add_argument("--64bit", action = "store_false", dest = "bit64", help = "Disable 64-bit scanning") - parser.add_argument("--pae", action = "store_false", dest = "pae", help = "Disable pae scanning") - parser.add_argument("-l", "--lime", action = "store_true", dest = "lime", help = "All files are LIME format") - parser.add_argument( - "-s", - "--selfref", - action = "store_true", - dest = "selfref", - help = "Run more generic self-referential tests scanner") - parser.add_argument( - "-v", "--verbose", action = "count", default = 0, help = "Increase the verbosity of the information returned") - - args = parser.parse_args() - - ctx = contexts.Context() - for filename in args.filenames: - ctx.config[interfaces.configuration.path_join('config' + str(args.filenames.index(filename)), - "filename")] = filename - data = layers.physical.FileLayer(ctx, 'config' + str(args.filenames.index(filename)), - 'data' + str(args.filenames.index(filename))) - ctx.layers.add_layer(data) - if args.lime: - ctx.config[interfaces.configuration.path_join('lime-config' + str(args.filenames.index(filename)), - "base_layer")] = 'data' + str(args.filenames.index(filename)) - data = layers.lime.LimeLayer(ctx, 'lime-config' + str(args.filenames.index(filename)), - 'lime-data' + str(args.filenames.index(filename))) - ctx.layers.add_layer(data) - - layername = 'data' - if args.lime: - layername = 'lime-data' - - tests = [] - if args.selfref: - if args.bit32: - tests.append(windows.DtbSelfRef32bit()) - if args.bit64: - tests.append(windows.DtbSelfRef64bit()) - else: - if args.bit32: - tests.append(windows.DtbTest32bit()) - if args.bit64: - tests.append(windows.DtbTest64bit()) - if args.pae: - tests.append(windows.DtbTestPae()) - - if tests: - for i in range(len(args.filenames)): - print("[*] Scanning " + args.filenames[i] + "...") - scan_results = ctx.layers[layername + str(i)].scan(ctx, windows.PageMapScanner(tests)) - - # Self-referential tests need post-processing to gather the most likely offset - if args.selfref: - selfref_results = dict([(test, dict()) for test in tests]) - for test, result in scan_results: - dtb, refs = result - test_dict = selfref_results[test] - for ref in refs: - # Initialize the value if necessary - tmp = test_dict.get(ref, set()) - tmp.add(dtb) - test_dict[ref] = tmp - selfref_results[test] = test_dict - scan_results = [] - print(" Self-referential data") - # Sort by largest pointer size, since the self-ref finder will find larger sizes when searching for smaller) - for test in sorted(selfref_results, key = lambda x: -x.ptr_size): - best_found = None - print(" " + test.layer_type.__name__ + ": ") - - test_dict = selfref_results[test] - for ref in sorted(test_dict, key = lambda x: -len(test_dict[x])): - # Most self-referential DTBs should turn up multiple times because multiple processes should have their own DTB - if len(test_dict[ref]) < 2: - continue - if best_found is None: - # Most processes are spread out across significantly different pages - # Therefore the standard deviation should be significant - if statistics.stdev(test_dict[ref]) > 0x100000: - for dtb in test_dict[ref]: - scan_results.append((test, dtb)) - best_found = ref - if args.verbose > 1 or best_found == ref: - print(" " + hex(ref) + ": " + ", ".join([hex(x) for x in sorted(test_dict[ref])])) - print(" Results") - - # Populate the guesses based on the scan_results - guesses = dict([(test.layer_type.__name__, set()) for test in tests]) - for test, dtb in scan_results: - guesses[test.layer_type.__name__].add(dtb) - - # Guesses should be a dictionary mapping tests to sets of most likely dtbs, the lowest of which is then chosen - arch = None - for guess_arch in sorted(guesses, key = lambda x: -len(guesses[x])): - if not arch and len(guesses[guess_arch]) > 0: - arch = guess_arch - if args.verbose: - print(" " + guess_arch + ": " + ", ".join([hex(x) for x in sorted(guesses[guess_arch])])) - if arch: - print("[!] OS Guess:", arch, "with DTB", hex(list(guesses[arch])[0])) - else: - print("[X] No DTBs found") - print() - else: - print("[X] No tests selected") diff --git a/development/jsonschema.schema b/development/jsonschema.schema new file mode 100644 index 000000000..85079d89c --- /dev/null +++ b/development/jsonschema.schema @@ -0,0 +1,168 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "http://json-schema.org/draft-07/schema#", + "title": "Core schema meta-schema", + "definitions": { + "schemaArray": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#" } + }, + "nonNegativeInteger": { + "type": "integer", + "minimum": 0 + }, + "nonNegativeIntegerDefault0": { + "allOf": [ + { "$ref": "#/definitions/nonNegativeInteger" }, + { "default": 0 } + ] + }, + "simpleTypes": { + "enum": [ + "array", + "boolean", + "integer", + "null", + "number", + "object", + "string" + ] + }, + "stringArray": { + "type": "array", + "items": { "type": "string" }, + "uniqueItems": true, + "default": [] + } + }, + "type": ["object", "boolean"], + "properties": { + "$id": { + "type": "string", + "format": "uri-reference" + }, + "$schema": { + "type": "string", + "format": "uri" + }, + "$ref": { + "type": "string", + "format": "uri-reference" + }, + "$comment": { + "type": "string" + }, + "title": { + "type": "string" + }, + "description": { + "type": "string" + }, + "default": true, + "readOnly": { + "type": "boolean", + "default": false + }, + "examples": { + "type": "array", + "items": true + }, + "multipleOf": { + "type": "number", + "exclusiveMinimum": 0 + }, + "maximum": { + "type": "number" + }, + "exclusiveMaximum": { + "type": "number" + }, + "minimum": { + "type": "number" + }, + "exclusiveMinimum": { + "type": "number" + }, + "maxLength": { "$ref": "#/definitions/nonNegativeInteger" }, + "minLength": { "$ref": "#/definitions/nonNegativeIntegerDefault0" }, + "pattern": { + "type": "string", + "format": "regex" + }, + "additionalItems": { "$ref": "#" }, + "items": { + "anyOf": [ + { "$ref": "#" }, + { "$ref": "#/definitions/schemaArray" } + ], + "default": true + }, + "maxItems": { "$ref": "#/definitions/nonNegativeInteger" }, + "minItems": { "$ref": "#/definitions/nonNegativeIntegerDefault0" }, + "uniqueItems": { + "type": "boolean", + "default": false + }, + "contains": { "$ref": "#" }, + "maxProperties": { "$ref": "#/definitions/nonNegativeInteger" }, + "minProperties": { "$ref": "#/definitions/nonNegativeIntegerDefault0" }, + "required": { "$ref": "#/definitions/stringArray" }, + "additionalProperties": { "$ref": "#" }, + "definitions": { + "type": "object", + "additionalProperties": { "$ref": "#" }, + "default": {} + }, + "properties": { + "type": "object", + "additionalProperties": { "$ref": "#" }, + "default": {} + }, + "patternProperties": { + "type": "object", + "additionalProperties": { "$ref": "#" }, + "propertyNames": { "format": "regex" }, + "default": {} + }, + "dependencies": { + "type": "object", + "additionalProperties": { + "anyOf": [ + { "$ref": "#" }, + { "$ref": "#/definitions/stringArray" } + ] + } + }, + "propertyNames": { "$ref": "#" }, + "const": true, + "enum": { + "type": "array", + "items": true, + "minItems": 1, + "uniqueItems": true + }, + "type": { + "anyOf": [ + { "$ref": "#/definitions/simpleTypes" }, + { + "type": "array", + "items": { "$ref": "#/definitions/simpleTypes" }, + "minItems": 1, + "uniqueItems": true + } + ] + }, + "format": { "type": "string" }, + "contentMediaType": { "type": "string" }, + "contentEncoding": { "type": "string" }, + "if": { "$ref": "#" }, + "then": { "$ref": "#" }, + "else": { "$ref": "#" }, + "allOf": { "$ref": "#/definitions/schemaArray" }, + "anyOf": { "$ref": "#/definitions/schemaArray" }, + "oneOf": { "$ref": "#/definitions/schemaArray" }, + "not": { "$ref": "#" } + }, + "default": true +} diff --git a/development/nlpdtbfinder.py b/development/nlpdtbfinder.py deleted file mode 100644 index adae682ef..000000000 --- a/development/nlpdtbfinder.py +++ /dev/null @@ -1,184 +0,0 @@ -"""nlpdtbfinder is an OS-agnostic DTB finder for Intel32e 4-layer paging. - -It should be extensible to other forms of paging. - -Some of the masks used are heuristics, but have been tested against recent -images from OSX, Windows, and Linux. - -Adapted from dtbfinder.py. -""" - -import sys - -# TODO: Remove this once we install volatility -sys.path += ".." - -from volatility.framework import contexts -from volatility.framework import interfaces -from volatility.framework import layers -from volatility.framework import exceptions - -import struct - -PAGE_SIZE = 0x1000 -PHYS_MASK = 0xfffffffffff -PML4_ENTRY_SIZE = int((2 ** 64) / 512) - - -class PML4EScanner(interfaces.layers.ScannerInterface): - overlap = 0x4000 - - def __call__(self, data, data_offset): - # print("offset: 0x%x len: %d" % (data_offset, len(data))) - - # go through each page in the data, look for signs of PML4 - for page_offset in range(0, len(data), PAGE_SIZE): - entries = struct.unpack('<512Q', data[page_offset:page_offset + PAGE_SIZE]) - valid_entries = [] - invalid_count = 0 - user_count = 0 - supervisor_count = 0 - - entry_num = 0 - for e in entries: - # print("PML4E: " + bin(e)) - if (e & 0b10111011) == 0b00100011: - # print("It's valid!") - valid_entries.append((entry_num, e)) - - if (e & 0b100): - user_count = user_count + 1 - else: - supervisor_count = supervisor_count + 1 - - elif e != 0: - invalid_count = invalid_count + 1 - - entry_num = entry_num + 1 - - # print("[%x] inv: %d val: %d" % (data_offset + page_offset, invalid_count, len(valid_entries))) - if invalid_count == 0 and len(valid_entries) > 4 and user_count != 0 and supervisor_count != 0: - if page_offset < self.chunk_size: - yield (data_offset + page_offset, valid_entries) - - -def find_pt_mapping(ctx, layer_name, entries): - for entry in entries: - # valid entry? - # print("pt: " + bin(entry)) - if (entry & 0b1111011) == 0b1100011: - return True - - return False - - -def find_pd_mapping(ctx, layer_name, entries): - for entry in entries: - # valid entry? - # print("pd: " + bin(entry)) - if not ((entry & 0b1111011) == 0b1100011): - continue - - # bit 7 is large page - if (entry & 0b10000000): - return True - - pt_offset = (entry & PHYS_MASK) >> 12 - try: - pt = ctx.layers.read(baselayer_name, pt_offset, PAGE_SIZE) - except exceptions.InvalidAddressException: - # print("page fault at " + hex(pt_offset)) - return False - - pt_entries = struct.unpack('<512Q', pt) - if find_pt_mapping(ctx, layer_name, pt_entries): - return True - return False - - -def find_pdpt_mapping(ctx, layer_name, entries): - for entry in entries: - # valid entry? - # print("pdpte: " + bin(entry)) - if not (entry & 1): - continue - - # bit 7 is large page - if (entry & 0b10000000): - return True - - pd_offset = (entry & PHYS_MASK) >> 12 - try: - pd = ctx.layers.read(baselayer_name, pd_offset, PAGE_SIZE) - except exceptions.InvalidAddressException: - # print("page fault at " + hex(pd_offset)) - return False - - pd_entries = struct.unpack('<512Q', pd) - if find_pd_mapping(ctx, layer_name, pd_entries): - return True - return False - - -def find_pml4_mapping(ctx, layer_name, entries): - """Walks through the list of PML4 (idx, entry) entries looking for the - first valid (and present) mapping""" - - for (idx, entry) in entries: - # print("PML4E [%d]: %s" % (idx, bin(entry))) - pdpte_offset = (entry & PHYS_MASK) >> 12 - - try: - pdpte = ctx.layers.read(baselayer_name, pdpte_offset, PAGE_SIZE) - except exceptions.InvalidAddressException: - # print("page fault at " + hex(pdpte_offset)) - return False - - pdpte_entries = struct.unpack('<512Q', pdpte) - if find_pdpt_mapping(ctx, layer_name, pdpte_entries): - return True - - return False - - -if __name__ == '__main__': - import argparse - - parser = argparse.ArgumentParser() - parser.add_argument("filenames", metavar = "FILE", nargs = "+", action = "store", help = "FILE to read for testing") - parser.add_argument("-l", "--lime", action = "store_true", dest = "lime", help = "All files are LIME format") - parser.add_argument( - "-v", "--verbose", action = "count", default = 0, help = "Increase the verbosity of the information returned") - - args = parser.parse_args() - - ctx = contexts.Context() - for filename in args.filenames: - ctx.config[interfaces.configuration.path_join('config' + str(args.filenames.index(filename)), - "filename")] = filename - data = layers.physical.FileLayer(ctx, 'config' + str(args.filenames.index(filename)), - 'data' + str(args.filenames.index(filename))) - ctx.layers.add_layer(data) - if args.lime: - ctx.config[interfaces.configuration.path_join('lime-config' + str(args.filenames.index(filename)), - "base_layer")] = 'data' + str(args.filenames.index(filename)) - data = layers.lime.LimeLayer(ctx, 'lime-config' + str(args.filenames.index(filename)), - 'lime-data' + str(args.filenames.index(filename))) - ctx.layers.add_layer(data) - - layername = 'data' - if args.lime: - layername = 'lime-data' - - print(str(ctx.config)) - - for i in range(len(args.filenames)): - print("[*] Scanning " + args.filenames[i] + "...") - baselayer_name = layername + str(i) - scan_results = ctx.layers[baselayer_name].scan(ctx, PML4EScanner()) - - for (dtb, entries) in scan_results: - # print("trying: " + hex(dtb)) - if find_pml4_mapping(ctx, baselayer_name, entries): - print("[!] %x" % dtb) - print() diff --git a/development/pdbconv.py b/development/pdbparse-to-json.py similarity index 100% rename from development/pdbconv.py rename to development/pdbparse-to-json.py diff --git a/development/pdbscanner.py b/development/pdbscanner.py deleted file mode 100644 index 127bc374b..000000000 --- a/development/pdbscanner.py +++ /dev/null @@ -1,57 +0,0 @@ -import struct -import sys - -# TODO: Rather nasty hack, when volatility's actually installed this would be unnecessary -sys.path += ".." - -from volatility.framework import interfaces, layers -from volatility.framework.automagic.pdbscan import scan - -if __name__ == '__main__': - import argparse - - from volatility.framework import contexts - - parser = argparse.ArgumentParser() - parser.add_argument("filenames", metavar = "FILE", nargs = "+", action = "store", help = "FILE to read for testing") - - args = parser.parse_args() - - ctx = contexts.Context() - scan_layers = [] - for filename in args.filenames: - - index = args.filenames.index(filename) - config_name = 'config' + str(index) - base_name = 'data' + str(index) - ctx.config[interfaces.configuration.path_join(config_name, "filename")] = filename - base = layers.physical.FileLayer(ctx, config_name, base_name) - - ctx.layers.add_layer(base) - - # XXX What's the right way to check for LiME? - (magic, ) = struct.unpack('