From bc4c386d6d09091cb433b5348deac82c7531197f Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 3 Jun 2017 12:41:18 +0100 Subject: [PATCH 1/8] Fix a multiprocessing issue (and tidy away the kwargs). By not passing the arguments from exceptions down to the base class Exception, it causes an issue when they are pickled to be passed between processes. For more information, see http://bugs.python.org/issue1692335. --- volatility/framework/exceptions.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/volatility/framework/exceptions.py b/volatility/framework/exceptions.py index d62ae85a2..8df4abce5 100644 --- a/volatility/framework/exceptions.py +++ b/volatility/framework/exceptions.py @@ -21,8 +21,8 @@ class SymbolError(VolatilityException): class InvalidAddressException(VolatilityException): """Thrown when an address is not valid in the space it was requested""" - def __init__(self, layer_name, invalid_address, *args, **kwargs): - super().__init__(*args, **kwargs) + def __init__(self, layer_name, invalid_address, *args): + super().__init__(layer_name, invalid_address, *args) self.invalid_address = invalid_address self.layer_name = layer_name @@ -33,8 +33,8 @@ class PagedInvalidAddressException(InvalidAddressException): Includes the invalid address and the number of bits of the address that are invalid """ - def __init__(self, layer_name, invalid_address, invalid_bits, *args, **kwargs): - super().__init__(layer_name, invalid_address, *args, **kwargs) + def __init__(self, layer_name, invalid_address, invalid_bits, *args): + super().__init__(layer_name, invalid_address, *args) self.invalid_bits = invalid_bits From 93a6efd6667198b2bfa353f1a2e6ef0c362d9839 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 3 Jun 2017 17:44:13 +0100 Subject: [PATCH 2/8] Ensure we don't get name confusion when dealing with variables. --- volatility/framework/interfaces/layers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility/framework/interfaces/layers.py b/volatility/framework/interfaces/layers.py index 476b0e9ac..8b94793b0 100644 --- a/volatility/framework/interfaces/layers.py +++ b/volatility/framework/interfaces/layers.py @@ -196,8 +196,8 @@ class DataLayerInterface(configuration.ConfigurableInterface, validity.ValidityR # Ensures we don't burn CPU cycles going round in a ready waiting loop # without delaying the user too long between progress updates/results result.wait(0.1) - for result in result.get(): - yield from result + for value in result.get(): + yield from value else: for value in scan_iterator(): if progress_callback: From 9a2bfe7c92c4ff138aaa4f175be18d8130c481bb Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 3 Jun 2017 17:44:59 +0100 Subject: [PATCH 3/8] Remove unused variable. --- volatility/framework/automagic/pdbscan.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility/framework/automagic/pdbscan.py b/volatility/framework/automagic/pdbscan.py index e90938370..5e782536f 100644 --- a/volatility/framework/automagic/pdbscan.py +++ b/volatility/framework/automagic/pdbscan.py @@ -265,7 +265,6 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): vlayer = context.memory[virtual_layer_name] physical_layer_name = context.config.get( interfaces.configuration.path_join(vlayer.config_path, 'memory_layer'), None) - found = False kvo_path = interfaces.configuration.path_join(virtual_config_path, 'kernel_virtual_offset') for kernel in kernels: # It seems the kernel is loaded at a fixed mapping (presumably because the memory manager hasn't started yet) From c862f389174647351cf7fcaf69fa79a5d8625ae1 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 3 Jun 2017 17:48:32 +0100 Subject: [PATCH 4/8] Add in flag to forcibly disable all multithreaded scanning. --- volatility/framework/constants.py | 1 + volatility/framework/interfaces/layers.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/volatility/framework/constants.py b/volatility/framework/constants.py index e841ea5c5..8dd7eedb4 100644 --- a/volatility/framework/constants.py +++ b/volatility/framework/constants.py @@ -9,6 +9,7 @@ import sys PLUGINS_PATH = [os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "plugins"))] BANG = "!" PACKAGE_VERSION = "3.0.0_alpha1" +DISABLE_MULTITHREADED_SCANNING = False LOGLEVEL_V = 9 LOGLEVEL_VV = 8 diff --git a/volatility/framework/interfaces/layers.py b/volatility/framework/interfaces/layers.py index 8b94793b0..9d0f528a3 100644 --- a/volatility/framework/interfaces/layers.py +++ b/volatility/framework/interfaces/layers.py @@ -8,7 +8,7 @@ import math import multiprocessing from abc import ABCMeta, abstractmethod, abstractproperty -from volatility.framework import exceptions, validity +from volatility.framework import constants, exceptions, validity from volatility.framework.interfaces import configuration, context vollog = logging.getLogger(__name__) @@ -186,7 +186,7 @@ class DataLayerInterface(configuration.ConfigurableInterface, validity.ValidityR scan_iterator = functools.partial(self._scan_iterator, scanner, min_address, max_address) scan_chunk = functools.partial(self._scan_chunk, scanner, min_address, max_address, progress) scan_metric = functools.partial(self._scan_metric, scanner, min_address, max_address) - if scanner.thread_safe: + if scanner.thread_safe and constants.DISABLE_MULTITHREADED_SCANNING: with multiprocessing.Pool() as pool: result = pool.map_async(scan_chunk, scan_iterator()) while not result.ready(): From 0675350346c9e77a45d561aa7b5f7724d0f9c0ed Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 3 Jun 2017 17:49:56 +0100 Subject: [PATCH 5/8] Fix up minor logical error in multithreaded scanning global flag. --- volatility/framework/interfaces/layers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility/framework/interfaces/layers.py b/volatility/framework/interfaces/layers.py index 9d0f528a3..db9cbfeff 100644 --- a/volatility/framework/interfaces/layers.py +++ b/volatility/framework/interfaces/layers.py @@ -186,7 +186,7 @@ class DataLayerInterface(configuration.ConfigurableInterface, validity.ValidityR scan_iterator = functools.partial(self._scan_iterator, scanner, min_address, max_address) scan_chunk = functools.partial(self._scan_chunk, scanner, min_address, max_address, progress) scan_metric = functools.partial(self._scan_metric, scanner, min_address, max_address) - if scanner.thread_safe and constants.DISABLE_MULTITHREADED_SCANNING: + if scanner.thread_safe and not constants.DISABLE_MULTITHREADED_SCANNING: with multiprocessing.Pool() as pool: result = pool.map_async(scan_chunk, scan_iterator()) while not result.ready(): From b72c3aaed55ffbb2b7079b7d49707b31c9e97f53 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 3 Jun 2017 20:26:21 +0100 Subject: [PATCH 6/8] Handle invalid translations lower than the top layer when scanning. --- volatility/framework/layers/intel.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/volatility/framework/layers/intel.py b/volatility/framework/layers/intel.py index d52381075..c3971c004 100644 --- a/volatility/framework/layers/intel.py +++ b/volatility/framework/layers/intel.py @@ -1,9 +1,12 @@ +import logging import math import struct from volatility.framework import exceptions, interfaces from volatility.framework.configuration import requirements +vollog = logging.getLogger(__name__) + class classproperty(object): """Class property decorator""" @@ -178,7 +181,7 @@ class Intel(interfaces.layers.TranslationLayerInterface): try: address, page_size, layer_name = self._translate(chunk_end) chunk_size = page_size - (address & (page_size - 1)) - except exceptions.PagedInvalidAddressException as e: + except exceptions.InvalidAddressException: address, chunk_size, layer_name = None, 1 << self._page_size_in_bits, '' # We've come to a break, so scan what we've seen so far if address is None or (previous, address) in scanned_pairs: @@ -194,7 +197,12 @@ class Intel(interfaces.layers.TranslationLayerInterface): data_to_scan, chunk_end = iterator_value data = b'' for layer_name, address, chunk_size in data_to_scan: - data += self.context.memory[layer_name].read(address, chunk_size) + try: + data += self.context.memory[layer_name].read(address, chunk_size) + except exceptions.InvalidAddressException: + vollog.debug( + "Invalid address in layer {} found scanning {} at address {:x}".format(layer_name, self.name, + address)) progress.value = chunk_end return list(scanner(data, chunk_end - len(data_to_scan))) From c05599d78a7ad354f7b453dd28b19c0f57906e88 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 3 Jun 2017 20:26:45 +0100 Subject: [PATCH 7/8] Minor clean-up of segmented layer. --- volatility/framework/layers/segmented.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/volatility/framework/layers/segmented.py b/volatility/framework/layers/segmented.py index c63c51815..812368c42 100644 --- a/volatility/framework/layers/segmented.py +++ b/volatility/framework/layers/segmented.py @@ -1,8 +1,7 @@ from abc import ABCMeta, abstractmethod from bisect import bisect_right -from volatility.framework import exceptions -from volatility.framework import interfaces +from volatility.framework import exceptions, interfaces from volatility.framework.configuration import requirements @@ -46,7 +45,7 @@ class SegmentedLayer(interfaces.layers.TranslationLayerInterface, metaclass = AB if not self._segments: self._load_segments() - 'Find rightmost value less than or equal to x' + # Find rightmost value less than or equal to x i = bisect_right(self._segments, (offset, self.context.memory[self._base_layer].maximum_address)) if i: if not next: From 761777c8ef5937c7a5c324fb2c40efb7f85f9190 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sat, 3 Jun 2017 20:57:19 +0100 Subject: [PATCH 8/8] Add commentary as to why we can't scan faster by ignoring surjective duplicates. --- volatility/framework/layers/intel.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/volatility/framework/layers/intel.py b/volatility/framework/layers/intel.py index c3971c004..3707c5e66 100644 --- a/volatility/framework/layers/intel.py +++ b/volatility/framework/layers/intel.py @@ -190,6 +190,17 @@ class Intel(interfaces.layers.TranslationLayerInterface): else: # TODO: We've already done the translation, so don't bother doing it again data_to_scan += [(layer_name, address, chunk_size)] + + # We can't actually use scanned_pairs because the user might want to find duplicate instances + # throughout the top layer, not just the one actual copy of the data in the bottom layer. + # We'd need to re-architect the scanner API to pass through multiple data_offsets to the scanners + # Then we'd also then need to batch all the data_offsets up until the end (so we know we're handing + # them a complete list) or we'd have to be able to add relevant offsets as they're found. + # All in all, massive complexity for little benefit in efficiency. + # + # At the moment, the following line is only good when you want *a* hit but don't care which one. + # scanned_pairs.add((previous, address)) + previous = address chunk_end += chunk_size