From 77be83edc419bc494f0c9de90f56b7aaeea7ae90 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 26 Jan 2020 15:35:54 +0000 Subject: [PATCH] Layers: Change mapping signature to return domain length --- volatility/framework/automagic/pdbscan.py | 2 +- volatility/framework/interfaces/layers.py | 110 +++++++----------- volatility/framework/layers/intel.py | 11 +- volatility/framework/layers/linear.py | 41 ++++++- volatility/framework/layers/msf.py | 11 +- volatility/framework/layers/registry.py | 7 +- volatility/framework/layers/segmented.py | 9 +- .../framework/plugins/windows/pslist.py | 2 +- .../framework/plugins/windows/pstree.py | 2 +- .../framework/plugins/windows/strings.py | 4 +- volatility/plugins/windows/statistics.py | 2 +- 11 files changed, 109 insertions(+), 92 deletions(-) diff --git a/volatility/framework/automagic/pdbscan.py b/volatility/framework/automagic/pdbscan.py index fcd07e465..745a7c74c 100644 --- a/volatility/framework/automagic/pdbscan.py +++ b/volatility/framework/automagic/pdbscan.py @@ -323,7 +323,7 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface): try: kvp = vlayer.mapping(kvo, 0) if (any([(p == kernel['mz_offset'] and layer_name == physical_layer_name) - for (_, p, _, layer_name) in kvp])): + for (_, _, p, _, layer_name) in kvp])): valid_kernels[virtual_layer_name] = (kvo, kernel) # Sit the virtual offset under the TranslationLayer it applies to context.config[kvo_path] = kvo diff --git a/volatility/framework/interfaces/layers.py b/volatility/framework/interfaces/layers.py index 8d80247d5..2454695ca 100644 --- a/volatility/framework/interfaces/layers.py +++ b/volatility/framework/interfaces/layers.py @@ -377,8 +377,9 @@ class TranslationLayerInterface(DataLayerInterface, metaclass = ABCMeta): """ @abstractmethod - def mapping(self, offset: int, length: int, ignore_errors: bool = False) -> Iterable[Tuple[int, int, int, str]]: - """Returns a sorted iterable of (offset, mapped_offset, length, layer) + def mapping(self, offset: int, length: int, + ignore_errors: bool = False) -> Iterable[Tuple[int, int, int, int, str]]: + """Returns a sorted iterable of (offset, sublength, mapped_offset, mapped_length, layer) mappings. ignore_errors will provide all available maps with gaps, but @@ -394,13 +395,32 @@ class TranslationLayerInterface(DataLayerInterface, metaclass = ABCMeta): """Returns a list of layer names that this layer translates onto.""" return [] - def _decode(self, data: bytes, mapped_offset: int, offset: int) -> bytes: - """Decodes any necessary data.""" - return data + def _decode(self, layer_name: str, mapped_offset: int, offset: int, output_length: int) -> bytes: + """Decodes any necessary data. - def _encode(self, data: bytes, mapped_offset: int, offset: int) -> bytes: - """Encodes any necessary data.""" - return data + Args: + layer_name: The layer from which the data should be taken + value: The new value to encode + mapped_offset: The offset in the underlying layer where the data would begin + offset: The offset in the higher-layer where the data would begin + output_length: The expected length of the returned data + + Returns: + The data to be read from the underlying layer.""" + return self._context.layers.read(layer_name, mapped_offset, output_length) + + def _encode(self, layer_name: str, mapped_offset: int, offset: int, value: bytes) -> bytes: + """Encodes any necessary data. + + Args: + layer_name: The layer from which the data should be taken + mapped_offset: The offset in the underlying layer where the data would begin + offset: The offset in the higher-layer where the data would begin + value: The new value to encode + + Returns: + The data to be rewritten at mapped_offset.""" + return value # ## Read/Write functions for mapped pages @@ -409,83 +429,41 @@ class TranslationLayerInterface(DataLayerInterface, metaclass = ABCMeta): """Reads an offset for length bytes and returns 'bytes' (not 'str') of length size.""" current_offset = offset - output = [] # type: List[bytes] - for (layer_offset, mapped_offset, mapped_length, layer) in self.mapping(offset, length, ignore_errors = pad): + output = b'' # type: bytes + for (layer_offset, sublength, mapped_offset, mapped_length, layer) in self.mapping(offset, + length, + ignore_errors = pad): if not pad and layer_offset > current_offset: raise exceptions.InvalidAddressException( self.name, current_offset, "Layer {} cannot map offset: {}".format(self.name, current_offset)) elif layer_offset > current_offset: - output += [b"\x00" * (layer_offset - current_offset)] + output += b"\x00" * (layer_offset - current_offset) current_offset = layer_offset # The layer_offset can be less than the current_offset in non-linearly mapped layers # it does not suggest an overlap, but that the data is in an encoded block if mapped_length > 0: - processed_data = self._decode(self._context.layers.read(layer, mapped_offset, mapped_length, pad), - mapped_offset, layer_offset) - # Chop off anything unnecessary at the start - processed_data = processed_data[current_offset - layer_offset:] - # Chop off anything unnecessary at the end - processed_data = processed_data[:length - (current_offset - offset)] - output += [processed_data] - current_offset += len(processed_data) - recovered_data = b"".join(output) - return recovered_data + b"\x00" * (length - len(recovered_data)) + processed_data = self._decode(layer, mapped_offset, layer_offset, sublength) + if len(processed_data) != sublength: + raise ValueError("ProcessedData length does not match expected length of chunk") + output += processed_data + current_offset += sublength + return output + (b"\x00" * (length - len(output))) def write(self, offset: int, value: bytes) -> None: """Writes a value at offset, distributing the writing across any underlying mapping.""" current_offset = offset length = len(value) - for (layer_offset, mapped_offset, mapped_length, layer) in self.mapping(offset, length): + for (layer_offset, sublength, mapped_offset, mapped_length, layer) in self.mapping(offset, length): if layer_offset > current_offset: raise exceptions.InvalidAddressException( self.name, current_offset, "Layer {} cannot map offset: {}".format(self.name, current_offset)) - original_data = self._context.layers.read(layer, mapped_offset, mapped_length) - # Always chunk the value based on the mapping - value_to_write = original_data[:current_offset - layer_offset] + value[:mapped_length - - (current_offset - layer_offset)] - value = value[mapped_length - (current_offset - layer_offset):] - encoded_value = self._encode(value_to_write, mapped_offset, layer_offset) - if len(encoded_value) != mapped_length: - raise exceptions.LayerException(self.name, - "Unable to write new value, does not map to the same dimensions") - self._context.layers.write(layer, mapped_offset, encoded_value) - current_offset += len(value_to_write) - # ## Scan implementation with knowledge of pages + value_chunk = value[layer_offset - offset:layer_offset - offset + sublength] + new_data = self._encode(layer, mapped_offset, layer_offset, value_chunk) + self._context.layers.write(layer, mapped_offset, new_data) - def _scan_iterator(self, scanner: 'ScannerInterface', - sections: Iterable[Tuple[int, int]]) -> Iterable[IteratorValue]: - """Essentially, for paged systems we take a bunch of pages and chunk them up into scanner.page_size or - as large a chunk as possible (if there are gaps).""" - for (section_start, section_length) in sections: - # For each section, split it into scan size chunks - for chunk_start in range(section_start, section_start + section_length, scanner.chunk_size): - # Shorten it, if we're at the end of the section - chunk_length = min(section_start + section_length - chunk_start, scanner.chunk_size + scanner.overlap) - - # Prev offset keeps track of the end of the previous subchunk - prev_offset = chunk_start - output = [] # type: List[Tuple[str, int, int]] - # We populate the response based on subchunks that may be mapped all over the place - for mapped in self.mapping(chunk_start, chunk_length, ignore_errors = True): - offset, mapped_offset, length, layer_name = mapped - - # We need to check if the offset is next to the end of the last one (contiguous) - if offset != prev_offset: - # Only yield if we've accumulated output - if len(output): - # Yield all the (joined) items so far - # and the ending point of that subchunk (where we'd gotten to previously) - yield output, prev_offset - output = [] - - # Shift the marker up to the end of what we just received and add it to the output - prev_offset = offset + length - output += [(layer_name, mapped_offset, length)] - # If there's still output left, output it - if len(output): - yield output, prev_offset + current_offset += len(value_chunk) class LayerContainer(collections.abc.Mapping): diff --git a/volatility/framework/layers/intel.py b/volatility/framework/layers/intel.py index 47197d2cd..565d48b82 100644 --- a/volatility/framework/layers/intel.py +++ b/volatility/framework/layers/intel.py @@ -166,13 +166,14 @@ class Intel(linear.LinearlyMappedLayer): # TODO: Consider reimplementing this, since calls to mapping can call is_valid return all([ self._context.layers[layer].is_valid(mapped_offset) - for _, mapped_offset, _, layer in self.mapping(offset, length) + for _, _, mapped_offset, _, layer in self.mapping(offset, length) ]) except exceptions.InvalidAddressException: return False - def mapping(self, offset: int, length: int, ignore_errors: bool = False) -> Iterable[Tuple[int, int, int, str]]: - """Returns a sorted iterable of (offset, mapped_offset, length, layer) + def mapping(self, offset: int, length: int, + ignore_errors: bool = False) -> Iterable[Tuple[int, int, int, int, str]]: + """Returns a sorted iterable of (offset, sublength, mapped_offset, mapped_length, layer) mappings. This allows translation layers to provide maps of contiguous @@ -187,7 +188,7 @@ class Intel(linear.LinearlyMappedLayer): if not ignore_errors: raise return - yield (offset, mapped_offset, length, layer_name) + yield offset, length, mapped_offset, length, layer_name return while length > 0: try: @@ -207,7 +208,7 @@ class Intel(linear.LinearlyMappedLayer): length -= length_diff offset += length_diff else: - yield (offset, chunk_offset, chunk_size, layer_name) + yield offset, length, chunk_offset, chunk_size, layer_name length -= chunk_size offset += chunk_size diff --git a/volatility/framework/layers/linear.py b/volatility/framework/layers/linear.py index 401c303d8..0497bc6d6 100644 --- a/volatility/framework/layers/linear.py +++ b/volatility/framework/layers/linear.py @@ -1,5 +1,5 @@ import functools -from typing import List, Optional, Tuple +from typing import List, Optional, Tuple, Iterable from volatility.framework import exceptions, interfaces @@ -13,7 +13,7 @@ class LinearlyMappedLayer(interfaces.layers.TranslationLayerInterface): def translate(self, offset: int, ignore_errors: bool = False) -> Tuple[Optional[int], Optional[str]]: mapping = list(self.mapping(offset, 0, ignore_errors)) if len(mapping) == 1: - original_offset, mapped_offset, _, layer = mapping[0] + original_offset, _, mapped_offset, _, layer = mapping[0] if original_offset != offset: raise exceptions.LayerException(self.name, "Layer {} claims to map linearly but does not".format(self.name)) @@ -34,7 +34,7 @@ class LinearlyMappedLayer(interfaces.layers.TranslationLayerInterface): length size.""" current_offset = offset output = [] # type: List[bytes] - for (offset, mapped_offset, mapped_length, layer) in self.mapping(offset, length, ignore_errors = pad): + for (offset, _, mapped_offset, mapped_length, layer) in self.mapping(offset, length, ignore_errors = pad): if not pad and offset > current_offset: raise exceptions.InvalidAddressException( self.name, current_offset, "Layer {} cannot map offset: {}".format(self.name, current_offset)) @@ -54,7 +54,7 @@ class LinearlyMappedLayer(interfaces.layers.TranslationLayerInterface): underlying mapping.""" current_offset = offset length = len(value) - for (offset, mapped_offset, length, layer) in self.mapping(offset, length): + for (offset, _, mapped_offset, length, layer) in self.mapping(offset, length): if offset > current_offset: raise exceptions.InvalidAddressException( self.name, current_offset, "Layer {} cannot map offset: {}".format(self.name, current_offset)) @@ -63,3 +63,36 @@ class LinearlyMappedLayer(interfaces.layers.TranslationLayerInterface): self._context.layers.write(layer, mapped_offset, value[:length]) value = value[length:] current_offset += length + + def _scan_iterator(self, scanner: 'interfaces.layers.ScannerInterface', + sections: Iterable[Tuple[int, int]]) -> Iterable[interfaces.layers.IteratorValue]: + """Essentially, for paged systems we take a bunch of pages and chunk them up into scanner.page_size or + as large a chunk as possible (if there are gaps).""" + for (section_start, section_length) in sections: + # For each section, split it into scan size chunks + for chunk_start in range(section_start, section_start + section_length, scanner.chunk_size): + # Shorten it, if we're at the end of the section + chunk_length = min(section_start + section_length - chunk_start, scanner.chunk_size + scanner.overlap) + + # Prev offset keeps track of the end of the previous subchunk + prev_offset = chunk_start + output = [] # type: List[Tuple[str, int, int]] + # We populate the response based on subchunks that may be mapped all over the place + for mapped in self.mapping(chunk_start, chunk_length, ignore_errors = True): + offset, _, mapped_offset, mapped_length, layer_name = mapped + + # We need to check if the offset is next to the end of the last one (contiguous) + if offset != prev_offset: + # Only yield if we've accumulated output + if len(output): + # Yield all the (joined) items so far + # and the ending point of that subchunk (where we'd gotten to previously) + yield output, prev_offset + output = [] + + # Shift the marker up to the end of what we just received and add it to the output + prev_offset = offset + mapped_length + output += [(layer_name, mapped_offset, mapped_length)] + # If there's still output left, output it + if len(output): + yield output, prev_offset diff --git a/volatility/framework/layers/msf.py b/volatility/framework/layers/msf.py index 5923c9367..783cda598 100644 --- a/volatility/framework/layers/msf.py +++ b/volatility/framework/layers/msf.py @@ -136,8 +136,9 @@ class PdbMultiStreamFormat(linear.LinearlyMappedLayer): def is_valid(self, offset: int, length: int = 1) -> bool: return self.context.layers[self._base_layer].is_valid(offset, length) - def mapping(self, offset: int, length: int, ignore_errors: bool = False) -> Iterable[Tuple[int, int, int, str]]: - yield (offset, offset, length, self._base_layer) + def mapping(self, offset: int, length: int, + ignore_errors: bool = False) -> Iterable[Tuple[int, int, int, int, str]]: + yield offset, length, offset, length, self._base_layer def get_stream(self, index) -> Optional['PdbMSFStream']: self.read_streams() @@ -182,7 +183,8 @@ class PdbMSFStream(linear.LinearlyMappedLayer): requirements.IntRequirement(name = 'maximum_size') ] - def mapping(self, offset: int, length: int, ignore_errors: bool = False) -> Iterable[Tuple[int, int, int, str]]: + def mapping(self, offset: int, length: int, + ignore_errors: bool = False) -> Iterable[Tuple[int, int, int, int, str]]: returned = 0 page_size = self._pdb_layer.page_size while length > 0: @@ -194,7 +196,8 @@ class PdbMSFStream(linear.LinearlyMappedLayer): raise exceptions.InvalidAddressException(layer_name = self.name, invalid_address = offset + returned) else: - yield (offset + returned, (self._pages[page] * page_size) + page_position, chunk_size, self._base_layer) + yield offset + returned, chunk_size, (self._pages[page] * + page_size) + page_position, chunk_size, self._base_layer returned += chunk_size length -= chunk_size diff --git a/volatility/framework/layers/registry.py b/volatility/framework/layers/registry.py index d54b65518..89ff601a6 100644 --- a/volatility/framework/layers/registry.py +++ b/volatility/framework/layers/registry.py @@ -209,7 +209,8 @@ class RegistryHive(linear.LinearlyMappedLayer): entry = table.Table[table_index] return entry.get_block_offset() + suboffset - def mapping(self, offset: int, length: int, ignore_errors: bool = False) -> Iterable[Tuple[int, int, int, str]]: + def mapping(self, offset: int, length: int, + ignore_errors: bool = False) -> Iterable[Tuple[int, int, int, int, str]]: if length < 0: raise ValueError("Mapping length of RegistryHive must be positive or zero") @@ -225,7 +226,7 @@ class RegistryHive(linear.LinearlyMappedLayer): chunk_size = min(chunk_size, remaining_length, self._page_size) try: translated_offset = self._translate(current_offset) - response.append((current_offset, translated_offset, chunk_size, self._base_layer)) + response.append((current_offset, chunk_size, translated_offset, chunk_size, self._base_layer)) except exceptions.LayerException: if not ignore_errors: raise @@ -245,7 +246,7 @@ class RegistryHive(linear.LinearlyMappedLayer): # Pass this to the lower layers for now return all([ self.context.layers[layer].is_valid(offset, length) - for (_, offset, length, layer) in self.mapping(offset, length) + for (_, _, offset, length, layer) in self.mapping(offset, length) ]) except exceptions.InvalidAddressException: return False diff --git a/volatility/framework/layers/segmented.py b/volatility/framework/layers/segmented.py index 838dd19b7..7a755320d 100644 --- a/volatility/framework/layers/segmented.py +++ b/volatility/framework/layers/segmented.py @@ -45,7 +45,7 @@ class SegmentedLayer(linear.LinearlyMappedLayer, metaclass = ABCMeta): try: base_layer = self._context.layers[self._base_layer] return all( - [base_layer.is_valid(mapped_offset) for _i, mapped_offset, _i, _s in self.mapping(offset, length)]) + [base_layer.is_valid(mapped_offset) for _i, _i, mapped_offset, _i, _s in self.mapping(offset, length)]) except exceptions.InvalidAddressException: return False @@ -69,8 +69,9 @@ class SegmentedLayer(linear.LinearlyMappedLayer, metaclass = ABCMeta): return self._segments[i] raise exceptions.InvalidAddressException(self.name, offset, "Invalid address at {:0x}".format(offset)) - def mapping(self, offset: int, length: int, ignore_errors: bool = False) -> Iterable[Tuple[int, int, int, str]]: - """Returns a sorted iterable of (offset, mapped_offset, length, layer) + def mapping(self, offset: int, length: int, + ignore_errors: bool = False) -> Iterable[Tuple[int, int, int, int, str]]: + """Returns a sorted iterable of (offset, length, mapped_offset, mapped_length, layer) mappings.""" done = False current_offset = offset @@ -100,7 +101,7 @@ class SegmentedLayer(linear.LinearlyMappedLayer, metaclass = ABCMeta): return # Crop it to the amount we need left chunk_size = min(size, length + offset - logical_offset) - yield (logical_offset, mapped_offset, chunk_size, self._base_layer) + yield logical_offset, chunk_size, mapped_offset, chunk_size, self._base_layer current_offset += chunk_size # Terminate if we've gone (or reached) our required limit if current_offset >= offset + length: diff --git a/volatility/framework/plugins/windows/pslist.py b/volatility/framework/plugins/windows/pslist.py index bc62b543b..255730fab 100644 --- a/volatility/framework/plugins/windows/pslist.py +++ b/volatility/framework/plugins/windows/pslist.py @@ -132,7 +132,7 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): if not self.config.get('physical', self.PHYSICAL_DEFAULT): offset = proc.vol.offset else: - (_, offset, _, _) = list(memory.mapping(offset = proc.vol.offset, length = 0))[0] + (_, _, offset, _, _) = list(memory.mapping(offset = proc.vol.offset, length = 0))[0] yield (0, (proc.UniqueProcessId, proc.InheritedFromUniqueProcessId, proc.ImageFileName.cast("string", max_length = proc.ImageFileName.vol.count, errors = 'replace'), diff --git a/volatility/framework/plugins/windows/pstree.py b/volatility/framework/plugins/windows/pstree.py index 9e0c4f496..43a6b8d95 100644 --- a/volatility/framework/plugins/windows/pstree.py +++ b/volatility/framework/plugins/windows/pstree.py @@ -42,7 +42,7 @@ class PsTree(pslist.PsList): else: layer_name = self.config['primary'] memory = self.context.layers[layer_name] - (_, offset, _, _) = list(memory.mapping(offset = proc.vol.offset, length = 0))[0] + (_, _, offset, _, _) = list(memory.mapping(offset = proc.vol.offset, length = 0))[0] self._processes[proc.UniqueProcessId] = proc diff --git a/volatility/framework/plugins/windows/strings.py b/volatility/framework/plugins/windows/strings.py index e961e45c5..79fa80845 100644 --- a/volatility/framework/plugins/windows/strings.py +++ b/volatility/framework/plugins/windows/strings.py @@ -84,7 +84,7 @@ class Strings(interfaces.plugins.PluginInterface): if isinstance(layer, intel.Intel): # We don't care about errors, we just wanted chunks that map correctly for mapval in layer.mapping(0x0, layer.maximum_address, ignore_errors = True): - vpage, kpage, page_size, maplayer = mapval + vpage, _, kpage, page_size, maplayer = mapval for val in range(kpage, kpage + page_size, 0x1000): cur_set = reverse_map.get(kpage >> 12, set()) cur_set.add(("kernel", vpage)) @@ -107,7 +107,7 @@ class Strings(interfaces.plugins.PluginInterface): proc_layer = self.context.layers[proc_layer_name] if isinstance(proc_layer, linear.LinearlyMappedLayer): for mapval in proc_layer.mapping(0x0, proc_layer.maximum_address, ignore_errors = True): - kpage, vpage, page_size, maplayer = mapval + kpage, _, vpage, page_size, maplayer = mapval for val in range(kpage, kpage + page_size, 0x1000): cur_set = reverse_map.get(kpage >> 12, set()) cur_set.add(("Process {}".format(process.UniqueProcessId), vpage)) diff --git a/volatility/plugins/windows/statistics.py b/volatility/plugins/windows/statistics.py index e9669c4c4..df01c71a6 100644 --- a/volatility/plugins/windows/statistics.py +++ b/volatility/plugins/windows/statistics.py @@ -34,7 +34,7 @@ class Statistics(plugins.PluginInterface): while page_addr < layer.maximum_address: try: - _, _, page_size, layer_name = list(layer.mapping(page_addr, 2 * expected_page_size))[0] + _, _, _, page_size, layer_name = list(layer.mapping(page_addr, 2 * expected_page_size))[0] if layer_name != layer.config['memory_layer']: swap_count += 1 else: