From 7a2ce9e9997df2abeed292cf7d576797e378cd85 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 27 Jan 2021 00:25:06 +0000 Subject: [PATCH 1/3] Layers: Make linear scanning more efficient Previously we'd chop the entire space up into scan_chunk sized blocks and then chop those up into mapped chunks. The mapping process can much more efficiently provide which blocks exist, so now we ask the layer to map itself in its entirety, and then any chunks of data we get, we cut into scan_chunk blocks. --- volatility3/framework/interfaces/layers.py | 64 +++++++++++++--------- 1 file changed, 37 insertions(+), 27 deletions(-) diff --git a/volatility3/framework/interfaces/layers.py b/volatility3/framework/interfaces/layers.py index 85c6d6c98..57208c629 100644 --- a/volatility3/framework/interfaces/layers.py +++ b/volatility3/framework/interfaces/layers.py @@ -477,39 +477,49 @@ class TranslationLayerInterface(DataLayerInterface, metaclass = ABCMeta): assumed to have no holes """ for (section_start, section_length) in sections: - # For each section, split it into scan size chunks - for chunk_start in range(section_start, section_start + section_length, scanner.chunk_size): - # Shorten it, if we're at the end of the section - chunk_length = min(section_start + section_length - chunk_start, scanner.chunk_size + scanner.overlap) + chunk_end = section_start + output = [] - # Prev offset keeps track of the end of the previous subchunk - prev_offset = chunk_start - output = [] # type: List[Tuple[str, int, int]] + # For each section, find out which bits of its exist and where they map to + # This is faster than cutting the entire space into scan_chunk sized blocks and then + # finding out what exists (particularly if most of the space isn't mapped) + for mapped in self.mapping(section_start, section_length, ignore_errors = True): + offset, sublength, mapped_offset, mapped_length, layer_name = mapped - # We populate the response based on subchunks that may be mapped all over the place - for mapped in self.mapping(chunk_start, chunk_length, ignore_errors = True): - # We don't bother with the other data in case the data's been processed by a lower layer - offset, sublength, mapped_offset, mapped_length, layer_name = mapped + # Check if this chunk and the previous aren't next to each other, + if len(output) and (offset != chunk_end): + # if so we can ship everything so far, because this must be a new chunk + chunk_end = offset + output[-1][1] + yield output, chunk_end + output = [] + else: + # Otherwise we're in a long run, and we can chunk it up in chunk_size blocks + current_chunk_size = min(sublength, scanner.chunk_size + scanner.overlap) - # We need to check if the offset is next to the end of the last one (contiguous) - if offset != prev_offset: - # Only yield if we've accumulated output - if len(output): - # Yield all the (joined) items so far - # and the ending point of that subchunk (where we'd gotten to previously) - yield output, prev_offset + # Cut it into scan_chunk + overlap sized chunks (each scan_chunk apart from each other) + while current_chunk_size == scanner.chunk_size + scanner.overlap and current_chunk_size > 0: + if current_chunk_size > 0: + output += [(self.name if not linear else layer_name, + offset if not linear else mapped_offset, current_chunk_size)] + chunk_end = offset + current_chunk_size + # Ship this scan_chunk size block + yield output, chunk_end output = [] + offset += scanner.chunk_size + mapped_offset += scanner.chunk_size + sublength -= scanner.chunk_size + current_chunk_size = min(sublength, scanner.chunk_size + scanner.overlap) - # Shift the marker up to the end of what we just received and add it to the output - prev_offset = offset + sublength + # We should now only have less than chunk_size data in it, but don't forget to ship it off too + if current_chunk_size > 0: + output += [(self.name if not linear else layer_name, offset if not linear else mapped_offset, + current_chunk_size)] - if not linear: - output += [(self.name, offset, sublength)] - else: - output += [(layer_name, mapped_offset, mapped_length)] - # If there's still output left, output it - if len(output): - yield output, prev_offset + # Ship anything we've accumulated + chunk_end = offset + current_chunk_size + + if len(output): + yield output, chunk_end class LayerContainer(collections.abc.Mapping): From f5ddb8880651e104d06db724c2e76a42df1236c4 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 27 Jan 2021 16:48:53 +0000 Subject: [PATCH 2/3] Layers: Scan by available and then chunk This changes the order of scanning to make it (potentially) more efficient. A layer can efficiently map (and ignore large segments that don't exist) and then have the available space chunked, rather than creating all the chunks and repeatedly asking the layer whether each can be mapped or not. --- volatility3/framework/interfaces/layers.py | 69 ++++++++++++---------- 1 file changed, 37 insertions(+), 32 deletions(-) diff --git a/volatility3/framework/interfaces/layers.py b/volatility3/framework/interfaces/layers.py index 57208c629..4f5314d74 100644 --- a/volatility3/framework/interfaces/layers.py +++ b/volatility3/framework/interfaces/layers.py @@ -477,49 +477,54 @@ class TranslationLayerInterface(DataLayerInterface, metaclass = ABCMeta): assumed to have no holes """ for (section_start, section_length) in sections: - chunk_end = section_start output = [] - # For each section, find out which bits of its exist and where they map to + # Hold the offsets of each chunk (including how much has been filled) + chunk_start = chunk_position = 0 + + # For each section, find out which bits of its exists and where they map to # This is faster than cutting the entire space into scan_chunk sized blocks and then # finding out what exists (particularly if most of the space isn't mapped) for mapped in self.mapping(section_start, section_length, ignore_errors = True): offset, sublength, mapped_offset, mapped_length, layer_name = mapped - # Check if this chunk and the previous aren't next to each other, - if len(output) and (offset != chunk_end): - # if so we can ship everything so far, because this must be a new chunk - chunk_end = offset + output[-1][1] - yield output, chunk_end + # Setup the variables for this block + block_start = offset + block_end = offset + sublength + conversion = mapped_offset - offset + + # If this isn't contiguous, start a new chunk + if chunk_position < block_start: + yield output, chunk_position output = [] - else: - # Otherwise we're in a long run, and we can chunk it up in chunk_size blocks - current_chunk_size = min(sublength, scanner.chunk_size + scanner.overlap) + chunk_start = chunk_position = block_start - # Cut it into scan_chunk + overlap sized chunks (each scan_chunk apart from each other) - while current_chunk_size == scanner.chunk_size + scanner.overlap and current_chunk_size > 0: - if current_chunk_size > 0: - output += [(self.name if not linear else layer_name, - offset if not linear else mapped_offset, current_chunk_size)] - chunk_end = offset + current_chunk_size - # Ship this scan_chunk size block - yield output, chunk_end + return_name = self.name if not linear else layer_name + + # Halfway through a chunk, finish the chunk, then take more + if chunk_position != chunk_start: + chunk_size = min(chunk_position - chunk_start, scanner.chunk_size + scanner.overlap) + output += [(return_name, chunk_position + conversion, chunk_size)] + chunk_start = chunk_position + chunk_size + chunk_position = chunk_start + + # Pack chunks, if we're enter the loop (starting a new chunk) and there's already chunk there, ship it + for chunk_start in range(chunk_position, block_end, scanner.chunk_size): + if output: + yield output, chunk_position output = [] - offset += scanner.chunk_size - mapped_offset += scanner.chunk_size - sublength -= scanner.chunk_size - current_chunk_size = min(sublength, scanner.chunk_size + scanner.overlap) + chunk_position = chunk_start + # Take from chunk_position as far as far as the block can go, + # or as much left of a scanner chunk as we can + chunk_size = min(block_end - chunk_position, + scanner.chunk_size + scanner.overlap - (chunk_position - chunk_start)) + output += [(return_name, chunk_position + conversion, chunk_size)] + chunk_start = chunk_position + chunk_size + chunk_position = chunk_start - # We should now only have less than chunk_size data in it, but don't forget to ship it off too - if current_chunk_size > 0: - output += [(self.name if not linear else layer_name, offset if not linear else mapped_offset, - current_chunk_size)] - - # Ship anything we've accumulated - chunk_end = offset + current_chunk_size - - if len(output): - yield output, chunk_end + # Ship anything that might be left + if output: + yield output, chunk_position class LayerContainer(collections.abc.Mapping): From c6181d0dcfdcdea00481d563edf35b1b43f9ffdc Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Wed, 27 Jan 2021 16:58:13 +0000 Subject: [PATCH 3/3] Layers: Group the linearity settings together --- volatility3/framework/interfaces/layers.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/interfaces/layers.py b/volatility3/framework/interfaces/layers.py index 4f5314d74..0cf7e087a 100644 --- a/volatility3/framework/interfaces/layers.py +++ b/volatility3/framework/interfaces/layers.py @@ -491,7 +491,14 @@ class TranslationLayerInterface(DataLayerInterface, metaclass = ABCMeta): # Setup the variables for this block block_start = offset block_end = offset + sublength - conversion = mapped_offset - offset + + # Setup the necessary bits for non-linear mappings + # For linear we give one layer down and mapped offsets (therefore the conversion) + # This saves an tiny amount of time not have to redo lookups we've already done + # For non-linear layers, we give the layer name and the offset in the layer name + # so that the read/conversion occurs properly + conversion = mapped_offset - offset if linear else 0 + return_name = layer_name if linear else self.name # If this isn't contiguous, start a new chunk if chunk_position < block_start: @@ -499,8 +506,6 @@ class TranslationLayerInterface(DataLayerInterface, metaclass = ABCMeta): output = [] chunk_start = chunk_position = block_start - return_name = self.name if not linear else layer_name - # Halfway through a chunk, finish the chunk, then take more if chunk_position != chunk_start: chunk_size = min(chunk_position - chunk_start, scanner.chunk_size + scanner.overlap)