layers: Fix up the maths around scanning chunks

This commit is contained in:
Mike Auty
2019-11-13 21:44:13 +00:00
committed by ikelos
parent ca8099e7d7
commit 19d46563ae
+14 -3
View File
@@ -458,22 +458,33 @@ class TranslationLayerInterface(DataLayerInterface, metaclass = ABCMeta):
"""Essentially, for paged systems we take a bunch of pages and chunk them up into scanner.page_size or
as large a chunk as possible (if there are gaps)."""
for (section_start, section_length) in sections:
# For each section, split it into scan size chunks
for chunk_start in range(section_start, section_start + section_length, scanner.chunk_size):
# Shorten it, if we're at the end of the section
chunk_length = min(section_start + section_length - chunk_start, scanner.chunk_size + scanner.overlap)
# Prev offset keeps track of the end of the previous subchunk
prev_offset = chunk_start
output = []
length = 0
# We populate the response based on subchunks that may be mapped all over the place
for mapped in self.mapping(chunk_start, chunk_length, ignore_errors = True):
offset, mapped_offset, length, layer_name = mapped
# We need to check if the offset is next to the end of the last one (contiguous)
if offset != prev_offset:
# Only yield if we've accumulated output
if len(output):
yield output, prev_offset + length
# Yield all the (joined) items so far
# and the ending point of that subchunk (where we'd gotten to previously)
yield output, prev_offset
output = []
# Shift the marker up to the end of what we just received and add it to the output
prev_offset = offset + length
output += [(layer_name, mapped_offset, length)]
# If there's still output left, output it
if len(output):
yield output, prev_offset + length
yield output, prev_offset
class LayerContainer(collections.abc.Mapping):