Document the reason for skipping entirely duplicated page tables

volatility3/framework/layers/intel.py

@@ -10,7 +10,7 @@ import struct
 from typing import Any, Dict, Iterable, List, Optional, Tuple
 
 from volatility3 import classproperty
-from volatility3.framework import exceptions, interfaces, constants
+from volatility3.framework import constants, exceptions, interfaces
 from volatility3.framework.configuration import requirements
 from volatility3.framework.layers import linear
 
@@ -276,7 +276,31 @@ class Intel(linear.LinearlyMappedLayer):
         except exceptions.InvalidAddressException:
             return None
 
+        ####
         # If the table is entirely duplicates, then mark the whole table as bad
+        # This is because Windows 10 onwards has a tendency to map unused pages as present
+        # This had the following consequences:
+        #  - Used very litle physical memory
+        #  - Exploded virtual memory
+        #  - Causes *scan plugins to take multiple hours to complete even on small images
+
+        # Previous versions of volatility would ignore a page during a scan when it matched
+        # the one directly preceding it in physical memory.
+        # This could trip if only two pages were identical and still required enumerating all
+        # the invalid pages (which itself was quite time consuming)
+
+        # For this reason, volatility 3 shifted to looking at entire page tables (1,024 pages)
+        # and if all the pages mapped to the same place the table wouuld be skipped
+        # This could also be applied to the Directory level as well as the Table level, allowing
+        # Volatility to skip huge sections of virtual memory very efficiently, without missing
+        # any pages that were distinct within a particular page table (or directory).
+
+        # In order to work at this level, the logic was moved out of the scanning component and
+        # directly into the layer logic itself.  This does have the side effect of preventing
+        # entirely duplicated page tables from reporting as present, however, the trade off between
+        # Windows 10+ reduced scanning times (common amongst scan plugins) versus incorrectly reporting
+        # entire page tables of identically mapped repeating *valid* data (rare) was accepted in favour
+        # of the more common occurance.
         if table == table[: self._entry_size] * self._entry_number:
             return None
         return table