Add an efficient multi-string search module (based on Wu-Manber 1994).

Empirically this seems far quicker than either Aho-Corasick or
Commentz-Walter for our dataset (very many long search terms).

volatility/framework/layers/scanners/__init__.py

@@ -1,6 +1,8 @@
 import re
 
 from volatility.framework.interfaces import layers
+from volatility.framework.layers.scanners import wumanber
+from volatility.framework.layers.scanners.suffix_tree import SuffixTree
 
 
 class BytesScanner(layers.ScannerInterface):
@@ -35,3 +37,29 @@ class RegExScanner(layers.ScannerInterface):
         for match in find_pos:
             offset = match.start()
             yield offset + data_offset
+
+
+class MultiStringScanner(layers.ScannerInterface):
+    thread_safe = True
+
+    def __init__(self, patterns):
+        super().__init__()
+        self._check_type(patterns, list)
+        self._patterns = wumanber.WuManber()
+        try:
+            for pattern in patterns:
+                self._check_type(pattern, bytes)
+                self._patterns.add_pattern(pattern)
+            self._patterns.preprocess()
+        except Exception as e:
+            print(repr(e))
+
+    def __call__(self, data, data_offset):
+        """Runs through the data looking for the needles"""
+        try:
+            for pattern, offset in self._patterns.search(data):
+                yield offset + data_offset, pattern
+        except Exception as e:
+            import pdb
+            pdb.post_mortem()
+            print("EXCEPTION", repr(e))

volatility/framework/layers/scanners/wumanber.py

@@ -0,0 +1,75 @@
+class WuManber(object):
+    """Algorithm for multi-string matching"""
+
+    def __init__(self, block_size = 3):
+        self.minimum_pattern_length = None
+
+        self._block_size = block_size
+        self._maximum_hash = 1 << 16  # This depends on the hash function used
+
+        self._patterns = []
+        self._shift = None  # This gets generated by preprocess
+        self._hashes = [set() for _ in range(self._maximum_hash)]
+
+    def add_pattern(self, pattern):
+        if not isinstance(pattern, bytes):
+            raise TypeError("Pattern must be a byte string")
+
+        if len(pattern) < self._block_size:
+            raise ValueError("Pattern legnth is too short")
+
+        self._patterns.append(pattern)
+
+    def preprocess(self):
+        """Preprocesses the patterns by populating the three arrays"""
+
+        # Set the minimun pattern length
+        self.minimum_pattern_length = min([len(pattern) for pattern in self._patterns])
+
+        max_jump = self.minimum_pattern_length - self._block_size + 1
+        self._shift = [max_jump] * self._maximum_hash
+        self.hashes = [set() for _ in range(self._maximum_hash)]
+
+        for pattern in self._patterns:
+            for i in range(self._block_size, self.minimum_pattern_length + 1):
+                hashval = self._hash_function(pattern[i - self._block_size:i])
+                self._shift[hashval] = min(self._shift[hashval], self.minimum_pattern_length - i)
+                # This will be left with the last
+                if self.minimum_pattern_length - i == 0:
+                    self._hashes[hashval].add(pattern)
+
+    def _hash_function(self, value_bytes):
+        return (value_bytes[0] << 5) + (value_bytes[1] << 3) + value_bytes[2]
+
+    def search(self, haystack):
+        """Search through a large body of data for patterns previously added with add_pattern"""
+        if not isinstance(haystack, bytes):
+            raise TypeError("Search haystack must be a byte string")
+        if self._shift is None:
+            raise KeyError("Preprocess has not been run on WuManber object yet")
+        index = self.minimum_pattern_length
+        while index < len(haystack):
+            hashval = self._hash_function(haystack[index - self._block_size:index])
+            shift = self._shift[hashval]
+            if shift < 1:
+                shift = 1
+                for pattern in self._hashes[hashval]:
+                    match_start = index - self.minimum_pattern_length
+                    if pattern == haystack[match_start:match_start + len(pattern)]:
+                        yield (match_start, pattern)
+            index += shift
+
+
+if __name__ == '__main__':
+    wm = WuManber()
+    print("Preprocessing")
+    for word in [b"quick bro", b"lazy do", b"abcd", b"fgh"]:
+        wm.add_pattern(word)
+    wm.preprocess()
+    print("Preprocessed")
+    print("Quick fox")
+    for result in wm.search(b"the quick brown fox jumped over the lazy dog"):
+        print("RESULT", repr(result))
+    print("ABC")
+    for result in wm.search(b"abcdefghijk"):
+        print("RESULT", repr(result))