Merge branch 'develop' into linux_pslist_dependencies_fix_1366

test/test_volatility.py

@@ -14,6 +14,7 @@ import tempfile
 import hashlib
 import ntpath
 import json
+import contextlib
 
 #
 # HELPER FUNCTIONS
@@ -275,6 +276,59 @@ def test_windows_devicetree(image, volatility, python):
     assert rc == 0
 
 
+def test_windows_vadyarascan_yara_rule(image, volatility, python):
+    yara_rule_01 = r"""
+        rule fullvadyarascan
+        {
+            strings:
+                $s1 = "!This program cannot be run in DOS mode."
+                $s2 = "Qw))Pw"
+                $s3 = "W_wD)Pw"
+                $s4 = "1Xw+2Xw"
+                $s5 = "xd`wh``w"
+                $s6 = "0g`w0g`w8g`w8g`w@g`w@g`wHg`wHg`wPg`wPg`wXg`wXg`w`g`w`g`whg`whg`wpg`wpg`wxg`wxg`w"
+            condition:
+                all of them
+        }
+    """
+
+    # FIXME: When the minimum Python version includes 3.12, replace the following with:
+    # with tempfile.NamedTemporaryFile(delete_on_close=False) as fd: ...
+    fd, filename = tempfile.mkstemp(suffix=".yar")
+    try:
+        with os.fdopen(fd, "w") as f:
+            f.write(yara_rule_01)
+
+        rc, out, _err = runvol_plugin(
+            "windows.vadyarascan.VadYaraScan",
+            image,
+            volatility,
+            python,
+            pluginargs=["--pid", "4012", "--yara-file", filename],
+        )
+    finally:
+        with contextlib.suppress(FileNotFoundError):
+            os.remove(filename)
+
+    out = out.lower()
+    assert out.count(b"\n") > 4
+    assert rc == 0
+
+
+def test_windows_vadyarascan(image, volatility, python):
+    rc, out, _err = runvol_plugin(
+        "windows.vadyarascan.VadYaraScan",
+        image,
+        volatility,
+        python,
+        pluginargs=["--pid", "4012", "--yara-string", "MZ"],
+    )
+    out = out.lower()
+
+    assert out.count(b"\n") > 10
+    assert rc == 0
+
+
 # LINUX
 
 
@@ -540,6 +594,42 @@ def test_linux_vmayarascan(image, volatility, python):
     assert rc == 0
 
 
+def test_linux_vmayarascan_yara_rule(image, volatility, python):
+    yara_rule_01 = r"""
+        rule fullvmayarascan
+        {
+            strings:
+                $s1 = "_nss_files_parse_grent"
+                $s2 = "/lib64/ld-linux-x86-64.so.2"
+                $s3 = "(bufferend - (char *) 0) % sizeof (char *) == 0"
+            condition:
+                all of them
+        }
+    """
+
+    # FIXME: When the minimum Python version includes 3.12, replace the following with:
+    # with tempfile.NamedTemporaryFile(delete_on_close=False) as fd: ...
+    fd, filename = tempfile.mkstemp(suffix=".yar")
+    try:
+        with os.fdopen(fd, "w") as f:
+            f.write(yara_rule_01)
+
+        rc, out, _err = runvol_plugin(
+            "linux.vmayarascan.VmaYaraScan",
+            image,
+            volatility,
+            python,
+            pluginargs=["--pid", "8600", "--yara-file", filename],
+        )
+    finally:
+        with contextlib.suppress(FileNotFoundError):
+            os.remove(filename)
+
+    out = out.lower()
+    assert out.count(b"\n") > 4
+    assert rc == 0
+
+
 # MAC
 
 

volatility3/framework/automagic/pdbscan.py

@@ -272,7 +272,7 @@ class KernelPDBScanner(interfaces.automagic.AutomagicInterface):
         for kernel in kernels:
             vollog.log(
                 constants.LOGLEVEL_VVVV,
-                f"Testing potential kernel for {kernel.get('pdb_name', 'Unknown')} at {kernel.get('signature_offset', -1)} with MZ offset at {kernel.get('mz_offset', -1)}",
+                f"Testing potential kernel for {kernel.get('pdb_name', 'Unknown')} at {kernel.get('signature_offset', -1):x} with MZ offset at {(kernel.get('mz_offset', -1) or -1):x}",
             )
             valid_kernel = test_kernel(physical_layer_name, virtual_layer_name, kernel)
             if valid_kernel is not None:

volatility3/framework/automagic/symbol_finder.py

@@ -142,11 +142,11 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface):
             )
 
         for _, banner in banner_list:
-            vollog.debug(f"Identified banner: {repr(banner)}")
-            symbol_files = self.banners.get(banner, None)
-            if symbol_files:
-                isf_path = symbol_files
-                vollog.debug(f"Using symbol library: {symbol_files}")
+            vollog.debug(f"Identified banner: {banner!r}")
+            symbols_file = self.banners.get(banner, None)
+            if symbols_file:
+                isf_path = symbols_file
+                vollog.debug(f"Using symbol library: {symbols_file}")
                 clazz = self.symbol_class
                 # Set the discovered options
                 path_join = interfaces.configuration.path_join
@@ -160,8 +160,31 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface):
                     path_join(config_path, requirement.name, "symbol_mask")
                 ] = layer.address_mask
 
+                # Keep track of the existing table names so we know which ones were added
+                old_table_names = set(context.symbol_space)
+
                 # Construct the appropriate symbol table
                 requirement.construct(context, config_path)
+
+                new_table_names = set(context.symbol_space) - old_table_names
+                # It should add only one symbol table. Ignore the next steps if it doesn't
+                if len(new_table_names) == 1:
+                    new_table_name = new_table_names.pop()
+                    symbol_table = context.symbol_space[new_table_name]
+                    producer_metadata = symbol_table.producer
+                    vollog.debug(
+                        f"producer_name: {producer_metadata.name}, producer_version: {producer_metadata.version_string}"
+                    )
+
+                    symbol_metadata = symbol_table.metadata
+                    vollog.debug("Types:")
+                    for types_source_dict in symbol_metadata.get_types_sources():
+                        vollog.debug(f"\t{types_source_dict}")
+
+                    vollog.debug("Symbols:")
+                    for symbol_source_dict in symbol_metadata.get_symbols_sources():
+                        vollog.debug(f"\t{symbol_source_dict}")
+
                 break
             else:
                 vollog.debug(f"Symbol library path not found for: {banner}")

volatility3/framework/plugins/linux/vmayarascan.py

@@ -2,6 +2,7 @@
 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0
 #
 
+import logging
 from typing import Iterable, List, Tuple
 
 from volatility3.framework import interfaces, renderers
@@ -10,6 +11,8 @@ from volatility3.framework.renderers import format_hints
 from volatility3.plugins import yarascan
 from volatility3.plugins.linux import pslist
 
+vollog = logging.getLogger(__name__)
+
 
 class VmaYaraScan(interfaces.plugins.PluginInterface):
     """Scans all virtual memory areas for tasks using yara."""
@@ -33,6 +36,9 @@ class VmaYaraScan(interfaces.plugins.PluginInterface):
             requirements.PluginRequirement(
                 name="yarascan", plugin=yarascan.YaraScan, version=(2, 0, 0)
             ),
+            requirements.VersionRequirement(
+                name="yarascanner", component=yarascan.YaraScanner, version=(2, 0, 0)
+            ),
             requirements.ModuleRequirement(
                 name="kernel",
                 description="Linux kernel",
@@ -50,6 +56,8 @@ class VmaYaraScan(interfaces.plugins.PluginInterface):
         # use yarascan to parse the yara options provided and create the rules
         rules = yarascan.YaraScan.process_yara_options(dict(self.config))
 
+        sanity_check = 1024 * 1024 * 1024  # 1 GB
+
         # filter based on the pid option if provided
         filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None))
         for task in pslist.PsList.list_tasks(
@@ -66,29 +74,36 @@ class VmaYaraScan(interfaces.plugins.PluginInterface):
             # get the proc_layer object from the context
             proc_layer = self.context.layers[proc_layer_name]
 
-            for start, end in self.get_vma_maps(task):
-                for match in rules.match(
-                    data=proc_layer.read(start, end - start, True)
+            max_vma_size = 0
+            vma_maps_to_scan = []
+            for start, size in self.get_vma_maps(task):
+                if size > sanity_check:
+                    vollog.debug(
+                        f"VMA at 0x{start:x} over sanity-check size, not scanning"
+                    )
+                    continue
+                max_vma_size = max(max_vma_size, size)
+                vma_maps_to_scan.append((start, size))
+
+            if not vma_maps_to_scan:
+                vollog.warning(f"No VMAs were found for task {task.tgid}, not scanning")
+                continue
+
+            scanner = yarascan.YaraScanner(rules=rules)
+            scanner.chunk_size = max_vma_size
+
+            # scan the VMA data (in one contiguous block) with the yarascanner
+            for start, size in vma_maps_to_scan:
+                for offset, rule_name, name, value in scanner(
+                    proc_layer.read(start, size, pad=True), start
                 ):
-                    if yarascan.YaraScan.yara_returns_instances():
-                        for match_string in match.strings:
-                            for instance in match_string.instances:
-                                yield 0, (
-                                    format_hints.Hex(instance.offset + start),
-                                    task.UniqueProcessId,
-                                    match.rule,
-                                    match_string.identifier,
-                                    instance.matched_data,
-                                )
-                    else:
-                        for offset, name, value in match.strings:
-                            yield 0, (
-                                format_hints.Hex(offset + start),
-                                task.tgid,
-                                match.rule,
-                                name,
-                                value,
-                            )
+                    yield 0, (
+                        format_hints.Hex(offset),
+                        task.tgid,
+                        rule_name,
+                        name,
+                        value,
+                    )
 
     @staticmethod
     def get_vma_maps(

volatility3/framework/plugins/windows/vadyarascan.py

@@ -32,6 +32,9 @@ class VadYaraScan(interfaces.plugins.PluginInterface):
             requirements.PluginRequirement(
                 name="pslist", plugin=pslist.PsList, version=(2, 0, 0)
             ),
+            requirements.VersionRequirement(
+                name="yarascanner", component=yarascan.YaraScanner, version=(2, 0, 0)
+            ),
             requirements.PluginRequirement(
                 name="yarascan", plugin=yarascan.YaraScan, version=(2, 0, 0)
             ),
@@ -66,49 +69,40 @@ class VadYaraScan(interfaces.plugins.PluginInterface):
         ):
             layer_name = task.add_process_layer()
             layer = self.context.layers[layer_name]
+
+            max_vad_size = 0
+            vad_maps_to_scan = []
+
             for start, size in self.get_vad_maps(task):
                 if size > sanity_check:
                     vollog.debug(
                         f"VAD at 0x{start:x} over sanity-check size, not scanning"
                     )
                     continue
+                max_vad_size = max(max_vad_size, size)
+                vad_maps_to_scan.append((start, size))
 
-                data = layer.read(start, size, True)
-                if not yarascan.YaraScan._yara_x:
-                    for match in rules.match(data=data):
-                        if yarascan.YaraScan.yara_returns_instances():
-                            for match_string in match.strings:
-                                for instance in match_string.instances:
-                                    yield 0, (
-                                        format_hints.Hex(instance.offset + start),
-                                        task.UniqueProcessId,
-                                        match.rule,
-                                        match_string.identifier,
-                                        instance.matched_data,
-                                    )
-                        else:
-                            for offset, name, value in match.strings:
-                                yield 0, (
-                                    format_hints.Hex(offset + start),
-                                    task.UniqueProcessId,
-                                    match.rule,
-                                    name,
-                                    value,
-                                )
-                else:
-                    for match in rules.scan(data).matching_rules:
-                        for match_string in match.patterns:
-                            for instance in match_string.matches:
-                                yield 0, (
-                                    format_hints.Hex(instance.offset + start),
-                                    task.UniqueProcessId,
-                                    f"{match.namespace}.{match.identifier}",
-                                    match_string.identifier,
-                                    data[
-                                        instance.offset : instance.offset
-                                        + instance.length
-                                    ],
-                                )
+            if not vad_maps_to_scan:
+                vollog.warning(
+                    f"No VADs were found for task {task.UniqueProcessID}, not scanning"
+                )
+                continue
+
+            scanner = yarascan.YaraScanner(rules=rules)
+            scanner.chunk_size = max_vad_size
+
+            # scan the VAD data (in one contiguous block) with the yarascanner
+            for start, size in vad_maps_to_scan:
+                for offset, rule_name, name, value in scanner(
+                    layer.read(start, size, pad=True), start
+                ):
+                    yield 0, (
+                        format_hints.Hex(offset),
+                        task.UniqueProcessId,
+                        rule_name,
+                        name,
+                        value,
+                    )
 
     @staticmethod
     def get_vad_maps(

volatility3/framework/symbols/intermed.py

@@ -738,10 +738,17 @@ class Version6Format(Version5Format):
     @property
     def metadata(self) -> Optional[interfaces.symbols.MetadataInterface]:
         """Returns a MetadataInterface object."""
-        if self._json_object.get("metadata", {}).get("windows"):
-            return metadata.WindowsMetadata(self._json_object["metadata"]["windows"])
-        if self._json_object.get("metadata", {}).get("linux"):
-            return metadata.LinuxMetadata(self._json_object["metadata"]["linux"])
+        if "metadata" not in self._json_object:
+            return None
+
+        json_metadata = self._json_object["metadata"]
+        if "windows" in json_metadata:
+            return metadata.WindowsMetadata(json_metadata["windows"])
+        if "linux" in json_metadata:
+            return metadata.LinuxMetadata(json_metadata["linux"])
+        if "mac" in json_metadata:
+            return metadata.MacMetadata(json_metadata["mac"])
+
         return None
 
 

volatility3/framework/symbols/metadata.py

@@ -4,8 +4,7 @@
 
 import datetime
 import logging
-from typing import Optional, Tuple, Union
-
+from typing import Optional, Tuple, Union, List, Dict
 from volatility3.framework import constants, interfaces
 
 vollog = logging.getLogger(__name__)
@@ -18,10 +17,17 @@ class ProducerMetadata(interfaces.symbols.MetadataInterface):
     def name(self) -> Optional[str]:
         return self._json_data.get("name", None)
 
+    @property
+    def version_string(self) -> str:
+        """Returns the ISF file producer's version as a string.
+        If no version is present, an empty string is returned.
+        """
+        return self._json_data.get("version", "")
+
     @property
     def version(self) -> Optional[Tuple[int]]:
         """Returns the version of the ISF file producer"""
-        version = self._json_data.get("version", None)
+        version = self.version_string()
         if not version:
             return None
         if all(x in "0123456789." for x in version):
@@ -79,5 +85,21 @@ class WindowsMetadata(interfaces.symbols.MetadataInterface):
         return self._json_data.get("pdb", {}).get("age", None)
 
 
-class LinuxMetadata(interfaces.symbols.MetadataInterface):
+class PosixMetadata(interfaces.symbols.MetadataInterface):
+    """Base class to handle metadata of Posix-based ISF sources"""
+
+    def get_types_sources(self) -> List[Optional[Dict]]:
+        """Returns the types sources metadata"""
+        return self._json_data.get("types", [])
+
+    def get_symbols_sources(self) -> List[Optional[Dict]]:
+        """Returns the symbols sources metadata"""
+        return self._json_data.get("symbols", [])
+
+
+class LinuxMetadata(PosixMetadata):
     """Class to handle the metadata from a Linux symbol table."""
+
+
+class MacMetadata(PosixMetadata):
+    """Class to handle the metadata from a Mac symbol table."""

volatility3/framework/symbols/windows/extensions/services.py

@@ -14,13 +14,16 @@ class SERVICE_RECORD(objects.StructType):
 
     def is_valid(self) -> bool:
         """Determine if the structure is valid."""
-        if self.Order < 0 or self.Order > 0xFFFF:
-            return False
-
         try:
-            _ = self.State.description
-            _ = self.Start.description
-        except ValueError:
+            if self.Order < 0 or self.Order > 0xFFFF:
+                return False
+
+            try:
+                _ = self.State.description
+                _ = self.Start.description
+            except ValueError:
+                return False
+        except exceptions.InvalidAddressException:
             return False
 
         return True