Merge branch 'volatilityfoundation:develop' into fix/unused-module

.style.yapf

@@ -107,7 +107,7 @@ each_dict_entry_on_separate_line=True
 i18n_comment=
 
 # The i18n function call names. The presence of this function stops
-# reformattting on that line, because the string it has cannot be moved
+# reformatting on that line, because the string it has cannot be moved
 # away from the i18n comment.
 i18n_function_call=
 

API_CHANGES.md

@@ -4,6 +4,14 @@ API Changes
 When an addition to the existing API is made, the minor version is bumped.
 When an API feature or function is removed or changed, the major version is bumped.
 
+2.4.0
+=====
+Add a `get_size()` method to Windows VAD structures and fix several off-by-one issues when calculating VAD sizes.
+
+2.3.1
+=====
+Update in the windows `_EPROCESS.owning_process` method to support Windows Vista and later versions.
+
 2.3.0
 =====
 Add in `child_template` to template class

LICENSE.txt

@@ -31,7 +31,7 @@ If you make any Additions available to others, such as by providing copies of th
 - You are responsible to ensure you have rights in Additions necessary to comply with this section.
 
 Contributing
-If you contribute (or offer to contribute) any materials to Volatility Foundation for the software, such as by submitting a pull request to the repository for the software or related content run by Volatility Foundation, you agree to contribute them under the under the BSD 2-Clause Plus Patent License (in the case of software) or the Creative Commons Zero Public Domain Dedication (in the case of content), unless you clearly mark them "Not a Contribution."
+If you contribute (or offer to contribute) any materials to Volatility Foundation for the software, such as by submitting a pull request to the repository for the software or related content run by Volatility Foundation, you agree to contribute them under the BSD 2-Clause Plus Patent License (in the case of software) or the Creative Commons Zero Public Domain Dedication (in the case of content), unless you clearly mark them "Not a Contribution."
 
 Trademarks
 This license grants you no rights to any trademarks or service marks.

volatility3/cli/text_renderer.py

@@ -224,7 +224,7 @@ class CSVRenderer(CLIRenderer):
             # Ignore the type because namedtuples don't realize they have accessible attributes
             header_list.append(f"{column.name}")
 
-        writer = csv.DictWriter(outfd, header_list)
+        writer = csv.DictWriter(outfd, header_list, lineterminator='\n')
         writer.writeheader()
 
         def visitor(node: interfaces.renderers.TreeNode, accumulator):

volatility3/framework/automagic/linux.py

@@ -3,6 +3,7 @@
 #
 
 import logging
+import os
 from typing import Optional, Tuple, Type
 
 from volatility3.framework import constants, interfaces
@@ -40,7 +41,8 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface):
         if isinstance(layer, intel.Intel):
             return None
 
-        linux_banners = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH).get_identifier_dictionary(
+        identifiers_path = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME)
+        linux_banners = symbol_cache.SqliteCache(identifiers_path).get_identifier_dictionary(
             operating_system = 'linux')
         # If we have no banners, don't bother scanning
         if not linux_banners:

volatility3/framework/automagic/mac.py

@@ -3,6 +3,7 @@
 #
 
 import logging
+import os
 import struct
 from typing import Optional
 
@@ -42,7 +43,8 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface):
         if isinstance(layer, intel.Intel):
             return None
 
-        mac_banners = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH).get_identifier_dictionary(
+        identifiers_path = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME)
+        mac_banners = symbol_cache.SqliteCache(identifiers_path).get_identifier_dictionary(
             operating_system = 'mac')
         # If we have no banners, don't bother scanning
         if not mac_banners:

volatility3/framework/automagic/symbol_cache.py

@@ -388,7 +388,8 @@ class SymbolCacheMagic(interfaces.automagic.AutomagicInterface):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self._cache = SqliteCache(constants.IDENTIFIERS_PATH)
+        identifiers_path = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME)
+        self._cache = SqliteCache(identifiers_path)
 
     def __call__(self, context, config_path, configurable, progress_callback = None):
         """Runs the automagic over the configurable."""

volatility3/framework/automagic/symbol_finder.py

@@ -3,6 +3,7 @@
 #
 
 import logging
+import os
 from typing import Any, Callable, Iterable, List, Optional, Tuple
 
 from volatility3.framework import constants, interfaces, layers
@@ -40,7 +41,8 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface):
         """Creates a cached copy of the results, but only it's been
         requested."""
         if not self._banners:
-            cache = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH)
+            identifiers_path = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME)
+            cache = symbol_cache.SqliteCache(identifiers_path)
             self._banners = cache.get_identifier_dictionary(operating_system = self.operating_system)
         return self._banners
 

volatility3/framework/constants/__init__.py

@@ -39,8 +39,8 @@ BANG = "!"
 
 # We use the SemVer 2.0.0 versioning scheme
 VERSION_MAJOR = 2  # Number of releases of the library with a breaking change
-VERSION_MINOR = 3  # Number of changes that only add to the interface
-VERSION_PATCH = 1  # Number of changes that do not change the interface
+VERSION_MINOR = 4  # Number of changes that only add to the interface
+VERSION_PATCH = 0  # Number of changes that do not change the interface
 VERSION_SUFFIX = ""
 
 # TODO: At version 2.0.0, remove the symbol_shift feature
@@ -67,13 +67,7 @@ if sys.platform == 'win32':
     CACHE_PATH = os.path.realpath(os.path.join(os.environ.get("APPDATA", os.path.expanduser("~")), "volatility3"))
 os.makedirs(CACHE_PATH, exist_ok = True)
 
-LINUX_BANNERS_PATH = os.path.join(CACHE_PATH, "linux_banners.cache")
-"""Default location to record information about available linux banners"""
-
-MAC_BANNERS_PATH = os.path.join(CACHE_PATH, "mac_banners.cache")
-"""Default location to record information about available mac banners"""
-
-IDENTIFIERS_PATH = os.path.join(CACHE_PATH, "identifiers.cache")
+IDENTIFIERS_FILENAME = "identifier.cache"
 """Default location to record information about available identifiers"""
 
 CACHE_SQLITE_SCHEMA_VERSION = 1

volatility3/framework/plugins/isfinfo.py

@@ -109,7 +109,8 @@ class IsfInfo(plugins.PluginInterface):
                         num_enums = len(data.get('enums', []))
                         num_bases = len(data.get('base_types', []))
 
-                        identifier_cache = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH)
+                        identifiers_path = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME)
+                        identifier_cache = symbol_cache.SqliteCache(identifiers_path)
                         identifier = identifier_cache.get_identifier(location = entry)
                         if identifier:
                             identifier = identifier.decode('utf-8', errors = 'replace')
@@ -120,7 +121,8 @@ class IsfInfo(plugins.PluginInterface):
                         vollog.warning(f"Invalid ISF: {entry}")
                 yield (0, (entry, valid, num_bases, num_types, num_symbols, num_enums, identifier))
         else:
-            cache = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH)
+            identifiers_path = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME)
+            cache = symbol_cache.SqliteCache(identifiers_path)
             valid = 'Unknown'
             for identifier, location in cache.get_identifier_dictionary().items():
                 num_bases, num_types, num_enums, num_symbols = cache.get_location_statistics(location)

volatility3/framework/plugins/windows/cachedump.py

@@ -46,7 +46,7 @@ class Cachedump(interfaces.plugins.PluginInterface):
             rc4 = ARC4.new(rc4key)
             data = rc4.encrypt(edata) # lgtm [py/weak-cryptographic-algorithm]
         else:
-            # based on  Based on code from http://lab.mediaservice.net/code/cachedump.rb
+            # Based on code from http://lab.mediaservice.net/code/cachedump.rb
             aes = AES.new(nlkm[16:32], AES.MODE_CBC, ch)
             data = b""
             for i in range(0, len(edata), 16):

volatility3/framework/plugins/windows/malfind.py

@@ -17,7 +17,7 @@ vollog = logging.getLogger(__name__)
 class Malfind(interfaces.plugins.PluginInterface):
     """Lists process memory ranges that potentially contain injected code."""
 
-    _required_framework_version = (2, 0, 0)
+    _required_framework_version = (2, 4, 0)
 
     @classmethod
     def get_requirements(cls):
@@ -56,7 +56,7 @@ class Malfind(interfaces.plugins.PluginInterface):
         all_zero_page = b"\x00" * CHUNK_SIZE
 
         offset = 0
-        vad_length = vad.get_end() - vad.get_start()
+        vad_length = vad.get_size()
 
         while offset < vad_length:
             next_addr = vad.get_start() + offset

volatility3/framework/plugins/windows/skeleton_key_check.py

@@ -41,7 +41,7 @@ vollog = logging.getLogger(__name__)
 class Skeleton_Key_Check(interfaces.plugins.PluginInterface):
     """ Looks for signs of Skeleton Key malware """
 
-    _required_framework_version = (2, 0, 0)
+    _required_framework_version = (2, 4, 0)
 
     @classmethod
     def get_requirements(cls):
@@ -262,7 +262,7 @@ class Skeleton_Key_Check(interfaces.plugins.PluginInterface):
 
             if isinstance(filename, str) and filename.lower().endswith("cryptdll.dll"):
                 base = vad.get_start()
-                return base, vad.get_end() - base
+                return base, vad.get_size()
 
         return None, None
 

volatility3/framework/plugins/windows/vadinfo.py

@@ -33,7 +33,7 @@ winnt_protections = {
 class VadInfo(interfaces.plugins.PluginInterface):
     """Lists process memory ranges."""
 
-    _required_framework_version = (2, 0, 0)
+    _required_framework_version = (2, 4, 0)
     _version = (2, 0, 0)
     MAXSIZE_DEFAULT = 1024 * 1024 * 1024  # 1 Gb
 
@@ -132,7 +132,7 @@ class VadInfo(interfaces.plugins.PluginInterface):
             vollog.debug("Unable to find the starting/ending VPN member")
             return None
 
-        if 0 < maxsize < (vad_end - vad_start):
+        if 0 < maxsize < vad.get_size():
             vollog.debug(f"Skip VAD dump {vad_start:#x}-{vad_end:#x} due to maxsize limit")
             return None
 
@@ -151,8 +151,9 @@ class VadInfo(interfaces.plugins.PluginInterface):
             file_handle = open_method(file_name)
             chunk_size = 1024 * 1024 * 10
             offset = vad_start
-            while offset < vad_end:
-                to_read = min(chunk_size, vad_end - offset)
+            vad_size = vad.get_size()
+            while offset < vad_start + vad_size:
+                to_read = min(chunk_size, vad_start + vad_size - offset)
                 data = proc_layer.read(offset, to_read, pad = True)
                 if not data:
                     break

volatility3/framework/plugins/windows/vadyarascan.py

@@ -17,7 +17,7 @@ vollog = logging.getLogger(__name__)
 class VadYaraScan(interfaces.plugins.PluginInterface):
     """Scans all the Virtual Address Descriptor memory maps using yara."""
 
-    _required_framework_version = (2, 0, 0)
+    _required_framework_version = (2, 4, 0)
     _version = (1, 0, 0)
 
     @classmethod
@@ -82,9 +82,7 @@ class VadYaraScan(interfaces.plugins.PluginInterface):
         """
         vad_root = task.get_vad_root()
         for vad in vad_root.traverse():
-            end = vad.get_end()
-            start = vad.get_start()
-            yield (start, end - start)
+            yield (vad.get_start(), vad.get_size())
 
     def run(self):
         return renderers.TreeGrid([('Offset', format_hints.Hex), ('PID', int), ('Rule', str), ('Component', str),

volatility3/framework/symbols/windows/extensions/__init__.py

@@ -197,8 +197,8 @@ class MMVAD_SHORT(objects.StructType):
 
         raise AttributeError("Unable to find the parent member")
 
-    def get_start(self):
-        """Get the VAD's starting virtual address."""
+    def get_start(self) -> int:
+        """Get the VAD's starting virtual address. This is the first accessible byte in the range."""
 
         if self.has_member("StartingVpn"):
 
@@ -216,8 +216,8 @@ class MMVAD_SHORT(objects.StructType):
 
         raise AttributeError("Unable to find the starting VPN member")
 
-    def get_end(self):
-        """Get the VAD's ending virtual address."""
+    def get_end(self) -> int:
+        """Get the VAD's ending virtual address. This is the last accessible byte in the range."""
 
         if self.has_member("EndingVpn"):
 
@@ -234,6 +234,10 @@ class MMVAD_SHORT(objects.StructType):
 
         raise AttributeError("Unable to find the ending VPN member")
 
+    def get_size(self) -> int:
+        """Get the size of the VAD region. The OS ensures page granularity."""
+        return (self.get_end() - self.get_start()) + 1
+
     def get_commit_charge(self):
         """Get the VAD's commit charge (number of committed pages)"""
 
@@ -488,7 +492,7 @@ class UNICODE_STRING(objects.StructType):
         # We manually construct an object rather than casting a dereferenced pointer in case
         # the buffer length is 0 and the pointer is a NULL pointer
         return self._context.object(self.vol.type_name.split(constants.BANG)[0] + constants.BANG + 'string',
-                                    layer_name = self.Buffer.vol.layer_name,
+                                    layer_name = self.Buffer.vol.native_layer_name,
                                     offset = self.Buffer,
                                     max_length = self.Length, errors = 'replace', encoding = 'utf16')
 

volatility3/framework/symbols/windows/pdbutil.py

@@ -80,7 +80,8 @@ class PDBUtility(interfaces.configuration.VersionableInterface):
             vollog.debug(f"Required version of SQLiteCache not found")
             return None
 
-        value = symbol_cache.SqliteCache(constants.IDENTIFIERS_PATH).find_location(
+        identifiers_path = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME)
+        value = symbol_cache.SqliteCache(identifiers_path).find_location(
             symbol_cache.WindowsIdentifier.generate(pdb_name.strip('\x00'), guid.upper(), age), 'windows')
 
         if value: