Merge branch 'volatilityfoundation:develop' into feature/vadwalk

doc/source/getting-started-linux-tutorial.rst

@@ -79,7 +79,7 @@ Thanks go to `stuxnet <https://github.com/stuxnet999/>`_ for providing this memo
 
 
 The above command helps us to find the memory dump's kernel version and the distribution version. Now using the above banner we can search for the needed ISF file from the ISF server.
-If ISF file cannt be found then, follow the instructions on :ref:`getting-started-linux-tutorial:Procedure to create symbol tables for linux`. After that, place the ISF file under the ``volatility3/symbols/linux`` directory.
+If ISF file cannot be found then, follow the instructions on :ref:`getting-started-linux-tutorial:Procedure to create symbol tables for linux`. After that, place the ISF file under the ``volatility3/symbols/linux`` directory.
 
 .. tip:: Use the banner text which is most repeated to search from ISF Server.
 

setup.py

@@ -40,7 +40,7 @@ setuptools.setup(name = "volatility3",
                      '': ['development', 'development.*'],
                      'development': ['*']
                  },
-                 packages = setuptools.find_packages(exclude = ["development", "development.*"]),
+                 packages = setuptools.find_namespace_packages(exclude = ["development", "development.*"]),
                  entry_points = {
                      'console_scripts': [
                          'vol = volatility3.cli:main',

volatility3/framework/automagic/symbol_cache.py

@@ -2,6 +2,7 @@
 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0
 #
 import base64
+import datetime
 import json
 import logging
 import os
@@ -157,10 +158,10 @@ class SqliteCache(CacheManagerInterface):
     _required_framework_version = (2, 0, 0)
     _version = (1, 0, 0)
 
-    cache_period = '-3 days'
 
     def __init__(self, filename: str):
         super().__init__(filename)
+        self.cache_period = constants.SQLITE_CACHE_PERIOD
         try:
             self._database = self._connect_storage(filename)
         except sqlite3.DatabaseError:
@@ -170,6 +171,7 @@ class SqliteCache(CacheManagerInterface):
     def _connect_storage(self, path: str) -> sqlite3.Connection:
         database = sqlite3.connect(path)
         database.row_factory = sqlite3.Row
+
         database.cursor().execute(
             f'CREATE TABLE IF NOT EXISTS database_info (schema_version INT DEFAULT {constants.CACHE_SQLITE_SCHEMA_VERSION})')
         schema_version = database.cursor().execute('SELECT schema_version FROM database_info').fetchone()
@@ -259,10 +261,31 @@ class SqliteCache(CacheManagerInterface):
         cache_update = set()
         files_to_timestamp = on_disk_locations.intersection(cached_locations)
         if files_to_timestamp:
-            result = self._database.cursor().execute("SELECT location FROM cache WHERE local = 1 "
+            result = self._database.cursor().execute("SELECT location, cached FROM cache WHERE local = 1 "
                                                      f"AND cached < date('now', '{self.cache_period}');")
             for row in result:
-                if row['location'] in files_to_timestamp:
+                location = row['location']
+                stored_timestamp = datetime.datetime.fromisoformat(row['cached'])
+                timestamp = stored_timestamp # Default to requiring update
+
+                # See if the file is a local URL type we can handle:
+                parsed = urllib.parse.urlparse(location)
+                pathname = None
+                if parsed.scheme == 'file':
+                    pathname = urllib.request.url2pathname(parsed.path)
+                if parsed.scheme == 'jar':
+                    inner_url = urllib.parse.urlparse(parsed.path)
+                    if inner_url.scheme == 'file':
+                        pathname = inner_url.path.split('!')[0]
+
+                if pathname:
+                    timestamp = datetime.datetime.fromtimestamp(os.stat(pathname).st_mtime)
+                else:
+                    vollog.log(constants.LOGLEVEL_VVVV,
+                               "File location in database classed as local but not file/jar URL")
+
+                # If we're supposed to include it, and our last check is older than (or equal to) the file timestamp
+                if row['location'] in files_to_timestamp and stored_timestamp < timestamp:
                     cache_update.add(row['location'])
 
         idextractors = list(framework.class_subclasses(IdentifierProcessor))

volatility3/framework/constants/__init__.py

@@ -40,7 +40,7 @@ BANG = "!"
 # We use the SemVer 2.0.0 versioning scheme
 VERSION_MAJOR = 2  # Number of releases of the library with a breaking change
 VERSION_MINOR = 4  # Number of changes that only add to the interface
-VERSION_PATCH = 0  # Number of changes that do not change the interface
+VERSION_PATCH = 1  # Number of changes that do not change the interface
 VERSION_SUFFIX = ""
 
 # TODO: At version 2.0.0, remove the symbol_shift feature
@@ -63,6 +63,9 @@ LOGLEVEL_VVVV = 6
 CACHE_PATH = os.path.join(os.path.expanduser("~"), ".cache", "volatility3")
 """Default path to store cached data"""
 
+SQLITE_CACHE_PERIOD = '-3 days'
+"""SQLite time modifier for how long each item is valid in the cache for"""
+
 if sys.platform == 'win32':
     CACHE_PATH = os.path.realpath(os.path.join(os.environ.get("APPDATA", os.path.expanduser("~")), "volatility3"))
 os.makedirs(CACHE_PATH, exist_ok = True)

volatility3/framework/symbols/mac/extensions/__init__.py

@@ -28,8 +28,11 @@ class proc(generic.GenericIntelProcess):
         if not isinstance(parent_layer, interfaces.layers.TranslationLayerInterface):
             raise TypeError("Parent layer is not a translation layer, unable to construct process layer")
 
-        with contextlib.suppress(exceptions.InvalidAddressException):
+        try:
             dtb = self.get_task().map.pmap.pm_cr3
+        except exceptions.InvalidAddressException:
+            # Bail out because we couldn't find the DTB
+            return None
 
         if preferred_name is None:
             preferred_name = self.vol.layer_name + f"_Process{self.p_pid}"
@@ -38,10 +41,8 @@ class proc(generic.GenericIntelProcess):
         return self._add_process_layer(self._context, dtb, config_prefix, preferred_name)
 
     def get_map_iter(self) -> Iterable[interfaces.objects.ObjectInterface]:
-        with contextlib.suppress(exceptions.InvalidAddressException):
-            task = self.get_task()
-
         try:
+            task = self.get_task()
             current_map = task.map.hdr.links.next
         except exceptions.InvalidAddressException:
             return