From afdd8b657eb104aaed653ae674862bb9f6892eee Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Sun, 30 Dec 2018 15:04:19 +0000 Subject: [PATCH] Fix memory problems with banner_caches. Since we stack using Mac layers as well as Linux layers, both sets of banner caches are required. As the cache builders are separate at the moment, we previously ran banner searches without any banners. This was bad. It inflated memory hugely and killed the buildbot. We now bomb out on stacking a Linux or Mac layer unless we have banners against which to match. --- volatility/framework/automagic/linux.py | 5 +++++ volatility/framework/automagic/mac.py | 11 +++++++---- volatility/framework/automagic/symbol_cache.py | 9 ++++++++- volatility/framework/layers/scanners/multiregexp.py | 2 ++ 4 files changed, 22 insertions(+), 5 deletions(-) diff --git a/volatility/framework/automagic/linux.py b/volatility/framework/automagic/linux.py index 3cff98b12..56fb7ccfb 100644 --- a/volatility/framework/automagic/linux.py +++ b/volatility/framework/automagic/linux.py @@ -66,6 +66,11 @@ class LintelStacker(interfaces.automagic.StackerLayerInterface): return None linux_banners = LinuxBannerCache.load_banners() + # If we have no banners, don't bother scanning + if not linux_banners: + vollog.info("No Linux banners found - if this is a linux plugin, please check your symbol files location") + return None + mss = scanners.MultiStringScanner([x for x in linux_banners if x is not None]) for _, banner in layer.scan(context = context, scanner = mss, progress_callback = progress_callback): dtb = None diff --git a/volatility/framework/automagic/mac.py b/volatility/framework/automagic/mac.py index 7eeac10a6..ff00eae64 100644 --- a/volatility/framework/automagic/mac.py +++ b/volatility/framework/automagic/mac.py @@ -20,7 +20,7 @@ import logging import struct -from typing import Optional, Tuple +from typing import Optional from volatility.framework import interfaces, constants, validity, layers from volatility.framework import symbols @@ -66,11 +66,14 @@ class MacintelStacker(interfaces.automagic.StackerLayerInterface): return None mac_banners = MacBannerCache.load_banners() + # If we have no banners, don't bother scanning + if not mac_banners: + vollog.info("No Mac banners found - if this is a mac plugin, please check your symbol files location") + return None + mss = scanners.MultiStringScanner([x for x in mac_banners if x]) for banner_offset, banner in layer.scan( - context = context, - scanner = scanners.MultiStringScanner([x for x in mac_banners if x]), - progress_callback = progress_callback): + context = context, scanner = mss, progress_callback = progress_callback): dtb = None vollog.debug("Identified banner: {}".format(repr(banner))) diff --git a/volatility/framework/automagic/symbol_cache.py b/volatility/framework/automagic/symbol_cache.py index 0784f8922..14ee06741 100644 --- a/volatility/framework/automagic/symbol_cache.py +++ b/volatility/framework/automagic/symbol_cache.py @@ -17,7 +17,7 @@ # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the # specific language governing rights and limitations under the License. # - +import gc import logging import os import pickle @@ -109,6 +109,7 @@ class SymbolBannerCache(interfaces.automagic.AutomagicInterface): progress_callback(current * 100 / total, "Building {} caches".format(self.os)) isf_url = cacheables[current] + isf = None try: # Loading the symbol table will be very slow until it's been validated isf = intermed.IntermediateSymbolTable(context, config_path, "temp", isf_url, validate = False) @@ -118,11 +119,17 @@ class SymbolBannerCache(interfaces.automagic.AutomagicInterface): # but we should check at least that the banner matches on load. banner = isf.get_symbol(self.symbol_name).constant_data vollog.log(constants.LOGLEVEL_V, "Caching banner {} for file {}".format(banner, isf_url)) + bannerlist = banners.get(banner, []) bannerlist.append(isf_url) banners[banner] = bannerlist except exceptions.SymbolError: pass + finally: + # Get rid of the loaded file, in case it sits in memory + if isf: + del isf + gc.collect() # Rewrite the cached banners each run, since writing is faster than the banner_cache validation portion self.save_banners(banners) diff --git a/volatility/framework/layers/scanners/multiregexp.py b/volatility/framework/layers/scanners/multiregexp.py index 7d8e42176..6e38ecbdc 100644 --- a/volatility/framework/layers/scanners/multiregexp.py +++ b/volatility/framework/layers/scanners/multiregexp.py @@ -39,6 +39,8 @@ class MultiRegexp(object): -> Generator[Tuple[int, Union[str, bytes]], None, None]: if not isinstance(haystack, bytes): raise TypeError("Search haystack must be a byte string") + if not self._regex: + raise ValueError("MultiRegexp cannot be used with an empty set of search strings") for match in re.finditer(self._regex, haystack): yield (match.start(0), match.group())