Caching: Only cache remote files

This commit is contained in:
Mike Auty
2020-07-28 21:03:16 +01:00
parent 123e9bc8a3
commit faaa3bf6d8
+8 -2
View File
@@ -74,6 +74,12 @@ class ResourceAccessor(object):
"Available URL handlers: {}".format(", ".join([x.__name__ for x in self._handlers])))
self.__class__.list_handlers = False
def uses_cache(self, url: str) -> bool:
"""Determines whether a URLs contents should be cached"""
parsed_url = urllib.parse.urlparse(url)
return not parsed_url.scheme in ['file', 'jar']
# Current urllib.request.urlopen returns Any, so we do the same
def open(self, url: str, mode: str = "rb") -> Any:
"""Returns a file-like object for a particular URL opened in mode.
@@ -103,9 +109,8 @@ class ResourceAccessor(object):
with contextlib.closing(fp) as fp:
# Cache the file locally
parsed_url = urllib.parse.urlparse(url)
if parsed_url.scheme == 'file':
if not self.uses_cache(url):
# ZipExtFiles (files in zips) cannot seek, so must be cached in order to use and/or decompress
curfile = urllib.request.urlopen(url, context = self._context)
else:
@@ -172,6 +177,7 @@ class ResourceAccessor(object):
if not IMPORTED_MAGIC:
# Somewhat of a hack, but prevents a hard dependency on the magic module
parsed_url = urllib.parse.urlparse(url)
url_path = parsed_url.path
stop = False
while not stop: