Source code for spack.util.file_cache

# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

import errno
import math
import os
import shutil

from llnl.util.filesystem import mkdirp, rename

from spack.error import SpackError
from spack.util.lock import Lock, ReadTransaction, WriteTransaction

[docs] class FileCache: """This class manages cached data in the filesystem. - Cache files are fetched and stored by unique keys. Keys can be relative paths, so that there can be some hierarchy in the cache. - The FileCache handles locking cache files for reading and writing, so client code need not manage locks for cache entries. """ def __init__(self, root, timeout=120): """Create a file cache object. This will create the cache directory if it does not exist yet. Args: root: specifies the root directory where the cache stores files timeout: when there is contention among multiple Spack processes for cache files, this specifies how long Spack should wait before assuming that there is a deadlock. """ self.root = root.rstrip(os.path.sep) if not os.path.exists(self.root): mkdirp(self.root) self._locks = {} self.lock_timeout = timeout
[docs] def destroy(self): """Remove all files under the cache root.""" for f in os.listdir(self.root): path = os.path.join(self.root, f) if os.path.isdir(path): shutil.rmtree(path, True) else: os.remove(path)
[docs] def cache_path(self, key): """Path to the file in the cache for a particular key.""" return os.path.join(self.root, key)
def _lock_path(self, key): """Path to the file in the cache for a particular key.""" keyfile = os.path.basename(key) keydir = os.path.dirname(key) return os.path.join(self.root, keydir, "." + keyfile + ".lock") def _get_lock(self, key): """Create a lock for a key, if necessary, and return a lock object.""" if key not in self._locks: self._locks[key] = Lock(self._lock_path(key), default_timeout=self.lock_timeout) return self._locks[key]
[docs] def init_entry(self, key): """Ensure we can access a cache file. Create a lock for it if needed. Return whether the cache file exists yet or not. """ cache_path = self.cache_path(key) exists = os.path.exists(cache_path) if exists: if not os.path.isfile(cache_path): raise CacheError("Cache file is not a file: %s" % cache_path) if not os.access(cache_path, os.R_OK): raise CacheError("Cannot access cache file: %s" % cache_path) else: # if the file is hierarchical, make parent directories parent = os.path.dirname(cache_path) if parent.rstrip(os.path.sep) != self.root: mkdirp(parent) if not os.access(parent, os.R_OK | os.W_OK): raise CacheError("Cannot access cache directory: %s" % parent) # ensure lock is created for this key self._get_lock(key) return exists
[docs] def read_transaction(self, key): """Get a read transaction on a file cache item. Returns a ReadTransaction context manager and opens the cache file for reading. You can use it like this: with file_cache_object.read_transaction(key) as cache_file: """ return ReadTransaction(self._get_lock(key), acquire=lambda: open(self.cache_path(key)))
[docs] def write_transaction(self, key): """Get a write transaction on a file cache item. Returns a WriteTransaction context manager that opens a temporary file for writing. Once the context manager finishes, if nothing went wrong, moves the file into place on top of the old file atomically. """ filename = self.cache_path(key) if os.path.exists(filename) and not os.access(filename, os.W_OK): raise CacheError( "Insufficient permissions to write to file cache at {0}".format(filename) ) # TODO: this nested context manager adds a lot of complexity and # TODO: is pretty hard to reason about in llnl.util.lock. At some # TODO: point we should just replace it with functions and simplify # TODO: the locking code. class WriteContextManager: def __enter__(cm): cm.orig_filename = self.cache_path(key) cm.orig_file = None if os.path.exists(cm.orig_filename): cm.orig_file = open(cm.orig_filename, "r") cm.tmp_filename = self.cache_path(key) + ".tmp" cm.tmp_file = open(cm.tmp_filename, "w") return cm.orig_file, cm.tmp_file def __exit__(cm, type, value, traceback): if cm.orig_file: cm.orig_file.close() cm.tmp_file.close() if value: os.remove(cm.tmp_filename) else: rename(cm.tmp_filename, cm.orig_filename) return WriteTransaction(self._get_lock(key), acquire=WriteContextManager)
[docs] def mtime(self, key) -> float: """Return modification time of cache file, or -inf if it does not exist. Time is in units returned by os.stat in the mtime field, which is platform-dependent. """ if not self.init_entry(key): return -math.inf else: return os.stat(self.cache_path(key)).st_mtime
[docs] def remove(self, key): file = self.cache_path(key) lock = self._get_lock(key) try: lock.acquire_write() os.unlink(file) except OSError as e: # File not found is OK, so remove is idempotent. if e.errno != errno.ENOENT: raise finally: lock.release_write()
[docs] class CacheError(SpackError): pass