# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import collections
import datetime
import errno
import inspect
import itertools
import json
import os
import os.path
import re
import shutil
import stat
import sys
import tempfile
import xml.etree.ElementTree
from typing import Dict # novm
import py
import pytest
import archspec.cpu.microarchitecture
import archspec.cpu.schema
import llnl.util.lang
from llnl.util.filesystem import mkdirp, remove_linked_tree, working_dir
import spack.binary_distribution
import spack.caches
import spack.compilers
import spack.config
import spack.database
import spack.directory_layout
import spack.environment as ev
import spack.package
import spack.package_prefs
import spack.paths
import spack.platforms
import spack.repo
import spack.stage
import spack.store
import spack.subprocess_context
import spack.test.cray_manifest
import spack.util.executable
import spack.util.gpg
import spack.util.spack_yaml as syaml
from spack.fetch_strategy import FetchError, FetchStrategyComposite, URLFetchStrategy
from spack.util.pattern import Bunch
is_windows = sys.platform == 'win32'
#
# Return list of shas for latest two git commits in local spack repo
#
[docs]@pytest.fixture(scope='session')
def last_two_git_commits():
git = spack.util.executable.which('git', required=True)
spack_git_path = spack.paths.prefix
with working_dir(spack_git_path):
git_log_out = git('log', '-n', '2', output=str, error=os.devnull)
regex = re.compile(r"^commit\s([^\s]+$)", re.MULTILINE)
yield regex.findall(git_log_out)
[docs]def write_file(filename, contents):
with open(filename, 'w') as f:
f.write(contents)
commit_counter = 0
[docs]@pytest.fixture
def override_git_repos_cache_path(tmpdir):
saved = spack.paths.user_repos_cache_path
tmp_path = tmpdir.mkdir('git-repo-cache-path-for-tests')
spack.paths.user_repos_cache_path = str(tmp_path)
yield
spack.paths.user_repos_cache_path = saved
[docs]@pytest.fixture
def mock_git_version_info(tmpdir, override_git_repos_cache_path,
scope="function"):
"""Create a mock git repo with known structure
The structure of commits in this repo is as follows::
| o fourth 1.x commit (1.2)
| o third 1.x commit
| |
o | fourth main commit (v2.0)
o | third main commit
| |
| o second 1.x commit (v1.1)
| o first 1.x commit
| /
|/
o second commit (v1.0)
o first commit
The repo consists of a single file, in which the Version._cmp representation
of each commit is expressed as a string.
Important attributes of the repo for test coverage are: multiple branches,
version tags on multiple branches, and version order is not equal to time
order or topological order.
"""
git = spack.util.executable.which('git', required=True)
repo_path = str(tmpdir.mkdir('git_repo'))
filename = 'file.txt'
def commit(message):
global commit_counter
git('commit', '--date', '2020-01-%02d 12:0:00 +0300' % commit_counter,
'-am', message)
commit_counter += 1
with working_dir(repo_path):
git("init")
git('config', 'user.name', 'Spack')
git('config', 'user.email', 'spack@spack.io')
commits = []
def latest_commit():
return git('rev-list', '-n1', 'HEAD', output=str, error=str).strip()
# Add two commits on main branch
write_file(filename, '[]')
git('add', filename)
commit('first commit')
commits.append(latest_commit())
# Get name of default branch (differs by git version)
main = git('rev-parse', '--abbrev-ref', 'HEAD', output=str, error=str).strip()
# Tag second commit as v1.0
write_file(filename, "[1, 0]")
commit('second commit')
commits.append(latest_commit())
git('tag', 'v1.0')
# Add two commits and a tag on 1.x branch
git('checkout', '-b', '1.x')
write_file(filename, "[1, 0, '', 1]")
commit('first 1.x commit')
commits.append(latest_commit())
write_file(filename, "[1, 1]")
commit('second 1.x commit')
commits.append(latest_commit())
git('tag', 'v1.1')
# Add two commits and a tag on main branch
git('checkout', main)
write_file(filename, "[1, 0, '', 1]")
commit('third main commit')
commits.append(latest_commit())
write_file(filename, "[2, 0]")
commit('fourth main commit')
commits.append(latest_commit())
git('tag', 'v2.0')
# Add two more commits on 1.x branch to ensure we aren't cheating by using time
git('checkout', '1.x')
write_file(filename, "[1, 1, '', 1]")
commit('third 1.x commit')
commits.append(latest_commit())
write_file(filename, "[1, 2]")
commit('fourth 1.x commit')
commits.append(latest_commit())
git('tag', '1.2') # test robust parsing to different syntax, no v
# The commits are ordered with the last commit first in the list
commits = list(reversed(commits))
# Return the git directory to install, the filename used, and the commits
yield repo_path, filename, commits
[docs]@pytest.fixture(autouse=True)
def clear_recorded_monkeypatches():
yield
spack.subprocess_context.clear_patches()
[docs]@pytest.fixture(scope='session', autouse=True)
def record_monkeypatch_setattr():
import _pytest
saved_setattr = _pytest.monkeypatch.MonkeyPatch.setattr
def record_setattr(cls, target, name, value, *args, **kwargs):
spack.subprocess_context.append_patch((target, name, value))
saved_setattr(cls, target, name, value, *args, **kwargs)
_pytest.monkeypatch.MonkeyPatch.setattr = record_setattr
try:
yield
finally:
_pytest.monkeypatch.MonkeyPatch.setattr = saved_setattr
def _can_access(path, perms):
return False
[docs]@pytest.fixture
def no_path_access(monkeypatch):
monkeypatch.setattr(os, 'access', _can_access)
#
# Disable any active Spack environment BEFORE all tests
#
[docs]@pytest.fixture(scope='session', autouse=True)
def clean_user_environment():
spack_env_value = os.environ.pop(ev.spack_env_var, None)
with ev.no_active_environment():
yield
if spack_env_value:
os.environ[ev.spack_env_var] = spack_env_value
#
# Make sure global state of active env does not leak between tests.
#
[docs]@pytest.fixture(scope='function', autouse=True)
def clean_test_environment():
yield
ev.deactivate()
def _verify_executables_noop(*args):
return None
#
# Disable checks on compiler executable existence
#
[docs]@pytest.fixture(scope='function', autouse=True)
def mock_compiler_executable_verification(request, monkeypatch):
"""Mock the compiler executable verification to allow missing executables.
This fixture can be disabled for tests of the compiler verification
functionality by::
@pytest.mark.enable_compiler_verification
If a test is marked in that way this is a no-op."""
if 'enable_compiler_verification' not in request.keywords:
monkeypatch.setattr(spack.compiler.Compiler,
'verify_executables',
_verify_executables_noop)
# Hooks to add command line options or set other custom behaviors.
# They must be placed here to be found by pytest. See:
#
# https://docs.pytest.org/en/latest/writing_plugins.html
#
[docs]def pytest_addoption(parser):
group = parser.getgroup("Spack specific command line options")
group.addoption(
'--fast', action='store_true', default=False,
help='runs only "fast" unit tests, instead of the whole suite')
[docs]def pytest_collection_modifyitems(config, items):
if not config.getoption('--fast'):
# --fast not given, run all the tests
return
slow_tests = ['db', 'network', 'maybeslow']
skip_as_slow = pytest.mark.skip(
reason='skipped slow test [--fast command line option given]'
)
for item in items:
if any(x in item.keywords for x in slow_tests):
item.add_marker(skip_as_slow)
#
# These fixtures are applied to all tests
#
[docs]@pytest.fixture(scope='function', autouse=True)
def no_chdir():
"""Ensure that no test changes Spack's working dirctory.
This prevents Spack tests (and therefore Spack commands) from
changing the working directory and causing other tests to fail
mysteriously. Tests should use ``working_dir`` or ``py.path``'s
``.as_cwd()`` instead of ``os.chdir`` to avoid failing this check.
We assert that the working directory hasn't changed, unless the
original wd somehow ceased to exist.
"""
original_wd = os.getcwd()
yield
if os.path.isdir(original_wd):
assert os.getcwd() == original_wd
[docs]@pytest.fixture(scope='function', autouse=True)
def reset_compiler_cache():
"""Ensure that the compiler cache is not shared across Spack tests
This cache can cause later tests to fail if left in a state incompatible
with the new configuration. Since tests can make almost unlimited changes
to their setup, default to not use the compiler cache across tests."""
spack.compilers._compiler_cache = {}
yield
spack.compilers._compiler_cache = {}
[docs]def onerror(func, path, error_info):
# Python on Windows is unable to remvove paths without
# write (IWUSR) permissions (such as those generated by Git on Windows)
# This method changes file permissions to allow removal by Python
os.chmod(path, stat.S_IWUSR)
func(path)
[docs]@pytest.fixture(scope='function', autouse=True)
def mock_stage(tmpdir_factory, monkeypatch, request):
"""Establish the temporary build_stage for the mock archive."""
# The approach with this autouse fixture is to set the stage root
# instead of using spack.config.override() to avoid configuration
# conflicts with dozens of tests that rely on other configuration
# fixtures, such as config.
if 'nomockstage' not in request.keywords:
# Set the build stage to the requested path
new_stage = tmpdir_factory.mktemp('mock-stage')
new_stage_path = str(new_stage)
# Ensure the source directory exists within the new stage path
source_path = os.path.join(new_stage_path,
spack.stage._source_path_subdir)
mkdirp(source_path)
monkeypatch.setattr(spack.stage, '_stage_root', new_stage_path)
yield new_stage_path
# Clean up the test stage directory
if os.path.isdir(new_stage_path):
shutil.rmtree(new_stage_path, onerror=onerror)
else:
# Must yield a path to avoid a TypeError on test teardown
yield str(tmpdir_factory)
[docs]@pytest.fixture(scope='session')
def ignore_stage_files():
"""Session-scoped helper for check_for_leftover_stage_files.
Used to track which leftover files in the stage have been seen.
"""
# to start with, ignore the .lock file at the stage root.
return set(['.lock', spack.stage._source_path_subdir, 'build_cache'])
[docs]def remove_whatever_it_is(path):
"""Type-agnostic remove."""
if os.path.isfile(path):
os.remove(path)
elif os.path.islink(path):
remove_linked_tree(path)
else:
shutil.rmtree(path, onerror=onerror)
[docs]@pytest.fixture
def working_env():
saved_env = os.environ.copy()
yield
# os.environ = saved_env doesn't work
# it causes module_parsing::test_module_function to fail
# when it's run after any test using this fixutre
os.environ.clear()
os.environ.update(saved_env)
[docs]@pytest.fixture(scope='function', autouse=True)
def check_for_leftover_stage_files(request, mock_stage, ignore_stage_files):
"""
Ensure that each (mock_stage) test leaves a clean stage when done.
Tests that are expected to dirty the stage can disable the check by
adding::
@pytest.mark.disable_clean_stage_check
and the associated stage files will be removed.
"""
stage_path = mock_stage
yield
files_in_stage = set()
try:
stage_files = os.listdir(stage_path)
files_in_stage = set(stage_files) - ignore_stage_files
except OSError as err:
if err.errno == errno.ENOENT or err.errno == errno.EINVAL:
pass
else:
raise
if 'disable_clean_stage_check' in request.keywords:
# clean up after tests that are expected to be dirty
for f in files_in_stage:
path = os.path.join(stage_path, f)
remove_whatever_it_is(path)
else:
ignore_stage_files |= files_in_stage
assert not files_in_stage
[docs]class MockCache(object):
[docs] def store(self, copy_cmd, relative_dest):
pass
[docs] def fetcher(self, target_path, digest, **kwargs):
return MockCacheFetcher()
[docs]class MockCacheFetcher(object):
[docs] def fetch(self):
raise FetchError('Mock cache always fails for tests')
def __str__(self):
return "[mock fetch cache]"
[docs]@pytest.fixture(autouse=True)
def mock_fetch_cache(monkeypatch):
"""Substitutes spack.paths.fetch_cache with a mock object that does nothing
and raises on fetch.
"""
monkeypatch.setattr(spack.caches, 'fetch_cache', MockCache())
[docs]@pytest.fixture()
def mock_binary_index(monkeypatch, tmpdir_factory):
"""Changes the directory for the binary index and creates binary index for
every test. Clears its own index when it's done.
"""
tmpdir = tmpdir_factory.mktemp('mock_binary_index')
index_path = tmpdir.join('binary_index').strpath
mock_index = spack.binary_distribution.BinaryCacheIndex(index_path)
monkeypatch.setattr(spack.binary_distribution, 'binary_index', mock_index)
yield
@pytest.fixture(autouse=True)
def _skip_if_missing_executables(request):
"""Permits to mark tests with 'require_executables' and skip the
tests if the executables passed as arguments are not found.
"""
if hasattr(request.node, 'get_marker'):
# TODO: Remove the deprecated API as soon as we drop support for Python 2.6
marker = request.node.get_marker('requires_executables')
else:
marker = request.node.get_closest_marker('requires_executables')
if marker:
required_execs = marker.args
missing_execs = [
x for x in required_execs if spack.util.executable.which(x) is None
]
if missing_execs:
msg = 'could not find executables: {0}'
pytest.skip(msg.format(', '.join(missing_execs)))
@pytest.fixture(autouse=True, scope='session')
def _use_test_platform(test_platform):
# This is the only context manager used at session scope (see note
# below for more insight) since we want to use the test platform as
# a default during tests.
with spack.platforms.use_platform(test_platform):
yield
#
# Note on context managers used by fixtures
#
# Because these context managers modify global state, they should really
# ONLY be used persistently (i.e., around yield statements) in
# function-scoped fixtures, OR in autouse session- or module-scoped
# fixtures.
#
# If they're used in regular tests or in module-scoped fixtures that are
# then injected as function arguments, weird things can happen, because
# the original state won't be restored until *after* the fixture is
# destroyed. This makes sense for an autouse fixture, where you know
# everything in the module/session is going to need the modified
# behavior, but modifying global state for one function in a way that
# won't be restored until after the module or session is done essentially
# leaves garbage behind for other tests.
#
# In general, we should module- or session-scope the *STATE* required for
# these global objects, but we shouldn't module- or session-scope their
# *USE*, or things can get really confusing.
#
#
# Test-specific fixtures
#
[docs]@pytest.fixture(scope='session')
def mock_repo_path():
yield spack.repo.Repo(spack.paths.mock_packages_path)
def _pkg_install_fn(pkg, spec, prefix):
# sanity_check_prefix requires something in the install directory
mkdirp(prefix.bin)
[docs]@pytest.fixture
def mock_pkg_install(monkeypatch):
monkeypatch.setattr(spack.package.PackageBase, 'install',
_pkg_install_fn, raising=False)
[docs]@pytest.fixture(scope='function')
def mock_packages(mock_repo_path, mock_pkg_install):
"""Use the 'builtin.mock' repository instead of 'builtin'"""
with spack.repo.use_repositories(mock_repo_path) as mock_repo:
yield mock_repo
[docs]@pytest.fixture(scope='function')
def mutable_mock_repo(mock_repo_path):
"""Function-scoped mock packages, for tests that need to modify them."""
mock_repo = spack.repo.Repo(spack.paths.mock_packages_path)
with spack.repo.use_repositories(mock_repo) as mock_repo_path:
yield mock_repo_path
[docs]@pytest.fixture(scope='session')
def linux_os():
"""Returns a named tuple with attributes 'name' and 'version'
representing the OS.
"""
platform = spack.platforms.host()
name, version = 'debian', '6'
if platform.name == 'linux':
current_os = platform.operating_system('default_os')
name, version = current_os.name, current_os.version
LinuxOS = collections.namedtuple('LinuxOS', ['name', 'version'])
return LinuxOS(name=name, version=version)
[docs]@pytest.fixture(scope='session')
def default_config():
"""Isolates the default configuration from the user configs.
This ensures we can test the real default configuration without having
tests fail when the user overrides the defaults that we test against."""
defaults_path = os.path.join(spack.paths.etc_path, 'defaults')
if is_windows:
defaults_path = os.path.join(defaults_path, "windows")
with spack.config.use_configuration(defaults_path) as defaults_config:
yield defaults_config
[docs]@pytest.fixture(scope='session')
def mock_uarch_json(tmpdir_factory):
"""Mock microarchitectures.json with test architecture descriptions."""
tmpdir = tmpdir_factory.mktemp('microarchitectures')
uarch_json = py.path.local(spack.paths.test_path).join(
"data", "microarchitectures", "microarchitectures.json")
uarch_json.copy(tmpdir)
yield str(tmpdir.join("microarchitectures.json"))
[docs]@pytest.fixture(scope='session')
def mock_uarch_configuration(mock_uarch_json):
"""Create mock dictionaries for the archspec.cpu."""
def load_json():
with open(mock_uarch_json) as f:
return json.load(f)
targets_json = load_json()
targets = archspec.cpu.microarchitecture._known_microarchitectures()
yield targets_json, targets
[docs]@pytest.fixture(scope='function')
def mock_targets(mock_uarch_configuration, monkeypatch):
"""Use this fixture to enable mock uarch targets for testing."""
targets_json, targets = mock_uarch_configuration
monkeypatch.setattr(archspec.cpu.schema, "TARGETS_JSON", targets_json)
monkeypatch.setattr(archspec.cpu.microarchitecture, "TARGETS", targets)
[docs]@pytest.fixture(scope='session')
def configuration_dir(tmpdir_factory, linux_os):
"""Copies mock configuration files in a temporary directory. Returns the
directory path.
"""
tmpdir = tmpdir_factory.mktemp('configurations')
# <test_path>/data/config has mock config yaml files in it
# copy these to the site config.
test_config = py.path.local(spack.paths.test_path).join('data', 'config')
test_config.copy(tmpdir.join('site'))
# Create temporary 'defaults', 'site' and 'user' folders
tmpdir.ensure('user', dir=True)
# Slightly modify config.yaml and compilers.yaml
if is_windows:
solver = 'original'
locks = False
else:
solver = os.environ.get('SPACK_TEST_SOLVER', 'clingo')
locks = True
config_yaml = test_config.join('config.yaml')
modules_root = tmpdir_factory.mktemp('share')
tcl_root = modules_root.ensure('modules', dir=True)
lmod_root = modules_root.ensure('lmod', dir=True)
content = ''.join(config_yaml.read()).format(
solver, locks, str(tcl_root), str(lmod_root)
)
t = tmpdir.join('site', 'config.yaml')
t.write(content)
compilers_yaml = test_config.join('compilers.yaml')
content = ''.join(compilers_yaml.read()).format(linux_os)
t = tmpdir.join('site', 'compilers.yaml')
t.write(content)
yield tmpdir
# Once done, cleanup the directory
shutil.rmtree(str(tmpdir))
def _create_mock_configuration_scopes(configuration_dir):
"""Create the configuration scopes used in `config` and `mutable_config`."""
scopes = [
spack.config.InternalConfigScope('_builtin', spack.config.config_defaults),
]
scopes += [
spack.config.ConfigScope(name, str(configuration_dir.join(name)))
for name in ['site', 'system', 'user']
]
scopes += [spack.config.InternalConfigScope('command_line')]
return scopes
[docs]@pytest.fixture(scope='session')
def mock_configuration_scopes(configuration_dir):
"""Create a persistent Configuration object from the configuration_dir."""
yield _create_mock_configuration_scopes(configuration_dir)
[docs]@pytest.fixture(scope='function')
def config(mock_configuration_scopes):
"""This fixture activates/deactivates the mock configuration."""
with spack.config.use_configuration(*mock_configuration_scopes) as config:
yield config
[docs]@pytest.fixture(scope='function')
def mutable_config(tmpdir_factory, configuration_dir):
"""Like config, but tests can modify the configuration."""
mutable_dir = tmpdir_factory.mktemp('mutable_config').join('tmp')
configuration_dir.copy(mutable_dir)
scopes = _create_mock_configuration_scopes(mutable_dir)
with spack.config.use_configuration(*scopes) as cfg:
yield cfg
[docs]@pytest.fixture(scope='function')
def mutable_empty_config(tmpdir_factory, configuration_dir):
"""Empty configuration that can be modified by the tests."""
mutable_dir = tmpdir_factory.mktemp('mutable_config').join('tmp')
scopes = [spack.config.ConfigScope(name, str(mutable_dir.join(name)))
for name in ['site', 'system', 'user']]
with spack.config.use_configuration(*scopes) as cfg:
yield cfg
[docs]@pytest.fixture
def no_compilers_yaml(mutable_config):
"""Creates a temporary configuration without compilers.yaml"""
for scope, local_config in mutable_config.scopes.items():
if not local_config.path: # skip internal scopes
continue
compilers_yaml = os.path.join(local_config.path, 'compilers.yaml')
if os.path.exists(compilers_yaml):
os.remove(compilers_yaml)
[docs]@pytest.fixture()
def mock_low_high_config(tmpdir):
"""Mocks two configuration scopes: 'low' and 'high'."""
scopes = [spack.config.ConfigScope(name, str(tmpdir.join(name)))
for name in ['low', 'high']]
with spack.config.use_configuration(*scopes) as config:
yield config
def _populate(mock_db):
r"""Populate a mock database with packages.
Here is what the mock DB looks like:
o mpileaks o mpileaks' o mpileaks''
|\ |\ |\
| o callpath | o callpath' | o callpath''
|/| |/| |/|
o | mpich o | mpich2 o | zmpi
| | o | fake
| | |
| |______________/
| .____________/
|/
o dyninst
|\
| o libdwarf
|/
o libelf
"""
def _install(spec):
s = spack.spec.Spec(spec).concretized()
pkg = spack.repo.get(s)
pkg.do_install(fake=True, explicit=True)
_install('mpileaks ^mpich')
_install('mpileaks ^mpich2')
_install('mpileaks ^zmpi')
_install('externaltest')
_install('trivial-smoke-test')
@pytest.fixture(scope='session')
def _store_dir_and_cache(tmpdir_factory):
"""Returns the directory where to build the mock database and
where to cache it.
"""
store = tmpdir_factory.mktemp('mock_store')
cache = tmpdir_factory.mktemp('mock_store_cache')
return store, cache
[docs]@pytest.fixture(scope='session')
def mock_store(tmpdir_factory, mock_repo_path, mock_configuration_scopes,
_store_dir_and_cache):
"""Creates a read-only mock database with some packages installed note
that the ref count for dyninst here will be 3, as it's recycled
across each install.
This does not actually activate the store for use by Spack -- see the
``database`` fixture for that.
"""
store_path, store_cache = _store_dir_and_cache
# If the cache does not exist populate the store and create it
if not os.path.exists(str(store_cache.join('.spack-db'))):
with spack.config.use_configuration(*mock_configuration_scopes):
with spack.store.use_store(str(store_path)) as store:
with spack.repo.use_repositories(mock_repo_path):
_populate(store.db)
store_path.copy(store_cache, mode=True, stat=True)
# Make the DB filesystem read-only to ensure we can't modify entries
store_path.join('.spack-db').chmod(mode=0o555, rec=1)
yield store_path
store_path.join('.spack-db').chmod(mode=0o755, rec=1)
[docs]@pytest.fixture(scope='function')
def database(mock_store, mock_packages, config):
"""This activates the mock store, packages, AND config."""
with spack.store.use_store(str(mock_store)) as store:
yield store.db
# Force reading the database again between tests
store.db.last_seen_verifier = ''
[docs]@pytest.fixture(scope='function')
def database_mutable_config(mock_store, mock_packages, mutable_config,
monkeypatch):
"""This activates the mock store, packages, AND config."""
with spack.store.use_store(str(mock_store)) as store:
yield store.db
store.db.last_seen_verifier = ''
[docs]@pytest.fixture(scope='function')
def mutable_database(database_mutable_config, _store_dir_and_cache):
"""Writeable version of the fixture, restored to its initial state
after each test.
"""
# Make the database writeable, as we are going to modify it
store_path, store_cache = _store_dir_and_cache
store_path.join('.spack-db').chmod(mode=0o755, rec=1)
yield database_mutable_config
# Restore the initial state by copying the content of the cache back into
# the store and making the database read-only
store_path.remove(rec=1)
store_cache.copy(store_path, mode=True, stat=True)
store_path.join('.spack-db').chmod(mode=0o555, rec=1)
[docs]@pytest.fixture()
def dirs_with_libfiles(tmpdir_factory):
lib_to_libfiles = {
'libstdc++': ['libstdc++.so', 'libstdc++.tbd'],
'libgfortran': ['libgfortran.a', 'libgfortran.dylib'],
'libirc': ['libirc.a', 'libirc.so']
}
root = tmpdir_factory.mktemp('root')
lib_to_dirs = {}
i = 0
for lib, libfiles in lib_to_libfiles.items():
dirs = []
for libfile in libfiles:
root.ensure(str(i), dir=True)
root.join(str(i)).ensure(libfile)
dirs.append(str(root.join(str(i))))
i += 1
lib_to_dirs[lib] = dirs
all_dirs = list(itertools.chain.from_iterable(lib_to_dirs.values()))
yield lib_to_dirs, all_dirs
def _compiler_link_paths_noop(*args):
return []
[docs]@pytest.fixture(scope='function', autouse=True)
def disable_compiler_execution(monkeypatch, request):
"""
This fixture can be disabled for tests of the compiler link path
functionality by::
@pytest.mark.enable_compiler_link_paths
If a test is marked in that way this is a no-op."""
if 'enable_compiler_link_paths' not in request.keywords:
# Compiler.determine_implicit_rpaths actually runs the compiler. So
# replace that function with a noop that simulates finding no implicit
# RPATHs
monkeypatch.setattr(
spack.compiler.Compiler,
'_get_compiler_link_paths',
_compiler_link_paths_noop
)
[docs]@pytest.fixture(scope='function')
def install_mockery(temporary_store, config, mock_packages):
"""Hooks a fake install directory, DB, and stage directory into Spack."""
# We use a fake package, so temporarily disable checksumming
with spack.config.override('config:checksum', False):
yield
# Also wipe out any cached prefix failure locks (associated with
# the session-scoped mock archive).
for pkg_id in list(temporary_store.db._prefix_failures.keys()):
lock = spack.store.db._prefix_failures.pop(pkg_id, None)
if lock:
try:
lock.release_write()
except Exception:
pass
[docs]@pytest.fixture(scope='function')
def temporary_store(tmpdir):
"""Hooks a temporary empty store for the test function."""
temporary_store_path = tmpdir.join('opt')
with spack.store.use_store(str(temporary_store_path)) as s:
yield s
temporary_store_path.remove()
[docs]@pytest.fixture(scope='function')
def install_mockery_mutable_config(
temporary_store, mutable_config, mock_packages
):
"""Hooks a fake install directory, DB, and stage directory into Spack.
This is specifically for tests which want to use 'install_mockery' but
also need to modify configuration (and hence would want to use
'mutable config'): 'install_mockery' does not support this.
"""
# We use a fake package, so temporarily disable checksumming
with spack.config.override('config:checksum', False):
yield
[docs]@pytest.fixture()
def mock_fetch(mock_archive, monkeypatch):
"""Fake the URL for a package so it downloads from a file."""
mock_fetcher = FetchStrategyComposite()
mock_fetcher.append(URLFetchStrategy(mock_archive.url))
monkeypatch.setattr(
spack.package.PackageBase, 'fetcher', mock_fetcher)
[docs]class MockLayout(object):
def __init__(self, root):
self.root = root
[docs] def path_for_spec(self, spec):
return os.path.sep.join([self.root, spec.name + '-' + spec.dag_hash()])
[docs] def ensure_installed(self, spec):
pass
[docs]@pytest.fixture()
def gen_mock_layout(tmpdir):
# Generate a MockLayout in a temporary directory. In general the prefixes
# specified by MockLayout should never be written to, but this ensures
# that even if they are, that it causes no harm
def create_layout(root):
subroot = tmpdir.mkdir(root)
return MockLayout(str(subroot))
yield create_layout
[docs]class MockConfig(object):
def __init__(self, configuration, writer_key):
self._configuration = configuration
self.writer_key = writer_key
[docs] def configuration(self, module_set_name):
return self._configuration
[docs] def writer_configuration(self, module_set_name):
return self.configuration(module_set_name)[self.writer_key]
[docs]class ConfigUpdate(object):
def __init__(self, root_for_conf, writer_mod, writer_key, monkeypatch):
self.root_for_conf = root_for_conf
self.writer_mod = writer_mod
self.writer_key = writer_key
self.monkeypatch = monkeypatch
def __call__(self, filename):
file = os.path.join(self.root_for_conf, filename + '.yaml')
with open(file) as f:
config_settings = syaml.load_config(f)
spack.config.set('modules:default', config_settings)
mock_config = MockConfig(config_settings, self.writer_key)
self.monkeypatch.setattr(
spack.modules.common,
'configuration',
mock_config.configuration
)
self.monkeypatch.setattr(
self.writer_mod,
'configuration',
mock_config.writer_configuration
)
self.monkeypatch.setattr(
self.writer_mod,
'configuration_registry',
{}
)
[docs]@pytest.fixture()
def module_configuration(monkeypatch, request):
"""Reads the module configuration file from the mock ones prepared
for tests and monkeypatches the right classes to hook it in.
"""
# Class of the module file writer
writer_cls = getattr(request.module, 'writer_cls')
# Module where the module file writer is defined
writer_mod = inspect.getmodule(writer_cls)
# Key for specific settings relative to this module type
writer_key = str(writer_mod.__name__).split('.')[-1]
# Root folder for configuration
root_for_conf = os.path.join(
spack.paths.test_path, 'data', 'modules', writer_key
)
return ConfigUpdate(root_for_conf, writer_mod, writer_key, monkeypatch)
[docs]@pytest.fixture()
def mock_gnupghome(monkeypatch):
# GNU PGP can't handle paths longer than 108 characters (wtf!@#$) so we
# have to make our own tmpdir with a shorter name than pytest's.
# This comes up because tmp paths on macOS are already long-ish, and
# pytest makes them longer.
try:
spack.util.gpg.init()
except spack.util.gpg.SpackGPGError:
if not spack.util.gpg.GPG:
pytest.skip('This test requires gpg')
short_name_tmpdir = tempfile.mkdtemp()
with spack.util.gpg.gnupghome_override(short_name_tmpdir):
yield short_name_tmpdir
# clean up, since we are doing this manually
# Ignore errors cause we seem to be hitting a bug similar to
# https://bugs.python.org/issue29699 in CI (FileNotFoundError: [Errno 2] No such
# file or directory: 'S.gpg-agent.extra').
shutil.rmtree(short_name_tmpdir, ignore_errors=True)
##########
# Fake archives and repositories
##########
[docs]@pytest.fixture(scope='session', params=[('.tar.gz', 'z')])
def mock_archive(request, tmpdir_factory):
"""Creates a very simple archive directory with a configure script and a
makefile that installs to a prefix. Tars it up into an archive.
"""
tar = spack.util.executable.which('tar', required=True)
tmpdir = tmpdir_factory.mktemp('mock-archive-dir')
tmpdir.ensure(spack.stage._source_path_subdir, dir=True)
repodir = tmpdir.join(spack.stage._source_path_subdir)
# Create the configure script
configure_path = str(tmpdir.join(spack.stage._source_path_subdir,
'configure'))
with open(configure_path, 'w') as f:
f.write(
"#!/bin/sh\n"
"prefix=$(echo $1 | sed 's/--prefix=//')\n"
"cat > Makefile <<EOF\n"
"all:\n"
"\techo Building...\n\n"
"install:\n"
"\tmkdir -p $prefix\n"
"\ttouch $prefix/dummy_file\n"
"EOF\n"
)
os.chmod(configure_path, 0o755)
# Archive it
with tmpdir.as_cwd():
archive_name = '{0}{1}'.format(spack.stage._source_path_subdir,
request.param[0])
tar('-c{0}f'.format(request.param[1]), archive_name,
spack.stage._source_path_subdir)
Archive = collections.namedtuple('Archive',
['url', 'path', 'archive_file',
'expanded_archive_basedir'])
archive_file = str(tmpdir.join(archive_name))
url = ('file://' + archive_file)
# Return the url
yield Archive(
url=url,
archive_file=archive_file,
path=str(repodir),
expanded_archive_basedir=spack.stage._source_path_subdir)
def _parse_cvs_date(line):
"""Turn a CVS log date into a datetime.datetime"""
# dates in CVS logs can have slashes or dashes and may omit the time zone:
# date: 2021-07-07 02:43:33 -0700; ...
# date: 2021-07-07 02:43:33; ...
# date: 2021/07/07 02:43:33; ...
m = re.search(r'date:\s+(\d+)[/-](\d+)[/-](\d+)\s+(\d+):(\d+):(\d+)', line)
if not m:
return None
year, month, day, hour, minute, second = [int(g) for g in m.groups()]
return datetime.datetime(year, month, day, hour, minute, second)
[docs]@pytest.fixture(scope='session')
def mock_cvs_repository(tmpdir_factory):
"""Creates a very simple CVS repository with two commits and a branch."""
cvs = spack.util.executable.which('cvs', required=True)
tmpdir = tmpdir_factory.mktemp('mock-cvs-repo-dir')
tmpdir.ensure(spack.stage._source_path_subdir, dir=True)
repodir = tmpdir.join(spack.stage._source_path_subdir)
cvsroot = str(repodir)
# The CVS repository and source tree need to live in a different directories
sourcedirparent = tmpdir_factory.mktemp('mock-cvs-source-dir')
module = spack.stage._source_path_subdir
url = cvsroot + "%module=" + module
sourcedirparent.ensure(module, dir=True)
sourcedir = sourcedirparent.join(module)
def format_date(date):
if date is None:
return None
return date.strftime('%Y-%m-%d %H:%M:%S')
def get_cvs_timestamp(output):
"""Find the most recent CVS time stamp in a `cvs log` output"""
latest_timestamp = None
for line in output.splitlines():
timestamp = _parse_cvs_date(line)
if timestamp:
if latest_timestamp is None:
latest_timestamp = timestamp
else:
latest_timestamp = max(latest_timestamp, timestamp)
return latest_timestamp
# We use this to record the time stamps for when we create CVS revisions,
# so that we can later check that we retrieve the proper commits when
# specifying a date. (CVS guarantees checking out the lastest revision
# before or on the specified date). As we create each revision, we
# separately record the time by querying CVS.
revision_date = {}
# Initialize the repository
with sourcedir.as_cwd():
cvs('-d', cvsroot, 'init')
cvs('-d', cvsroot, 'import', '-m', 'initial mock repo commit',
module, 'mockvendor', 'mockrelease')
with sourcedirparent.as_cwd():
cvs('-d', cvsroot, 'checkout', module)
# Commit file r0
r0_file = 'r0_file'
sourcedir.ensure(r0_file)
cvs('-d', cvsroot, 'add', r0_file)
cvs('-d', cvsroot, 'commit', '-m', 'revision 0', r0_file)
output = cvs('log', '-N', r0_file, output=str)
revision_date['1.1'] = format_date(get_cvs_timestamp(output))
# Commit file r1
r1_file = 'r1_file'
sourcedir.ensure(r1_file)
cvs('-d', cvsroot, 'add', r1_file)
cvs('-d', cvsroot, 'commit', '-m' 'revision 1', r1_file)
output = cvs('log', '-N', r0_file, output=str)
revision_date['1.2'] = format_date(get_cvs_timestamp(output))
# Create branch 'mock-branch'
cvs('-d', cvsroot, 'tag', 'mock-branch-root')
cvs('-d', cvsroot, 'tag', '-b', 'mock-branch')
# CVS does not have the notion of a unique branch; branches and revisions
# are managed separately for every file
def get_branch():
"""Return the branch name if all files are on the same branch, else
return None. Also return None if all files are on the trunk."""
lines = cvs('-d', cvsroot, 'status', '-v', output=str).splitlines()
branch = None
for line in lines:
m = re.search(r'(\S+)\s+[(]branch:', line)
if m:
tag = m.group(1)
if branch is None:
# First branch name found
branch = tag
elif tag == branch:
# Later branch name found; all branch names found so far
# agree
pass
else:
# Later branch name found; branch names differ
branch = None
break
return branch
# CVS does not have the notion of a unique revision; usually, one uses
# commit dates instead
def get_date():
"""Return latest date of the revisions of all files"""
output = cvs('log', '-N', r0_file, output=str)
timestamp = get_cvs_timestamp(output)
if timestamp is None:
return None
return format_date(timestamp)
checks = {
'default': Bunch(
file=r1_file,
branch=None,
date=None,
args={'cvs': url},
),
'branch': Bunch(
file=r1_file,
branch='mock-branch',
date=None,
args={'cvs': url, 'branch': 'mock-branch'},
),
'date': Bunch(
file=r0_file,
branch=None,
date=revision_date['1.1'],
args={'cvs': url,
'date': revision_date['1.1']},
),
}
test = Bunch(
checks=checks,
url=url,
get_branch=get_branch,
get_date=get_date,
path=str(repodir),
)
yield test
[docs]@pytest.fixture(scope='session')
def mock_git_repository(tmpdir_factory):
"""Creates a git repository multiple commits, branches, submodules, and
a tag. Visual representation of the commit history (starting with the
earliest commit at c0)::
c3 c1 (test-branch, r1) c2 (tag-branch)
|______/_____________________/
c0 (r0)
We used to test with 'master', but git has since developed the ability to
have differently named default branches, so now we query the user's config to
determine what the default branch should be.
There are two branches aside from 'default': 'test-branch' and 'tag-branch';
each has one commit; the tag-branch has a tag referring to its commit
(c2 in the diagram).
Two submodules are added as part of the very first commit on 'default'; each
of these refers to a repository with a single commit.
c0, c1, and c2 include information to define explicit versions in the
associated builtin.mock package 'git-test'. c3 is a commit in the
repository but does not have an associated explicit package version.
"""
git = spack.util.executable.which('git', required=True)
suburls = []
# Create two git repositories which will be used as submodules in the
# main repository
for submodule_count in range(2):
tmpdir = tmpdir_factory.mktemp('mock-git-repo-submodule-dir-{0}'
.format(submodule_count))
tmpdir.ensure(spack.stage._source_path_subdir, dir=True)
repodir = tmpdir.join(spack.stage._source_path_subdir)
suburls.append((submodule_count, 'file://' + str(repodir)))
with repodir.as_cwd():
git('init')
git('config', 'user.name', 'Spack')
git('config', 'user.email', 'spack@spack.io')
# r0 is just the first commit
submodule_file = 'r0_file_{0}'.format(submodule_count)
repodir.ensure(submodule_file)
git('add', submodule_file)
git('-c', 'commit.gpgsign=false', 'commit',
'-m', 'mock-git-repo r0 {0}'.format(submodule_count))
tmpdir = tmpdir_factory.mktemp('mock-git-repo-dir')
tmpdir.ensure(spack.stage._source_path_subdir, dir=True)
repodir = tmpdir.join(spack.stage._source_path_subdir)
# Create the main repository
with repodir.as_cwd():
git('init')
git('config', 'user.name', 'Spack')
git('config', 'user.email', 'spack@spack.io')
url = 'file://' + str(repodir)
for number, suburl in suburls:
git('submodule', 'add', suburl,
'third_party/submodule{0}'.format(number))
# r0 is the first commit: it consists of one file and two submodules
r0_file = 'r0_file'
repodir.ensure(r0_file)
git('add', r0_file)
git('-c', 'commit.gpgsign=false', 'commit', '-m', 'mock-git-repo r0')
branch = 'test-branch'
branch_file = 'branch_file'
git('branch', branch)
tag_branch = 'tag-branch'
tag_file = 'tag_file'
git('branch', tag_branch)
# Check out test branch and add one commit
git('checkout', branch)
repodir.ensure(branch_file)
git('add', branch_file)
git('-c', 'commit.gpgsign=false', 'commit', '-m' 'r1 test branch')
# Check out the tag branch, add one commit, and then add a tag for it
git('checkout', tag_branch)
repodir.ensure(tag_file)
git('add', tag_file)
git('-c', 'commit.gpgsign=false', 'commit', '-m' 'tag test branch')
tag = 'test-tag'
git('tag', tag)
try:
default_branch = git(
'config',
'--get',
'init.defaultBranch',
output=str,
).strip()
except Exception:
default_branch = 'master'
git('checkout', default_branch)
r2_file = 'r2_file'
repodir.ensure(r2_file)
git('add', r2_file)
git('-c', 'commit.gpgsign=false', 'commit', '-m', 'mock-git-repo r2')
rev_hash = lambda x: git('rev-parse', x, output=str).strip()
r2 = rev_hash(default_branch)
# Record the commit hash of the (only) commit from test-branch and
# the file added by that commit
r1 = rev_hash(branch)
r1_file = branch_file
# Map of version -> bunch. Each bunch includes; all the args
# that must be specified as part of a version() declaration (used to
# manufacture a version for the 'git-test' package); the associated
# revision for the version; a file associated with (and particular to)
# that revision/branch.
checks = {
'default': Bunch(
revision=default_branch, file=r0_file, args={'git': url}
),
'branch': Bunch(
revision=branch, file=branch_file, args={
'git': url, 'branch': branch
}
),
'tag-branch': Bunch(
revision=tag_branch, file=tag_file, args={
'git': url, 'branch': tag_branch
}
),
'tag': Bunch(
revision=tag, file=tag_file, args={'git': url, 'tag': tag}
),
'commit': Bunch(
revision=r1, file=r1_file, args={'git': url, 'commit': r1}
),
# In this case, the version() args do not include a 'git' key:
# this is the norm for packages, so this tests how the fetching logic
# would most-commonly assemble a Git fetcher
'default-no-per-version-git': Bunch(
revision=default_branch, file=r0_file, args={'branch': default_branch}
)
}
t = Bunch(checks=checks, url=url, hash=rev_hash,
path=str(repodir), git_exe=git, unversioned_commit=r2)
yield t
[docs]@pytest.fixture(scope='session')
def mock_hg_repository(tmpdir_factory):
"""Creates a very simple hg repository with two commits."""
hg = spack.util.executable.which('hg', required=True)
tmpdir = tmpdir_factory.mktemp('mock-hg-repo-dir')
tmpdir.ensure(spack.stage._source_path_subdir, dir=True)
repodir = tmpdir.join(spack.stage._source_path_subdir)
get_rev = lambda: hg('id', '-i', output=str).strip()
# Initialize the repository
with repodir.as_cwd():
url = 'file://' + str(repodir)
hg('init')
# Commit file r0
r0_file = 'r0_file'
repodir.ensure(r0_file)
hg('add', r0_file)
hg('commit', '-m', 'revision 0', '-u', 'test')
r0 = get_rev()
# Commit file r1
r1_file = 'r1_file'
repodir.ensure(r1_file)
hg('add', r1_file)
hg('commit', '-m' 'revision 1', '-u', 'test')
r1 = get_rev()
checks = {
'default': Bunch(
revision=r1, file=r1_file, args={'hg': str(repodir)}
),
'rev0': Bunch(
revision=r0, file=r0_file, args={
'hg': str(repodir), 'revision': r0
}
)
}
t = Bunch(checks=checks, url=url, hash=get_rev, path=str(repodir))
yield t
[docs]@pytest.fixture(scope='session')
def mock_svn_repository(tmpdir_factory):
"""Creates a very simple svn repository with two commits."""
svn = spack.util.executable.which('svn', required=True)
svnadmin = spack.util.executable.which('svnadmin', required=True)
tmpdir = tmpdir_factory.mktemp('mock-svn-stage')
tmpdir.ensure(spack.stage._source_path_subdir, dir=True)
repodir = tmpdir.join(spack.stage._source_path_subdir)
url = 'file://' + str(repodir)
# Initialize the repository
with repodir.as_cwd():
# NOTE: Adding --pre-1.5-compatible works for NERSC
# Unknown if this is also an issue at other sites.
svnadmin('create', '--pre-1.5-compatible', str(repodir))
# Import a structure (first commit)
r0_file = 'r0_file'
tmpdir.ensure('tmp-path', r0_file)
tmp_path = tmpdir.join('tmp-path')
svn('import',
str(tmp_path),
url,
'-m',
'Initial import r0')
tmp_path.remove()
# Second commit
r1_file = 'r1_file'
svn('checkout', url, str(tmp_path))
tmpdir.ensure('tmp-path', r1_file)
with tmp_path.as_cwd():
svn('add', str(tmpdir.ensure('tmp-path', r1_file)))
svn('ci', '-m', 'second revision r1')
tmp_path.remove()
r0 = '1'
r1 = '2'
checks = {
'default': Bunch(
revision=r1, file=r1_file, args={'svn': url}),
'rev0': Bunch(
revision=r0, file=r0_file, args={
'svn': url, 'revision': r0})
}
def get_rev():
output = svn('info', '--xml', output=str)
info = xml.etree.ElementTree.fromstring(output)
return info.find('entry/commit').get('revision')
t = Bunch(checks=checks, url=url, hash=get_rev, path=str(repodir))
yield t
[docs]@pytest.fixture()
def mutable_mock_env_path(tmpdir_factory):
"""Fixture for mocking the internal spack environments directory."""
saved_path = ev.environment.env_path
mock_path = tmpdir_factory.mktemp('mock-env-path')
ev.environment.env_path = str(mock_path)
yield mock_path
ev.environment.env_path = saved_path
##########
# Specs of various kind
##########
[docs]@pytest.fixture(
params=[
'conflict%clang+foo',
'conflict-parent@0.9^conflict~foo'
]
)
def conflict_spec(request):
"""Specs which violate constraints specified with the "conflicts"
directive in the "conflict" package.
"""
return request.param
[docs]@pytest.fixture(
params=[
'conflict%~'
]
)
def invalid_spec(request):
"""Specs that do not parse cleanly due to invalid formatting.
"""
return request.param
[docs]@pytest.fixture(scope='module')
def mock_test_repo(tmpdir_factory):
"""Create an empty repository."""
repo_namespace = 'mock_test_repo'
repodir = tmpdir_factory.mktemp(repo_namespace)
repodir.ensure(spack.repo.packages_dir_name, dir=True)
yaml = repodir.join('repo.yaml')
yaml.write("""
repo:
namespace: mock_test_repo
""")
with spack.repo.use_repositories(str(repodir)) as repo:
yield repo, repodir
shutil.rmtree(str(repodir))
##########
# Class and fixture to work around problems raising exceptions in directives,
# which cause tests like test_from_list_url to hang for Python 2.x metaclass
# processing.
#
# At this point only version and patch directive handling has been addressed.
##########
[docs]class MockBundle(object):
has_code = False
name = 'mock-bundle'
versions = {} # type: Dict
[docs]@pytest.fixture
def mock_directive_bundle():
"""Return a mock bundle package for directive tests."""
return MockBundle()
[docs]@pytest.fixture
def clear_directive_functions():
"""Clear all overidden directive functions for subsequent tests."""
yield
# Make sure any directive functions overidden by tests are cleared before
# proceeding with subsequent tests that may depend on the original
# functions.
spack.directives.DirectiveMeta._directives_to_be_executed = []
[docs]@pytest.fixture
def mock_executable(tmpdir):
"""Factory to create a mock executable in a temporary directory that
output a custom string when run.
"""
import jinja2
shebang = '#!/bin/bash\n' if not is_windows else '@ECHO OFF'
def _factory(name, output, subdir=('bin',)):
f = tmpdir.ensure(*subdir, dir=True).join(name)
if is_windows:
f += '.bat'
t = jinja2.Template('{{ shebang }}{{ output }}\n')
f.write(t.render(shebang=shebang, output=output))
f.chmod(0o755)
return str(f)
return _factory
[docs]@pytest.fixture()
def mock_test_stage(mutable_config, tmpdir):
# NOTE: This fixture MUST be applied after any fixture that uses
# the config fixture under the hood
# No need to unset because we use mutable_config
tmp_stage = str(tmpdir.join('test_stage'))
mutable_config.set('config:test_stage', tmp_stage)
yield tmp_stage
[docs]@pytest.fixture(autouse=True)
def brand_new_binary_cache():
yield
spack.binary_distribution.binary_index = llnl.util.lang.Singleton(
spack.binary_distribution._binary_index)
[docs]@pytest.fixture
def directory_with_manifest(tmpdir):
"""Create a manifest file in a directory. Used by 'spack external'.
"""
with tmpdir.as_cwd():
test_db_fname = 'external-db.json'
with open(test_db_fname, 'w') as db_file:
json.dump(spack.test.cray_manifest.create_manifest_content(),
db_file)
yield str(tmpdir)
[docs]@pytest.fixture()
def noncyclical_dir_structure(tmpdir):
"""
Create some non-trivial directory structure with
symlinks to dirs and dangling symlinks, but no cycles::
.
|-- a/
| |-- d/
| |-- file_1
| |-- to_file_1 -> file_1
| `-- to_c -> ../c
|-- b -> a
|-- c/
| |-- dangling_link -> nowhere
| `-- file_2
`-- file_3
"""
d, j = tmpdir.mkdir('nontrivial-dir'), os.path.join
with d.as_cwd():
os.mkdir(j('a'))
os.mkdir(j('a', 'd'))
with open(j('a', 'file_1'), 'wb'):
pass
os.symlink(j('file_1'), j('a', 'to_file_1'))
os.symlink(j('..', 'c'), j('a', 'to_c'))
os.symlink(j('a'), j('b'))
os.mkdir(j('c'))
os.symlink(j('nowhere'), j('c', 'dangling_link'))
with open(j('c', 'file_2'), 'wb'):
pass
with open(j('file_3'), 'wb'):
pass
yield d