# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import argparse
import os
import shutil
import sys
import textwrap
import llnl.util.filesystem as fs
import llnl.util.tty as tty
import spack.build_environment
import spack.cmd
import spack.cmd.common.arguments as arguments
import spack.environment as ev
import spack.fetch_strategy
import spack.monitor
import spack.paths
import spack.report
from spack.error import SpackError
from spack.installer import PackageInstaller
description = "build and install packages"
section = "build"
level = "short"
[docs]def update_kwargs_from_args(args, kwargs):
"""Parse cli arguments and construct a dictionary
that will be passed to the package installer."""
kwargs.update({
'fail_fast': args.fail_fast,
'keep_prefix': args.keep_prefix,
'keep_stage': args.keep_stage,
'restage': not args.dont_restage,
'install_source': args.install_source,
'verbose': args.verbose,
'fake': args.fake,
'dirty': args.dirty,
'use_cache': args.use_cache,
'cache_only': args.cache_only,
'include_build_deps': args.include_build_deps,
'explicit': True, # Always true for install command
'stop_at': args.until,
'unsigned': args.unsigned,
'full_hash_match': args.full_hash_match,
})
kwargs.update({
'install_deps': ('dependencies' in args.things_to_install),
'install_package': ('package' in args.things_to_install)
})
if hasattr(args, 'setup'):
setups = set()
for arglist_s in args.setup:
for arg in [x.strip() for x in arglist_s.split(',')]:
setups.add(arg)
kwargs['setup'] = setups
tty.msg('Setup={0}'.format(kwargs['setup']))
[docs]def setup_parser(subparser):
subparser.add_argument(
'--only',
default='package,dependencies',
dest='things_to_install',
choices=['package', 'dependencies'],
help="""select the mode of installation.
the default is to install the package along with all its dependencies.
alternatively one can decide to install only the package or only
the dependencies"""
)
subparser.add_argument(
'-u', '--until', type=str, dest='until', default=None,
help="phase to stop after when installing (default None)")
arguments.add_common_arguments(subparser, ['jobs', 'reuse'])
subparser.add_argument(
'--overwrite', action='store_true',
help="reinstall an existing spec, even if it has dependents")
subparser.add_argument(
'--fail-fast', action='store_true',
help="stop all builds if any build fails (default is best effort)")
subparser.add_argument(
'--keep-prefix', action='store_true',
help="don't remove the install prefix if installation fails")
subparser.add_argument(
'--keep-stage', action='store_true',
help="don't remove the build stage if installation succeeds")
subparser.add_argument(
'--dont-restage', action='store_true',
help="if a partial install is detected, don't delete prior state")
cache_group = subparser.add_mutually_exclusive_group()
cache_group.add_argument(
'--use-cache', action='store_true', dest='use_cache', default=True,
help="check for pre-built Spack packages in mirrors (default)")
cache_group.add_argument(
'--no-cache', action='store_false', dest='use_cache', default=True,
help="do not check for pre-built Spack packages in mirrors")
cache_group.add_argument(
'--cache-only', action='store_true', dest='cache_only', default=False,
help="only install package from binary mirrors")
monitor_group = spack.monitor.get_monitor_group(subparser) # noqa
subparser.add_argument(
'--include-build-deps', action='store_true', dest='include_build_deps',
default=False, help="""include build deps when installing from cache,
which is useful for CI pipeline troubleshooting""")
subparser.add_argument(
'--no-check-signature', action='store_true',
dest='unsigned', default=False,
help="do not check signatures of binary packages")
subparser.add_argument(
'--require-full-hash-match', action='store_true',
dest='full_hash_match', default=False, help="""when installing from
binary mirrors, do not install binary package unless the full hash of the
remote spec matches that of the local spec""")
subparser.add_argument(
'--show-log-on-error', action='store_true',
help="print full build log to stderr if build fails")
subparser.add_argument(
'--source', action='store_true', dest='install_source',
help="install source files in prefix")
arguments.add_common_arguments(subparser, ['no_checksum', 'deprecated'])
subparser.add_argument(
'-v', '--verbose', action='store_true',
help="display verbose build output while installing")
subparser.add_argument(
'--fake', action='store_true',
help="fake install for debug purposes.")
subparser.add_argument(
'--only-concrete', action='store_true', default=False,
help='(with environment) only install already concretized specs')
subparser.add_argument(
'--no-add', action='store_true', default=False,
help="""(with environment) only install specs provided as argument
if they are already in the concretized environment""")
subparser.add_argument(
'-f', '--file', action='append', default=[],
dest='specfiles', metavar='SPEC_YAML_FILE',
help="install from file. Read specs to install from .yaml files")
cd_group = subparser.add_mutually_exclusive_group()
arguments.add_common_arguments(cd_group, ['clean', 'dirty'])
testing = subparser.add_mutually_exclusive_group()
testing.add_argument(
'--test', default=None,
choices=['root', 'all'],
help="""If 'root' is chosen, run package tests during
installation for top-level packages (but skip tests for dependencies).
if 'all' is chosen, run package tests during installation for all
packages. If neither are chosen, don't run tests for any packages."""
)
testing.add_argument(
'--run-tests', action='store_true',
help='run package tests during installation (same as --test=all)'
)
subparser.add_argument(
'--log-format',
default=None,
choices=spack.report.valid_formats,
help="format to be used for log files"
)
subparser.add_argument(
'--log-file',
default=None,
help="filename for the log file. if not passed a default will be used"
)
subparser.add_argument(
'--help-cdash',
action='store_true',
help="Show usage instructions for CDash reporting"
)
arguments.add_cdash_args(subparser, False)
arguments.add_common_arguments(subparser, ['yes_to_all', 'spec'])
[docs]def default_log_file(spec):
"""Computes the default filename for the log file and creates
the corresponding directory if not present
"""
fmt = 'test-{x.name}-{x.version}-{hash}.xml'
basename = fmt.format(x=spec, hash=spec.dag_hash())
dirname = fs.os.path.join(spack.paths.reports_path, 'junit')
fs.mkdirp(dirname)
return fs.os.path.join(dirname, basename)
[docs]def install_specs(cli_args, kwargs, specs):
"""Do the actual installation.
Args:
cli_args (argparse.Namespace): argparse namespace with command arguments
kwargs (dict): keyword arguments
specs (list): list of (abstract, concrete) spec tuples
"""
# handle active environment, if any
env = ev.active_environment()
try:
if env:
specs_to_install = []
specs_to_add = []
for abstract, concrete in specs:
# This won't find specs added to the env since last
# concretize, therefore should we consider enforcing
# concretization of the env before allowing to install
# specs?
m_spec = env.matching_spec(abstract)
# If there is any ambiguity in the above call to matching_spec
# (i.e. if more than one spec in the environment matches), then
# SpackEnvironmentError is raised, with a message listing the
# the matches. Getting to this point means there were either
# no matches or exactly one match.
if not m_spec:
tty.debug('{0} matched nothing in the env'.format(
abstract.name))
# no matches in the env
if cli_args.no_add:
msg = ('You asked to install {0} without adding it ' +
'(--no-add), but no such spec exists in ' +
'environment').format(abstract.name)
tty.die(msg)
else:
tty.debug('adding {0} as a root'.format(abstract.name))
specs_to_add.append((abstract, concrete))
continue
tty.debug('exactly one match for {0} in env -> {1}'.format(
m_spec.name, m_spec.dag_hash()))
if m_spec in env.roots() or cli_args.no_add:
# either the single match is a root spec (and --no-add is
# the default for roots) or --no-add was stated explicitly
tty.debug('just install {0}'.format(m_spec.name))
specs_to_install.append(m_spec)
else:
# the single match is not a root (i.e. it's a dependency),
# and --no-add was not specified, so we'll add it as a
# root before installing
tty.debug('add {0} then install it'.format(m_spec.name))
specs_to_add.append((abstract, concrete))
if specs_to_add:
tty.debug('Adding the following specs as roots:')
for abstract, concrete in specs_to_add:
tty.debug(' {0}'.format(abstract.name))
with env.write_transaction():
specs_to_install.append(
env.concretize_and_add(abstract, concrete))
env.write(regenerate=False)
# Install the validated list of cli specs
if specs_to_install:
tty.debug('Installing the following cli specs:')
for s in specs_to_install:
tty.debug(' {0}'.format(s.name))
env.install_specs(specs_to_install, args=cli_args, **kwargs)
else:
installs = [(concrete.package, kwargs) for _, concrete in specs]
builder = PackageInstaller(installs)
builder.install()
except spack.build_environment.InstallError as e:
if cli_args.show_log_on_error:
e.print_context()
if not os.path.exists(e.pkg.build_log_path):
tty.error("'spack install' created no log.")
else:
sys.stderr.write('Full build log:\n')
with open(e.pkg.build_log_path) as log:
shutil.copyfileobj(log, sys.stderr)
raise
[docs]def install(parser, args, **kwargs):
if args.help_cdash:
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog=textwrap.dedent('''\
environment variables:
SPACK_CDASH_AUTH_TOKEN
authentication token to present to CDash
'''))
arguments.add_cdash_args(parser, True)
parser.print_help()
return
# The user wants to monitor builds using github.com/spack/spack-monitor
if args.use_monitor:
monitor = spack.monitor.get_client(
host=args.monitor_host,
prefix=args.monitor_prefix,
disable_auth=args.monitor_disable_auth,
tags=args.monitor_tags,
save_local=args.monitor_save_local,
)
reporter = spack.report.collect_info(
spack.package.PackageInstaller, '_install_task', args.log_format, args)
if args.log_file:
reporter.filename = args.log_file
if args.run_tests:
tty.warn("Deprecated option: --run-tests: use --test=all instead")
def get_tests(specs):
if args.test == 'all' or args.run_tests:
return True
elif args.test == 'root':
return [spec.name for spec in specs]
else:
return False
# Parse cli arguments and construct a dictionary
# that will be passed to the package installer
update_kwargs_from_args(args, kwargs)
if not args.spec and not args.specfiles:
# if there are no args but an active environment
# then install the packages from it.
env = ev.active_environment()
if env:
tests = get_tests(env.user_specs)
kwargs['tests'] = tests
if not args.only_concrete:
with env.write_transaction():
concretized_specs = env.concretize(tests=tests, reuse=args.reuse)
ev.display_specs(concretized_specs)
# save view regeneration for later, so that we only do it
# once, as it can be slow.
env.write(regenerate=False)
specs = env.all_specs()
if not args.log_file and not reporter.filename:
reporter.filename = default_log_file(specs[0])
reporter.specs = specs
# Tell the monitor about the specs
if args.use_monitor and specs:
monitor.new_configuration(specs)
tty.msg("Installing environment {0}".format(env.name))
with reporter('build'):
env.install_all(**kwargs)
tty.debug("Regenerating environment views for {0}"
.format(env.name))
with env.write_transaction():
# write env to trigger view generation and modulefile
# generation
env.write()
return
else:
msg = "install requires a package argument or active environment"
if 'spack.yaml' in os.listdir(os.getcwd()):
# There's a spack.yaml file in the working dir, the user may
# have intended to use that
msg += "\n\n"
msg += "Did you mean to install using the `spack.yaml`"
msg += " in this directory? Try: \n"
msg += " spack env activate .\n"
msg += " spack install\n"
msg += " OR\n"
msg += " spack --env . install"
tty.die(msg)
if args.no_checksum:
spack.config.set('config:checksum', False, scope='command_line')
if args.deprecated:
spack.config.set('config:deprecated', True, scope='command_line')
# 1. Abstract specs from cli
abstract_specs = spack.cmd.parse_specs(args.spec)
tests = get_tests(abstract_specs)
kwargs['tests'] = tests
try:
specs = spack.cmd.parse_specs(
args.spec, concretize=True, tests=tests, reuse=args.reuse
)
except SpackError as e:
tty.debug(e)
reporter.concretization_report(e.message)
raise
# 2. Concrete specs from yaml files
for file in args.specfiles:
with open(file, 'r') as f:
if file.endswith('yaml') or file.endswith('yml'):
s = spack.spec.Spec.from_yaml(f)
else:
s = spack.spec.Spec.from_json(f)
concretized = s.concretized()
if concretized.dag_hash() != s.dag_hash():
msg = 'skipped invalid file "{0}". '
msg += 'The file does not contain a concrete spec.'
tty.warn(msg.format(file))
continue
abstract_specs.append(s)
specs.append(concretized)
if len(specs) == 0:
tty.die('The `spack install` command requires a spec to install.')
if not args.log_file and not reporter.filename:
reporter.filename = default_log_file(specs[0])
reporter.specs = specs
with reporter('build'):
if args.overwrite:
installed = list(filter(lambda x: x,
map(spack.store.db.query_one, specs)))
if not args.yes_to_all:
display_args = {
'long': True,
'show_flags': True,
'variants': True
}
if installed:
tty.msg('The following package specs will be '
'reinstalled:\n')
spack.cmd.display_specs(installed, **display_args)
not_installed = list(filter(lambda x: x not in installed,
specs))
if not_installed:
tty.msg('The following package specs are not installed and'
' the --overwrite flag was given. The package spec'
' will be newly installed:\n')
spack.cmd.display_specs(not_installed, **display_args)
# We have some specs, so one of the above must have been true
answer = tty.get_yes_or_no(
'Do you want to proceed?', default=False
)
if not answer:
tty.die('Reinstallation aborted.')
# overwrite all concrete explicit specs from this build
kwargs['overwrite'] = [spec.dag_hash() for spec in specs]
# Update install_args with the monitor args, needed for build task
kwargs.update({
"monitor_disable_auth": args.monitor_disable_auth,
"monitor_keep_going": args.monitor_keep_going,
"monitor_host": args.monitor_host,
"use_monitor": args.use_monitor,
"monitor_prefix": args.monitor_prefix,
})
# If we are using the monitor, we send configs. and create build
# The full_hash is the main package id, the build_hash for others
if args.use_monitor and specs:
monitor.new_configuration(specs)
install_specs(args, kwargs, zip(abstract_specs, specs))