Source code for spack.cmd.checksum

# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

import re
import sys
from typing import Dict, Optional

import llnl.string
import llnl.util.lang
from llnl.util import tty

import spack.cmd
import spack.repo
import spack.spec
import spack.stage
import spack.util.crypto
import spack.util.web as web_util
from spack.cmd.common import arguments
from spack.package_base import (
    ManualDownloadRequiredError,
    PackageBase,
    deprecated_version,
    preferred_version,
)
from spack.util.editor import editor
from spack.util.format import get_version_lines
from spack.version import StandardVersion, Version

description = "checksum available versions of a package"
section = "packaging"
level = "long"


[docs] def setup_parser(subparser): subparser.add_argument( "--keep-stage", action="store_true", default=False, help="don't clean up staging area when command completes", ) subparser.add_argument( "--batch", "-b", action="store_true", default=False, help="don't ask which versions to checksum", ) subparser.add_argument( "--latest", "-l", action="store_true", default=False, help="checksum the latest available version", ) subparser.add_argument( "--preferred", "-p", action="store_true", default=False, help="checksum the known Spack preferred version", ) modes_parser = subparser.add_mutually_exclusive_group() modes_parser.add_argument( "--add-to-package", "-a", action="store_true", default=False, help="add new versions to package", ) modes_parser.add_argument( "--verify", action="store_true", default=False, help="verify known package checksums" ) subparser.add_argument("package", help="name or spec (e.g. `cmake` or `cmake@3.18`)") subparser.add_argument( "versions", nargs="*", help="checksum these specific versions (if omitted, Spack searches for remote versions)", ) arguments.add_common_arguments(subparser, ["jobs"]) subparser.epilog = ( "examples:\n" " `spack checksum zlib@1.2` autodetects versions 1.2.0 to 1.2.13 from the remote\n" " `spack checksum zlib 1.2.13` checksums exact version 1.2.13 directly without search\n" )
[docs] def checksum(parser, args): spec = spack.spec.Spec(args.package) # Get the package we're going to generate checksums for pkg: PackageBase = spack.repo.PATH.get_pkg_class(spec.name)(spec) # Skip manually downloaded packages if pkg.manual_download: raise ManualDownloadRequiredError(pkg.download_instr) versions = [StandardVersion.from_string(v) for v in args.versions] # Define placeholder for remote versions. This'll help reduce redundant work if we need to # check for the existence of remote versions more than once. remote_versions: Optional[Dict[StandardVersion, str]] = None # Add latest version if requested if args.latest: remote_versions = pkg.fetch_remote_versions(concurrency=args.jobs) if len(remote_versions) > 0: versions.append(max(remote_versions.keys())) # Add preferred version if requested (todo: exclude git versions) if args.preferred: versions.append(preferred_version(pkg)) # Store a dict of the form version -> URL url_dict: Dict[StandardVersion, str] = {} for version in versions: if deprecated_version(pkg, version): tty.warn(f"Version {version} is deprecated") url = pkg.find_valid_url_for_version(version) if url is not None: url_dict[version] = url continue # If we get here, it's because no valid url was provided by the package. Do expensive # fallback to try to recover if remote_versions is None: remote_versions = pkg.fetch_remote_versions(concurrency=args.jobs) if version in remote_versions: url_dict[version] = remote_versions[version] if len(versions) <= 0: if remote_versions is None: remote_versions = pkg.fetch_remote_versions(concurrency=args.jobs) url_dict = remote_versions # A spidered URL can differ from the package.py *computed* URL, pointing to different tarballs. # For example, GitHub release pages sometimes have multiple tarballs with different shasum: # - releases/download/1.0/<pkg>-1.0.tar.gz (uploaded tarball) # - archive/refs/tags/1.0.tar.gz (generated tarball) # We wanna ensure that `spack checksum` and `spack install` ultimately use the same URL, so # here we check whether the crawled and computed URLs disagree, and if so, prioritize the # former if that URL exists (just sending a HEAD request that is). url_changed_for_version = set() for version, url in url_dict.items(): possible_urls = pkg.all_urls_for_version(version) if url not in possible_urls: for possible_url in possible_urls: if web_util.url_exists(possible_url): url_dict[version] = possible_url break else: url_changed_for_version.add(version) if not url_dict: tty.die(f"Could not find any remote versions for {pkg.name}") elif len(url_dict) > 1 and not args.batch and sys.stdin.isatty(): filtered_url_dict = spack.stage.interactive_version_filter( url_dict, pkg.versions, url_changes=url_changed_for_version, initial_verion_filter=spec.versions, ) if not filtered_url_dict: exit(0) url_dict = filtered_url_dict else: tty.info(f"Found {llnl.string.plural(len(url_dict), 'version')} of {pkg.name}") version_hashes = spack.stage.get_checksums_for_versions( url_dict, pkg.name, keep_stage=args.keep_stage, fetch_options=pkg.fetch_options ) if args.verify: print_checksum_status(pkg, version_hashes) sys.exit(0) # convert dict into package.py version statements version_lines = get_version_lines(version_hashes, url_dict) print() print(version_lines) print() if args.add_to_package: add_versions_to_package(pkg, version_lines, args.batch)
[docs] def add_versions_to_package(pkg: PackageBase, version_lines: str, is_batch: bool): """ Add checksumed versions to a package's instructions and open a user's editor so they may double check the work of the function. Args: pkg (spack.package_base.PackageBase): A package class for a given package in Spack. version_lines (str): A string of rendered version lines. """ # Get filename and path for package filename = spack.repo.PATH.filename_for_package_name(pkg.name) num_versions_added = 0 version_statement_re = re.compile(r"([\t ]+version\([^\)]*\))") version_re = re.compile(r'[\t ]+version\(\s*"([^"]+)"[^\)]*\)') # Split rendered version lines into tuple of (version, version_line) # We reverse sort here to make sure the versions match the version_lines new_versions = [] for ver_line in version_lines.split("\n"): match = version_re.match(ver_line) if match: new_versions.append((Version(match.group(1)), ver_line)) with open(filename, "r+") as f: contents = f.read() split_contents = version_statement_re.split(contents) for i, subsection in enumerate(split_contents): # If there are no more versions to add we should exit if len(new_versions) <= 0: break # Check if the section contains a version contents_version = version_re.match(subsection) if contents_version is not None: parsed_version = Version(contents_version.group(1)) if parsed_version < new_versions[0][0]: split_contents[i:i] = [new_versions.pop(0)[1], " # FIXME", "\n"] num_versions_added += 1 elif parsed_version == new_versions[0][0]: new_versions.pop(0) # Seek back to the start of the file so we can rewrite the file contents. f.seek(0) f.writelines("".join(split_contents)) tty.msg(f"Added {num_versions_added} new versions to {pkg.name}") tty.msg(f"Open {filename} to review the additions.") if sys.stdout.isatty() and not is_batch: editor(filename)