Source code for spack.container.writers

# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
"""Writers for different kind of recipes and related
convenience functions.
import copy
from collections import namedtuple
from typing import Optional

import spack.environment as ev
import spack.error
import spack.schema.env
import spack.tengine as tengine
import spack.util.spack_yaml as syaml

from ..images import (

#: Caches all the writers that are currently supported
_writer_factory = {}

[docs] def writer(name): """Decorator to register a factory for a recipe writer. Each factory should take a configuration dictionary and return a properly configured writer that, when called, prints the corresponding recipe. """ def _decorator(factory): _writer_factory[name] = factory return factory return _decorator
[docs] def create(configuration, last_phase=None): """Returns a writer that conforms to the configuration passed as input. Args: configuration (dict): how to generate the current recipe last_phase (str): last phase to be printed or None to print them all """ name = configuration[ev.TOP_LEVEL_KEY]["container"]["format"] return _writer_factory[name](configuration, last_phase)
[docs] def recipe(configuration, last_phase=None): """Returns a recipe that conforms to the configuration passed as input. Args: configuration (dict): how to generate the current recipe last_phase (str): last phase to be printed or None to print them all """ return create(configuration, last_phase)()
def _stage_base_images(images_config): """Return a tuple with the base images to be used at the various stages. Args: images_config (dict): configuration under container:images """ # If we have custom base images, just return them verbatim. build_stage = images_config.get("build", None) if build_stage: final_stage = images_config["final"] return None, build_stage, final_stage # Check the operating system: this will be the base of the bootstrap # stage, if there, and of the final stage. operating_system = images_config.get("os", None) # Check the OS is mentioned in the internal data stored in a JSON file images_json = data()["images"] if not any(os_name == operating_system for os_name in images_json): msg = 'invalid operating system name "{0}". ' "[Allowed values are {1}]" msg = msg.format(operating_system, ", ".join(data()["images"])) raise ValueError(msg) # Retrieve the build stage spack_info = images_config["spack"] if isinstance(spack_info, dict): build_stage = "bootstrap" else: spack_version = images_config["spack"] image_name, tag = build_info(operating_system, spack_version) build_stage = "bootstrap" if image_name: build_stage = ":".join([image_name, tag]) # Retrieve the bootstrap stage bootstrap_stage = None if build_stage == "bootstrap": bootstrap_stage = images_json[operating_system]["bootstrap"].get("image", operating_system) # Retrieve the final stage final_stage = images_json[operating_system].get("final", {"image": operating_system})["image"] return bootstrap_stage, build_stage, final_stage def _spack_checkout_config(images_config): spack_info = images_config["spack"] url = "" ref = "develop" resolve_sha, verify = False, False # Config specific values may override defaults if isinstance(spack_info, dict): url = spack_info.get("url", url) ref = spack_info.get("ref", ref) resolve_sha = spack_info.get("resolve_sha", resolve_sha) verify = spack_info.get("verify", verify) else: ref = spack_info return url, ref, resolve_sha, verify
[docs] class PathContext(tengine.Context): """Generic context used to instantiate templates of recipes that install software in a common location and make it available directly via PATH. """ # Must be set by derived classes template_name: Optional[str] = None def __init__(self, config, last_phase): self.config = config[ev.TOP_LEVEL_KEY] self.container_config = self.config["container"] # Operating system tag as written in the configuration file self.operating_system_key = self.container_config["images"].get("os") # Get base images and verify the OS bootstrap, build, final = _stage_base_images(self.container_config["images"]) self.bootstrap_image = bootstrap self.build_image = build self.final_image = final # Record the last phase self.last_phase = last_phase @tengine.context_property def depfile(self): return self.container_config.get("depfile", False) @tengine.context_property def run(self): """Information related to the run image.""" Run = namedtuple("Run", ["image"]) return Run(image=self.final_image) @tengine.context_property def build(self): """Information related to the build image.""" Build = namedtuple("Build", ["image"]) return Build(image=self.build_image) @tengine.context_property def strip(self): """Whether or not to strip binaries in the image""" return self.container_config.get("strip", True) @tengine.context_property def paths(self): """Important paths in the image""" Paths = namedtuple("Paths", ["environment", "store", "view_parent", "view", "former_view"]) return Paths( environment="/opt/spack-environment", store="/opt/software", view_parent="/opt/views", view="/opt/views/view", former_view="/opt/view", # /opt/view -> /opt/views/view for backward compatibility ) @tengine.context_property def manifest(self): """The spack.yaml file that should be used in the image""" import jsonschema # Copy in the part of spack.yaml prescribed in the configuration file manifest = copy.deepcopy(self.config) manifest.pop("container") # Ensure that a few paths are where they need to be manifest.setdefault("config", syaml.syaml_dict()) manifest["config"]["install_tree"] = manifest["view"] = self.paths.view manifest = {"spack": manifest} # Validate the manifest file jsonschema.validate(manifest, schema=spack.schema.env.schema) return syaml.dump(manifest, default_flow_style=False).strip() @tengine.context_property def os_packages_final(self): """Additional system packages that are needed at run-time.""" try: return self._os_packages_for_stage("final") except Exception as e: msg = f"an error occurred while rendering the 'final' stage of the image: {e}" raise spack.error.SpackError(msg) from e @tengine.context_property def os_packages_build(self): """Additional system packages that are needed at build-time.""" try: return self._os_packages_for_stage("build") except Exception as e: msg = f"an error occurred while rendering the 'build' stage of the image: {e}" raise spack.error.SpackError(msg) from e @tengine.context_property def os_package_update(self): """Whether or not to update the OS package manager cache.""" os_packages = self.container_config.get("os_packages", {}) return os_packages.get("update", True) def _os_packages_for_stage(self, stage): os_packages = self.container_config.get("os_packages", {}) package_list = os_packages.get(stage, None) return self._package_info_from(package_list) def _package_info_from(self, package_list): """Helper method to pack a list of packages with the additional information required by the template. Args: package_list: list of packages Returns: Enough information to know how to update the cache, install a list opf packages, and clean in the end. """ if not package_list: return package_list image_config = self.container_config["images"] image = image_config.get("build", None) if image is None: os_pkg_manager = os_package_manager_for(image_config["os"]) else: os_pkg_manager = self._os_pkg_manager() update, install, clean = commands_for(os_pkg_manager) Packages = namedtuple("Packages", ["update", "install", "list", "clean"]) return Packages(update=update, install=install, list=package_list, clean=clean) def _os_pkg_manager(self): try: os_pkg_manager = self.container_config["os_packages"]["command"] except KeyError: msg = ( "cannot determine the OS package manager to use.\n\n\tPlease add an " "appropriate 'os_packages:command' entry to the spack.yaml manifest file\n" ) raise spack.error.SpackError(msg) return os_pkg_manager @tengine.context_property def extra_instructions(self): Extras = namedtuple("Extra", ["build", "final"]) extras = self.container_config.get("extra_instructions", {}) build, final = extras.get("build", None), extras.get("final", None) return Extras(build=build, final=final) @tengine.context_property def labels(self): return self.container_config.get("labels", {}) @tengine.context_property def bootstrap(self): """Information related to the build image.""" images_config = self.container_config["images"] bootstrap_recipe = None if self.bootstrap_image: config_args = _spack_checkout_config(images_config) command = checkout_command(*config_args) template_path = bootstrap_template_for(self.operating_system_key) env = tengine.make_environment() context = {"bootstrap": {"image": self.bootstrap_image, "spack_checkout": command}} bootstrap_recipe = env.get_template(template_path).render(**context) Bootstrap = namedtuple("Bootstrap", ["image", "recipe"]) return Bootstrap(image=self.bootstrap_image, recipe=bootstrap_recipe) @tengine.context_property def render_phase(self): render_bootstrap = bool(self.bootstrap_image) render_build = not (self.last_phase == "bootstrap") render_final = self.last_phase in (None, "final") Render = namedtuple("Render", ["bootstrap", "build", "final"]) return Render(bootstrap=render_bootstrap, build=render_build, final=render_final) def __call__(self): """Returns the recipe as a string""" env = tengine.make_environment() template_name = self.container_config.get("template", self.template_name) t = env.get_template(template_name) return t.render(**self.to_dict())
import spack.container.writers.docker # noqa: E402 # Import after function definition all the modules in this package, # so that registration of writers will happen automatically import spack.container.writers.singularity # noqa: E402