Source code for spack.container.writers

# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
"""Writers for different kind of recipes and related
convenience functions.
"""
import collections
import copy

import spack.environment as ev
import spack.schema.env
import spack.tengine as tengine
import spack.util.spack_yaml as syaml
from spack.container.images import (
    bootstrap_template_for,
    build_info,
    checkout_command,
    commands_for,
    data,
    os_package_manager_for,
)

#: Caches all the writers that are currently supported
_writer_factory = {}


[docs]def writer(name): """Decorator to register a factory for a recipe writer. Each factory should take a configuration dictionary and return a properly configured writer that, when called, prints the corresponding recipe. """ def _decorator(factory): _writer_factory[name] = factory return factory return _decorator
[docs]def create(configuration, last_phase=None): """Returns a writer that conforms to the configuration passed as input. Args: configuration (dict): how to generate the current recipe last_phase (str): last phase to be printed or None to print them all """ name = ev.config_dict(configuration)["container"]["format"] return _writer_factory[name](configuration, last_phase)
[docs]def recipe(configuration, last_phase=None): """Returns a recipe that conforms to the configuration passed as input. Args: configuration (dict): how to generate the current recipe last_phase (str): last phase to be printed or None to print them all """ return create(configuration, last_phase)()
def _stage_base_images(images_config): """Return a tuple with the base images to be used at the various stages. Args: images_config (dict): configuration under container:images """ # If we have custom base images, just return them verbatim. build_stage = images_config.get("build", None) if build_stage: final_stage = images_config["final"] return None, build_stage, final_stage # Check the operating system: this will be the base of the bootstrap # stage, if there, and of the final stage. operating_system = images_config.get("os", None) # Check the OS is mentioned in the internal data stored in a JSON file images_json = data()["images"] if not any(os_name == operating_system for os_name in images_json): msg = 'invalid operating system name "{0}". ' "[Allowed values are {1}]" msg = msg.format(operating_system, ", ".join(data()["images"])) raise ValueError(msg) # Retrieve the build stage spack_info = images_config["spack"] if isinstance(spack_info, dict): build_stage = "bootstrap" else: spack_version = images_config["spack"] image_name, tag = build_info(operating_system, spack_version) build_stage = "bootstrap" if image_name: build_stage = ":".join([image_name, tag]) # Retrieve the bootstrap stage bootstrap_stage = None if build_stage == "bootstrap": bootstrap_stage = images_json[operating_system]["bootstrap"].get("image", operating_system) # Retrieve the final stage final_stage = images_json[operating_system].get("final", {"image": operating_system})["image"] return bootstrap_stage, build_stage, final_stage def _spack_checkout_config(images_config): spack_info = images_config["spack"] url = "https://github.com/spack/spack.git" ref = "develop" resolve_sha, verify = False, False # Config specific values may override defaults if isinstance(spack_info, dict): url = spack_info.get("url", url) ref = spack_info.get("ref", ref) resolve_sha = spack_info.get("resolve_sha", resolve_sha) verify = spack_info.get("verify", verify) else: ref = spack_info return url, ref, resolve_sha, verify
[docs]class PathContext(tengine.Context): """Generic context used to instantiate templates of recipes that install software in a common location and make it available directly via PATH. """ def __init__(self, config, last_phase): self.config = ev.config_dict(config) self.container_config = self.config["container"] # Operating system tag as written in the configuration file self.operating_system_key = self.container_config["images"].get("os") # Get base images and verify the OS bootstrap, build, final = _stage_base_images(self.container_config["images"]) self.bootstrap_image = bootstrap self.build_image = build self.final_image = final # Record the last phase self.last_phase = last_phase @tengine.context_property def run(self): """Information related to the run image.""" Run = collections.namedtuple("Run", ["image"]) return Run(image=self.final_image) @tengine.context_property def build(self): """Information related to the build image.""" Build = collections.namedtuple("Build", ["image"]) return Build(image=self.build_image) @tengine.context_property def strip(self): """Whether or not to strip binaries in the image""" return self.container_config.get("strip", True) @tengine.context_property def paths(self): """Important paths in the image""" Paths = collections.namedtuple("Paths", ["environment", "store", "hidden_view", "view"]) return Paths( environment="/opt/spack-environment", store="/opt/software", hidden_view="/opt/._view", view="/opt/view", ) @tengine.context_property def manifest(self): """The spack.yaml file that should be used in the image""" import jsonschema # Copy in the part of spack.yaml prescribed in the configuration file manifest = copy.deepcopy(self.config) manifest.pop("container") # Ensure that a few paths are where they need to be manifest.setdefault("config", syaml.syaml_dict()) manifest["config"]["install_tree"] = self.paths.store manifest["view"] = self.paths.view manifest = {"spack": manifest} # Validate the manifest file jsonschema.validate(manifest, schema=spack.schema.env.schema) return syaml.dump(manifest, default_flow_style=False).strip() @tengine.context_property def os_packages_final(self): """Additional system packages that are needed at run-time.""" return self._os_packages_for_stage("final") @tengine.context_property def os_packages_build(self): """Additional system packages that are needed at build-time.""" return self._os_packages_for_stage("build") @tengine.context_property def os_package_update(self): """Whether or not to update the OS package manager cache.""" os_packages = self.container_config.get("os_packages", {}) return os_packages.get("update", True) def _os_packages_for_stage(self, stage): os_packages = self.container_config.get("os_packages", {}) package_list = os_packages.get(stage, None) return self._package_info_from(package_list) def _package_info_from(self, package_list): """Helper method to pack a list of packages with the additional information required by the template. Args: package_list: list of packages Returns: Enough information to know how to update the cache, install a list opf packages, and clean in the end. """ if not package_list: return package_list image_config = self.container_config["images"] image = image_config.get("build", None) if image is None: os_pkg_manager = os_package_manager_for(image_config["os"]) else: os_pkg_manager = self.container_config["os_packages"]["command"] update, install, clean = commands_for(os_pkg_manager) Packages = collections.namedtuple("Packages", ["update", "install", "list", "clean"]) return Packages(update=update, install=install, list=package_list, clean=clean) @tengine.context_property def extra_instructions(self): Extras = collections.namedtuple("Extra", ["build", "final"]) extras = self.container_config.get("extra_instructions", {}) build, final = extras.get("build", None), extras.get("final", None) return Extras(build=build, final=final) @tengine.context_property def labels(self): return self.container_config.get("labels", {}) @tengine.context_property def bootstrap(self): """Information related to the build image.""" images_config = self.container_config["images"] bootstrap_recipe = None if self.bootstrap_image: config_args = _spack_checkout_config(images_config) command = checkout_command(*config_args) template_path = bootstrap_template_for(self.operating_system_key) env = tengine.make_environment() context = {"bootstrap": {"image": self.bootstrap_image, "spack_checkout": command}} bootstrap_recipe = env.get_template(template_path).render(**context) Bootstrap = collections.namedtuple("Bootstrap", ["image", "recipe"]) return Bootstrap(image=self.bootstrap_image, recipe=bootstrap_recipe) @tengine.context_property def render_phase(self): render_bootstrap = bool(self.bootstrap_image) render_build = not (self.last_phase == "bootstrap") render_final = self.last_phase in (None, "final") Render = collections.namedtuple("Render", ["bootstrap", "build", "final"]) return Render(bootstrap=render_bootstrap, build=render_build, final=render_final) def __call__(self): """Returns the recipe as a string""" env = tengine.make_environment() t = env.get_template(self.template_name) return t.render(**self.to_dict())
import spack.container.writers.docker # noqa: E402 # Import after function definition all the modules in this package, # so that registration of writers will happen automatically import spack.container.writers.singularity # noqa: E402