Source code for spack.test.spec_yaml

# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

"""Test YAML and JSON serialization for specs.

The YAML and JSON formats preserve DAG information in the spec.

from __future__ import print_function

import ast
import collections
import inspect
import os

import pytest

from llnl.util.compat import Iterable, Mapping

import spack.hash_types as ht
import spack.paths
import spack.spec
import spack.util.spack_json as sjson
import spack.util.spack_yaml as syaml
import spack.version
from spack import repo
from spack.spec import Spec, save_dependency_specfiles
from spack.util.mock_package import MockPackageMultiRepo
from spack.util.spack_yaml import SpackYAMLError, syaml_dict

[docs]def check_yaml_round_trip(spec): yaml_text = spec.to_yaml() spec_from_yaml = Spec.from_yaml(yaml_text) assert spec.eq_dag(spec_from_yaml)
[docs]def check_json_round_trip(spec): json_text = spec.to_json() spec_from_json = Spec.from_json(json_text) assert spec.eq_dag(spec_from_json)
[docs]def test_simple_spec(): spec = Spec('mpileaks') check_yaml_round_trip(spec) check_json_round_trip(spec)
[docs]def test_read_spec_from_signed_json(): spec_dir = os.path.join( spack.paths.test_path, 'data', 'mirrors', 'signed_json') file_name = ( 'linux-ubuntu18.04-haswell-gcc-8.4.0-' 'zlib-1.2.12-g7otk5dra3hifqxej36m5qzm7uyghqgb.spec.json.sig') spec_path = os.path.join(spec_dir, file_name) def check_spec(spec_to_check): assert( == 'zlib') assert(spec_to_check._hash == 'g7otk5dra3hifqxej36m5qzm7uyghqgb') with open(spec_path) as fd: s = Spec.from_signed_json(fd) check_spec(s) with open(spec_path) as fd: s = Spec.from_signed_json( check_spec(s)
[docs]def test_normal_spec(mock_packages): spec = Spec('mpileaks+debug~opt') spec.normalize() check_yaml_round_trip(spec) check_json_round_trip(spec)
[docs]@pytest.mark.parametrize( "invalid_yaml", [ "playing_playlist: {{ action }} playlist {{ playlist_name }}" ] ) def test_invalid_yaml_spec(invalid_yaml): with pytest.raises(SpackYAMLError) as e: Spec.from_yaml(invalid_yaml) exc_msg = str(e.value) assert exc_msg.startswith("error parsing YAML spec:") assert invalid_yaml in exc_msg
[docs]@pytest.mark.parametrize( "invalid_json, error_message", [ ("{13:", "Expecting property name") ] ) def test_invalid_json_spec(invalid_json, error_message): with pytest.raises(sjson.SpackJSONError) as e: Spec.from_json(invalid_json) exc_msg = str(e.value) assert exc_msg.startswith("error parsing JSON spec:") assert error_message in exc_msg
[docs]def test_external_spec(config, mock_packages): spec = Spec('externaltool') spec.concretize() check_yaml_round_trip(spec) check_json_round_trip(spec) spec = Spec('externaltest') spec.concretize() check_yaml_round_trip(spec) check_json_round_trip(spec)
[docs]def test_ambiguous_version_spec(mock_packages): spec = Spec('mpileaks@1.0:5.0,6.1,7.3+debug~opt') spec.normalize() check_yaml_round_trip(spec) check_json_round_trip(spec)
[docs]def test_concrete_spec(config, mock_packages): spec = Spec('mpileaks+debug~opt') spec.concretize() check_yaml_round_trip(spec) check_json_round_trip(spec)
[docs]def test_yaml_multivalue(config, mock_packages): spec = Spec('multivalue-variant foo="bar,baz"') spec.concretize() check_yaml_round_trip(spec) check_json_round_trip(spec)
[docs]def test_yaml_subdag(config, mock_packages): spec = Spec('mpileaks^mpich+debug') spec.concretize() yaml_spec = Spec.from_yaml(spec.to_yaml()) json_spec = Spec.from_json(spec.to_json()) for dep in ('callpath', 'mpich', 'dyninst', 'libdwarf', 'libelf'): assert spec[dep].eq_dag(yaml_spec[dep]) assert spec[dep].eq_dag(json_spec[dep])
[docs]def test_using_ordered_dict(mock_packages): """ Checks that dicts are ordered Necessary to make sure that dag_hash is stable across python versions and processes. """ def descend_and_check(iterable, level=0): if isinstance(iterable, Mapping): assert isinstance(iterable, syaml_dict) return descend_and_check(iterable.values(), level=level + 1) max_level = level for value in iterable: if isinstance(value, Iterable) and not isinstance(value, str): nlevel = descend_and_check(value, level=level + 1) if nlevel > max_level: max_level = nlevel return max_level specs = ['mpileaks ^zmpi', 'dttop', 'dtuse'] for spec in specs: dag = Spec(spec) dag.normalize() level = descend_and_check(dag.to_node_dict()) # level just makes sure we are doing something here assert level >= 5
[docs]def test_ordered_read_not_required_for_consistent_dag_hash( config, mock_packages ): """Make sure ordered serialization isn't required to preserve hashes. For consistent hashes, we require that YAML and json documents have their keys serialized in a deterministic order. However, we don't want to require them to be serialized in order. This ensures that is not required. """ specs = ['mpileaks ^zmpi', 'dttop', 'dtuse'] for spec in specs: spec = Spec(spec) spec.concretize() # # Dict & corresponding YAML & JSON from the original spec. # spec_dict = spec.to_dict() spec_yaml = spec.to_yaml() spec_json = spec.to_json() # # Make a spec with reversed OrderedDicts for every # OrderedDict in the original. # reversed_spec_dict = reverse_all_dicts(spec.to_dict()) # # Dump to YAML and JSON # yaml_string = syaml.dump(spec_dict, default_flow_style=False) reversed_yaml_string = syaml.dump(reversed_spec_dict, default_flow_style=False) json_string = sjson.dump(spec_dict) reversed_json_string = sjson.dump(reversed_spec_dict) # # Do many consistency checks # # spec yaml is ordered like the spec dict assert yaml_string == spec_yaml assert json_string == spec_json # reversed string is different from the original, so it # *would* generate a different hash assert yaml_string != reversed_yaml_string assert json_string != reversed_json_string # build specs from the "wrongly" ordered data round_trip_yaml_spec = Spec.from_yaml(yaml_string) round_trip_json_spec = Spec.from_json(json_string) round_trip_reversed_yaml_spec = Spec.from_yaml( reversed_yaml_string ) round_trip_reversed_json_spec = Spec.from_yaml( reversed_json_string ) # Strip spec if we stripped the yaml spec = spec.copy(deps=ht.dag_hash.deptype) # specs are equal to the original assert spec == round_trip_yaml_spec assert spec == round_trip_json_spec assert spec == round_trip_reversed_yaml_spec assert spec == round_trip_reversed_json_spec assert round_trip_yaml_spec == round_trip_reversed_yaml_spec assert round_trip_json_spec == round_trip_reversed_json_spec # dag_hashes are equal assert spec.dag_hash() == round_trip_yaml_spec.dag_hash() assert spec.dag_hash() == round_trip_json_spec.dag_hash() assert spec.dag_hash() == round_trip_reversed_yaml_spec.dag_hash() assert spec.dag_hash() == round_trip_reversed_json_spec.dag_hash() # dag_hash is equal after round-trip by dag_hash spec.concretize() round_trip_yaml_spec.concretize() round_trip_json_spec.concretize() round_trip_reversed_yaml_spec.concretize() round_trip_reversed_json_spec.concretize() assert spec.dag_hash() == round_trip_yaml_spec.dag_hash() assert spec.dag_hash() == round_trip_json_spec.dag_hash() assert spec.dag_hash() == round_trip_reversed_yaml_spec.dag_hash() assert spec.dag_hash() == round_trip_reversed_json_spec.dag_hash()
[docs]@pytest.mark.parametrize("module", [ spack.spec, spack.version, ]) def test_hashes_use_no_python_dicts(module): """Coarse check to make sure we don't use dicts in Spec.to_node_dict(). Python dicts are not guaranteed to iterate in a deterministic order (at least not in all python versions) so we need to use lists and syaml_dicts. syaml_dicts are ordered and ensure that hashes in Spack are deterministic. This test is intended to handle cases that are not covered by the consistency checks above, or that would be missed by a dynamic check. This test traverses the ASTs of functions that are used in our hash algorithms, finds instances of dictionaries being constructed, and prints out the line numbers where they occur. """ class FindFunctions(ast.NodeVisitor): """Find a function definition called to_node_dict.""" def __init__(self): self.nodes = [] def visit_FunctionDef(self, node): # noqa if in ("to_node_dict", "to_dict", "to_dict_or_value"): self.nodes.append(node) class FindDicts(ast.NodeVisitor): """Find source locations of dicts in an AST.""" def __init__(self, filename): self.nodes = [] self.filename = filename def add_error(self, node): self.nodes.append( "Use syaml_dict instead of dict at %s:%s:%s" % (self.filename, node.lineno, node.col_offset) ) def visit_Dict(self, node): # noqa self.add_error(node) def visit_Call(self, node): # noqa name = None if isinstance(node.func, ast.Name): name = elif isinstance(node.func, ast.Attribute): name = node.func.attr if name == 'dict': self.add_error(node) find_functions = FindFunctions() module_ast = ast.parse(inspect.getsource(module)) find_functions.visit(module_ast) find_dicts = FindDicts(module.__file__) for node in find_functions.nodes: find_dicts.visit(node) # fail with offending lines if we found some dicts. assert [] == find_dicts.nodes
[docs]def reverse_all_dicts(data): """Descend into data and reverse all the dictionaries""" if isinstance(data, dict): return syaml_dict(reversed( [(reverse_all_dicts(k), reverse_all_dicts(v)) for k, v in data.items()])) elif isinstance(data, (list, tuple)): return type(data)(reverse_all_dicts(elt) for elt in data) else: return data
[docs]def check_specs_equal(original_spec, spec_yaml_path): with open(spec_yaml_path, 'r') as fd: spec_yaml = spec_from_yaml = Spec.from_yaml(spec_yaml) return original_spec.eq_dag(spec_from_yaml)
[docs]def test_save_dependency_spec_jsons_subset(tmpdir, config): output_path = str(tmpdir.mkdir('spec_jsons')) default = ('build', 'link') mock_repo = MockPackageMultiRepo() g = mock_repo.add_package('g', [], []) f = mock_repo.add_package('f', [], []) e = mock_repo.add_package('e', [], []) d = mock_repo.add_package('d', [f, g], [default, default]) c = mock_repo.add_package('c', [], []) b = mock_repo.add_package('b', [d, e], [default, default]) mock_repo.add_package('a', [b, c], [default, default]) with repo.use_repositories(mock_repo): spec_a = Spec('a') spec_a.concretize() b_spec = spec_a['b'] c_spec = spec_a['c'] spec_a_json = spec_a.to_json() save_dependency_specfiles(spec_a_json, output_path, ['b', 'c']) assert check_specs_equal(b_spec, os.path.join(output_path, 'b.json')) assert check_specs_equal(c_spec, os.path.join(output_path, 'c.json'))
[docs]def test_legacy_yaml(tmpdir, install_mockery, mock_packages): """Tests a simple legacy YAML with a dependency and ensures spec survives concretization.""" yaml = """ spec: - a: version: '2.0' arch: platform: linux platform_os: rhel7 target: x86_64 compiler: name: gcc version: 8.3.0 namespace: builtin.mock parameters: bvv: true foo: - bar foobar: bar cflags: [] cppflags: [] cxxflags: [] fflags: [] ldflags: [] ldlibs: [] dependencies: b: hash: iaapywazxgetn6gfv2cfba353qzzqvhn type: - build - link hash: obokmcsn3hljztrmctbscmqjs3xclazz full_hash: avrk2tqsnzxeabmxa6r776uq7qbpeufv build_hash: obokmcsn3hljztrmctbscmqjs3xclazy - b: version: '1.0' arch: platform: linux platform_os: rhel7 target: x86_64 compiler: name: gcc version: 8.3.0 namespace: builtin.mock parameters: cflags: [] cppflags: [] cxxflags: [] fflags: [] ldflags: [] ldlibs: [] hash: iaapywazxgetn6gfv2cfba353qzzqvhn full_hash: qvsxvlmjaothtpjluqijv7qfnni3kyyg build_hash: iaapywazxgetn6gfv2cfba353qzzqvhy """ spec = Spec.from_yaml(yaml) concrete_spec = spec.concretized() assert concrete_spec.eq_dag(spec)
#: A well ordered Spec dictionary, using ``OrderdDict``. #: Any operation that transforms Spec dictionaries should #: preserve this order. ordered_spec = collections.OrderedDict([ ("arch", collections.OrderedDict([ ("platform", "darwin"), ("platform_os", "bigsur"), ("target", collections.OrderedDict([ ("features", [ "adx", "aes", "avx", "avx2", "bmi1", "bmi2", "clflushopt", "f16c", "fma", "mmx", "movbe", "pclmulqdq", "popcnt", "rdrand", "rdseed", "sse", "sse2", "sse4_1", "sse4_2", "ssse3", "xsavec", "xsaveopt" ]), ("generation", 0), ("name", "skylake"), ("parents", ["broadwell"]), ("vendor", "GenuineIntel"), ])), ])), ("compiler", collections.OrderedDict([ ("name", "apple-clang"), ("version", "13.0.0"), ])), ("name", "zlib"), ("namespace", "builtin"), ("parameters", collections.OrderedDict([ ("cflags", []), ("cppflags", []), ("cxxflags", []), ("fflags", []), ("ldflags", []), ("ldlibs", []), ("optimize", True), ("pic", True), ("shared", True), ])), ("version", "1.2.11"), ])
[docs]@pytest.mark.regression("31092") def test_strify_preserves_order(): """Ensure that ``spack_json._strify()`` dumps dictionaries in the right order. ``_strify()`` is used in ``spack_json.dump()``, which is used in ``Spec.dag_hash()``, so if this goes wrong, ``Spec`` hashes can vary between python versions. """ strified = sjson._strify(ordered_spec) assert list(ordered_spec.items()) == list(strified.items())