# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import codecs
import collections
import hashlib
import os.path
import platform
import posixpath
import re
import socket
import time
import xml.sax.saxutils
from typing import Dict, Optional
from urllib.parse import urlencode
from urllib.request import HTTPSHandler, Request, build_opener
import llnl.util.tty as tty
from llnl.util.filesystem import working_dir
import spack.build_environment
import spack.fetch_strategy
import spack.package_base
import spack.platforms
import spack.util.git
from spack.error import SpackError
from spack.util.crypto import checksum
from spack.util.log_parse import parse_log_events
from spack.util.web import urllib_ssl_cert_handler
from .base import Reporter
from .extract import extract_test_parts
# Mapping Spack phases to the corresponding CTest/CDash phase.
# TODO: Some of the phases being lumped into configure in the CDash tables
# TODO: really belong in a separate column, such as "Setup".
# TODO: Would also be nice to have `stage` as a separate phase that could
# TODO: be lumped into that new column instead of configure, for example.
MAP_PHASES_TO_CDASH = {
"autoreconf": "configure", # AutotoolsBuilder
"bootstrap": "configure", # CMakeBuilder
"build": "build",
"build_processes": "build", # Openloops
"cmake": "configure", # CMakeBuilder
"configure": "configure",
"edit": "configure", # MakefileBuilder
"generate_luarocks_config": "configure", # LuaBuilder
"hostconfig": "configure", # Lvarray
"initconfig": "configure", # CachedCMakeBuilder
"install": "build",
"meson": "configure", # MesonBuilder
"preprocess": "configure", # LuaBuilder
"qmake": "configure", # QMakeBuilder
"unpack": "configure", # LuaBuilder
}
# Initialize data structures common to each phase's report.
CDASH_PHASES = set(MAP_PHASES_TO_CDASH.values())
CDASH_PHASES.add("update")
CDashConfiguration = collections.namedtuple(
"CDashConfiguration", ["upload_url", "packages", "build", "site", "buildstamp", "track"]
)
[docs]
def build_stamp(track, timestamp):
buildstamp_format = "%Y%m%d-%H%M-{0}".format(track)
return time.strftime(buildstamp_format, time.localtime(timestamp))
[docs]
class CDash(Reporter):
"""Generate reports of spec installations for CDash.
To use this reporter, pass the ``--cdash-upload-url`` argument to
``spack install``::
spack install --cdash-upload-url=\\
https://mydomain.com/cdash/submit.php?project=Spack <spec>
In this example, results will be uploaded to the *Spack* project on the
CDash instance hosted at https://mydomain.com/cdash.
"""
def __init__(self, configuration: CDashConfiguration):
#: Set to False if any error occurs when building the CDash report
self.success = True
# Jinja2 expects `/` path separators
self.template_dir = "reports/cdash"
self.cdash_upload_url = configuration.upload_url
if self.cdash_upload_url:
self.buildid_regexp = re.compile("<buildId>([0-9]+)</buildId>")
self.phase_regexp = re.compile(r"Executing phase: '(.*)'")
self.authtoken = None
if "SPACK_CDASH_AUTH_TOKEN" in os.environ:
tty.verbose("Using CDash auth token from environment")
self.authtoken = os.environ.get("SPACK_CDASH_AUTH_TOKEN")
self.install_command = " ".join(configuration.packages)
self.base_buildname = configuration.build or self.install_command
self.site = configuration.site or socket.gethostname()
self.osname = platform.system()
self.osrelease = platform.release()
self.target = spack.platforms.host().target("default_target")
self.starttime = int(time.time())
self.endtime = self.starttime
self.buildstamp = (
configuration.buildstamp
if configuration.buildstamp
else build_stamp(configuration.track, self.starttime)
)
self.buildIds: Dict[str, str] = {}
self.revision = ""
git = spack.util.git.git()
with working_dir(spack.paths.spack_root):
self.revision = git("rev-parse", "HEAD", output=str).strip()
self.generator = "spack-{0}".format(spack.main.get_version())
self.multiple_packages = False
[docs]
def report_build_name(self, pkg_name):
return (
"{0} - {1}".format(self.base_buildname, pkg_name)
if self.multiple_packages
else self.base_buildname
)
[docs]
def build_report_for_package(self, report_dir, package, duration):
if "stdout" not in package:
# Skip reporting on packages that do not generate output.
return
self.current_package_name = package["name"]
self.buildname = self.report_build_name(self.current_package_name)
report_data = self.initialize_report(report_dir)
for phase in CDASH_PHASES:
report_data[phase] = {}
report_data[phase]["loglines"] = []
report_data[phase]["status"] = 0
report_data[phase]["starttime"] = self.starttime
# Track the phases we perform so we know what reports to create.
# We always report the update step because this is how we tell CDash
# what revision of Spack we are using.
phases_encountered = ["update"]
# Generate a report for this package.
current_phase = ""
cdash_phase = ""
for line in package["stdout"].splitlines():
match = None
if line.find("Executing phase: '") != -1:
match = self.phase_regexp.search(line)
if match:
current_phase = match.group(1)
if current_phase not in MAP_PHASES_TO_CDASH:
current_phase = ""
continue
cdash_phase = MAP_PHASES_TO_CDASH[current_phase]
if cdash_phase not in phases_encountered:
phases_encountered.append(cdash_phase)
report_data[cdash_phase]["loglines"].append(
str("{0} output for {1}:".format(cdash_phase, package["name"]))
)
elif cdash_phase:
report_data[cdash_phase]["loglines"].append(xml.sax.saxutils.escape(line))
# something went wrong pre-cdash "configure" phase b/c we have an exception and only
# "update" was encounterd.
# dump the report in the configure line so teams can see what the issue is
if len(phases_encountered) == 1 and package["exception"]:
# TODO this mapping is not ideal since these are pre-configure errors
# we need to determine if a more appropriate cdash phase can be utilized
# for now we will add a message to the log explaining this
cdash_phase = "configure"
phases_encountered.append(cdash_phase)
log_message = (
"Pre-configure errors occured in Spack's process that terminated the "
"build process prematurely.\nSpack output::\n{0}".format(
xml.sax.saxutils.escape(package["exception"])
)
)
report_data[cdash_phase]["loglines"].append(log_message)
# Move the build phase to the front of the list if it occurred.
# This supports older versions of CDash that expect this phase
# to be reported before all others.
if "build" in phases_encountered:
build_pos = phases_encountered.index("build")
phases_encountered.insert(0, phases_encountered.pop(build_pos))
self.endtime = self.starttime + duration
for phase in phases_encountered:
report_data[phase]["endtime"] = self.endtime
report_data[phase]["log"] = "\n".join(report_data[phase]["loglines"])
errors, warnings = parse_log_events(report_data[phase]["loglines"])
# Convert errors to warnings if the package reported success.
if package["result"] == "success":
warnings = errors + warnings
errors = []
# Cap the number of errors and warnings at 50 each.
errors = errors[:50]
warnings = warnings[:50]
nerrors = len(errors)
if nerrors > 0:
self.success = False
if phase == "configure":
report_data[phase]["status"] = 1
if phase == "build":
# Convert log output from ASCII to Unicode and escape for XML.
def clean_log_event(event):
event = vars(event)
event["text"] = xml.sax.saxutils.escape(event["text"])
event["pre_context"] = xml.sax.saxutils.escape("\n".join(event["pre_context"]))
event["post_context"] = xml.sax.saxutils.escape(
"\n".join(event["post_context"])
)
# source_file and source_line_no are either strings or
# the tuple (None,). Distinguish between these two cases.
if event["source_file"][0] is None:
event["source_file"] = ""
event["source_line_no"] = ""
else:
event["source_file"] = xml.sax.saxutils.escape(event["source_file"])
return event
report_data[phase]["errors"] = []
report_data[phase]["warnings"] = []
for error in errors:
report_data[phase]["errors"].append(clean_log_event(error))
for warning in warnings:
report_data[phase]["warnings"].append(clean_log_event(warning))
if phase == "update":
report_data[phase]["revision"] = self.revision
# Write the report.
report_name = phase.capitalize() + ".xml"
if self.multiple_packages:
report_file_name = package["name"] + "_" + report_name
else:
report_file_name = report_name
phase_report = os.path.join(report_dir, report_file_name)
with codecs.open(phase_report, "w", "utf-8") as f:
env = spack.tengine.make_environment()
if phase != "update":
# Update.xml stores site information differently
# than the rest of the CTest XML files.
site_template = posixpath.join(self.template_dir, "Site.xml")
t = env.get_template(site_template)
f.write(t.render(report_data))
phase_template = posixpath.join(self.template_dir, report_name)
t = env.get_template(phase_template)
f.write(t.render(report_data))
self.upload(phase_report)
[docs]
def build_report(self, report_dir, specs):
# Do an initial scan to determine if we are generating reports for more
# than one package. When we're only reporting on a single package we
# do not explicitly include the package's name in the CDash build name.
self.multiple_packages = False
num_packages = 0
for spec in specs:
# Do not generate reports for packages that were installed
# from the binary cache.
spec["packages"] = [
x
for x in spec["packages"]
if "installed_from_binary_cache" not in x or not x["installed_from_binary_cache"]
]
for package in spec["packages"]:
if "stdout" in package:
num_packages += 1
if num_packages > 1:
self.multiple_packages = True
break
if self.multiple_packages:
break
# Generate reports for each package in each spec.
for spec in specs:
duration = 0
if "time" in spec:
duration = int(spec["time"])
for package in spec["packages"]:
self.build_report_for_package(report_dir, package, duration)
self.finalize_report()
[docs]
def extract_standalone_test_data(self, package, phases, report_data):
"""Extract stand-alone test outputs for the package."""
testing = {}
report_data["testing"] = testing
testing["starttime"] = self.starttime
testing["endtime"] = self.starttime
testing["generator"] = self.generator
testing["parts"] = extract_test_parts(package["name"], package["stdout"].splitlines())
[docs]
def report_test_data(self, report_dir, package, phases, report_data):
"""Generate and upload the test report(s) for the package."""
for phase in phases:
# Write the report.
report_name = phase.capitalize() + ".xml"
report_file_name = "_".join([package["name"], package["id"], report_name])
phase_report = os.path.join(report_dir, report_file_name)
with codecs.open(phase_report, "w", "utf-8") as f:
env = spack.tengine.make_environment()
if phase not in ["update", "testing"]:
# Update.xml stores site information differently
# than the rest of the CTest XML files.
site_template = posixpath.join(self.template_dir, "Site.xml")
t = env.get_template(site_template)
f.write(t.render(report_data))
phase_template = posixpath.join(self.template_dir, report_name)
t = env.get_template(phase_template)
f.write(t.render(report_data))
tty.debug("Preparing to upload {0}".format(phase_report))
self.upload(phase_report)
[docs]
def test_report_for_package(self, report_dir, package, duration):
if "stdout" not in package:
# Skip reporting on packages that did not generate any output.
tty.debug("Skipping report for {0}: No generated output".format(package["name"]))
return
self.current_package_name = package["name"]
if self.base_buildname == self.install_command:
# The package list is NOT all that helpful in this case
self.buildname = "{0}-{1}".format(self.current_package_name, package["id"])
else:
self.buildname = self.report_build_name(self.current_package_name)
self.endtime = self.starttime + duration
report_data = self.initialize_report(report_dir)
report_data["hostname"] = socket.gethostname()
phases = ["testing"]
self.extract_standalone_test_data(package, phases, report_data)
self.report_test_data(report_dir, package, phases, report_data)
[docs]
def test_report(self, report_dir, specs):
"""Generate reports for each package in each spec."""
tty.debug("Processing test report")
for spec in specs:
duration = 0
if "time" in spec:
duration = int(spec["time"])
for package in spec["packages"]:
self.test_report_for_package(report_dir, package, duration)
self.finalize_report()
[docs]
def test_skipped_report(
self, report_dir: str, spec: spack.spec.Spec, reason: Optional[str] = None
):
"""Explicitly report spec as being skipped (e.g., CI).
Examples are the installation failed or the package is known to have
broken tests.
Args:
report_dir: directory where the report is to be written
spec: spec being tested
reason: optional reason the test is being skipped
"""
output = "Skipped {0} package".format(spec.name)
if reason:
output += "\n{0}".format(reason)
package = {"name": spec.name, "id": spec.dag_hash(), "result": "skipped", "stdout": output}
self.test_report_for_package(report_dir, package, duration=0.0)
[docs]
def concretization_report(self, report_dir, msg):
self.buildname = self.base_buildname
report_data = self.initialize_report(report_dir)
report_data["update"] = {}
report_data["update"]["starttime"] = self.starttime
report_data["update"]["endtime"] = self.endtime
report_data["update"]["revision"] = self.revision
report_data["update"]["log"] = msg
env = spack.tengine.make_environment()
update_template = posixpath.join(self.template_dir, "Update.xml")
t = env.get_template(update_template)
output_filename = os.path.join(report_dir, "Update.xml")
with open(output_filename, "w") as f:
f.write(t.render(report_data))
# We don't have a current package when reporting on concretization
# errors so refer to this report with the base buildname instead.
self.current_package_name = self.base_buildname
self.upload(output_filename)
self.success = False
self.finalize_report()
[docs]
def initialize_report(self, report_dir):
if not os.path.exists(report_dir):
os.mkdir(report_dir)
report_data = {}
report_data["buildname"] = self.buildname
report_data["buildstamp"] = self.buildstamp
report_data["install_command"] = self.install_command
report_data["generator"] = self.generator
report_data["osname"] = self.osname
report_data["osrelease"] = self.osrelease
report_data["site"] = self.site
report_data["target"] = self.target
return report_data
[docs]
def upload(self, filename):
if not self.cdash_upload_url:
print("Cannot upload {0} due to missing upload url".format(filename))
return
# Compute md5 checksum for the contents of this file.
md5sum = checksum(hashlib.md5, filename, block_size=8192)
opener = build_opener(HTTPSHandler(context=urllib_ssl_cert_handler()))
with open(filename, "rb") as f:
params_dict = {
"build": self.buildname,
"site": self.site,
"stamp": self.buildstamp,
"MD5": md5sum,
}
encoded_params = urlencode(params_dict)
url = "{0}&{1}".format(self.cdash_upload_url, encoded_params)
request = Request(url, data=f)
request.add_header("Content-Type", "text/xml")
request.add_header("Content-Length", os.path.getsize(filename))
if self.authtoken:
request.add_header("Authorization", "Bearer {0}".format(self.authtoken))
try:
# By default, urllib2 only support GET and POST.
# CDash expects this file to be uploaded via PUT.
request.get_method = lambda: "PUT"
response = opener.open(request)
if self.current_package_name not in self.buildIds:
resp_value = response.read()
if isinstance(resp_value, bytes):
resp_value = resp_value.decode("utf-8")
match = self.buildid_regexp.search(resp_value)
if match:
buildid = match.group(1)
self.buildIds[self.current_package_name] = buildid
except Exception as e:
print("Upload to CDash failed: {0}".format(e))
[docs]
def finalize_report(self):
if self.buildIds:
tty.msg("View your build results here:")
for package_name, buildid in self.buildIds.items():
# Construct and display a helpful link if CDash responded with
# a buildId.
build_url = self.cdash_upload_url
build_url = build_url[0 : build_url.find("submit.php")]
build_url += "buildSummary.php?buildid={0}".format(buildid)
tty.msg("{0}: {1}".format(package_name, build_url))
if not self.success:
raise SpackError("Errors encountered, see above for more details")