# Copyright Spack Project Developers. See COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import argparse
import ast
import os
import re
import sys
from pathlib import Path
from typing import Dict, List, Optional, Set, Union
import spack.paths
import spack.repo
import spack.util.git
from spack.cmd.common.spec_strings import (
_check_spec_strings,
_spec_str_default_handler,
_spec_str_fix_handler,
)
from spack.util import tty
from spack.util.executable import Executable, which
from spack.util.filesystem import working_dir
from spack.util.tty import color
description = "runs source code style checks on spack"
section = "developer"
level = "long"
#: List of paths to exclude from checks -- relative to spack root
exclude_paths = [os.path.relpath(spack.paths.vendor_path, spack.paths.prefix)]
#: Order in which tools should be run.
#: The list maps an executable name to a method to ensure the tool is
#: bootstrapped or present in the environment.
tool_names = ["import", "ruff-format", "ruff-check", "mypy"]
#: warnings to ignore in mypy
mypy_ignores = [
# same as `disable_error_code = "annotation-unchecked"` in pyproject.toml, which
# doesn't exist in mypy 0.971 for Python 3.6
"[annotation-unchecked]"
]
#: decorator for adding tools to the list
#: tools we run in spack style
tools: Dict[str, tool] = {}
[docs]
def changed_files(base="develop", untracked=True, all_files=False, root=None) -> List[Path]:
"""Get list of changed files in the Spack repository.
Arguments:
base (str): name of base branch to evaluate differences with.
untracked (bool): include untracked files in the list.
all_files (bool): list all files in the repository.
root (str): use this directory instead of the Spack prefix.
"""
if root is None:
root = spack.paths.prefix
git = spack.util.git.git(required=True)
# ensure base is in the repo
base_sha = git(
"rev-parse", "--quiet", "--verify", "--revs-only", base, fail_on_error=False, output=str
)
if git.returncode != 0:
tty.die(
"This repository does not have a '%s' revision." % base,
"spack style needs this branch to determine which files changed.",
"Ensure that '%s' exists, or specify files to check explicitly." % base,
)
range = "{0}...".format(base_sha.strip())
git_args = [
# Add changed files committed since branching off of develop
["diff", "--name-only", "--diff-filter=ACMR", range],
# Add changed files that have been staged but not yet committed
["diff", "--name-only", "--diff-filter=ACMR", "--cached"],
# Add changed files that are unstaged
["diff", "--name-only", "--diff-filter=ACMR"],
]
# Add new files that are untracked
if untracked:
git_args.append(["ls-files", "--exclude-standard", "--other"])
# add everything if the user asked for it
if all_files:
git_args.append(["ls-files", "--exclude-standard"])
excludes = [os.path.realpath(os.path.join(root, f)) for f in exclude_paths]
changed = set()
for arg_list in git_args:
files = git(*arg_list, output=str).split("\n")
for f in files:
# Ignore non-Python files
if not (f.endswith(".py") or f == "bin/spack"):
continue
# Ignore files in the exclude locations
if any(os.path.realpath(f).startswith(e) for e in excludes):
continue
changed.add(Path(f))
return sorted(changed)
[docs]
def setup_parser(subparser: argparse.ArgumentParser) -> None:
subparser.add_argument(
"-b",
"--base",
action="store",
default="develop",
help="branch to compare against to determine changed files (default: develop)",
)
subparser.add_argument(
"-a",
"--all",
action="store_true",
help="check all files, not just changed files (applies only to Import Check)",
)
subparser.add_argument(
"-r",
"--root-relative",
action="store_true",
default=False,
help="print root-relative paths (default: cwd-relative)",
)
subparser.add_argument(
"-U",
"--no-untracked",
dest="untracked",
action="store_false",
default=True,
help="exclude untracked files from checks",
)
subparser.add_argument(
"-f",
"--fix",
action="store_true",
default=False,
help="format automatically if possible (e.g., with isort, black)",
)
subparser.add_argument(
"--root", action="store", default=None, help="style check a different spack instance"
)
tool_group = subparser.add_mutually_exclusive_group()
tool_group.add_argument(
"-t",
"--tool",
action="append",
help="specify which tools to run (default: %s)" % ", ".join(tool_names),
)
tool_group.add_argument(
"-s",
"--skip",
metavar="TOOL",
action="append",
help="specify tools to skip (choose from %s)" % ", ".join(tool_names),
)
subparser.add_argument(
"--spec-strings",
action="store_true",
help="upgrade spec strings in Python, JSON and YAML files for compatibility with Spack "
"v1.0 and v0.x. Example: spack style ``--spec-strings $(git ls-files)``. Note: must be "
"used only on specs from spack v0.X.",
)
subparser.add_argument("files", nargs=argparse.REMAINDER, help="specific files to check")
[docs]
def cwd_relative(path: Path, root: Union[Path, str], initial_working_dir: Path) -> Path:
"""Translate prefix-relative path to current working directory-relative."""
if path.is_absolute():
return path
return Path(os.path.relpath((root / path), initial_working_dir))
[docs]
def rewrite_and_print_output(
output,
root,
working_dir,
root_relative,
re_obj=re.compile(r"^(.+):([0-9]+):"),
replacement=r"{0}:{1}:",
):
"""rewrite output with <file>:<line>: format to respect path args"""
# print results relative to current working directory
def translate(match):
return replacement.format(
cwd_relative(Path(match.group(1)), root, working_dir), *list(match.groups()[1:])
)
for line in output.split("\n"):
if not line:
continue
if any(ignore in line for ignore in mypy_ignores):
# some mypy annotations can't be disabled in older mypys (e.g. .971, which
# is the only mypy that supports python 3.6), so we filter them here.
continue
if not root_relative and re_obj:
line = re_obj.sub(translate, line)
print(line)
[docs]
@tool("ruff-check", cmd="ruff")
def ruff_check(file_list, args):
"""Run the ruff-check command. Handles config and non generic ruff argument logic"""
cmd_args = ["--config", os.path.join(spack.paths.prefix, "pyproject.toml"), "--quiet"]
if args.fix:
cmd_args += ["--fix", "--no-unsafe-fixes"]
else:
cmd_args += ["--no-fix"]
return run_ruff(
file_list, "check", cmd_args, args.root, args.initial_working_dir, args.root_relative
)
[docs]
def run_ruff(
file_list: List[Path],
cmd: str,
args: List[str],
root: Path,
working_dir: Path,
root_relative: bool,
):
"""Run the ruff tool"""
ruff_cmd = tools[f"ruff-{cmd}"].executable
if not ruff_cmd:
tty.warn("Cannot execute requested tool: ruff\nCannot find tool")
return -1
files = (str(x) for x in file_list)
if color.get_color_when():
args += ("--color", "auto")
pat = re.compile("would reformat +(.*)")
replacement = "would reformat {0}"
packed_args = (cmd,) + (*args,) + tuple(files)
output = ruff_cmd(*packed_args, fail_on_error=False, output=str, error=str)
returncode = ruff_cmd.returncode
rewrite_and_print_output(output, root, working_dir, root_relative, pat, replacement)
print_tool_result(f"ruff-{cmd}", returncode)
return returncode
[docs]
@tool("mypy")
def run_mypy(file_list, args):
mypy_cmd = tools["mypy"].executable
if not mypy_cmd:
tty.warn("Cannot execute requested tool: mypy\nCannot find tool")
return -1
# always run with config from running spack prefix
common_mypy_args = [
"--config-file",
os.path.join(spack.paths.prefix, "pyproject.toml"),
"--show-error-codes",
]
mypy_arg_sets = [common_mypy_args + ["--package", "spack"]]
if "SPACK_MYPY_CHECK_PACKAGES" in os.environ:
mypy_arg_sets.append(
common_mypy_args + ["--package", "packages", "--disable-error-code", "no-redef"]
)
returncode = 0
for mypy_args in mypy_arg_sets:
output = mypy_cmd(*mypy_args, fail_on_error=False, output=str)
returncode |= mypy_cmd.returncode
rewrite_and_print_output(output, args.root, args.initial_working_dir, args.root_relative)
print_tool_result("mypy", returncode)
return returncode
def _module_part(root: Path, expr: str):
parts = expr.split(".")
# spack.pkg is for repositories, don't try to resolve it here.
if expr.startswith(spack.repo.PKG_MODULE_PREFIX_V1) or expr == "spack.pkg":
return None
while parts:
f1 = (root / "lib" / "spack").joinpath(*parts).with_suffix(".py")
f2 = (root / "lib" / "spack").joinpath(*parts, "__init__.py")
if (
f1.exists()
# ensure case sensitive match
and any(p.name == f"{parts[-1]}.py" for p in f1.parent.iterdir())
or f2.exists()
):
return ".".join(parts)
parts.pop()
return None
def _run_import_check(
file_list: List[Path],
*,
fix: bool,
root_relative: bool,
root: Path,
working_dir: Path,
out=sys.stdout,
base="develop",
all=False,
):
if sys.version_info < (3, 9):
print("import check requires Python 3.9 or later")
return 0
is_use = re.compile(r"(?<!from )(?<!import )spack\.[a-zA-Z0-9_\.]+")
exit_code = 0
files = file_list or changed_files(root=root, base=base, all_files=all)
for file in files:
to_add: Set[str] = set()
to_remove: List[str] = []
pretty_path = file if root_relative else cwd_relative(file, root, working_dir)
try:
with open(file, "r", encoding="utf-8") as f:
contents = f.read()
parsed = ast.parse(contents)
except Exception:
exit_code = 1
print(f"{pretty_path}: could not parse", file=out)
continue
imported_modules: Set[str] = set()
potential_redundant_imports: List[str] = []
for node in ast.walk(parsed):
# Clear strings to make sure usages in strings are not counted
if isinstance(node, ast.Constant) and isinstance(node.value, str):
node.value = ""
elif isinstance(node, ast.Import):
# Track `import ...` without aliases
for name in node.names:
if name.asname is None:
imported_modules.add(name.name)
# Track top-level imports for redundancy check
if (
node.col_offset == 0
and len(node.names) == 1
and node.names[0].asname is None
and node.names[0].name.startswith("spack.")
):
potential_redundant_imports.append(node.names[0].name)
# Convert back to code after clearing strings
filtered_contents = ast.unparse(parsed) # novermin
# Check for redundant imports
for module_name in potential_redundant_imports:
usage_regex = rf"(?<!from )(?<!import ){re.escape(module_name)}(?!\w)"
if re.search(usage_regex, filtered_contents):
continue
statement = f"import {module_name}"
# redundant imports followed by a `# comment` are ignored, cause there can be
# legitimate reason to import a module: execute module scope init code, or to deal
# with circular imports.
if re.search(rf"^{re.escape(statement)}$", contents, re.MULTILINE):
to_remove.append(statement)
exit_code = 1
print(f"{pretty_path}: redundant import: {module_name}", file=out)
# Check for missing imports
for m in is_use.finditer(filtered_contents):
module = _module_part(root, m.group(0))
if not module or module in to_add:
continue
if module in imported_modules:
continue
to_add.add(module)
exit_code = 1
print(f"{pretty_path}: missing import: {module} ({m.group(0)})", file=out)
if not fix or not to_add and not to_remove:
continue
with open(file, "r", encoding="utf-8") as f:
lines = f.readlines()
if to_add:
# insert missing imports before the first import, delegate ordering to isort
for node in parsed.body:
if isinstance(node, (ast.Import, ast.ImportFrom)):
first_line = node.lineno
break
else:
print(f"{pretty_path}: could not fix", file=out)
continue
lines.insert(first_line, "\n".join(f"import {x}" for x in to_add) + "\n")
new_contents = "".join(lines)
# remove redundant imports
for statement in to_remove:
new_contents = new_contents.replace(f"{statement}\n", "")
with open(file, "w", encoding="utf-8") as f:
f.write(new_contents)
return exit_code
[docs]
@tool("import", external=False)
def run_import_check(file_list, args):
exit_code = _run_import_check(
file_list,
fix=args.fix,
root_relative=args.root_relative,
root=args.root,
working_dir=args.initial_working_dir,
base=args.base,
all=args.all,
)
print_tool_result("import", exit_code)
return exit_code
def _bootstrap_dev_dependencies():
import spack.bootstrap
with spack.bootstrap.ensure_bootstrap_configuration():
spack.bootstrap.ensure_environment_dependencies()
[docs]
def style(parser, args):
if args.spec_strings:
if not args.files:
tty.die("No files provided to check spec strings.")
handler = _spec_str_fix_handler if args.fix else _spec_str_default_handler
return _check_spec_strings(args.files, handler)
# save initial working directory for relativizing paths later
args.initial_working_dir = Path.cwd()
# ensure that the config files we need actually exist in the spack prefix.
# assertions b/c users should not ever see these errors -- they're checked in CI.
assert (Path(spack.paths.prefix) / "pyproject.toml").is_file()
# validate spack root if the user provided one
args.root = Path(args.root).resolve() if args.root else Path(spack.paths.prefix)
spack_script = args.root / "bin" / "spack"
if not spack_script.exists():
tty.die("This does not look like a valid spack root.", "No such file: '%s'" % spack_script)
def prefix_relative(path: Union[Path, str]) -> Path:
return Path(os.path.relpath(os.path.abspath(os.path.realpath(path)), args.root))
file_list = [prefix_relative(file) for file in args.files]
# process --tool and --skip arguments
selected = set(tool_names)
if args.tool is not None:
selected = validate_toolset(args.tool)
if args.skip is not None:
selected -= validate_toolset(args.skip)
if not selected:
tty.msg("Nothing to run.")
return
tools_to_run = [t for t in tool_names if t in selected]
if missing_tools(tools_to_run):
_bootstrap_dev_dependencies()
return_code = 0
with working_dir(str(args.root)):
print_style_header(file_list, args, tools_to_run)
for tool_name in tools_to_run:
tool = tools[tool_name]
tty.msg(f"Running {tool.name} checks")
return_code |= tool.fun(file_list, args)
if return_code == 0:
tty.msg(color.colorize("@*{spack style checks were clean}"))
else:
tty.error(color.colorize("@*{spack style found errors}"))
return return_code