Source code for spack.hooks.sbang

# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

import filecmp
import os
import re
import shutil
import stat
import sys
import tempfile

import llnl.util.filesystem as fs
import llnl.util.tty as tty

import spack.error
import spack.package_prefs
import spack.paths
import spack.spec

#: OS-imposed character limit for shebang line: 127 for Linux; 511 for Mac.
#: Different Linux distributions have different limits, but 127 is the
#: smallest among all modern versions.
if sys.platform == 'darwin':
    system_shebang_limit = 511
    system_shebang_limit = 127

#: Groupdb does not exist on Windows, prevent imports
#: on supported systems
is_windows = sys.platform == 'win32'
if not is_windows:
    import grp

#: Spack itself also limits the shebang line to at most 4KB, which should be plenty.
spack_shebang_limit = 4096

interpreter_regex = re.compile(b'#![ \t]*?([^ \t\0\n]+)')

[docs]def sbang_install_path(): """Location sbang should be installed within Spack's ``install_tree``.""" sbang_root = str( install_path = os.path.join(sbang_root, "bin", "sbang") path_length = len(install_path) if path_length > system_shebang_limit: msg = ('Install tree root is too long. Spack cannot patch shebang lines' ' when script path length ({0}) exceeds limit ({1}).\n {2}') msg = msg.format(path_length, system_shebang_limit, install_path) raise SbangPathError(msg) return install_path
[docs]def sbang_shebang_line(): """Full shebang line that should be prepended to files to use sbang. The line returned does not have a final newline (caller should add it if needed). This should be the only place in Spack that knows about what interpreter we use for ``sbang``. """ return '#!/bin/sh %s' % sbang_install_path()
[docs]def get_interpreter(binary_string): # The interpreter may be preceded with ' ' and \t, is itself any byte that # follows until the first occurrence of ' ', \t, \0, \n or end of file. match = interpreter_regex.match(binary_string) return None if match is None else
[docs]def filter_shebang(path): """ Adds a second shebang line, using sbang, at the beginning of a file, if necessary. Note: Spack imposes a relaxed shebang line limit, meaning that a newline or end of file must occur before ``spack_shebang_limit`` bytes. If not, the file is not patched. """ with open(path, 'rb') as original: # If there is no shebang, we shouldn't replace anything. old_shebang_line = if old_shebang_line != b'#!': return False # Stop reading after b'\n'. Note that old_shebang_line includes the first b'\n'. old_shebang_line += original.readline(spack_shebang_limit - 2) # If the shebang line is short, we don't have to do anything. if len(old_shebang_line) <= system_shebang_limit: return False # Whenever we can't find a newline within the maximum number of bytes, we will # not attempt to rewrite it. In principle we could still get the interpreter if # only the arguments are truncated, but note that for PHP we need the full line # since we have to append `?>` to it. Since our shebang limit is already very # generous, it's unlikely to happen, and it should be fine to ignore. if ( len(old_shebang_line) == spack_shebang_limit and old_shebang_line[-1] != b'\n' ): return False # This line will be prepended to file new_sbang_line = (sbang_shebang_line() + '\n').encode('utf-8') # Skip files that are already using sbang. if old_shebang_line == new_sbang_line: return interpreter = get_interpreter(old_shebang_line) # If there was only whitespace we don't have to do anything. if not interpreter: return False # Store the file permissions, the patched version needs the same. saved_mode = os.stat(path).st_mode # Change non-writable files to be writable if needed. if not os.access(path, os.W_OK): os.chmod(path, saved_mode | stat.S_IWUSR) # No need to delete since we'll move it and overwrite the original. patched = tempfile.NamedTemporaryFile('wb', delete=False) patched.write(new_sbang_line) # Note that in Python this does not go out of bounds even if interpreter is a # short byte array. # Note: if the interpreter string was encoded with UTF-16, there would have # been a \0 byte between all characters of lua, node, php; meaning that it would # lead to truncation of the interpreter. So we don't have to worry about weird # encodings here, and just looking at bytes is justified. if interpreter[-4:] == b'/lua' or interpreter[-7:] == b'/luajit': # Use --! instead of #! on second line for lua. patched.write(b'--!' + old_shebang_line[2:]) elif interpreter[-5:] == b'/node': # Use //! instead of #! on second line for node.js. patched.write(b'//!' + old_shebang_line[2:]) elif interpreter[-4:] == b'/php': # Use <?php #!... ?> instead of #!... on second line for php. patched.write(b'<?php ' + old_shebang_line + b' ?>') else: patched.write(old_shebang_line) # After copying the remainder of the file, we can close the original shutil.copyfileobj(original, patched) # And close the temporary file so we can move it. patched.close() # Overwrite original file with patched file, and keep the original mode shutil.move(, path) os.chmod(path, saved_mode) return True
[docs]def filter_shebangs_in_directory(directory, filenames=None): if filenames is None: filenames = os.listdir(directory) is_exe = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH for file in filenames: path = os.path.join(directory, file) # Only look at executable, non-symlink files. try: st = os.lstat(path) except (IOError, OSError): continue if (stat.S_ISLNK(st.st_mode) or stat.S_ISDIR(st.st_mode) or not st.st_mode & is_exe): continue # test the file for a long shebang, and filter if filter_shebang(path): tty.debug("Patched overlong shebang in %s" % path)
[docs]def install_sbang(): """Ensure that ``sbang`` is installed in the root of Spack's install_tree. This is the shortest known publicly accessible path, and installing ``sbang`` here ensures that users can access the script and that ``sbang`` itself is in a short path. """ # copy in a new version of sbang if it differs from what's in spack sbang_path = sbang_install_path() if os.path.exists(sbang_path) and filecmp.cmp( spack.paths.sbang_script, sbang_path): return # make $install_tree/bin sbang_bin_dir = os.path.dirname(sbang_path) fs.mkdirp(sbang_bin_dir) # get permissions for bin dir from configuration files group_name = spack.package_prefs.get_package_group(spack.spec.Spec("all")) config_mode = spack.package_prefs.get_package_dir_permissions( spack.spec.Spec("all") ) if group_name: os.chmod(sbang_bin_dir, config_mode) # Use package directory permissions else: fs.set_install_permissions(sbang_bin_dir) # set group on sbang_bin_dir if not already set (only if set in configuration) if group_name and grp.getgrgid(os.stat(sbang_bin_dir).st_gid).gr_name != group_name: os.chown( sbang_bin_dir, os.stat(sbang_bin_dir).st_uid, grp.getgrnam(group_name).gr_gid ) # copy over the fresh copy of `sbang` sbang_tmp_path = os.path.join( os.path.dirname(sbang_path), ".%s.tmp" % os.path.basename(sbang_path), ) shutil.copy(spack.paths.sbang_script, sbang_tmp_path) # set permissions on `sbang` (including group if set in configuration) os.chmod(sbang_tmp_path, config_mode) if group_name: os.chown( sbang_tmp_path, os.stat(sbang_tmp_path).st_uid, grp.getgrnam(group_name).gr_gid ) # Finally, move the new `sbang` into place atomically os.rename(sbang_tmp_path, sbang_path)
[docs]def post_install(spec): """This hook edits scripts so that they call /bin/bash $spack_prefix/bin/sbang instead of something longer than the shebang limit. """ if spec.external: tty.debug('SKIP: shebang filtering [external package]') return install_sbang() for directory, _, filenames in os.walk(spec.prefix): filter_shebangs_in_directory(directory, filenames)
[docs]class SbangPathError(spack.error.SpackError): """Raised when the install tree root is too long for sbang to work."""