# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import itertools
import re
import shlex
import sys
from six import string_types
import spack.error
import spack.util.path as sp
[docs]class Token(object):
"""Represents tokens; generated from input by lexer and fed to parse()."""
def __init__(self, type, value='', start=0, end=0):
self.type = type
self.value = value
self.start = start
self.end = end
def __repr__(self):
return str(self)
def __str__(self):
return "<%d: '%s'>" % (self.type, self.value)
[docs] def is_a(self, type):
return self.type == type
def __eq__(self, other):
return (self.type == other.type) and (self.value == other.value)
[docs]class Lexer(object):
"""Base class for Lexers that keep track of line numbers."""
def __init__(self, lexicon0, mode_switches_01=[],
lexicon1=[], mode_switches_10=[]):
self.scanner0 = re.Scanner(lexicon0)
self.mode_switches_01 = mode_switches_01
self.scanner1 = re.Scanner(lexicon1)
self.mode_switches_10 = mode_switches_10
self.mode = 0
[docs] def token(self, type, value=''):
if self.mode == 0:
return Token(type, value,
self.scanner0.match.start(0),
self.scanner0.match.end(0))
else:
return Token(type, value,
self.scanner1.match.start(0),
self.scanner1.match.end(0))
[docs] def lex_word(self, word):
scanner = self.scanner0
mode_switches = self.mode_switches_01
if self.mode == 1:
scanner = self.scanner1
mode_switches = self.mode_switches_10
tokens, remainder = scanner.scan(word)
remainder_used = 0
for i, t in enumerate(tokens):
if t.type in mode_switches:
# Combine post-switch tokens with remainder and
# scan in other mode
self.mode = 1 - self.mode # swap 0/1
remainder_used = 1
tokens = tokens[:i + 1] + self.lex_word(
word[word.index(t.value) + len(t.value):])
break
if remainder and not remainder_used:
raise LexError("Invalid character", word, word.index(remainder))
return tokens
[docs] def lex(self, text):
lexed = []
for word in text:
tokens = self.lex_word(word)
lexed.extend(tokens)
return lexed
[docs]class Parser(object):
"""Base class for simple recursive descent parsers."""
def __init__(self, lexer):
self.tokens = iter([]) # iterators over tokens, handled in order.
self.token = Token(None) # last accepted token
self.next = None # next token
self.lexer = lexer
self.text = None
[docs] def gettok(self):
"""Puts the next token in the input stream into self.next."""
try:
self.next = next(self.tokens)
except StopIteration:
self.next = None
[docs] def push_tokens(self, iterable):
"""Adds all tokens in some iterable to the token stream."""
self.tokens = itertools.chain(
iter(iterable), iter([self.next]), self.tokens)
self.gettok()
[docs] def accept(self, id):
"""Put the next symbol in self.token if accepted, then call gettok()"""
if self.next and self.next.is_a(id):
self.token = self.next
self.gettok()
return True
return False
[docs] def next_token_error(self, message):
"""Raise an error about the next token in the stream."""
raise ParseError(message, self.text[0], self.token.end)
[docs] def last_token_error(self, message):
"""Raise an error about the previous token in the stream."""
raise ParseError(message, self.text[0], self.token.start)
[docs] def unexpected_token(self):
self.next_token_error("Unexpected token: '%s'" % self.next.value)
[docs] def expect(self, id):
"""Like accept(), but fails if we don't like the next token."""
if self.accept(id):
return True
else:
if self.next:
self.unexpected_token()
else:
self.next_token_error("Unexpected end of input")
sys.exit(1)
[docs] def setup(self, text):
if isinstance(text, string_types):
# shlex does not handle Windows path
# separators, so we must normalize to posix
text = sp.convert_to_posix_path(text)
text = shlex.split(str(text))
self.text = text
self.push_tokens(self.lexer.lex(text))
[docs] def parse(self, text):
self.setup(text)
return self.do_parse()
[docs]class ParseError(spack.error.SpackError):
"""Raised when we don't hit an error while parsing."""
def __init__(self, message, string, pos):
super(ParseError, self).__init__(message)
self.string = string
self.pos = pos
[docs]class LexError(ParseError):
"""Raised when we don't know how to lex something."""
def __init__(self, message, string, pos):
super(LexError, self).__init__(message, string, pos)