John Doty
56d24c5fb9
- Tables are split into `actions` and `goto` now to make formatting nicer - Token is renamed Terminal - Likes are now Florps - Lexer now loaded dynamically (badly)
380 lines
11 KiB
Python
380 lines
11 KiB
Python
import bisect
|
|
import importlib
|
|
import inspect
|
|
import enum
|
|
import os
|
|
import select
|
|
import sys
|
|
import termios
|
|
import time
|
|
import traceback
|
|
import tty
|
|
import types
|
|
import typing
|
|
from dataclasses import dataclass
|
|
|
|
import parser
|
|
|
|
# from parser import Token, Grammar, rule, seq
|
|
|
|
|
|
def trace_state(stack, input, input_index, action):
|
|
print(
|
|
"{stack: <20} {input: <50} {action: <5}".format(
|
|
stack=repr([s[0] for s in stack]),
|
|
input=repr(input[input_index : input_index + 4]),
|
|
action=repr(action),
|
|
)
|
|
)
|
|
|
|
|
|
@dataclass
|
|
class Tree:
|
|
name: str | None
|
|
children: typing.Tuple["Tree | str", ...]
|
|
|
|
|
|
def parse(table: parser.ParseTable, tokens, trace=None) -> typing.Tuple[Tree | None, list[str]]:
|
|
"""Parse the input with the generated parsing table and return the
|
|
concrete syntax tree.
|
|
|
|
The parsing table can be generated by GenerateLR0.gen_table() or by any
|
|
of the other generators below. The parsing mechanism never changes, only
|
|
the table generation mechanism.
|
|
|
|
input is a list of tokens. Don't stick an end-of-stream marker, I'll stick
|
|
one on for you.
|
|
|
|
This is not a *great* parser, it's really just a demo for what you can
|
|
do with the table.
|
|
"""
|
|
input_tokens = tokens.tokens()
|
|
input: list[str] = [t.value for (t, _, _) in input_tokens]
|
|
|
|
assert "$" not in input
|
|
input = input + ["$"]
|
|
input_index = 0
|
|
|
|
# Our stack is a stack of tuples, where the first entry is the state number
|
|
# and the second entry is the 'value' that was generated when the state was
|
|
# pushed.
|
|
stack: list[typing.Tuple[int, str | Tree | None]] = [(0, None)]
|
|
while True:
|
|
current_state = stack[-1][0]
|
|
current_token = input[input_index]
|
|
|
|
action = table.actions[current_state].get(current_token, parser.Error())
|
|
if trace:
|
|
trace(stack, input, input_index, action)
|
|
|
|
match action:
|
|
case parser.Accept():
|
|
result = stack[-1][1]
|
|
assert isinstance(result, Tree)
|
|
return (result, [])
|
|
|
|
case parser.Reduce(name=name, count=size, transparent=transparent):
|
|
children: list[str | Tree] = []
|
|
for _, c in stack[-size:]:
|
|
if c is None:
|
|
continue
|
|
elif isinstance(c, Tree) and c.name is None:
|
|
children.extend(c.children)
|
|
else:
|
|
children.append(c)
|
|
|
|
value = Tree(name=name if not transparent else None, children=tuple(children))
|
|
stack = stack[:-size]
|
|
|
|
goto = table.gotos[stack[-1][0]].get(name)
|
|
assert goto is not None
|
|
stack.append((goto, value))
|
|
|
|
case parser.Shift(state):
|
|
stack.append((state, current_token))
|
|
input_index += 1
|
|
|
|
case parser.Error():
|
|
if input_index >= len(input_tokens):
|
|
message = "Unexpected end of file"
|
|
start = input_tokens[-1][1]
|
|
else:
|
|
message = f"Syntax error: unexpected symbol {current_token}"
|
|
(_, start, _) = input_tokens[input_index]
|
|
|
|
line_index = bisect.bisect_left(tokens.lines, start)
|
|
if line_index == 0:
|
|
col_start = 0
|
|
else:
|
|
col_start = tokens.lines[line_index - 1] + 1
|
|
column_index = start - col_start
|
|
line_index += 1
|
|
|
|
error = f"{line_index}:{column_index}: {message}"
|
|
return (None, [error])
|
|
|
|
case _:
|
|
raise ValueError(f"Unknown action type: {action}")
|
|
|
|
|
|
# https://en.wikipedia.org/wiki/ANSI_escape_code
|
|
# https://gist.github.com/fnky/458719343aabd01cfb17a3a4f7296797
|
|
|
|
|
|
class CharColor(enum.IntEnum):
|
|
CHAR_COLOR_DEFAULT = 0
|
|
CHAR_COLOR_BLACK = 30
|
|
CHAR_COLOR_RED = enum.auto()
|
|
CHAR_COLOR_GREEN = enum.auto()
|
|
CHAR_COLOR_YELLOW = enum.auto()
|
|
CHAR_COLOR_BLUE = enum.auto()
|
|
CHAR_COLOR_MAGENTA = enum.auto()
|
|
CHAR_COLOR_CYAN = enum.auto()
|
|
CHAR_COLOR_WHITE = enum.auto() # Really light gray
|
|
CHAR_COLOR_BRIGHT_BLACK = 90 # Really dark gray
|
|
CHAR_COLOR_BRIGHT_RED = enum.auto()
|
|
CHAR_COLOR_BRIGHT_GREEN = enum.auto()
|
|
CHAR_COLOR_BRIGHT_YELLOW = enum.auto()
|
|
CHAR_COLOR_BRIGHT_BLUE = enum.auto()
|
|
CHAR_COLOR_BRIGHT_MAGENTA = enum.auto()
|
|
CHAR_COLOR_BRIGHT_CYAN = enum.auto()
|
|
CHAR_COLOR_BRIGHT_WHITE = enum.auto()
|
|
|
|
|
|
def ESC(x: bytes) -> bytes:
|
|
return b"\033" + x
|
|
|
|
|
|
def CSI(x: bytes) -> bytes:
|
|
return ESC(b"[" + x)
|
|
|
|
|
|
CLEAR = CSI(b"H") + CSI(b"J")
|
|
|
|
|
|
def enter_alt_screen():
|
|
sys.stdout.buffer.write(CSI(b"?1049h"))
|
|
|
|
|
|
def leave_alt_screen():
|
|
sys.stdout.buffer.write(CSI(b"?1049l"))
|
|
|
|
|
|
class DynamicModule:
|
|
file_name: str
|
|
member_name: str | None
|
|
|
|
last_time: float | None
|
|
module: types.ModuleType | None
|
|
|
|
def __init__(self, file_name, member_name):
|
|
self.file_name = file_name
|
|
self.member_name = member_name
|
|
|
|
self.last_time = None
|
|
self.module = None
|
|
self.value = None
|
|
|
|
def _predicate(self, member) -> bool:
|
|
if not inspect.isclass(member):
|
|
return False
|
|
|
|
assert self.module is not None
|
|
if member.__module__ != self.module.__name__:
|
|
return False
|
|
|
|
return True
|
|
|
|
def _transform(self, value):
|
|
return value
|
|
|
|
def get(self):
|
|
st = os.stat(self.file_name)
|
|
if self.last_time == st.st_mtime:
|
|
assert self.value is not None
|
|
return self.value
|
|
|
|
self.value = None
|
|
|
|
if self.module is None:
|
|
mod_name = inspect.getmodulename(self.file_name)
|
|
if mod_name is None:
|
|
raise Exception(f"{self.file_name} does not seem to be a module")
|
|
self.module = importlib.import_module(mod_name)
|
|
else:
|
|
importlib.reload(self.module)
|
|
|
|
if self.member_name is None:
|
|
classes = inspect.getmembers(self.module, self._predicate)
|
|
if len(classes) == 0:
|
|
raise Exception(f"No grammars found in {self.file_name}")
|
|
if len(classes) > 1:
|
|
raise Exception(
|
|
f"{len(classes)} grammars found in {self.file_name}: {', '.join(c[0] for c in classes)}"
|
|
)
|
|
cls = classes[0][1]
|
|
else:
|
|
cls = getattr(self.module, self.member_name)
|
|
if cls is None:
|
|
raise Exception(f"Cannot find {self.member_name} in {self.file_name}")
|
|
if not self._predicate(cls):
|
|
raise Exception(f"{self.member_name} in {self.file_name} is not suitable")
|
|
|
|
self.value = self._transform(cls)
|
|
self.last_time = st.st_mtime
|
|
return self.value
|
|
|
|
|
|
class DynamicGrammarModule(DynamicModule):
|
|
def __init__(self, file_name, member_name, start_rule, generator):
|
|
super().__init__(file_name, member_name)
|
|
|
|
self.start_rule = start_rule
|
|
self.generator = generator
|
|
|
|
def _predicate(self, member) -> bool:
|
|
if not super()._predicate(member):
|
|
return False
|
|
|
|
if getattr(member, "build_table", None):
|
|
return True
|
|
|
|
return False
|
|
|
|
def _transform(self, value):
|
|
return value().build_table(start=self.start_rule, generator=self.generator)
|
|
|
|
|
|
class DynamicLexerModule(DynamicModule):
|
|
def _predicate(self, member) -> bool:
|
|
if not super()._predicate(member):
|
|
return False
|
|
|
|
if getattr(member, "tokens", None):
|
|
return True
|
|
|
|
return False
|
|
|
|
|
|
class Harness:
|
|
source: str | None
|
|
table: parser.ParseTable | None
|
|
tree: Tree | None
|
|
|
|
def __init__(self, start_rule, source_path):
|
|
self.start_rule = start_rule
|
|
self.source_path = source_path
|
|
|
|
self.source = None
|
|
self.table = None
|
|
self.tokens = None
|
|
self.tree = None
|
|
self.errors = None
|
|
|
|
self.grammar_module = DynamicGrammarModule(
|
|
"./grammar.py", None, self.start_rule, generator=parser.GenerateLALR
|
|
)
|
|
|
|
self.lexer_module = DynamicLexerModule("./grammar.py", None)
|
|
|
|
def run(self):
|
|
while True:
|
|
i, _, _ = select.select([sys.stdin], [], [], 1)
|
|
if i:
|
|
k = sys.stdin.read(1)
|
|
print(f"Key {k}\r")
|
|
return
|
|
|
|
self.update()
|
|
|
|
def load_grammar(self) -> parser.ParseTable:
|
|
return self.grammar_module.get()
|
|
|
|
def update(self):
|
|
start_time = time.time()
|
|
try:
|
|
table = self.load_grammar()
|
|
lexer_func = self.lexer_module.get()
|
|
|
|
with open(self.source_path, "r", encoding="utf-8") as f:
|
|
self.source = f.read()
|
|
|
|
self.tokens = lexer_func(self.source)
|
|
lex_time = time.time()
|
|
|
|
# print(f"{tokens.lines}")
|
|
# tokens.dump(end=5)
|
|
(tree, errors) = parse(table, self.tokens, trace=None)
|
|
parse_time = time.time()
|
|
self.tree = tree
|
|
self.errors = errors
|
|
parse_elapsed = parse_time - lex_time
|
|
|
|
except Exception as e:
|
|
self.tree = None
|
|
self.errors = ["Error loading grammar:"] + [
|
|
" " + l.rstrip() for fl in traceback.format_exception(e) for l in fl.splitlines()
|
|
]
|
|
parse_elapsed = time.time() - start_time
|
|
table = None
|
|
|
|
sys.stdout.buffer.write(CLEAR)
|
|
rows, cols = termios.tcgetwinsize(sys.stdout.fileno())
|
|
|
|
if table is not None:
|
|
states = table.actions
|
|
average_entries = sum(len(row) for row in states) / len(states)
|
|
max_entries = max(len(row) for row in states)
|
|
print(
|
|
f"{len(states)} states - {average_entries:.3} average, {max_entries} max - {parse_elapsed:.3}s \r"
|
|
)
|
|
else:
|
|
print("No table\r\n")
|
|
|
|
if self.tree is not None:
|
|
lines = []
|
|
self.format_node(lines, self.tree)
|
|
for line in lines[: rows - 2]:
|
|
print(line[:cols] + "\r")
|
|
else:
|
|
for error in self.errors[: rows - 2]:
|
|
print(error[:cols] + "\r")
|
|
|
|
sys.stdout.flush()
|
|
sys.stdout.buffer.flush()
|
|
|
|
def format_node(self, lines, node: Tree | str, indent=0):
|
|
"""Print out an indented concrete syntax tree, from parse()."""
|
|
match node:
|
|
case Tree(name, children):
|
|
lines.append((" " * indent) + (name or "???"))
|
|
for child in children:
|
|
self.format_node(lines, child, indent + 2)
|
|
case _:
|
|
lines.append((" " * indent) + str(node))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
source_path = None
|
|
if len(sys.argv) == 2:
|
|
source_path = sys.argv[1]
|
|
|
|
fd = sys.stdin.fileno()
|
|
old_settings = termios.tcgetattr(fd)
|
|
try:
|
|
tty.setraw(fd)
|
|
enter_alt_screen()
|
|
|
|
h = Harness(
|
|
start_rule="file",
|
|
source_path=source_path,
|
|
)
|
|
h.run()
|
|
|
|
finally:
|
|
leave_alt_screen()
|
|
termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)
|
|
|
|
# print(parser_faster.format_table(gen, table))
|
|
# print()
|
|
# tree = parse(table, ["id", "+", "(", "id", "[", "id", "]", ")"])
|