DeCarabas/lrparsers
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Prettier harness, start logging properly

Browse source
This commit is contained in:
John Doty 2024-06-08 18:12:16 -07:00
parent a439e1456a
commit 1dcc86e9fe
1 changed files with 81 additions and 54 deletions
Download patch file Download diff file Expand all files Collapse all files

135
harness.py
View file

@ -1,9 +1,10 @@
import argparse import argparse
import bisect import bisect
import enum import enum
import enum
import importlib import importlib
import inspect import inspect
import enum import logging
import os import os
import select import select
import sys import sys
@ -26,17 +27,6 @@ import parser
############################################################################### ###############################################################################
def trace_state(id, stack, token, action):
print(
"{id: <04}: {stack: <20} {input: <50} {action: <5}".format(
id=id,
stack=repr([s[0] for s in stack]),
input=token.kind,
action=repr(action),
)
)
@dataclass @dataclass
class TokenValue: class TokenValue:
kind: str kind: str
@ -62,17 +52,7 @@ class ParseError:
ParseStack = list[typing.Tuple[int, TokenValue | Tree | None]] ParseStack = list[typing.Tuple[int, TokenValue | Tree | None]]
RECOVER_TRACE: list[str] = [] recover_log = logging.getLogger("parser.recovery")
def clear_recover_trace():
RECOVER_TRACE.clear()
def recover_trace(s: str):
# RECOVER_TRACE.append(s)
del s
pass
class RepairAction(enum.Enum): class RepairAction(enum.Enum):
@ -121,6 +101,8 @@ class RepairStack(typing.NamedTuple):
def handle_token( def handle_token(
self, table: parser.ParseTable, token: str self, table: parser.ParseTable, token: str
) -> typing.Tuple["RepairStack | None", bool]: ) -> typing.Tuple["RepairStack | None", bool]:
rl = recover_log
stack = self stack = self
while True: while True:
action = table.actions[stack.state].get(token) action = table.actions[stack.state].get(token)
@ -129,19 +111,19 @@ class RepairStack(typing.NamedTuple):
match action: match action:
case parser.Shift(): case parser.Shift():
recover_trace(f" {stack.state}: SHIFT -> {action.state}") rl.info(f"{stack.state}: SHIFT -> {action.state}")
return stack.push(action.state), False return stack.push(action.state), False
case parser.Accept(): case parser.Accept():
recover_trace(f" {stack.state}: ACCEPT") rl.info(f"{stack.state}: ACCEPT")
return stack, True # ? return stack, True # ?
case parser.Reduce(): case parser.Reduce():
recover_trace(f" {stack.state}: REDUCE {action.name} {action.count} ") rl.info(f"{stack.state}: REDUCE {action.name} {action.count} ")
new_stack = stack.pop(action.count) new_stack = stack.pop(action.count)
recover_trace(f" -> {new_stack.state}") rl.info(f" -> {new_stack.state}")
new_state = table.gotos[new_stack.state][action.name] new_state = table.gotos[new_stack.state][action.name]
recover_trace(f" goto {new_state}") rl.info(f" goto {new_state}")
stack = new_stack.push(new_state) stack = new_stack.push(new_state)
case parser.Error(): case parser.Error():
@ -182,13 +164,16 @@ class Repair:
input: list[TokenValue], input: list[TokenValue],
start: int, start: int,
): ):
rl = recover_log
input_index = start + self.advance input_index = start + self.advance
if input_index >= len(input): if input_index >= len(input):
return return
valstr = f"({self.value})" if self.value is not None else "" if rl.isEnabledFor(logging.INFO):
recover_trace(f"{self.repair.value}{valstr} @ {self.cost} input:{input_index}") valstr = f"({self.value})" if self.value is not None else ""
recover_trace(f" {','.join(str(s) for s in self.stack.flatten())}") rl.info(f"{self.repair.value}{valstr} @ {self.cost} input:{input_index}")
rl.info(f" {','.join(str(s) for s in self.stack.flatten())}")
state = self.stack.state state = self.stack.state
@ -202,7 +187,7 @@ class Repair:
# necessary, producing a new version of the stack. Count up the # necessary, producing a new version of the stack. Count up the
# number of successful shifts. # number of successful shifts.
for token in table.actions[state].keys(): for token in table.actions[state].keys():
recover_trace(f" token: {token}") rl.info(f" token: {token}")
new_stack, success = self.stack.handle_token(table, token) new_stack, success = self.stack.handle_token(table, token)
if new_stack is None: if new_stack is None:
# Not clear why this is necessary, but I think state merging # Not clear why this is necessary, but I think state merging
@ -211,7 +196,7 @@ class Repair:
continue continue
if token == input[input_index].kind: if token == input[input_index].kind:
recover_trace(f" generate shift {token}") rl.info(f" generate shift {token}")
yield Repair( yield Repair(
repair=RepairAction.Shift, repair=RepairAction.Shift,
parent=self, parent=self,
@ -220,7 +205,7 @@ class Repair:
advance=1, # Move forward by one. advance=1, # Move forward by one.
) )
recover_trace(f" generate insert {token}") rl.info(f" generate insert {token}")
yield Repair( yield Repair(
repair=RepairAction.Insert, repair=RepairAction.Insert,
value=token, value=token,
@ -237,7 +222,7 @@ class Repair:
# delete-insert pairs, not insert-delete, because they are # delete-insert pairs, not insert-delete, because they are
# symmetrical and therefore a waste of time and memory.) # symmetrical and therefore a waste of time and memory.)
if self.repair != RepairAction.Insert: if self.repair != RepairAction.Insert:
recover_trace(f" generate delete") rl.info(f" generate delete")
yield Repair( yield Repair(
repair=RepairAction.Delete, repair=RepairAction.Delete,
parent=self, parent=self,
@ -284,6 +269,9 @@ def recover(table: parser.ParseTable, input: list[TokenValue], start: int, stack
level += 1 level += 1
action_log = logging.getLogger("parser.action")
class Parser: class Parser:
# Our stack is a stack of tuples, where the first entry is the state # Our stack is a stack of tuples, where the first entry is the state
# number and the second entry is the 'value' that was generated when the # number and the second entry is the 'value' that was generated when the
@ -295,8 +283,6 @@ class Parser:
self.table = table self.table = table
def parse(self, tokens) -> typing.Tuple[Tree | None, list[str]]: def parse(self, tokens) -> typing.Tuple[Tree | None, list[str]]:
clear_recover_trace()
input_tokens = tokens.tokens() input_tokens = tokens.tokens()
input: list[TokenValue] = [ input: list[TokenValue] = [
TokenValue(kind=kind.value, start=start, end=start + length) TokenValue(kind=kind.value, start=start, end=start + length)
@ -311,13 +297,20 @@ class Parser:
result: Tree | None = None result: Tree | None = None
errors: list[ParseError] = [] errors: list[ParseError] = []
al = action_log
while True: while True:
current_token = input[input_index] current_token = input[input_index]
current_state = stack[-1][0] current_state = stack[-1][0]
action = self.table.actions[current_state].get(current_token.kind, parser.Error()) action = self.table.actions[current_state].get(current_token.kind, parser.Error())
if self.trace: if al.isEnabledFor(logging.INFO):
self.trace(stack, current_token, action) al.info(
"{stack: <20} {input: <50} {action: <5}".format(
stack=repr([s[0] for s in stack[-5:]]),
input=current_token.kind,
action=repr(action),
)
)
match action: match action:
case parser.Accept(): case parser.Accept():
@ -327,8 +320,6 @@ class Parser:
break break
case parser.Reduce(name=name, count=size, transparent=transparent): case parser.Reduce(name=name, count=size, transparent=transparent):
recover_trace(f" {current_token.kind}")
recover_trace(f" {current_state}: REDUCE {name} {size}")
children: list[TokenValue | Tree] = [] children: list[TokenValue | Tree] = []
for _, c in stack[-size:]: for _, c in stack[-size:]:
if c is None: if c is None:
@ -345,10 +336,8 @@ class Parser:
children=tuple(children), children=tuple(children),
) )
del stack[-size:] del stack[-size:]
recover_trace(f" -> {stack[-1][0]}")
goto = self.table.gotos[stack[-1][0]].get(name) goto = self.table.gotos[stack[-1][0]].get(name)
assert goto is not None assert goto is not None
recover_trace(f" -> {goto}")
stack.append((goto, value)) stack.append((goto, value))
case parser.Shift(): case parser.Shift():
@ -474,6 +463,12 @@ def leave_alt_screen():
sys.stdout.buffer.write(CSI(b"?1049l")) sys.stdout.buffer.write(CSI(b"?1049l"))
def goto_cursor(x: int, y: int):
sx = str(x).encode("utf-8")
sy = str(y).encode("utf-8")
sys.stdout.buffer.write(CSI(sy + b";" + sx + b"H"))
############################################################################### ###############################################################################
# Dynamic Modules: Detect and Reload Modules when they Change # Dynamic Modules: Detect and Reload Modules when they Change
############################################################################### ###############################################################################
@ -574,6 +569,12 @@ class DynamicLexerModule(DynamicModule):
return False return False
class DisplayMode(enum.Enum):
TREE = 0
ERRORS = 1
LOG = 2
class Harness: class Harness:
grammar_file: str grammar_file: str
grammar_member: str | None grammar_member: str | None
@ -583,6 +584,7 @@ class Harness:
source: str | None source: str | None
table: parser.ParseTable | None table: parser.ParseTable | None
tree: Tree | None tree: Tree | None
mode: DisplayMode
def __init__( def __init__(
self, grammar_file, grammar_member, lexer_file, lexer_member, start_rule, source_path self, grammar_file, grammar_member, lexer_file, lexer_member, start_rule, source_path
@ -594,6 +596,8 @@ class Harness:
self.start_rule = start_rule self.start_rule = start_rule
self.source_path = source_path self.source_path = source_path
self.mode = DisplayMode.TREE
self.source = None self.source = None
self.table = None self.table = None
self.tokens = None self.tokens = None
@ -614,9 +618,15 @@ class Harness:
while True: while True:
i, _, _ = select.select([sys.stdin], [], [], 1) i, _, _ = select.select([sys.stdin], [], [], 1)
if i: if i:
k = sys.stdin.read(1) k = sys.stdin.read(1).lower()
print(f"Key {k}\r") if k == "q":
return return
elif k == "t":
self.mode = DisplayMode.TREE
elif k == "e":
self.mode = DisplayMode.ERRORS
elif k == "l":
self.mode = DisplayMode.LOG
self.update() self.update()
self.render() self.render()
@ -671,19 +681,34 @@ class Harness:
print(f"No table\r") print(f"No table\r")
print(("\u2500" * cols) + "\r") print(("\u2500" * cols) + "\r")
lines = list(RECOVER_TRACE) lines = []
if self.errors is not None: match self.mode:
wrapper = textwrap.TextWrapper(width=cols, drop_whitespace=False) case DisplayMode.ERRORS:
lines.extend(line for error in self.errors for line in wrapper.wrap(error)) if self.errors is not None:
lines.append("") wrapper = textwrap.TextWrapper(width=cols, drop_whitespace=False)
lines.extend(line for error in self.errors for line in wrapper.wrap(error))
if self.tree is not None: case DisplayMode.TREE:
self.format_node(lines, self.tree) if self.tree is not None:
self.format_node(lines, self.tree)
for line in lines[: rows - 3]: case DisplayMode.LOG:
pass
case _:
typing.assert_never(self.mode)
for line in lines[: rows - 4]:
print(line[:cols] + "\r") print(line[:cols] + "\r")
has_errors = "*" if self.errors else " "
has_tree = "*" if self.tree else " "
has_log = " "
goto_cursor(0, rows - 1)
print(("\u2500" * cols) + "\r")
print(f"(e)rrors{has_errors} | (t)ree{has_tree} | (l)og{has_log} | (q)uit\r", end="")
sys.stdout.flush() sys.stdout.flush()
sys.stdout.buffer.flush() sys.stdout.buffer.flush()
@ -742,6 +767,7 @@ def main(args: list[str]):
try: try:
tty.setraw(fd) tty.setraw(fd)
enter_alt_screen() enter_alt_screen()
sys.stdout.buffer.write(CSI(b"?25l"))
h = Harness( h = Harness(
grammar_file=parsed.grammar, grammar_file=parsed.grammar,
@ -754,6 +780,7 @@ def main(args: list[str]):
h.run() h.run()
finally: finally:
sys.stdout.buffer.write(CSI(b"?25h"))
leave_alt_screen() leave_alt_screen()
termios.tcsetattr(fd, termios.TCSADRAIN, old_settings) termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)