DeCarabas/lrparsers
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Better tree output

Browse source
This commit is contained in:
John Doty 2024-05-30 08:24:36 -07:00
parent 56d24c5fb9
commit dfef449c33
2 changed files with 20 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

5
grammar.py
View file

@ -553,6 +553,5 @@ class FineTokens:
else: else:
col_start = self.lines[line_index - 1] + 1 col_start = self.lines[line_index - 1] + 1
column_index = start - col_start column_index = start - col_start
print( value = self.src[start : start + length]
f"{start:04} {kind.value:12} {self.src[start:start+length]} ({line_index}, {column_index})" print(f"{start:04} {kind.value:12} {value} ({line_index}, {column_index})")
)

25
harness.py
View file

@ -28,10 +28,17 @@ def trace_state(stack, input, input_index, action):
) )
@dataclass
class TokenValue:
kind: str
start: int
end: int
@dataclass @dataclass
class Tree: class Tree:
name: str | None name: str | None
children: typing.Tuple["Tree | str", ...] children: typing.Tuple["Tree | TokenValue", ...]
def parse(table: parser.ParseTable, tokens, trace=None) -> typing.Tuple[Tree | None, list[str]]: def parse(table: parser.ParseTable, tokens, trace=None) -> typing.Tuple[Tree | None, list[str]]:
@ -58,7 +65,7 @@ def parse(table: parser.ParseTable, tokens, trace=None) -> typing.Tuple[Tree | N
# Our stack is a stack of tuples, where the first entry is the state number # Our stack is a stack of tuples, where the first entry is the state number
# and the second entry is the 'value' that was generated when the state was # and the second entry is the 'value' that was generated when the state was
# pushed. # pushed.
stack: list[typing.Tuple[int, str | Tree | None]] = [(0, None)] stack: list[typing.Tuple[int, TokenValue | Tree | None]] = [(0, None)]
while True: while True:
current_state = stack[-1][0] current_state = stack[-1][0]
current_token = input[input_index] current_token = input[input_index]
@ -74,7 +81,7 @@ def parse(table: parser.ParseTable, tokens, trace=None) -> typing.Tuple[Tree | N
return (result, []) return (result, [])
case parser.Reduce(name=name, count=size, transparent=transparent): case parser.Reduce(name=name, count=size, transparent=transparent):
children: list[str | Tree] = [] children: list[TokenValue | Tree] = []
for _, c in stack[-size:]: for _, c in stack[-size:]:
if c is None: if c is None:
continue continue
@ -91,7 +98,9 @@ def parse(table: parser.ParseTable, tokens, trace=None) -> typing.Tuple[Tree | N
stack.append((goto, value)) stack.append((goto, value))
case parser.Shift(state): case parser.Shift(state):
stack.append((state, current_token)) (kind, start, length) = input_tokens[input_index]
tval = TokenValue(kind=kind.value, start=start, end=start + length)
stack.append((state, tval))
input_index += 1 input_index += 1
case parser.Error(): case parser.Error():
@ -343,15 +352,17 @@ class Harness:
sys.stdout.flush() sys.stdout.flush()
sys.stdout.buffer.flush() sys.stdout.buffer.flush()
def format_node(self, lines, node: Tree | str, indent=0): def format_node(self, lines, node: Tree | TokenValue, indent=0):
"""Print out an indented concrete syntax tree, from parse().""" """Print out an indented concrete syntax tree, from parse()."""
match node: match node:
case Tree(name, children): case Tree(name, children):
lines.append((" " * indent) + (name or "???")) lines.append((" " * indent) + (name or "???"))
for child in children: for child in children:
self.format_node(lines, child, indent + 2) self.format_node(lines, child, indent + 2)
case _: case TokenValue(kind=kind, start=start, end=end):
lines.append((" " * indent) + str(node)) assert self.source is not None
value = self.source[start:end]
lines.append((" " * indent) + f"{kind}:'{value}'")
if __name__ == "__main__": if __name__ == "__main__":