Specify and honor trivia tokens
e.g. "this is how machine-generated parsers know to skip blanks and comments" The run time implementation could be better; we don't really want to just discard trivia because it's useful for e.g. doc comments and the like. BUT for now this is fine.
This commit is contained in:
parent
8e22c59aa8
commit
7a5f17f74b
3 changed files with 43 additions and 7 deletions
|
|
@ -288,10 +288,13 @@ class Parser:
|
|||
self.table = table
|
||||
|
||||
def parse(self, tokens: TokenStream) -> typing.Tuple[Tree | None, list[str]]:
|
||||
# TODO: If this were a for reals for reals parser we would keep the trivia
|
||||
# accessible in the tree.
|
||||
input_tokens = tokens.tokens()
|
||||
input: list[TokenValue] = [
|
||||
TokenValue(kind=kind.value, start=start, end=start + length)
|
||||
for (kind, start, length) in input_tokens
|
||||
if kind.value is not None and kind.value not in self.table.trivia
|
||||
]
|
||||
|
||||
eof = 0 if len(input) == 0 else input[-1].end
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue