Specify and honor trivia tokens

e.g. "this is how machine-generated parsers know to skip blanks and
comments"

The run time implementation could be better; we don't really want to
just discard trivia because it's useful for e.g. doc comments and the
like. BUT for now this is fine.
This commit is contained in:
John Doty 2024-08-24 10:01:40 -07:00
parent 8e22c59aa8
commit 7a5f17f74b
3 changed files with 43 additions and 7 deletions

View file

@ -18,6 +18,8 @@ class FineGrammar(Grammar):
# generator = parser.GenerateLR1
start = "File"
trivia = ["BLANKS", "COMMENT"]
def __init__(self):
super().__init__(
precedence=[
@ -330,14 +332,14 @@ class FineGrammar(Grammar):
def field_value(self) -> Rule:
return self.IDENTIFIER | seq(self.IDENTIFIER, self.COLON, self.expression)
BLANK = Terminal(Re.set(" ", "\t", "\r", "\n").plus())
BLANKS = Terminal(Re.set(" ", "\t", "\r", "\n").plus())
COMMENT = Terminal(Re.seq(Re.literal("//"), Re.set("\n").invert().star()))
ARROW = Terminal("->")
AS = Terminal("as")
BAR = Terminal("bar")
CLASS = Terminal("class")
COLON = Terminal("colon")
COMMENT = Terminal("comment")
ELSE = Terminal("else")
FOR = Terminal("for")
FUN = Terminal("fun")