grammar.py

@@ -18,8 +18,6 @@ class FineGrammar(Grammar):
     # generator = parser.GenerateLR1
     start = "File"
 
-    trivia = ["BLANKS", "COMMENT"]
-
     def __init__(self):
         super().__init__(
             precedence=[
@@ -332,14 +330,14 @@ class FineGrammar(Grammar):
     def field_value(self) -> Rule:
         return self.IDENTIFIER | seq(self.IDENTIFIER, self.COLON, self.expression)
 
-    BLANKS = Terminal(Re.set(" ", "\t", "\r", "\n").plus())
-    COMMENT = Terminal(Re.seq(Re.literal("//"), Re.set("\n").invert().star()))
+    BLANK = Terminal(Re.set(" ", "\t", "\r", "\n").plus())
 
     ARROW = Terminal("->")
     AS = Terminal("as")
     BAR = Terminal("bar")
     CLASS = Terminal("class")
     COLON = Terminal("colon")
+    COMMENT = Terminal("comment")
     ELSE = Terminal("else")
     FOR = Terminal("for")
     FUN = Terminal("fun")
@@ -395,12 +393,12 @@ class FineGrammar(Grammar):
             Re.set(("0", "9")).plus(),
             Re.seq(
                 Re.literal("."),
-                Re.set(("0", "9")).plus(),
-            ).question(),
-            Re.seq(
-                Re.set("e", "E"),
-                Re.set("+", "-").question(),
-                Re.set(("0", "9")).plus(),
+                Re.set(("0", "9")),
+                Re.seq(
+                    Re.set("e", "E"),
+                    Re.set("+", "-").question(),
+                    Re.set(("0", "9")).plus(),
+                ).question(),
             ).question(),
         )
     )

parser/parser.py

@@ -561,7 +561,6 @@ class ErrorCollection:
 class ParseTable:
     actions: list[dict[str, ParseAction]]
     gotos: list[dict[str, int]]
-    trivia: set[str]
 
     def format(self):
         """Format a parser table so pretty."""
@@ -652,7 +651,7 @@ class TableBuilder(object):
         if error is not None:
             raise error
 
-        return ParseTable(actions=self.actions, gotos=self.gotos, trivia=set())
+        return ParseTable(actions=self.actions, gotos=self.gotos)
 
     def new_row(self, config_set: ConfigSet):
         """Start a new row, processing the given config set. Call this before
@@ -1795,14 +1794,12 @@ class Grammar:
     _start: str
     _generator: type[GenerateLR0]
     _terminals: list[Terminal]
-    _trivia: list[Terminal]
 
     def __init__(
         self,
         start: str | None = None,
         precedence: PrecedenceList | None = None,
         generator: type[GenerateLR0] | None = None,
-        trivia: list[str | Terminal] | None = None,
     ):
         if start is None:
             start = getattr(self, "start", None)
@@ -1820,30 +1817,12 @@ class Grammar:
             generator = getattr(self, "generator", GenerateLALR)
         assert generator is not None
 
-        if trivia is None:
-            trivia = getattr(self, "trivia", [])
-        assert trivia is not None
-
         # Fixup terminal names with the name of the member that declared it.
-        terminals = {}
+        terminals = []
         for n, t in inspect.getmembers(self, lambda x: isinstance(x, Terminal)):
             if t.value is None:
                 t.value = n
-
-            if n in terminals:
-                raise ValueError(f"More than one terminal has the name '{n}'")
-            terminals[n] = t
-
-        # Resolve the trivia declarations correctly.
-        resolved_trivia: list[Terminal] = []
-        for t in trivia:
-            if isinstance(t, str):
-                resolved = terminals.get(t)
-                if resolved is None:
-                    raise ValueError(f"The trivia '{t}' is not a terminal name")
-                resolved_trivia.append(resolved)
-            else:
-                resolved_trivia.append(t)
+            terminals.append(t)
 
         # Fix up the precedence table.
         precedence_table = {}
@@ -1861,17 +1840,12 @@ class Grammar:
         self._precedence = precedence_table
         self._start = start
         self._generator = generator
-        self._terminals = list(terminals.values())
-        self._trivia = resolved_trivia
+        self._terminals = terminals
 
     @property
     def terminals(self) -> list[Terminal]:
         return self._terminals
 
-    @property
-    def resolved_trivia(self) -> list[Terminal]:
-        return self._trivia
-
     def generate_nonterminal_dict(
         self, start: str | None = None
     ) -> typing.Tuple[dict[str, list[list[str | Terminal]]], set[str]]:
@@ -1945,7 +1919,7 @@ class Grammar:
 
         return grammar, transparents
 
-    def build_table(self, start: str | None = None, generator=None) -> ParseTable:
+    def build_table(self, start: str | None = None, generator=None):
         """Construct a parse table for this grammar, starting at the named
         nonterminal rule.
         """
@@ -1957,11 +1931,6 @@ class Grammar:
             generator = self._generator
         gen = generator(start, desugared, precedence=self._precedence, transparents=transparents)
         table = gen.gen_table()
-
-        for t in self._trivia:
-            assert t.value is not None
-            table.trivia.add(t.value)
-
         return table
 
 

parser/runtime.py

@@ -288,13 +288,10 @@ class Parser:
         self.table = table
 
     def parse(self, tokens: TokenStream) -> typing.Tuple[Tree | None, list[str]]:
-        # TODO: If this were a for reals for reals parser we would keep the trivia
-        #       accessible in the tree.
         input_tokens = tokens.tokens()
         input: list[TokenValue] = [
             TokenValue(kind=kind.value, start=start, end=start + length)
             for (kind, start, length) in input_tokens
-            if kind.value is not None and kind.value not in self.table.trivia
         ]
 
         eof = 0 if len(input) == 0 else input[-1].end