Extra marks, fields, whatnot

grammar.py

@@ -52,8 +52,8 @@ class FineGrammar(Grammar):
     def class_declaration(self) -> Rule:
         return seq(
             self.CLASS,
-            mark(self.IDENTIFIER, highlight=highlight.entity.name.type),
-            self._class_body,
+            mark(self.IDENTIFIER, field="name", highlight=highlight.entity.name.type),
+            mark(self._class_body, field="body"),
         )
 
     @rule
@@ -107,10 +107,10 @@ class FineGrammar(Grammar):
     def function_declaration(self) -> Rule:
         return seq(
             self.FUN,
-            mark(self.IDENTIFIER, highlight=highlight.entity.name.function),
-            self.function_parameters,
-            opt(self.ARROW, self.type_expression),
-            self.block,
+            mark(self.IDENTIFIER, field="name", highlight=highlight.entity.name.function),
+            mark(self.function_parameters, field="parameters"),
+            mark(opt(self.ARROW, self.type_expression), field="return_type"),
+            mark(self.block, field="body"),
         )
 
     @rule("ParamList")

parser/parser.py

@@ -2729,6 +2729,9 @@ class Grammar:
     def terminals(self) -> list[Terminal]:
         return self._terminals
 
+    def trivia_terminals(self) -> list[Terminal]:
+        return self._trivia
+
     def non_terminals(self) -> list[NonTerminal]:
         return [nt for _, nt in inspect.getmembers(self, lambda x: isinstance(x, NonTerminal))]
 

parser/tree_sitter.py

@@ -99,6 +99,17 @@ def to_javascript_regex(re: parser.Re) -> str:
     raise Exception(f"Regex node {re} not supported for tree-sitter")
 
 
+def terminal_to_tree_sitter(rule: parser.Terminal) -> str:
+    if isinstance(rule.pattern, parser.Re):
+        regex = to_javascript_regex(rule.pattern)
+        regex = regex.replace("/", "\\/")
+        result = f"/{regex}/"
+    else:
+        string = to_js_string(rule.pattern)
+        result = f'"{string}"'
+    return result
+
+
 def apply_precedence(js: str, name: str, grammar: parser.Grammar) -> str:
     prec = grammar.get_precedence(name)
     if prec is not None:
@@ -119,14 +130,7 @@ def convert_to_tree_sitter(rule: parser.Rule, grammar: parser.Grammar) -> str:
         return method(grammar)
 
     if isinstance(rule, parser.Terminal):
-        if isinstance(rule.pattern, parser.Re):
-            regex = to_javascript_regex(rule.pattern)
-            result = f"/{regex}/"
-        else:
-            string = to_js_string(rule.pattern)
-            result = f'"{string}"'
-
-        return result
+        return terminal_to_tree_sitter(rule)
 
     elif isinstance(rule, parser.AlternativeRule):
         final = []
@@ -205,7 +209,11 @@ def convert_to_tree_sitter(rule: parser.Rule, grammar: parser.Grammar) -> str:
         return f"$['{target_name}']"
 
     elif isinstance(rule, parser.MetadataRule):
-        return convert_to_tree_sitter(rule.rule, grammar)
+        result = convert_to_tree_sitter(rule.rule, grammar)
+        field = rule.metadata.get("field")
+        if field is not None:
+            result = f"field('{field}', {result})"
+        return result
 
     else:
         raise ValueError(f"Rule {rule} not supported for tree-sitter")
@@ -221,6 +229,10 @@ def emit_tree_sitter_grammar(grammar: parser.Grammar, path: pathlib.Path | str):
         f.write("\n")
         f.write("module.exports = grammar({\n")
         f.write(f"  name: '{grammar.name}',\n")
+
+        extras = ", ".join([terminal_to_tree_sitter(t) for t in grammar.trivia_terminals()])
+        f.write(f"  extras: $ => [{extras}],\n")
+
         f.write("  rules: {\n")
         f.write(f"    source_file: $ => $['{grammar.start}'],\n")
         for rule in grammar.non_terminals():