From 94f59580877bbcde68ad02596123c89dfe95cf92 Mon Sep 17 00:00:00 2001
From: John Doty <john@d0ty.me>
Date: Thu, 5 Sep 2024 06:30:55 -0700
Subject: [PATCH 1/3] Field propagation

---
 parser/tree_sitter.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/parser/tree_sitter.py b/parser/tree_sitter.py
index fbba486..4b71eca 100644
--- a/parser/tree_sitter.py
+++ b/parser/tree_sitter.py
@@ -205,7 +205,11 @@ def convert_to_tree_sitter(rule: parser.Rule, grammar: parser.Grammar) -> str:
         return f"$['{target_name}']"
 
     elif isinstance(rule, parser.MetadataRule):
-        return convert_to_tree_sitter(rule.rule, grammar)
+        result = convert_to_tree_sitter(rule.rule, grammar)
+        field = rule.metadata.get("field")
+        if field is not None:
+            result = f"field('{field}', {result})"
+        return result
 
     else:
         raise ValueError(f"Rule {rule} not supported for tree-sitter")

From be8e017fd94c3aa617ef750e3bb29386ae4e53d8 Mon Sep 17 00:00:00 2001
From: John Doty <john@d0ty.me>
Date: Thu, 5 Sep 2024 06:31:17 -0700
Subject: [PATCH 2/3] Fix regex generation, extras

---
 parser/parser.py      |  3 +++
 parser/tree_sitter.py | 24 ++++++++++++++++--------
 2 files changed, 19 insertions(+), 8 deletions(-)

diff --git a/parser/parser.py b/parser/parser.py
index cb47ea2..86e4e15 100644
--- a/parser/parser.py
+++ b/parser/parser.py
@@ -2729,6 +2729,9 @@ class Grammar:
     def terminals(self) -> list[Terminal]:
         return self._terminals
 
+    def trivia_terminals(self) -> list[Terminal]:
+        return self._trivia
+
     def non_terminals(self) -> list[NonTerminal]:
         return [nt for _, nt in inspect.getmembers(self, lambda x: isinstance(x, NonTerminal))]
 
diff --git a/parser/tree_sitter.py b/parser/tree_sitter.py
index 4b71eca..daf4940 100644
--- a/parser/tree_sitter.py
+++ b/parser/tree_sitter.py
@@ -99,6 +99,17 @@ def to_javascript_regex(re: parser.Re) -> str:
     raise Exception(f"Regex node {re} not supported for tree-sitter")
 
 
+def terminal_to_tree_sitter(rule: parser.Terminal) -> str:
+    if isinstance(rule.pattern, parser.Re):
+        regex = to_javascript_regex(rule.pattern)
+        regex = regex.replace("/", "\\/")
+        result = f"/{regex}/"
+    else:
+        string = to_js_string(rule.pattern)
+        result = f'"{string}"'
+    return result
+
+
 def apply_precedence(js: str, name: str, grammar: parser.Grammar) -> str:
     prec = grammar.get_precedence(name)
     if prec is not None:
@@ -119,14 +130,7 @@ def convert_to_tree_sitter(rule: parser.Rule, grammar: parser.Grammar) -> str:
         return method(grammar)
 
     if isinstance(rule, parser.Terminal):
-        if isinstance(rule.pattern, parser.Re):
-            regex = to_javascript_regex(rule.pattern)
-            result = f"/{regex}/"
-        else:
-            string = to_js_string(rule.pattern)
-            result = f'"{string}"'
-
-        return result
+        return terminal_to_tree_sitter(rule)
 
     elif isinstance(rule, parser.AlternativeRule):
         final = []
@@ -225,6 +229,10 @@ def emit_tree_sitter_grammar(grammar: parser.Grammar, path: pathlib.Path | str):
         f.write("\n")
         f.write("module.exports = grammar({\n")
         f.write(f"  name: '{grammar.name}',\n")
+
+        extras = ", ".join([terminal_to_tree_sitter(t) for t in grammar.trivia_terminals()])
+        f.write(f"  extras: $ => [{extras}],\n")
+
         f.write("  rules: {\n")
         f.write(f"    source_file: $ => $['{grammar.start}'],\n")
         for rule in grammar.non_terminals():

From 5e12af9f31629c31a49b69d4d192e931336f7b2a Mon Sep 17 00:00:00 2001
From: John Doty <john@d0ty.me>
Date: Thu, 5 Sep 2024 06:31:42 -0700
Subject: [PATCH 3/3] Extra marks, fields, whatnot

---
 grammar.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/grammar.py b/grammar.py
index ed0d3b8..3aac841 100644
--- a/grammar.py
+++ b/grammar.py
@@ -52,8 +52,8 @@ class FineGrammar(Grammar):
     def class_declaration(self) -> Rule:
         return seq(
             self.CLASS,
-            mark(self.IDENTIFIER, highlight=highlight.entity.name.type),
-            self._class_body,
+            mark(self.IDENTIFIER, field="name", highlight=highlight.entity.name.type),
+            mark(self._class_body, field="body"),
         )
 
     @rule
@@ -107,10 +107,10 @@ class FineGrammar(Grammar):
     def function_declaration(self) -> Rule:
         return seq(
             self.FUN,
-            mark(self.IDENTIFIER, highlight=highlight.entity.name.function),
-            self.function_parameters,
-            opt(self.ARROW, self.type_expression),
-            self.block,
+            mark(self.IDENTIFIER, field="name", highlight=highlight.entity.name.function),
+            mark(self.function_parameters, field="parameters"),
+            mark(opt(self.ARROW, self.type_expression), field="return_type"),
+            mark(self.block, field="body"),
         )
 
     @rule("ParamList")