DeCarabas/lrparsers
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Fix regex generation, extras

Browse source
This commit is contained in:
John Doty 2024-09-05 06:31:17 -07:00
parent 94f5958087
commit be8e017fd9
2 changed files with 19 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

3
parser/parser.py
View file

@ -2729,6 +2729,9 @@ class Grammar:
def terminals(self) -> list[Terminal]: def terminals(self) -> list[Terminal]:
return self._terminals return self._terminals
def trivia_terminals(self) -> list[Terminal]:
return self._trivia
def non_terminals(self) -> list[NonTerminal]: def non_terminals(self) -> list[NonTerminal]:
return [nt for _, nt in inspect.getmembers(self, lambda x: isinstance(x, NonTerminal))] return [nt for _, nt in inspect.getmembers(self, lambda x: isinstance(x, NonTerminal))]

24
parser/tree_sitter.py
View file

@ -99,6 +99,17 @@ def to_javascript_regex(re: parser.Re) -> str:
raise Exception(f"Regex node {re} not supported for tree-sitter") raise Exception(f"Regex node {re} not supported for tree-sitter")
def terminal_to_tree_sitter(rule: parser.Terminal) -> str:
if isinstance(rule.pattern, parser.Re):
regex = to_javascript_regex(rule.pattern)
regex = regex.replace("/", "\\/")
result = f"/{regex}/"
else:
string = to_js_string(rule.pattern)
result = f'"{string}"'
return result
def apply_precedence(js: str, name: str, grammar: parser.Grammar) -> str: def apply_precedence(js: str, name: str, grammar: parser.Grammar) -> str:
prec = grammar.get_precedence(name) prec = grammar.get_precedence(name)
if prec is not None: if prec is not None:
@ -119,14 +130,7 @@ def convert_to_tree_sitter(rule: parser.Rule, grammar: parser.Grammar) -> str:
return method(grammar) return method(grammar)
if isinstance(rule, parser.Terminal): if isinstance(rule, parser.Terminal):
if isinstance(rule.pattern, parser.Re): return terminal_to_tree_sitter(rule)
regex = to_javascript_regex(rule.pattern)
result = f"/{regex}/"
else:
string = to_js_string(rule.pattern)
result = f'"{string}"'
return result
elif isinstance(rule, parser.AlternativeRule): elif isinstance(rule, parser.AlternativeRule):
final = [] final = []
@ -225,6 +229,10 @@ def emit_tree_sitter_grammar(grammar: parser.Grammar, path: pathlib.Path | str):
f.write("\n") f.write("\n")
f.write("module.exports = grammar({\n") f.write("module.exports = grammar({\n")
f.write(f" name: '{grammar.name}',\n") f.write(f" name: '{grammar.name}',\n")
extras = ", ".join([terminal_to_tree_sitter(t) for t in grammar.trivia_terminals()])
f.write(f" extras: $ => [{extras}],\n")
f.write(" rules: {\n") f.write(" rules: {\n")
f.write(f" source_file: $ => $['{grammar.start}'],\n") f.write(f" source_file: $ => $['{grammar.start}'],\n")
for rule in grammar.non_terminals(): for rule in grammar.non_terminals():