From 591da0c97180bbdc339ee57da9ec0ef66a7a3e53 Mon Sep 17 00:00:00 2001
From: John Doty <john@d0ty.me>
Date: Mon, 2 Sep 2024 08:50:36 -0700
Subject: [PATCH] Rework highlighting metadata GOD

---
 grammar.py       | 103 ++++++++--------
 parser/parser.py | 312 ++++++++++++++++++++++++++++++++++++++---------
 2 files changed, 304 insertions(+), 111 deletions(-)

diff --git a/grammar.py b/grammar.py
index 6fed257..ed0d3b8 100644
--- a/grammar.py
+++ b/grammar.py
@@ -1,5 +1,5 @@
 # This is an example grammar.
-from parser import alt, Assoc, Grammar, rule, seq, Rule, Terminal, Re, Highlight, mark, opt
+from parser import alt, Assoc, Grammar, rule, seq, Rule, Terminal, Re, highlight, mark, opt
 
 
 class FineGrammar(Grammar):
@@ -52,7 +52,7 @@ class FineGrammar(Grammar):
     def class_declaration(self) -> Rule:
         return seq(
             self.CLASS,
-            mark(self.IDENTIFIER, highlight=Highlight.Entity.Name.Type),
+            mark(self.IDENTIFIER, highlight=highlight.entity.name.type),
             self._class_body,
         )
 
@@ -107,7 +107,7 @@ class FineGrammar(Grammar):
     def function_declaration(self) -> Rule:
         return seq(
             self.FUN,
-            mark(self.IDENTIFIER, highlight=Highlight.Entity.Name.Function),
+            mark(self.IDENTIFIER, highlight=highlight.entity.name.function),
             self.function_parameters,
             opt(self.ARROW, self.type_expression),
             self.block,
@@ -330,32 +330,31 @@ class FineGrammar(Grammar):
     BLANKS = Terminal(Re.set(" ", "\t", "\r", "\n").plus())
     COMMENT = Terminal(
         Re.seq(Re.literal("//"), Re.set("\n").invert().star()),
-        highlight=Highlight.Comment.Line,
+        highlight=highlight.comment.line,
     )
 
-    ARROW = Terminal("->", highlight=Highlight.Keyword.Operator)
-    AS = Terminal("as", highlight=Highlight.Keyword.Operator.Expression)
-    BAR = Terminal("|", highlight=Highlight.Keyword.Operator.Expression)
-    CLASS = Terminal("class", highlight=Highlight.Storage.Type.Class)
-    COLON = Terminal(":", highlight=Highlight.Punctuation.Separator)
-    ELSE = Terminal("else", highlight=Highlight.Keyword.Control.Conditional)
-    FOR = Terminal("for", highlight=Highlight.Keyword.Control)
-    FUN = Terminal("fun", highlight=Highlight.Storage.Type.Function)
+    ARROW = Terminal("->", highlight=highlight.keyword.operator)
+    AS = Terminal("as", highlight=highlight.keyword.operator.expression)
+    BAR = Terminal("|", highlight=highlight.keyword.operator.expression)
+    CLASS = Terminal("class", highlight=highlight.storage.type.klass)
+    COLON = Terminal(":", highlight=highlight.punctuation.separator)
+    ELSE = Terminal("else", highlight=highlight.keyword.control.conditional)
+    FOR = Terminal("for", highlight=highlight.keyword.control)
+    FUN = Terminal("fun", highlight=highlight.storage.type.function)
     IDENTIFIER = Terminal(
         Re.seq(
             Re.set(("a", "z"), ("A", "Z"), "_"),
             Re.set(("a", "z"), ("A", "Z"), ("0", "9"), "_").star(),
         ),
-        # highlight=Highlight.Variable, #?
     )
-    IF = Terminal("if", highlight=Highlight.Keyword.Control.Conditional)
-    IMPORT = Terminal("import", highlight=Highlight.Keyword.Other)
-    IN = Terminal("in", highlight=Highlight.Keyword.Operator)
-    LCURLY = Terminal("{", highlight=Highlight.Punctuation.CurlyBrace.Open)
-    RCURLY = Terminal("}", highlight=Highlight.Punctuation.CurlyBrace.Close)
-    LET = Terminal("let", highlight=Highlight.Keyword.Other)
-    RETURN = Terminal("return", highlight=Highlight.Keyword.Control)
-    SEMICOLON = Terminal(";", highlight=Highlight.Punctuation.Separator)
+    IF = Terminal("if", highlight=highlight.keyword.control.conditional)
+    IMPORT = Terminal("import", highlight=highlight.keyword.other)
+    IN = Terminal("in", highlight=highlight.keyword.operator)
+    LCURLY = Terminal("{", highlight=highlight.punctuation.curly_brace.open)
+    RCURLY = Terminal("}", highlight=highlight.punctuation.curly_brace.close)
+    LET = Terminal("let", highlight=highlight.keyword.other)
+    RETURN = Terminal("return", highlight=highlight.keyword.control)
+    SEMICOLON = Terminal(";", highlight=highlight.punctuation.separator)
     STRING = Terminal(
         # Double-quoted string.
         Re.seq(
@@ -369,27 +368,27 @@ class FineGrammar(Grammar):
             (~Re.set("'", "\\") | (Re.set("\\") + Re.any())).star(),
             Re.literal("'"),
         ),
-        highlight=Highlight.String.Quoted,
+        highlight=highlight.string.quoted,
     )
-    WHILE = Terminal("while", highlight=Highlight.Keyword.Control)
-    EQUAL = Terminal("=", highlight=Highlight.Keyword.Operator.Expression)
-    LPAREN = Terminal("(", highlight=Highlight.Punctuation.Parenthesis.Open)
-    RPAREN = Terminal(")", highlight=Highlight.Punctuation.Parenthesis.Close)
-    COMMA = Terminal(",", highlight=Highlight.Punctuation.Separator)
-    SELF = Terminal("self", name="SELFF", highlight=Highlight.Variable.Language)
-    OR = Terminal("or", highlight=Highlight.Keyword.Operator.Expression)
-    IS = Terminal("is", highlight=Highlight.Keyword.Operator.Expression)
-    AND = Terminal("and", highlight=Highlight.Keyword.Operator.Expression)
-    EQUALEQUAL = Terminal("==", highlight=Highlight.Keyword.Operator.Expression)
-    BANGEQUAL = Terminal("!=", highlight=Highlight.Keyword.Operator.Expression)
-    LESS = Terminal("<", highlight=Highlight.Keyword.Operator.Expression)
-    GREATER = Terminal(">", highlight=Highlight.Keyword.Operator.Expression)
-    LESSEQUAL = Terminal("<=", highlight=Highlight.Keyword.Operator.Expression)
-    GREATEREQUAL = Terminal(">=", highlight=Highlight.Keyword.Operator.Expression)
-    PLUS = Terminal("+", highlight=Highlight.Keyword.Operator.Expression)
-    MINUS = Terminal("-", highlight=Highlight.Keyword.Operator.Expression)
-    STAR = Terminal("*", highlight=Highlight.Keyword.Operator.Expression)
-    SLASH = Terminal("/", highlight=Highlight.Keyword.Operator.Expression)
+    WHILE = Terminal("while", highlight=highlight.keyword.control)
+    EQUAL = Terminal("=", highlight=highlight.keyword.operator.expression)
+    LPAREN = Terminal("(", highlight=highlight.punctuation.parenthesis.open)
+    RPAREN = Terminal(")", highlight=highlight.punctuation.parenthesis.close)
+    COMMA = Terminal(",", highlight=highlight.punctuation.separator)
+    SELF = Terminal("self", name="SELFF", highlight=highlight.variable.language)
+    OR = Terminal("or", highlight=highlight.keyword.operator.expression)
+    IS = Terminal("is", highlight=highlight.keyword.operator.expression)
+    AND = Terminal("and", highlight=highlight.keyword.operator.expression)
+    EQUALEQUAL = Terminal("==", highlight=highlight.keyword.operator.expression)
+    BANGEQUAL = Terminal("!=", highlight=highlight.keyword.operator.expression)
+    LESS = Terminal("<", highlight=highlight.keyword.operator.expression)
+    GREATER = Terminal(">", highlight=highlight.keyword.operator.expression)
+    LESSEQUAL = Terminal("<=", highlight=highlight.keyword.operator.expression)
+    GREATEREQUAL = Terminal(">=", highlight=highlight.keyword.operator.expression)
+    PLUS = Terminal("+", highlight=highlight.keyword.operator.expression)
+    MINUS = Terminal("-", highlight=highlight.keyword.operator.expression)
+    STAR = Terminal("*", highlight=highlight.keyword.operator.expression)
+    SLASH = Terminal("/", highlight=highlight.keyword.operator.expression)
     NUMBER = Terminal(
         Re.seq(
             Re.set(("0", "9")).plus(),
@@ -403,18 +402,18 @@ class FineGrammar(Grammar):
                 Re.set(("0", "9")).plus(),
             ).question(),
         ),
-        highlight=Highlight.Constant.Numeric,
+        highlight=highlight.constant.numeric,
     )
-    TRUE = Terminal("true", highlight=Highlight.Constant.Language)
-    FALSE = Terminal("false", highlight=Highlight.Constant.Language)
-    BANG = Terminal("!", highlight=Highlight.Keyword.Operator.Expression)
-    DOT = Terminal(".", highlight=Highlight.Punctuation.Separator)
-    MATCH = Terminal("match", highlight=Highlight.Keyword.Other)
-    EXPORT = Terminal("export", highlight=Highlight.Keyword.Other)
-    UNDERSCORE = Terminal("_", highlight=Highlight.Variable.Language)
-    NEW = Terminal("new", highlight=Highlight.Keyword.Operator)
-    LSQUARE = Terminal("[", highlight=Highlight.Punctuation.SquareBracket.Open)
-    RSQUARE = Terminal("]", highlight=Highlight.Punctuation.SquareBracket.Close)
+    TRUE = Terminal("true", highlight=highlight.constant.language)
+    FALSE = Terminal("false", highlight=highlight.constant.language)
+    BANG = Terminal("!", highlight=highlight.keyword.operator.expression)
+    DOT = Terminal(".", highlight=highlight.punctuation.separator)
+    MATCH = Terminal("match", highlight=highlight.keyword.other)
+    EXPORT = Terminal("export", highlight=highlight.keyword.other)
+    UNDERSCORE = Terminal("_", highlight=highlight.variable.language)
+    NEW = Terminal("new", highlight=highlight.keyword.operator)
+    LSQUARE = Terminal("[", highlight=highlight.punctuation.square_bracket.open)
+    RSQUARE = Terminal("]", highlight=highlight.punctuation.square_bracket.close)
 
 
 if __name__ == "__main__":
diff --git a/parser/parser.py b/parser/parser.py
index 86f5058..cb47ea2 100644
--- a/parser/parser.py
+++ b/parser/parser.py
@@ -2334,90 +2334,284 @@ def dump_lexer_table(table: LexerTable, name: str = "lexer.dot"):
 #       This here might be enough to produce extremely basic TextMate
 #       grammars but anything more complicated will want tree patterns
 #       anyway, and we can only do tree patterns by influencing the grammar.
+#
+#       Here's the info on textmate grammars:
+#           https://macromates.com/manual/en/language_grammars
 class SyntaxMeta:
     pass
 
 
-class Highlight(SyntaxMeta):
-    class Comment(SyntaxMeta):
-        class Block(SyntaxMeta):
-            pass
+class HighlightMeta(SyntaxMeta):
+    scope: str
 
-        class Line(SyntaxMeta):
-            pass
+    def __init__(self, *scope: str):
+        self.scope = ".".join(scope)
 
-    class Constant(SyntaxMeta):
-        class Language(SyntaxMeta):
-            pass
 
-        class Numeric(SyntaxMeta):
-            pass
+class CommentHighlight(HighlightMeta):
+    def __init__(self, *scope: str):
+        super().__init__("comment", *scope)
 
-    class Entity(SyntaxMeta):
-        class Name(SyntaxMeta):
-            class Function(SyntaxMeta):
-                pass
 
-            class Type(SyntaxMeta):
-                pass
+class BlockCommentHighlight(CommentHighlight):
+    def __init__(self, *scope: str):
+        super().__init__("block", *scope)
 
-    class Keyword(SyntaxMeta):
-        class Control(SyntaxMeta):
-            class Conditional(SyntaxMeta):
-                pass
 
-        class Operator(SyntaxMeta):
-            class Expression(SyntaxMeta):
-                pass
+class LineCommentHighlight(CommentHighlight):
+    def __init__(self, *scope: str):
+        super().__init__("line", *scope)
 
-        class Other(SyntaxMeta):
-            pass
 
-    class Punctuation(SyntaxMeta):
-        class Separator(SyntaxMeta):
-            pass
+class ConstantHighlight(HighlightMeta):
+    def __init__(self, *scope: str):
+        super().__init__("constant", *scope)
 
-        class Parenthesis(SyntaxMeta):
-            class Open(SyntaxMeta):
-                pass
 
-            class Close(SyntaxMeta):
-                pass
+class LanguageConstantHighlight(ConstantHighlight):
+    def __init__(self, *scope: str):
+        super().__init__("language", *scope)
 
-        class CurlyBrace(SyntaxMeta):
-            class Open(SyntaxMeta):
-                pass
 
-            class Close(SyntaxMeta):
-                pass
+class NumericConstantHighlight(ConstantHighlight):
+    def __init__(self, *scope: str):
+        super().__init__("numeric", *scope)
 
-        class SquareBracket(SyntaxMeta):
-            class Open(SyntaxMeta):
-                pass
 
-            class Close(SyntaxMeta):
-                pass
+class EntityHighlight(HighlightMeta):
+    def __init__(self, *scope: str):
+        super().__init__("entity", *scope)
 
-    class Storage(SyntaxMeta):
-        class Type(SyntaxMeta):
-            class Class(SyntaxMeta):
-                pass
 
-            class Function(SyntaxMeta):
-                pass
+class NameEntityHighlight(EntityHighlight):
+    def __init__(self, *scope: str):
+        super().__init__("name", *scope)
 
-    class String(SyntaxMeta):
-        class Quoted(SyntaxMeta):
-            class Single(SyntaxMeta):
-                pass
 
-            class Double(SyntaxMeta):
-                pass
+class FunctionNameEntityHighlight(NameEntityHighlight):
+    def __init__(self, *scope: str):
+        super().__init__("function", *scope)
 
-    class Variable(SyntaxMeta):
-        class Language(SyntaxMeta):
-            pass
 
+class TypeNameEntityHighlight(NameEntityHighlight):
+    def __init__(self, *scope: str):
+        super().__init__("type", *scope)
+
+
+class KeywordHighlight(HighlightMeta):
+    def __init__(self, *scope: str):
+        super().__init__("keyword", *scope)
+
+
+class ControlKeywordHighlight(KeywordHighlight):
+    def __init__(self, *scope: str):
+        super().__init__("control", *scope)
+
+
+class ConditionalControlKeywordHighlight(ControlKeywordHighlight):
+    def __init__(self, *scope: str):
+        super().__init__("conditional", *scope)
+
+
+class OperatorKeywordHighlight(KeywordHighlight):
+    def __init__(self, *scope: str):
+        super().__init__("operator", *scope)
+
+
+class ExpressionOperatorKeywordHighlight(OperatorKeywordHighlight):
+    def __init__(self, *scope: str):
+        super().__init__("expression", *scope)
+
+
+class OtherKeywordHighlight(KeywordHighlight):
+    def __init__(self, *scope: str):
+        super().__init__("other", *scope)
+
+
+class PunctuationHighlight(HighlightMeta):
+    def __init__(self, *scope: str):
+        super().__init__("punctuation", *scope)
+
+
+class SeparatorPunctuationHighlight(PunctuationHighlight):
+    def __init__(self, *scope: str):
+        super().__init__("separator", *scope)
+
+
+class ParenthesisPunctuationHighlight(PunctuationHighlight):
+    def __init__(self, *scope: str):
+        super().__init__("parenthesis", *scope)
+
+
+class OpenParenthesisPunctuationHighlight(ParenthesisPunctuationHighlight):
+    def __init__(self, *scope: str):
+        super().__init__("open", *scope)
+
+
+class CloseParenthesisPunctuationHighlight(ParenthesisPunctuationHighlight):
+    def __init__(self, *scope: str):
+        super().__init__("close", *scope)
+
+
+class CurlyBracePunctuationHighlight(PunctuationHighlight):
+    def __init__(self, *scope: str):
+        super().__init__("curlybrace", *scope)
+
+
+class OpenCurlyBracePunctuationHighlight(CurlyBracePunctuationHighlight):
+    def __init__(self, *scope: str):
+        super().__init__("open", *scope)
+
+
+class CloseCurlyBracePunctuationHighlight(CurlyBracePunctuationHighlight):
+    def __init__(self, *scope: str):
+        super().__init__("close", *scope)
+
+
+class SquareBracketPunctuationHighlight(PunctuationHighlight):
+    def __init__(self, *scope: str):
+        super().__init__("squarebracket", *scope)
+
+
+class OpenSquareBracketPunctuationHighlight(SquareBracketPunctuationHighlight):
+    def __init__(self, *scope: str):
+        super().__init__("open", *scope)
+
+
+class CloseSquareBracketPunctuationHighlight(SquareBracketPunctuationHighlight):
+    def __init__(self, *scope: str):
+        super().__init__("close", *scope)
+
+
+class StorageHighlight(HighlightMeta):
+    def __init__(self, *scope: str):
+        super().__init__("storage", *scope)
+
+
+class TypeStorageHighlight(StorageHighlight):
+    def __init__(self, *scope: str):
+        super().__init__("type", *scope)
+
+
+class ClassTypeStorageHighlight(TypeStorageHighlight):
+    def __init__(self, *scope: str):
+        super().__init__("class", *scope)
+
+
+class FunctionTypeStorageHighlight(TypeStorageHighlight):
+    def __init__(self, *scope: str):
+        super().__init__("function", *scope)
+
+
+class StructTypeStorageHighlight(TypeStorageHighlight):
+    def __init__(self, *scope: str):
+        super().__init__("struct", *scope)
+
+
+class StringHighlight(HighlightMeta):
+    def __init__(self, *scope: str):
+        super().__init__("string", *scope)
+
+
+class QuotedStringHighlight(StringHighlight):
+    def __init__(self, *scope: str):
+        super().__init__("quoted", *scope)
+
+
+class SingleQuotedStringHighlight(QuotedStringHighlight):
+    def __init__(self, *scope: str):
+        super().__init__("single", *scope)
+
+
+class DoubleQuotedStringHighlight(QuotedStringHighlight):
+    def __init__(self, *scope: str):
+        super().__init__("double", *scope)
+
+
+class VariableHighlight(HighlightMeta):
+    def __init__(self, *scope: str):
+        super().__init__("variable", *scope)
+
+
+class LanguageVariableHighlight(VariableHighlight):
+    def __init__(self, *scope: str):
+        super().__init__("language", *scope)
+
+
+class _Highlight:
+    class _Comment(CommentHighlight):
+        line = LineCommentHighlight()
+
+    class _Constant(ConstantHighlight):
+        language = LanguageConstantHighlight()
+        numeric = NumericConstantHighlight()
+
+    class _Entity(EntityHighlight):
+        class _Name(NameEntityHighlight):
+            function = FunctionNameEntityHighlight()
+            type = TypeNameEntityHighlight()
+
+        name = _Name()
+
+    class _Keyword(KeywordHighlight):
+        class _Control(ControlKeywordHighlight):
+            conditional = ConditionalControlKeywordHighlight()
+
+        class _Operator(OperatorKeywordHighlight):
+            expression = ExpressionOperatorKeywordHighlight()
+
+        control = _Control()
+        operator = _Operator()
+        other = OtherKeywordHighlight()
+
+    class _Punctuation:
+        class _Parenthesis:
+            open = OpenParenthesisPunctuationHighlight()
+            close = CloseParenthesisPunctuationHighlight()
+
+        class _CurlyBrace:
+            open = OpenCurlyBracePunctuationHighlight()
+            close = CloseCurlyBracePunctuationHighlight()
+
+        class _SquareBracket:
+            open = OpenSquareBracketPunctuationHighlight()
+            close = CloseSquareBracketPunctuationHighlight()
+
+        parenthesis = _Parenthesis()
+        curly_brace = _CurlyBrace()
+        square_bracket = _SquareBracket()
+        separator = SeparatorPunctuationHighlight()
+
+    class _Storage(StorageHighlight):
+        class _Type(TypeStorageHighlight):
+
+            klass = ClassTypeStorageHighlight()  # Sorry.
+            function = FunctionTypeStorageHighlight()
+            struct = StructTypeStorageHighlight()
+
+        type = _Type()
+
+    class _String(StringHighlight):
+        class _Quoted(QuotedStringHighlight):
+            single = SingleQuotedStringHighlight()
+            double = DoubleQuotedStringHighlight()
+
+        quoted = _Quoted()
+
+    class _Variable(VariableHighlight):
+        language = LanguageVariableHighlight()
+
+    comment = _Comment()
+    constant = _Constant()
+    entity = _Entity()
+    keyword = _Keyword()
+    punctuation = _Punctuation()
+    storage = _Storage()
+    string = _String()
+    variable = _Variable()
+
+
+highlight = _Highlight()
 
 ###############################################################################
 # Finally, the base class for grammars