# This is an example grammar. import re import typing from parser import Assoc, Grammar, Nothing, rule, seq, Rule, Terminal, Re, TerminalKind class FineGrammar(Grammar): # generator = parser.GenerateLR1 start = "File" trivia = ["BLANKS", "COMMENT"] def __init__(self): super().__init__( precedence=[ (Assoc.RIGHT, [self.EQUAL]), (Assoc.LEFT, [self.OR]), (Assoc.LEFT, [self.IS]), (Assoc.LEFT, [self.AND]), (Assoc.LEFT, [self.EQUALEQUAL, self.BANGEQUAL]), (Assoc.LEFT, [self.LESS, self.GREATER, self.GREATEREQUAL, self.LESSEQUAL]), (Assoc.LEFT, [self.PLUS, self.MINUS]), (Assoc.LEFT, [self.STAR, self.SLASH]), (Assoc.LEFT, [self.primary_expression]), (Assoc.LEFT, [self.LPAREN]), (Assoc.LEFT, [self.DOT]), # # If there's a confusion about whether to make an IF # statement or an expression, prefer the statement. # (Assoc.NONE, [self.if_statement]), ], ) @rule("File") def file(self) -> Rule: return self._file_statement_list @rule def _file_statement_list(self) -> Rule: return self._file_statement | (self._file_statement_list + self._file_statement) @rule def _file_statement(self) -> Rule: return ( self.import_statement | self.class_declaration | self.export_statement | self._statement ) @rule def import_statement(self) -> Rule: return seq(self.IMPORT, self.STRING, self.AS, self.IDENTIFIER, self.SEMICOLON) @rule("ClassDeclaration") def class_declaration(self) -> Rule: return seq(self.CLASS, self.IDENTIFIER, self._class_body) @rule def _class_body(self) -> Rule: return seq(self.LCURLY, self.RCURLY) | seq(self.LCURLY, self._class_members, self.RCURLY) @rule def _class_members(self) -> Rule: return self._class_member | seq(self._class_members, self._class_member) @rule def _class_member(self) -> Rule: return self.field_declaration | self.function_declaration @rule("FieldDecl") def field_declaration(self) -> Rule: return seq(self.IDENTIFIER, self.COLON, self.type_expression, self.SEMICOLON) # Types @rule("TypeExpression") def type_expression(self) -> Rule: return self.alternate_type | self.type_identifier @rule("AlternateType") def alternate_type(self) -> Rule: return seq(self.type_expression, self.OR, self.type_identifier) @rule("TypeIdentifier") def type_identifier(self) -> Rule: return self.IDENTIFIER @rule def export_statement(self) -> Rule: return ( seq(self.EXPORT, self.class_declaration) | seq(self.EXPORT, self.function_declaration) | seq(self.EXPORT, self.let_statement) | seq(self.EXPORT, self.export_list, self.SEMICOLON) ) @rule def export_list(self) -> Rule: return Nothing | self.IDENTIFIER | seq(self.IDENTIFIER, self.COMMA, self.export_list) # Functions @rule("FunctionDecl") def function_declaration(self) -> Rule: return seq(self.FUN, self.IDENTIFIER, self.function_parameters, self.block) | seq( self.FUN, self.IDENTIFIER, self.function_parameters, self.ARROW, self.type_expression, self.block, ) @rule("ParamList") def function_parameters(self) -> Rule: return ( seq(self.LPAREN, self.RPAREN) | seq(self.LPAREN, self._first_parameter, self.RPAREN) | seq(self.LPAREN, self._first_parameter, self.COMMA, self._parameter_list, self.RPAREN) ) @rule def _first_parameter(self) -> Rule: return self.SELF | self.parameter @rule def _parameter_list(self) -> Rule: return Nothing | self.parameter | seq(self.parameter, self.COMMA, self._parameter_list) @rule("Parameter") def parameter(self) -> Rule: return seq(self.IDENTIFIER, self.COLON, self.type_expression) # Block @rule("Block") def block(self) -> Rule: return ( seq(self.LCURLY, self.RCURLY) | seq(self.LCURLY, self.expression, self.RCURLY) | seq(self.LCURLY, self._statement_list, self.RCURLY) | seq(self.LCURLY, self._statement_list, self.expression, self.RCURLY) ) @rule def _statement_list(self) -> Rule: return self._statement | seq(self._statement_list, self._statement) @rule def _statement(self) -> Rule: return ( self.function_declaration | self.let_statement | self.return_statement | self.for_statement | self.if_statement | self.while_statement | self.expression_statement ) @rule("LetStatement") def let_statement(self) -> Rule: return seq(self.LET, self.IDENTIFIER, self.EQUAL, self.expression, self.SEMICOLON) @rule("ReturnStatement") def return_statement(self) -> Rule: return seq(self.RETURN, self.expression, self.SEMICOLON) | seq(self.RETURN, self.SEMICOLON) @rule("ForStatement") def for_statement(self) -> Rule: return seq(self.FOR, self.iterator_variable, self.IN, self.expression, self.block) @rule("IteratorVariable") def iterator_variable(self) -> Rule: return self.IDENTIFIER @rule("IfStatement") def if_statement(self) -> Rule: return self.conditional_expression @rule def while_statement(self) -> Rule: return seq(self.WHILE, self.expression, self.block) @rule def expression_statement(self) -> Rule: return seq(self.expression, self.SEMICOLON) # Expressions @rule(transparent=True) def expression(self) -> Rule: return self.binary_expression | self.is_expression | self.primary_expression @rule("BinaryExpression") def binary_expression(self) -> Rule: return ( seq(self.expression, self.EQUAL, self.expression) | seq(self.expression, self.OR, self.expression) | seq(self.expression, self.AND, self.expression) | seq(self.expression, self.EQUALEQUAL, self.expression) | seq(self.expression, self.BANGEQUAL, self.expression) | seq(self.expression, self.LESS, self.expression) | seq(self.expression, self.LESSEQUAL, self.expression) | seq(self.expression, self.GREATER, self.expression) | seq(self.expression, self.GREATEREQUAL, self.expression) | seq(self.expression, self.PLUS, self.expression) | seq(self.expression, self.MINUS, self.expression) | seq(self.expression, self.STAR, self.expression) | seq(self.expression, self.SLASH, self.expression) ) @rule("IsExpression") def is_expression(self) -> Rule: return seq(self.expression, self.IS, self.pattern) @rule def primary_expression(self) -> Rule: return ( self.identifier_expression | self.literal_expression | self.SELF | seq(self.BANG, self.primary_expression) | seq(self.MINUS, self.primary_expression) | self.block | self.conditional_expression | self.list_constructor_expression | self.object_constructor_expression | self.match_expression | seq(self.primary_expression, self.LPAREN, self.RPAREN) | seq(self.primary_expression, self.LPAREN, self._expression_list, self.RPAREN) | seq(self.primary_expression, self.DOT, self.IDENTIFIER) | seq(self.LPAREN, self.expression, self.RPAREN) ) @rule("IdentifierExpression") def identifier_expression(self): return self.IDENTIFIER @rule("Literal") def literal_expression(self): return self.NUMBER | self.STRING | self.TRUE | self.FALSE @rule("ConditionalExpression") def conditional_expression(self) -> Rule: return ( seq(self.IF, self.expression, self.block) | seq(self.IF, self.expression, self.block, self.ELSE, self.conditional_expression) | seq(self.IF, self.expression, self.block, self.ELSE, self.block) ) @rule def list_constructor_expression(self) -> Rule: return seq(self.LSQUARE, self.RSQUARE) | seq( self.LSQUARE, self._expression_list, self.RSQUARE ) @rule def _expression_list(self) -> Rule: return ( self.expression | seq(self.expression, self.COMMA) | seq(self.expression, self.COMMA, self._expression_list) ) @rule def match_expression(self) -> Rule: return seq(self.MATCH, self.expression, self.match_body) @rule("MatchBody") def match_body(self) -> Rule: return seq(self.LCURLY, self.RCURLY) | seq(self.LCURLY, self._match_arms, self.RCURLY) @rule def _match_arms(self) -> Rule: return ( self.match_arm | seq(self.match_arm, self.COMMA) | seq(self.match_arm, self.COMMA, self._match_arms) ) @rule("MatchArm") def match_arm(self) -> Rule: return seq(self.pattern, self.ARROW, self.expression) @rule("Pattern") def pattern(self) -> Rule: return ( seq(self.variable_binding, self._pattern_core, self._pattern_predicate) | seq(self.variable_binding, self._pattern_core) | self._pattern_core ) @rule def _pattern_predicate(self) -> Rule: return seq(self.AND, self.expression) @rule def _pattern_core(self) -> Rule: return self.type_expression | self.wildcard_pattern @rule("WildcardPattern") def wildcard_pattern(self) -> Rule: return self.UNDERSCORE @rule("VariableBinding") def variable_binding(self) -> Rule: return seq(self.IDENTIFIER, self.COLON) @rule def object_constructor_expression(self) -> Rule: return seq(self.NEW, self.type_identifier, self.field_list) @rule def field_list(self) -> Rule: return seq(self.LCURLY, self.RCURLY) | seq(self.LCURLY, self.field_values, self.RCURLY) @rule def field_values(self) -> Rule: return ( self.field_value | seq(self.field_value, self.COMMA) | seq(self.field_value, self.COMMA, self.field_values) ) @rule def field_value(self) -> Rule: return self.IDENTIFIER | seq(self.IDENTIFIER, self.COLON, self.expression) BLANKS = Terminal(Re.set(" ", "\t", "\r", "\n").plus()) COMMENT = Terminal( Re.seq(Re.literal("//"), Re.set("\n").invert().star()), kind=TerminalKind.Comment.Line, ) ARROW = Terminal("->", kind=TerminalKind.Keyword.Operator) AS = Terminal("as", kind=TerminalKind.Keyword.Operator.Expression) BAR = Terminal("|", kind=TerminalKind.Keyword.Operator.Expression) CLASS = Terminal("class", kind=TerminalKind.Storage.Type.Class) COLON = Terminal(":", kind=TerminalKind.Punctuation.Separator) ELSE = Terminal("else", kind=TerminalKind.Keyword.Control.Conditional) FOR = Terminal("for", kind=TerminalKind.Keyword.Control) FUN = Terminal("fun", kind=TerminalKind.Storage.Type.Function) IDENTIFIER = Terminal( Re.seq( Re.set(("a", "z"), ("A", "Z"), "_"), Re.set(("a", "z"), ("A", "Z"), ("0", "9"), "_").star(), ), # kind=TerminalKind.Variable, #? ) IF = Terminal("if", kind=TerminalKind.Keyword.Control.Conditional) IMPORT = Terminal("import", kind=TerminalKind.Keyword.Other) IN = Terminal("in", kind=TerminalKind.Keyword.Operator) LCURLY = Terminal("{", kind=TerminalKind.Punctuation.CurlyBrace.Open) RCURLY = Terminal("}", kind=TerminalKind.Punctuation.CurlyBrace.Close) LET = Terminal("Let", kind=TerminalKind.Keyword.Other) RETURN = Terminal("return", kind=TerminalKind.Keyword.Control) SEMICOLON = Terminal(";", kind=TerminalKind.Punctuation.Separator) STRING = Terminal( # Double-quoted string. Re.seq( Re.literal('"'), (~Re.set('"', "\\") | (Re.set("\\") + Re.any())).star(), Re.literal('"'), ) # Single-quoted string. | Re.seq( Re.literal("'"), (~Re.set("'", "\\") | (Re.set("\\") + Re.any())).star(), Re.literal("'"), ), kind=TerminalKind.String.Quoted, ) WHILE = Terminal("while", kind=TerminalKind.Keyword.Control) EQUAL = Terminal("=", kind=TerminalKind.Keyword.Operator.Expression) LPAREN = Terminal("(", kind=TerminalKind.Punctuation.Parenthesis.Open) RPAREN = Terminal(")", kind=TerminalKind.Punctuation.Parenthesis.Close) COMMA = Terminal(",", kind=TerminalKind.Punctuation.Separator) SELF = Terminal("self", name="SELFF", kind=TerminalKind.Variable.Language) OR = Terminal("or", kind=TerminalKind.Keyword.Operator.Expression) IS = Terminal("is", kind=TerminalKind.Keyword.Operator.Expression) AND = Terminal("and", kind=TerminalKind.Keyword.Operator.Expression) EQUALEQUAL = Terminal("==", kind=TerminalKind.Keyword.Operator.Expression) BANGEQUAL = Terminal("!=", kind=TerminalKind.Keyword.Operator.Expression) LESS = Terminal("<", kind=TerminalKind.Keyword.Operator.Expression) GREATER = Terminal(">", kind=TerminalKind.Keyword.Operator.Expression) LESSEQUAL = Terminal("<=", kind=TerminalKind.Keyword.Operator.Expression) GREATEREQUAL = Terminal(">=", kind=TerminalKind.Keyword.Operator.Expression) PLUS = Terminal("+", kind=TerminalKind.Keyword.Operator.Expression) MINUS = Terminal("-", kind=TerminalKind.Keyword.Operator.Expression) STAR = Terminal("*", kind=TerminalKind.Keyword.Operator.Expression) SLASH = Terminal("/", kind=TerminalKind.Keyword.Operator.Expression) NUMBER = Terminal( Re.seq( Re.set(("0", "9")).plus(), Re.seq( Re.literal("."), Re.set(("0", "9")).plus(), ).question(), Re.seq( Re.set("e", "E"), Re.set("+", "-").question(), Re.set(("0", "9")).plus(), ).question(), ), kind=TerminalKind.Constant.Numeric, ) TRUE = Terminal("true", kind=TerminalKind.Constant.Language) FALSE = Terminal("false", kind=TerminalKind.Constant.Language) BANG = Terminal("!", kind=TerminalKind.Keyword.Operator.Expression) DOT = Terminal(".", kind=TerminalKind.Punctuation.Separator) MATCH = Terminal("match", kind=TerminalKind.Keyword.Other) EXPORT = Terminal("export", kind=TerminalKind.Keyword.Other) UNDERSCORE = Terminal("_", kind=TerminalKind.Variable.Language) NEW = Terminal("new", kind=TerminalKind.Keyword.Operator) LSQUARE = Terminal("[", kind=TerminalKind.Punctuation.SquareBracket.Open) RSQUARE = Terminal("]", kind=TerminalKind.Punctuation.SquareBracket.Close) # ----------------------------------------------------------------------------- # DORKY LEXER # ----------------------------------------------------------------------------- import bisect NUMBER_RE = re.compile("[0-9]+(\\.[0-9]*([eE][-+]?[0-9]+)?)?") IDENTIFIER_RE = re.compile("[_A-Za-z][_A-Za-z0-9]*") KEYWORD_TABLE = { "_": FineGrammar.UNDERSCORE, "and": FineGrammar.AND, "as": FineGrammar.AS, "class": FineGrammar.CLASS, "else": FineGrammar.ELSE, "export": FineGrammar.EXPORT, "false": FineGrammar.FALSE, "for": FineGrammar.FOR, "fun": FineGrammar.FUN, "if": FineGrammar.IF, "import": FineGrammar.IMPORT, "in": FineGrammar.IN, "is": FineGrammar.IS, "let": FineGrammar.LET, "match": FineGrammar.MATCH, "new": FineGrammar.NEW, "or": FineGrammar.OR, "return": FineGrammar.RETURN, "self": FineGrammar.SELF, "true": FineGrammar.TRUE, "while": FineGrammar.WHILE, } def tokenize(src: str): pos = 0 while pos < len(src): ch = src[pos] if ch.isspace(): pos += 1 continue token = None if ch == "-": if src[pos : pos + 2] == "->": token = (FineGrammar.ARROW, pos, 2) else: token = (FineGrammar.MINUS, pos, 1) elif ch == "|": token = (FineGrammar.BAR, pos, 1) elif ch == ":": token = (FineGrammar.COLON, pos, 1) elif ch == "{": token = (FineGrammar.LCURLY, pos, 1) elif ch == "}": token = (FineGrammar.RCURLY, pos, 1) elif ch == ";": token = (FineGrammar.SEMICOLON, pos, 1) elif ch == "=": if src[pos : pos + 2] == "==": token = (FineGrammar.EQUALEQUAL, pos, 2) else: token = (FineGrammar.EQUAL, pos, 1) elif ch == "(": token = (FineGrammar.LPAREN, pos, 1) elif ch == ")": token = (FineGrammar.RPAREN, pos, 1) elif ch == ",": token = (FineGrammar.COMMA, pos, 1) elif ch == "!": if src[pos : pos + 2] == "!=": token = (FineGrammar.BANGEQUAL, pos, 2) else: token = (FineGrammar.BANG, pos, 1) elif ch == "<": if src[pos : pos + 2] == "<=": token = (FineGrammar.LESSEQUAL, pos, 2) else: token = (FineGrammar.LESS, pos, 1) elif ch == ">": if src[pos : pos + 2] == ">=": token = (FineGrammar.GREATEREQUAL, pos, 2) else: token = (FineGrammar.GREATER, pos, 1) elif ch == "+": token = (FineGrammar.PLUS, pos, 1) elif ch == "*": token = (FineGrammar.STAR, pos, 1) elif ch == "/": if src[pos : pos + 2] == "//": while pos < len(src) and src[pos] != "\n": pos = pos + 1 continue token = (FineGrammar.SLASH, pos, 1) elif ch == ".": token = (FineGrammar.DOT, pos, 1) elif ch == "[": token = (FineGrammar.LSQUARE, pos, 1) elif ch == "]": token = (FineGrammar.RSQUARE, pos, 1) elif ch == '"' or ch == "'": end = pos + 1 while end < len(src) and src[end] != ch: if src[end] == "\\": end += 1 end += 1 if end == len(src): raise Exception(f"Unterminated string constant at {pos}") end += 1 token = (FineGrammar.STRING, pos, end - pos) else: number_match = NUMBER_RE.match(src, pos) if number_match: token = (FineGrammar.NUMBER, pos, number_match.end() - pos) else: id_match = IDENTIFIER_RE.match(src, pos) if id_match: fragment = src[pos : id_match.end()] keyword = KEYWORD_TABLE.get(fragment) if keyword: token = (keyword, pos, len(fragment)) else: token = (FineGrammar.IDENTIFIER, pos, len(fragment)) if token is None: raise Exception("Token error") yield token pos += token[2] class FineTokens: def __init__(self, src: str): self.src = src self._tokens: list[typing.Tuple[Terminal, int, int]] = list(tokenize(src)) self._lines = [m.start() for m in re.finditer("\n", src)] def tokens(self): return self._tokens def lines(self): return self._lines def dump(self, *, start=None, end=None): if start is None: start = 0 if end is None: end = len(self._tokens) for token in self._tokens[start:end]: (kind, start, length) = token line_index = bisect.bisect_left(self._lines, start) if line_index == 0: col_start = 0 else: col_start = self._lines[line_index - 1] + 1 column_index = start - col_start value = self.src[start : start + length] print(f"{start:04} {kind.value:12} {value} ({line_index}, {column_index})") if __name__ == "__main__": from parser.parser import compile_lexer, dump_lexer_table grammar = FineGrammar() grammar.build_table() lexer = compile_lexer(grammar) dump_lexer_table(lexer)