From ece5576fb2f0db79e1d04d7acb16368dfcf8b4c1 Mon Sep 17 00:00:00 2001 From: John Doty Date: Mon, 1 Jan 2024 08:07:29 -0800 Subject: [PATCH] [fine] Starting to parse (ugh) --- oden-script/src/lib.rs | 1 + oden-script/src/parser.rs | 360 ++++++++++++++++++++++++++++++++++++++ oden-script/src/tokens.rs | 358 ++++++++++++++++++------------------- 3 files changed, 534 insertions(+), 185 deletions(-) create mode 100644 oden-script/src/parser.rs diff --git a/oden-script/src/lib.rs b/oden-script/src/lib.rs index 5c766355..4144a208 100644 --- a/oden-script/src/lib.rs +++ b/oden-script/src/lib.rs @@ -1 +1,2 @@ +pub mod parser; pub mod tokens; diff --git a/oden-script/src/parser.rs b/oden-script/src/parser.rs new file mode 100644 index 00000000..61283f73 --- /dev/null +++ b/oden-script/src/parser.rs @@ -0,0 +1,360 @@ +use crate::tokens::{Token, TokenKind, Tokens}; +use std::fmt; + +#[derive(PartialEq, Eq)] +pub struct SyntaxError { + pub line: usize, + pub column: usize, + pub message: String, +} + +impl SyntaxError { + pub fn new(line: usize, column: usize, message: String) -> Self { + SyntaxError { + line, + column, + message, + } + } +} + +impl fmt::Debug for SyntaxError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}:{}: {}", self.line, self.column, self.message) + } +} + +impl fmt::Display for SyntaxError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}:{}: {}", self.line, self.column, self.message) + } +} + +pub enum Literal { + Float64(f64), +} + +pub enum UnaryOp { + Negate, +} + +pub enum BinaryOp { + Add, + Subtract, + Mutiply, + Divide, + And, + Or, +} + +pub enum Expr { + Literal(Literal), + Unary(UnaryOp, ExprRef), + Binary(BinaryOp, ExprRef, ExprRef), +} + +pub struct ExprRef(Option); + +impl ExprRef { + pub fn error() -> Self { + ExprRef(None) + } +} + +pub struct SyntaxTree { + pub errors: Vec, + expressions: Vec, +} + +impl SyntaxTree { + pub fn new() -> Self { + SyntaxTree { + errors: Vec::new(), + expressions: Vec::new(), + } + } + + pub fn add_error(&mut self, error: SyntaxError) { + self.errors.push(error); + } + + pub fn add_expr(&mut self, expr: Expr) -> ExprRef { + let index = self.expressions.len(); + self.expressions.push(expr); + ExprRef(Some(index)) + } + + pub fn dump_expr(&self, expr: &ExprRef) -> String { + match expr.0 { + Some(idx) => { + let expr = &self.expressions[idx]; + match expr { + Expr::Literal(lit) => match lit { + Literal::Float64(f) => f.to_string(), + }, + Expr::Unary(op, e) => { + let op = match op { + UnaryOp::Negate => "-", + }; + format!("({op} {})", self.dump_expr(e)) + } + Expr::Binary(op, l, r) => { + let op = match op { + BinaryOp::Add => "+", + BinaryOp::Subtract => "-", + BinaryOp::Mutiply => "*", + BinaryOp::Divide => "/", + BinaryOp::And => "and", + BinaryOp::Or => "or", + }; + format!("({op} {} {})", self.dump_expr(l), self.dump_expr(r)) + } + } + } + None => "<|EOF|>".to_string(), + } + } +} + +// BINDING POWERS. When parsing expressions we only accept expressions that +// meet a minimum binding power. (This is like "precedence" but I just super +// don't like that terminology.) +const ASSIGNMENT_POWER: u8 = 0; // = +const OR_POWER: u8 = 1; // or +const AND_POWER: u8 = 2; // and +const EQUALITY_POWER: u8 = 3; // == != +const COMPARISON_POWER: u8 = 4; // < > <= >= +const TERM_POWER: u8 = 5; // + - +const FACTOR_POWER: u8 = 6; // * / +const UNARY_POWER: u8 = 7; // ! - + +// const CALL_POWER: u8 = 8; // . () +// const PRIMARY_POWER: u8 = 9; + +fn token_power<'a>(token: &Option>) -> Option { + let token = match token { + Some(t) => t, + None => return None, + }; + + match token.kind() { + TokenKind::Equal => Some(ASSIGNMENT_POWER), + TokenKind::Or => Some(OR_POWER), + TokenKind::And => Some(AND_POWER), + TokenKind::EqualEqual | TokenKind::BangEqual => Some(EQUALITY_POWER), + TokenKind::Less | TokenKind::Greater | TokenKind::GreaterEqual | TokenKind::LessEqual => { + Some(COMPARISON_POWER) + } + TokenKind::Plus | TokenKind::Minus => Some(TERM_POWER), + TokenKind::Star | TokenKind::Slash => Some(FACTOR_POWER), + _ => None, + } +} + +pub struct Parser<'a> { + tokens: Tokens<'a>, + tree: SyntaxTree, + current: Option>, + previous: Option>, + + panic_mode: bool, +} + +impl<'a> Parser<'a> { + pub fn new(source: &'a str) -> Self { + let mut parser = Parser { + tokens: Tokens::new(source), + tree: SyntaxTree::new(), + current: None, + previous: None, + panic_mode: false, + }; + parser.advance(); + parser + } + + pub fn parse(mut self) -> (SyntaxTree, ExprRef) { + let expr = self.expression(); + self.consume(None, "expected end of expression"); + (self.tree, expr) + } + + fn expression(&mut self) -> ExprRef { + self.expression_with_power(0) + } + + fn expression_with_power(&mut self, minimum_power: u8) -> ExprRef { + self.advance(); + let mut expr = self.prefix_expression(); + loop { + let power = match token_power(&self.current) { + Some(p) => p, + None => break, // EOF, end of expression? + }; + + if power < minimum_power { + break; + } + + self.advance(); + expr = self.infix_expression(power, expr); + } + expr + } + + fn prefix_expression(&mut self) -> ExprRef { + let token = self.previous.as_ref(); + match token { + Some(token) => match token.kind() { + TokenKind::LeftParen => self.grouping(), + TokenKind::Number => self.number(), + TokenKind::Minus => self.unary(), + _ => { + self.error("expected an expression"); + ExprRef::error() + } + }, + None => { + self.error("expected an expression"); + ExprRef::error() + } + } + } + + fn infix_expression(&mut self, power: u8, left: ExprRef) -> ExprRef { + let kind = self.previous.as_ref().unwrap().kind(); + match kind { + TokenKind::Plus | TokenKind::Minus | TokenKind::Star | TokenKind::Slash => { + self.binary(power, left) + } + _ => panic!("Unknown infix operator, dispatch error?"), + } + } + + fn number(&mut self) -> ExprRef { + let token = self.previous.as_ref().unwrap(); + // What kind is it? For now let's just ... make it good. + + match token.as_str().parse::() { + Ok(v) => self.tree.add_expr(Expr::Literal(Literal::Float64(v))), + Err(e) => { + self.error(format!("invalid f64: {e}")); + ExprRef::error() + } + } + } + + fn grouping(&mut self) -> ExprRef { + let result = self.number(); + self.consume( + Some(TokenKind::RightParen), + "expected ')' after an expression", + ); + result + } + + fn unary(&mut self) -> ExprRef { + let kind = self.previous.as_ref().unwrap().kind(); + let expr = self.expression_with_power(UNARY_POWER); + let op = match kind { + TokenKind::Minus => UnaryOp::Negate, + _ => panic!("unsuitable unary: {:?}: no op", kind), + }; + self.tree.add_expr(Expr::Unary(op, expr)) + } + + fn binary(&mut self, power: u8, left: ExprRef) -> ExprRef { + let right = self.expression_with_power(power + 1); + let op = match self.previous.as_ref().unwrap().kind() { + TokenKind::Plus => BinaryOp::Add, + TokenKind::Minus => BinaryOp::Subtract, + TokenKind::Star => BinaryOp::Mutiply, + TokenKind::Slash => BinaryOp::Divide, + TokenKind::And => BinaryOp::And, + TokenKind::Or => BinaryOp::Or, + _ => panic!("unsuitable binary: {:?}: no op", self.previous), + }; + + self.tree.add_expr(Expr::Binary(op, left, right)) + } + + fn advance(&mut self) { + self.previous = self.current.take(); + loop { + self.current = self.tokens.next(); + match &self.current { + Some(token) if token.kind() == TokenKind::Error => { + self.error_at_current(token.clone()) + } + _ => break, + } + } + } + + fn consume(&mut self, kind: Option, error: &str) { + match (&self.current, kind) { + (Some(token), Some(kind)) if token.kind() == kind => self.advance(), + (None, None) => (), + _ => { + self.error_at_current(error); + } + } + } + + fn error(&mut self, message: T) + where + T: Into, + { + self.error_at(self.previous.clone(), message) + } + + fn error_at_current(&mut self, message: T) + where + T: Into, + { + self.error_at(self.current.clone(), message) + } + + fn error_at(&mut self, token: Option>, message: T) + where + T: Into, + { + if self.panic_mode { + return; + } + self.panic_mode = true; + + let message: String = message.into(); + let (line, column) = self.tokens.token_position(&token); + let mut final_message = "Error ".to_string(); + match token { + None => final_message.push_str("at end"), + Some(t) => { + if t.kind() != TokenKind::Error { + final_message.push_str("at '"); + final_message.push_str(t.as_str()); + final_message.push_str("'"); + } + } + } + final_message.push_str(": "); + final_message.push_str(&message); + + self.tree + .add_error(SyntaxError::new(line, column, final_message)); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + + #[test] + pub fn number_expressions() { + // How am I going to test this? + let (tree, expr) = Parser::new("23.5").parse(); + assert_eq!(Vec::::new(), tree.errors); + assert_eq!("23.5", tree.dump_expr(&expr)); + } +} diff --git a/oden-script/src/tokens.rs b/oden-script/src/tokens.rs index 02979fe5..d0df5685 100644 --- a/oden-script/src/tokens.rs +++ b/oden-script/src/tokens.rs @@ -1,5 +1,5 @@ -#[derive(Debug, PartialEq, Eq)] -pub enum TokenKind<'a> { +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +pub enum TokenKind { LeftBrace, RightBrace, LeftBracket, @@ -23,9 +23,9 @@ pub enum TokenKind<'a> { Less, LessEqual, - Identifier(&'a str), // TODO - String(&'a str), - Number(&'a str), + Identifier, + String, + Number, And, Async, @@ -47,80 +47,54 @@ pub enum TokenKind<'a> { While, Yield, - Error(String), + Error, } -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq, Clone)] pub struct Token<'a> { - kind: TokenKind<'a>, + kind: TokenKind, start: usize, + value: Result<&'a str, String>, } impl<'a> Token<'a> { - pub fn new(start: usize, kind: TokenKind<'a>) -> Self { - Token { kind, start } + pub fn new(kind: TokenKind, start: usize, value: &'a str) -> Self { + Token { + kind, + start, + value: Ok(value), + } + } + + pub fn error(start: usize, message: String) -> Self { + Token { + kind: TokenKind::Error, + start, + value: Err(message), + } + } + + pub fn kind(&self) -> TokenKind { + self.kind } pub fn as_str<'b>(&'b self) -> &'a str where 'b: 'a, { - use TokenKind::*; - match &self.kind { - LeftBrace => "{", - RightBrace => "}", - LeftBracket => "[", - RightBracket => "]", - - LeftParen => "(", - RightParen => ")", - Comma => ",", - Dot => ".", - Minus => "-", - - Plus => "+", - Semicolon => ";", - Slash => "/", - Star => "*", - - Bang => "+", - BangEqual => "!=", - Equal => "=", - EqualEqual => "==", - Greater => ">", - GreaterEqual => ">=", - Less => "<", - LessEqual => "<=", - - Identifier(v) => v, - String(v) => v, - Number(v) => v, - - And => "and", - Async => "async", - Await => "await", - Class => "class", - Else => "else", - False => "false", - For => "for", - From => "from", - Fun => "fun", - If => "if", - Let => "let", - Or => "or", - Print => "print", - Return => "return", - Select => "select", - This => "this", - True => "true", - While => "while", - Yield => "yield", - - Error(e) => e, + match &self.value { + Ok(v) => v, + Err(e) => &e, } } } +impl<'a> Into for Token<'a> { + fn into(self) -> String { + self.as_str().to_string() + } +} + pub struct Tokens<'a> { source: &'a str, chars: std::str::CharIndices<'a>, @@ -140,8 +114,17 @@ impl<'a> Tokens<'a> { result } - pub fn token_position(&self, token: &Token) -> (usize, usize) { - let line_end_index = match self.newlines.binary_search(&token.start) { + /// Return the position of the given token as a (line, column) pair. By + /// convention, lines are 1-based and columns are 0-based. Also, in + /// keeping with the iterator-nature of the tokenizer, `None` here + /// indicates end-of-file, and will return the position of the end of the + /// file. + pub fn token_position(&self, token: &Option) -> (usize, usize) { + let start = match token { + Some(t) => t.start, + None => self.source.len(), + }; + let line_end_index = match self.newlines.binary_search(&start) { Ok(index) => index, Err(index) => index, }; @@ -151,15 +134,16 @@ impl<'a> Tokens<'a> { self.newlines[line_end_index - 1] + 1 }; let line_number = line_end_index + 1; - let column_offset = token.start - line_start_pos; + let column_offset = start - line_start_pos; (line_number, column_offset) } - fn token(&self, start: usize, kind: TokenKind<'a>) -> Token<'a> { - Token::new(start, kind) + fn token(&self, start: usize, kind: TokenKind) -> Token<'a> { + let value = &self.source[start..self.pos()]; + Token::new(kind, start, value) } - fn number(&mut self, start: usize) -> TokenKind<'a> { + fn number(&mut self, start: usize) -> Token<'a> { // First, the main part. loop { if !self.matches_digit() { @@ -198,9 +182,10 @@ impl<'a> Tokens<'a> { if !saw_digit { // This is just a broken number. let slice = &self.source[start..self.pos()]; - return TokenKind::Error(format!( - "Invalid floating-point literal: {slice}" - )); + return Token::error( + start, + format!("Invalid floating-point literal: {slice}"), + ); } } } else { @@ -209,13 +194,13 @@ impl<'a> Tokens<'a> { } } - TokenKind::Number(&self.source[start..self.pos()]) + self.token(start, TokenKind::Number) } - fn string(&mut self, start: usize, delimiter: char) -> TokenKind<'a> { + fn string(&mut self, start: usize, delimiter: char) -> Token<'a> { while !self.matches(delimiter) { if self.eof() { - return TokenKind::Error("Unterminated string constant".to_string()); + return Token::error(start, "Unterminated string constant".to_string()); } if self.matches('\\') { self.advance(); @@ -224,20 +209,12 @@ impl<'a> Tokens<'a> { } } - TokenKind::String(&self.source[start..self.pos()]) + self.token(start, TokenKind::String) } - fn identifier(&mut self, start: usize) -> TokenKind<'a> { - loop { - // TODO: Use unicode identifier classes instead - if !self.matches_next(|c| c.is_ascii_alphanumeric() || c == '_') { - break; - } - } - - let ident = &self.source[start..self.pos()]; - match ident.chars().nth(0) { - Some('a') => { + fn identifier_token_kind(ident: &str) -> TokenKind { + match ident.chars().nth(0).unwrap() { + 'a' => { if ident == "and" { return TokenKind::And; } @@ -248,17 +225,17 @@ impl<'a> Tokens<'a> { return TokenKind::Await; } } - Some('c') => { + 'c' => { if ident == "class" { return TokenKind::Class; } } - Some('e') => { + 'e' => { if ident == "else" { return TokenKind::Else; } } - Some('f') => { + 'f' => { if ident == "false" { return TokenKind::False; } @@ -272,37 +249,37 @@ impl<'a> Tokens<'a> { return TokenKind::Fun; } } - Some('i') => { + 'i' => { if ident == "if" { return TokenKind::If; } } - Some('l') => { + 'l' => { if ident == "let" { return TokenKind::Let; } } - Some('o') => { + 'o' => { if ident == "or" { return TokenKind::Or; } } - Some('p') => { + 'p' => { if ident == "print" { return TokenKind::Print; } } - Some('r') => { + 'r' => { if ident == "return" { return TokenKind::Return; } } - Some('s') => { + 's' => { if ident == "select" { return TokenKind::Select; } } - Some('t') => { + 't' => { if ident == "this" { return TokenKind::This; } @@ -310,12 +287,12 @@ impl<'a> Tokens<'a> { return TokenKind::True; } } - Some('w') => { + 'w' => { if ident == "while" { return TokenKind::While; } } - Some('y') => { + 'y' => { if ident == "yield" { return TokenKind::Yield; } @@ -323,7 +300,20 @@ impl<'a> Tokens<'a> { _ => (), } - TokenKind::Identifier(ident) + TokenKind::Identifier + } + + fn identifier(&mut self, start: usize) -> Token<'a> { + loop { + // TODO: Use unicode identifier classes instead + if !self.matches_next(|c| c.is_ascii_alphanumeric() || c == '_') { + break; + } + } + + let ident = &self.source[start..self.pos()]; + let kind = Self::identifier_token_kind(ident); + Token::new(kind, start, ident) } fn matches(&mut self, ch: char) -> bool { @@ -342,14 +332,9 @@ impl<'a> Tokens<'a> { { if let Some((_, next_ch)) = self.next_char { if f(next_ch) { - eprintln!("MATCHES NEXT: {next_ch}"); self.advance(); return true; - } else { - eprintln!("NOT MATCHES NEXT: {next_ch}"); } - } else { - eprintln!("E O F"); } false } @@ -361,7 +346,6 @@ impl<'a> Tokens<'a> { fn advance(&mut self) -> Option<(usize, char)> { let result = self.next_char; self.next_char = self.chars.next(); - eprintln!("NEXT: {:?}", self.next_char); result } @@ -399,57 +383,57 @@ impl<'a> std::iter::Iterator for Tokens<'a> { }; let token = match c { - '{' => TokenKind::LeftBrace, - '}' => TokenKind::RightBrace, - '[' => TokenKind::LeftBracket, - ']' => TokenKind::RightBracket, - '(' => TokenKind::LeftParen, - ')' => TokenKind::RightParen, - ',' => TokenKind::Comma, - '.' => TokenKind::Dot, + '{' => self.token(pos, TokenKind::LeftBrace), + '}' => self.token(pos, TokenKind::RightBrace), + '[' => self.token(pos, TokenKind::LeftBracket), + ']' => self.token(pos, TokenKind::RightBracket), + '(' => self.token(pos, TokenKind::LeftParen), + ')' => self.token(pos, TokenKind::RightParen), + ',' => self.token(pos, TokenKind::Comma), + '.' => self.token(pos, TokenKind::Dot), '-' => { if self.matches_next(|c| c.is_ascii_digit()) { self.number(pos) } else { - TokenKind::Minus + self.token(pos, TokenKind::Minus) } } '+' => { if self.matches_next(|c| c.is_ascii_digit()) { self.number(pos) } else { - TokenKind::Plus + self.token(pos, TokenKind::Plus) } } - ';' => TokenKind::Semicolon, - '/' => TokenKind::Slash, - '*' => TokenKind::Star, + ';' => self.token(pos, TokenKind::Semicolon), + '/' => self.token(pos, TokenKind::Slash), + '*' => self.token(pos, TokenKind::Star), '!' => { if self.matches('=') { - TokenKind::BangEqual + self.token(pos, TokenKind::BangEqual) } else { - TokenKind::Bang + self.token(pos, TokenKind::Bang) } } '=' => { if self.matches('=') { - TokenKind::EqualEqual + self.token(pos, TokenKind::EqualEqual) } else { - TokenKind::Equal + self.token(pos, TokenKind::Equal) } } '>' => { if self.matches('=') { - TokenKind::GreaterEqual + self.token(pos, TokenKind::GreaterEqual) } else { - TokenKind::Greater + self.token(pos, TokenKind::Greater) } } '<' => { if self.matches('=') { - TokenKind::LessEqual + self.token(pos, TokenKind::LessEqual) } else { - TokenKind::Less + self.token(pos, TokenKind::Less) } } '\'' => self.string(pos, '\''), @@ -460,11 +444,10 @@ impl<'a> std::iter::Iterator for Tokens<'a> { } else if c.is_ascii_alphabetic() || c == '_' { self.identifier(pos) } else { - TokenKind::Error(format!("Unexpected character '{c}'")) + Token::error(pos, format!("Unexpected character '{c}'")) } } }; - let token = self.token(pos, token); Some(token) } } @@ -480,7 +463,12 @@ mod tests { fn $name() { use TokenKind::*; let tokens: Vec<_> = Tokens::new($input).collect(); - let expected = vec![$($s),*]; + + let expected: Vec = (vec![$($s),*]) + .into_iter() + .map(|t| Token::new(t.1, t.0, t.2)) + .collect(); + assert_eq!(expected, tokens); } } @@ -489,81 +477,81 @@ mod tests { test_tokens!( numbers, "1 1.0 1.2e7 2.3e+7 3.3E-06 7_6 8.0e_8", - Token::new(0, Number("1")), - Token::new(2, Number("1.0")), - Token::new(6, Number("1.2e7")), - Token::new(12, Number("2.3e+7")), - Token::new(19, Number("3.3E-06")), - Token::new(27, Number("7_6")), - Token::new(31, Number("8.0e_8")) + (0, Number, "1"), + (2, Number, "1.0"), + (6, Number, "1.2e7"), + (12, Number, "2.3e+7"), + (19, Number, "3.3E-06"), + (27, Number, "7_6"), + (31, Number, "8.0e_8") ); test_tokens!( identifiers, "asdf x _123 a_23 x3a and or yield async await class else false for from", - Token::new(0, Identifier("asdf")), - Token::new(5, Identifier("x")), - Token::new(7, Identifier("_123")), - Token::new(12, Identifier("a_23")), - Token::new(17, Identifier("x3a")), - Token::new(21, And), - Token::new(25, Or), - Token::new(28, Yield), - Token::new(34, Async), - Token::new(40, Await), - Token::new(46, Class), - Token::new(52, Else), - Token::new(57, False), - Token::new(63, For), - Token::new(67, From) + (0, Identifier, "asdf"), + (5, Identifier, "x"), + (7, Identifier, "_123"), + (12, Identifier, "a_23"), + (17, Identifier, "x3a"), + (21, And, "and"), + (25, Or, "or"), + (28, Yield, "yield"), + (34, Async, "async"), + (40, Await, "await"), + (46, Class, "class"), + (52, Else, "else"), + (57, False, "false"), + (63, For, "for"), + (67, From, "from") ); test_tokens!( more_keywords, "fun if let print return select this true while truewhile", - Token::new(0, Fun), - Token::new(4, If), - Token::new(7, Let), - Token::new(11, Print), - Token::new(17, Return), - Token::new(24, Select), - Token::new(31, This), - Token::new(36, True), - Token::new(41, While), - Token::new(47, Identifier("truewhile")) + (0, Fun, "fun"), + (4, If, "if"), + (7, Let, "let"), + (11, Print, "print"), + (17, Return, "return"), + (24, Select, "select"), + (31, This, "this"), + (36, True, "true"), + (41, While, "while"), + (47, Identifier, "truewhile") ); test_tokens!( strings, r#"'this is a string that\'s great!\r\n' "foo's" 'bar"s' "#, - Token::new(0, String(r#"'this is a string that\'s great!\r\n'"#)), - Token::new(38, String(r#""foo's""#)), - Token::new(46, String("'bar\"s'")) + (0, String, r#"'this is a string that\'s great!\r\n'"#), + (38, String, r#""foo's""#), + (46, String, "'bar\"s'") ); test_tokens!( symbols, "{ } ( ) [ ] . ! != < <= > >= = == , - + * / ;", - Token::new(0, LeftBrace), - Token::new(2, RightBrace), - Token::new(4, LeftParen), - Token::new(6, RightParen), - Token::new(8, LeftBracket), - Token::new(10, RightBracket), - Token::new(12, Dot), - Token::new(14, Bang), - Token::new(16, BangEqual), - Token::new(19, Less), - Token::new(21, LessEqual), - Token::new(24, Greater), - Token::new(26, GreaterEqual), - Token::new(29, Equal), - Token::new(31, EqualEqual), - Token::new(34, Comma), - Token::new(36, Minus), - Token::new(38, Plus), - Token::new(40, Star), - Token::new(42, Slash), - Token::new(44, Semicolon) + (0, LeftBrace, "{"), + (2, RightBrace, "}"), + (4, LeftParen, "("), + (6, RightParen, ")"), + (8, LeftBracket, "["), + (10, RightBracket, "]"), + (12, Dot, "."), + (14, Bang, "!"), + (16, BangEqual, "!="), + (19, Less, "<"), + (21, LessEqual, "<="), + (24, Greater, ">"), + (26, GreaterEqual, ">="), + (29, Equal, "="), + (31, EqualEqual, "=="), + (34, Comma, ","), + (36, Minus, "-"), + (38, Plus, "+"), + (40, Star, "*"), + (42, Slash, "/"), + (44, Semicolon, ";") ); }