[fine] Starting to parse (ugh)

oden-script/src/lib.rs

@@ -1 +1,2 @@
+pub mod parser;
 pub mod tokens;

oden-script/src/parser.rs

@@ -0,0 +1,360 @@
+use crate::tokens::{Token, TokenKind, Tokens};
+use std::fmt;
+
+#[derive(PartialEq, Eq)]
+pub struct SyntaxError {
+    pub line: usize,
+    pub column: usize,
+    pub message: String,
+}
+
+impl SyntaxError {
+    pub fn new(line: usize, column: usize, message: String) -> Self {
+        SyntaxError {
+            line,
+            column,
+            message,
+        }
+    }
+}
+
+impl fmt::Debug for SyntaxError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}:{}: {}", self.line, self.column, self.message)
+    }
+}
+
+impl fmt::Display for SyntaxError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}:{}: {}", self.line, self.column, self.message)
+    }
+}
+
+pub enum Literal {
+    Float64(f64),
+}
+
+pub enum UnaryOp {
+    Negate,
+}
+
+pub enum BinaryOp {
+    Add,
+    Subtract,
+    Mutiply,
+    Divide,
+    And,
+    Or,
+}
+
+pub enum Expr {
+    Literal(Literal),
+    Unary(UnaryOp, ExprRef),
+    Binary(BinaryOp, ExprRef, ExprRef),
+}
+
+pub struct ExprRef(Option<usize>);
+
+impl ExprRef {
+    pub fn error() -> Self {
+        ExprRef(None)
+    }
+}
+
+pub struct SyntaxTree {
+    pub errors: Vec<SyntaxError>,
+    expressions: Vec<Expr>,
+}
+
+impl SyntaxTree {
+    pub fn new() -> Self {
+        SyntaxTree {
+            errors: Vec::new(),
+            expressions: Vec::new(),
+        }
+    }
+
+    pub fn add_error(&mut self, error: SyntaxError) {
+        self.errors.push(error);
+    }
+
+    pub fn add_expr(&mut self, expr: Expr) -> ExprRef {
+        let index = self.expressions.len();
+        self.expressions.push(expr);
+        ExprRef(Some(index))
+    }
+
+    pub fn dump_expr(&self, expr: &ExprRef) -> String {
+        match expr.0 {
+            Some(idx) => {
+                let expr = &self.expressions[idx];
+                match expr {
+                    Expr::Literal(lit) => match lit {
+                        Literal::Float64(f) => f.to_string(),
+                    },
+                    Expr::Unary(op, e) => {
+                        let op = match op {
+                            UnaryOp::Negate => "-",
+                        };
+                        format!("({op} {})", self.dump_expr(e))
+                    }
+                    Expr::Binary(op, l, r) => {
+                        let op = match op {
+                            BinaryOp::Add => "+",
+                            BinaryOp::Subtract => "-",
+                            BinaryOp::Mutiply => "*",
+                            BinaryOp::Divide => "/",
+                            BinaryOp::And => "and",
+                            BinaryOp::Or => "or",
+                        };
+                        format!("({op} {} {})", self.dump_expr(l), self.dump_expr(r))
+                    }
+                }
+            }
+            None => "<|EOF|>".to_string(),
+        }
+    }
+}
+
+// BINDING POWERS. When parsing expressions we only accept expressions that
+// meet a minimum binding power. (This is like "precedence" but I just super
+// don't like that terminology.)
+const ASSIGNMENT_POWER: u8 = 0; // =
+const OR_POWER: u8 = 1; // or
+const AND_POWER: u8 = 2; // and
+const EQUALITY_POWER: u8 = 3; // == !=
+const COMPARISON_POWER: u8 = 4; // < > <= >=
+const TERM_POWER: u8 = 5; // + -
+const FACTOR_POWER: u8 = 6; // * /
+const UNARY_POWER: u8 = 7; // ! -
+
+// const CALL_POWER: u8 = 8; // . ()
+// const PRIMARY_POWER: u8 = 9;
+
+fn token_power<'a>(token: &Option<Token<'a>>) -> Option<u8> {
+    let token = match token {
+        Some(t) => t,
+        None => return None,
+    };
+
+    match token.kind() {
+        TokenKind::Equal => Some(ASSIGNMENT_POWER),
+        TokenKind::Or => Some(OR_POWER),
+        TokenKind::And => Some(AND_POWER),
+        TokenKind::EqualEqual | TokenKind::BangEqual => Some(EQUALITY_POWER),
+        TokenKind::Less | TokenKind::Greater | TokenKind::GreaterEqual | TokenKind::LessEqual => {
+            Some(COMPARISON_POWER)
+        }
+        TokenKind::Plus | TokenKind::Minus => Some(TERM_POWER),
+        TokenKind::Star | TokenKind::Slash => Some(FACTOR_POWER),
+        _ => None,
+    }
+}
+
+pub struct Parser<'a> {
+    tokens: Tokens<'a>,
+    tree: SyntaxTree,
+    current: Option<Token<'a>>,
+    previous: Option<Token<'a>>,
+
+    panic_mode: bool,
+}
+
+impl<'a> Parser<'a> {
+    pub fn new(source: &'a str) -> Self {
+        let mut parser = Parser {
+            tokens: Tokens::new(source),
+            tree: SyntaxTree::new(),
+            current: None,
+            previous: None,
+            panic_mode: false,
+        };
+        parser.advance();
+        parser
+    }
+
+    pub fn parse(mut self) -> (SyntaxTree, ExprRef) {
+        let expr = self.expression();
+        self.consume(None, "expected end of expression");
+        (self.tree, expr)
+    }
+
+    fn expression(&mut self) -> ExprRef {
+        self.expression_with_power(0)
+    }
+
+    fn expression_with_power(&mut self, minimum_power: u8) -> ExprRef {
+        self.advance();
+        let mut expr = self.prefix_expression();
+        loop {
+            let power = match token_power(&self.current) {
+                Some(p) => p,
+                None => break, // EOF, end of expression?
+            };
+
+            if power < minimum_power {
+                break;
+            }
+
+            self.advance();
+            expr = self.infix_expression(power, expr);
+        }
+        expr
+    }
+
+    fn prefix_expression(&mut self) -> ExprRef {
+        let token = self.previous.as_ref();
+        match token {
+            Some(token) => match token.kind() {
+                TokenKind::LeftParen => self.grouping(),
+                TokenKind::Number => self.number(),
+                TokenKind::Minus => self.unary(),
+                _ => {
+                    self.error("expected an expression");
+                    ExprRef::error()
+                }
+            },
+            None => {
+                self.error("expected an expression");
+                ExprRef::error()
+            }
+        }
+    }
+
+    fn infix_expression(&mut self, power: u8, left: ExprRef) -> ExprRef {
+        let kind = self.previous.as_ref().unwrap().kind();
+        match kind {
+            TokenKind::Plus | TokenKind::Minus | TokenKind::Star | TokenKind::Slash => {
+                self.binary(power, left)
+            }
+            _ => panic!("Unknown infix operator, dispatch error?"),
+        }
+    }
+
+    fn number(&mut self) -> ExprRef {
+        let token = self.previous.as_ref().unwrap();
+        // What kind is it? For now let's just ... make it good.
+
+        match token.as_str().parse::<f64>() {
+            Ok(v) => self.tree.add_expr(Expr::Literal(Literal::Float64(v))),
+            Err(e) => {
+                self.error(format!("invalid f64: {e}"));
+                ExprRef::error()
+            }
+        }
+    }
+
+    fn grouping(&mut self) -> ExprRef {
+        let result = self.number();
+        self.consume(
+            Some(TokenKind::RightParen),
+            "expected ')' after an expression",
+        );
+        result
+    }
+
+    fn unary(&mut self) -> ExprRef {
+        let kind = self.previous.as_ref().unwrap().kind();
+        let expr = self.expression_with_power(UNARY_POWER);
+        let op = match kind {
+            TokenKind::Minus => UnaryOp::Negate,
+            _ => panic!("unsuitable unary: {:?}: no op", kind),
+        };
+        self.tree.add_expr(Expr::Unary(op, expr))
+    }
+
+    fn binary(&mut self, power: u8, left: ExprRef) -> ExprRef {
+        let right = self.expression_with_power(power + 1);
+        let op = match self.previous.as_ref().unwrap().kind() {
+            TokenKind::Plus => BinaryOp::Add,
+            TokenKind::Minus => BinaryOp::Subtract,
+            TokenKind::Star => BinaryOp::Mutiply,
+            TokenKind::Slash => BinaryOp::Divide,
+            TokenKind::And => BinaryOp::And,
+            TokenKind::Or => BinaryOp::Or,
+            _ => panic!("unsuitable binary: {:?}: no op", self.previous),
+        };
+
+        self.tree.add_expr(Expr::Binary(op, left, right))
+    }
+
+    fn advance(&mut self) {
+        self.previous = self.current.take();
+        loop {
+            self.current = self.tokens.next();
+            match &self.current {
+                Some(token) if token.kind() == TokenKind::Error => {
+                    self.error_at_current(token.clone())
+                }
+                _ => break,
+            }
+        }
+    }
+
+    fn consume(&mut self, kind: Option<TokenKind>, error: &str) {
+        match (&self.current, kind) {
+            (Some(token), Some(kind)) if token.kind() == kind => self.advance(),
+            (None, None) => (),
+            _ => {
+                self.error_at_current(error);
+            }
+        }
+    }
+
+    fn error<T>(&mut self, message: T)
+    where
+        T: Into<String>,
+    {
+        self.error_at(self.previous.clone(), message)
+    }
+
+    fn error_at_current<T>(&mut self, message: T)
+    where
+        T: Into<String>,
+    {
+        self.error_at(self.current.clone(), message)
+    }
+
+    fn error_at<T>(&mut self, token: Option<Token<'a>>, message: T)
+    where
+        T: Into<String>,
+    {
+        if self.panic_mode {
+            return;
+        }
+        self.panic_mode = true;
+
+        let message: String = message.into();
+        let (line, column) = self.tokens.token_position(&token);
+        let mut final_message = "Error ".to_string();
+        match token {
+            None => final_message.push_str("at end"),
+            Some(t) => {
+                if t.kind() != TokenKind::Error {
+                    final_message.push_str("at '");
+                    final_message.push_str(t.as_str());
+                    final_message.push_str("'");
+                }
+            }
+        }
+        final_message.push_str(": ");
+        final_message.push_str(&message);
+
+        self.tree
+            .add_error(SyntaxError::new(line, column, final_message));
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use pretty_assertions::assert_eq;
+
+    #[test]
+    pub fn number_expressions() {
+        // How am I going to test this?
+        let (tree, expr) = Parser::new("23.5").parse();
+        assert_eq!(Vec::<SyntaxError>::new(), tree.errors);
+        assert_eq!("23.5", tree.dump_expr(&expr));
+    }
+}

oden-script/src/tokens.rs

@@ -1,5 +1,5 @@
-#[derive(Debug, PartialEq, Eq)]
-pub enum TokenKind<'a> {
+#[derive(Debug, PartialEq, Eq, Clone, Copy)]
+pub enum TokenKind {
     LeftBrace,
     RightBrace,
     LeftBracket,
@@ -23,9 +23,9 @@ pub enum TokenKind<'a> {
     Less,
     LessEqual,
 
-    Identifier(&'a str), // TODO
-    String(&'a str),
-    Number(&'a str),
+    Identifier,
+    String,
+    Number,
 
     And,
     Async,
@@ -47,80 +47,54 @@ pub enum TokenKind<'a> {
     While,
     Yield,
 
-    Error(String),
+    Error,
 }
 
-#[derive(Debug, PartialEq, Eq)]
+#[derive(Debug, PartialEq, Eq, Clone)]
 pub struct Token<'a> {
-    kind: TokenKind<'a>,
+    kind: TokenKind,
     start: usize,
+    value: Result<&'a str, String>,
 }
 
 impl<'a> Token<'a> {
-    pub fn new(start: usize, kind: TokenKind<'a>) -> Self {
-        Token { kind, start }
+    pub fn new(kind: TokenKind, start: usize, value: &'a str) -> Self {
+        Token {
+            kind,
+            start,
+            value: Ok(value),
+        }
+    }
+
+    pub fn error(start: usize, message: String) -> Self {
+        Token {
+            kind: TokenKind::Error,
+            start,
+            value: Err(message),
+        }
+    }
+
+    pub fn kind(&self) -> TokenKind {
+        self.kind
     }
 
     pub fn as_str<'b>(&'b self) -> &'a str
     where
         'b: 'a,
     {
-        use TokenKind::*;
-        match &self.kind {
-            LeftBrace => "{",
-            RightBrace => "}",
-            LeftBracket => "[",
-            RightBracket => "]",
-
-            LeftParen => "(",
-            RightParen => ")",
-            Comma => ",",
-            Dot => ".",
-            Minus => "-",
-
-            Plus => "+",
-            Semicolon => ";",
-            Slash => "/",
-            Star => "*",
-
-            Bang => "+",
-            BangEqual => "!=",
-            Equal => "=",
-            EqualEqual => "==",
-            Greater => ">",
-            GreaterEqual => ">=",
-            Less => "<",
-            LessEqual => "<=",
-
-            Identifier(v) => v,
-            String(v) => v,
-            Number(v) => v,
-
-            And => "and",
-            Async => "async",
-            Await => "await",
-            Class => "class",
-            Else => "else",
-            False => "false",
-            For => "for",
-            From => "from",
-            Fun => "fun",
-            If => "if",
-            Let => "let",
-            Or => "or",
-            Print => "print",
-            Return => "return",
-            Select => "select",
-            This => "this",
-            True => "true",
-            While => "while",
-            Yield => "yield",
-
-            Error(e) => e,
+        match &self.value {
+            Ok(v) => v,
+            Err(e) => &e,
         }
     }
 }
 
+impl<'a> Into<String> for Token<'a> {
+    fn into(self) -> String {
+        self.as_str().to_string()
+    }
+}
+
 pub struct Tokens<'a> {
     source: &'a str,
     chars: std::str::CharIndices<'a>,
@@ -140,8 +114,17 @@ impl<'a> Tokens<'a> {
         result
     }
 
-    pub fn token_position(&self, token: &Token) -> (usize, usize) {
-        let line_end_index = match self.newlines.binary_search(&token.start) {
+    /// Return the position of the given token as a (line, column) pair. By
+    /// convention, lines are 1-based and columns are 0-based. Also, in
+    /// keeping with the iterator-nature of the tokenizer, `None` here
+    /// indicates end-of-file, and will return the position of the end of the
+    /// file.
+    pub fn token_position(&self, token: &Option<Token>) -> (usize, usize) {
+        let start = match token {
+            Some(t) => t.start,
+            None => self.source.len(),
+        };
+        let line_end_index = match self.newlines.binary_search(&start) {
             Ok(index) => index,
             Err(index) => index,
         };
@@ -151,15 +134,16 @@ impl<'a> Tokens<'a> {
             self.newlines[line_end_index - 1] + 1
         };
         let line_number = line_end_index + 1;
-        let column_offset = token.start - line_start_pos;
+        let column_offset = start - line_start_pos;
         (line_number, column_offset)
     }
 
-    fn token(&self, start: usize, kind: TokenKind<'a>) -> Token<'a> {
-        Token::new(start, kind)
+    fn token(&self, start: usize, kind: TokenKind) -> Token<'a> {
+        let value = &self.source[start..self.pos()];
+        Token::new(kind, start, value)
     }
 
-    fn number(&mut self, start: usize) -> TokenKind<'a> {
+    fn number(&mut self, start: usize) -> Token<'a> {
         // First, the main part.
         loop {
             if !self.matches_digit() {
@@ -198,9 +182,10 @@ impl<'a> Tokens<'a> {
                     if !saw_digit {
                         // This is just a broken number.
                         let slice = &self.source[start..self.pos()];
-                        return TokenKind::Error(format!(
-                            "Invalid floating-point literal: {slice}"
-                        ));
+                        return Token::error(
+                            start,
+                            format!("Invalid floating-point literal: {slice}"),
+                        );
                     }
                 }
             } else {
@@ -209,13 +194,13 @@ impl<'a> Tokens<'a> {
             }
         }
 
-        TokenKind::Number(&self.source[start..self.pos()])
+        self.token(start, TokenKind::Number)
     }
 
-    fn string(&mut self, start: usize, delimiter: char) -> TokenKind<'a> {
+    fn string(&mut self, start: usize, delimiter: char) -> Token<'a> {
         while !self.matches(delimiter) {
             if self.eof() {
-                return TokenKind::Error("Unterminated string constant".to_string());
+                return Token::error(start, "Unterminated string constant".to_string());
             }
             if self.matches('\\') {
                 self.advance();
@@ -224,20 +209,12 @@ impl<'a> Tokens<'a> {
             }
         }
 
-        TokenKind::String(&self.source[start..self.pos()])
+        self.token(start, TokenKind::String)
     }
 
-    fn identifier(&mut self, start: usize) -> TokenKind<'a> {
-        loop {
-            // TODO: Use unicode identifier classes instead
-            if !self.matches_next(|c| c.is_ascii_alphanumeric() || c == '_') {
-                break;
-            }
-        }
-
-        let ident = &self.source[start..self.pos()];
-        match ident.chars().nth(0) {
-            Some('a') => {
+    fn identifier_token_kind(ident: &str) -> TokenKind {
+        match ident.chars().nth(0).unwrap() {
+            'a' => {
                 if ident == "and" {
                     return TokenKind::And;
                 }
@@ -248,17 +225,17 @@ impl<'a> Tokens<'a> {
                     return TokenKind::Await;
                 }
             }
-            Some('c') => {
+            'c' => {
                 if ident == "class" {
                     return TokenKind::Class;
                 }
             }
-            Some('e') => {
+            'e' => {
                 if ident == "else" {
                     return TokenKind::Else;
                 }
             }
-            Some('f') => {
+            'f' => {
                 if ident == "false" {
                     return TokenKind::False;
                 }
@@ -272,37 +249,37 @@ impl<'a> Tokens<'a> {
                     return TokenKind::Fun;
                 }
             }
-            Some('i') => {
+            'i' => {
                 if ident == "if" {
                     return TokenKind::If;
                 }
             }
-            Some('l') => {
+            'l' => {
                 if ident == "let" {
                     return TokenKind::Let;
                 }
             }
-            Some('o') => {
+            'o' => {
                 if ident == "or" {
                     return TokenKind::Or;
                 }
             }
-            Some('p') => {
+            'p' => {
                 if ident == "print" {
                     return TokenKind::Print;
                 }
             }
-            Some('r') => {
+            'r' => {
                 if ident == "return" {
                     return TokenKind::Return;
                 }
             }
-            Some('s') => {
+            's' => {
                 if ident == "select" {
                     return TokenKind::Select;
                 }
             }
-            Some('t') => {
+            't' => {
                 if ident == "this" {
                     return TokenKind::This;
                 }
@@ -310,12 +287,12 @@ impl<'a> Tokens<'a> {
                     return TokenKind::True;
                 }
             }
-            Some('w') => {
+            'w' => {
                 if ident == "while" {
                     return TokenKind::While;
                 }
             }
-            Some('y') => {
+            'y' => {
                 if ident == "yield" {
                     return TokenKind::Yield;
                 }
@@ -323,7 +300,20 @@ impl<'a> Tokens<'a> {
             _ => (),
         }
 
-        TokenKind::Identifier(ident)
+        TokenKind::Identifier
+    }
+
+    fn identifier(&mut self, start: usize) -> Token<'a> {
+        loop {
+            // TODO: Use unicode identifier classes instead
+            if !self.matches_next(|c| c.is_ascii_alphanumeric() || c == '_') {
+                break;
+            }
+        }
+
+        let ident = &self.source[start..self.pos()];
+        let kind = Self::identifier_token_kind(ident);
+        Token::new(kind, start, ident)
     }
 
     fn matches(&mut self, ch: char) -> bool {
@@ -342,14 +332,9 @@ impl<'a> Tokens<'a> {
     {
         if let Some((_, next_ch)) = self.next_char {
             if f(next_ch) {
-                eprintln!("MATCHES NEXT: {next_ch}");
                 self.advance();
                 return true;
-            } else {
-                eprintln!("NOT MATCHES NEXT: {next_ch}");
             }
-        } else {
-            eprintln!("E O F");
         }
         false
     }
@@ -361,7 +346,6 @@ impl<'a> Tokens<'a> {
     fn advance(&mut self) -> Option<(usize, char)> {
         let result = self.next_char;
         self.next_char = self.chars.next();
-        eprintln!("NEXT: {:?}", self.next_char);
         result
     }
 
@@ -399,57 +383,57 @@ impl<'a> std::iter::Iterator for Tokens<'a> {
         };
 
         let token = match c {
-            '{' => TokenKind::LeftBrace,
-            '}' => TokenKind::RightBrace,
-            '[' => TokenKind::LeftBracket,
-            ']' => TokenKind::RightBracket,
-            '(' => TokenKind::LeftParen,
-            ')' => TokenKind::RightParen,
-            ',' => TokenKind::Comma,
-            '.' => TokenKind::Dot,
+            '{' => self.token(pos, TokenKind::LeftBrace),
+            '}' => self.token(pos, TokenKind::RightBrace),
+            '[' => self.token(pos, TokenKind::LeftBracket),
+            ']' => self.token(pos, TokenKind::RightBracket),
+            '(' => self.token(pos, TokenKind::LeftParen),
+            ')' => self.token(pos, TokenKind::RightParen),
+            ',' => self.token(pos, TokenKind::Comma),
+            '.' => self.token(pos, TokenKind::Dot),
             '-' => {
                 if self.matches_next(|c| c.is_ascii_digit()) {
                     self.number(pos)
                 } else {
-                    TokenKind::Minus
+                    self.token(pos, TokenKind::Minus)
                 }
             }
             '+' => {
                 if self.matches_next(|c| c.is_ascii_digit()) {
                     self.number(pos)
                 } else {
-                    TokenKind::Plus
+                    self.token(pos, TokenKind::Plus)
                 }
             }
-            ';' => TokenKind::Semicolon,
-            '/' => TokenKind::Slash,
-            '*' => TokenKind::Star,
+            ';' => self.token(pos, TokenKind::Semicolon),
+            '/' => self.token(pos, TokenKind::Slash),
+            '*' => self.token(pos, TokenKind::Star),
             '!' => {
                 if self.matches('=') {
-                    TokenKind::BangEqual
+                    self.token(pos, TokenKind::BangEqual)
                 } else {
-                    TokenKind::Bang
+                    self.token(pos, TokenKind::Bang)
                 }
             }
             '=' => {
                 if self.matches('=') {
-                    TokenKind::EqualEqual
+                    self.token(pos, TokenKind::EqualEqual)
                 } else {
-                    TokenKind::Equal
+                    self.token(pos, TokenKind::Equal)
                 }
             }
             '>' => {
                 if self.matches('=') {
-                    TokenKind::GreaterEqual
+                    self.token(pos, TokenKind::GreaterEqual)
                 } else {
-                    TokenKind::Greater
+                    self.token(pos, TokenKind::Greater)
                 }
             }
             '<' => {
                 if self.matches('=') {
-                    TokenKind::LessEqual
+                    self.token(pos, TokenKind::LessEqual)
                 } else {
-                    TokenKind::Less
+                    self.token(pos, TokenKind::Less)
                 }
             }
             '\'' => self.string(pos, '\''),
@@ -460,11 +444,10 @@ impl<'a> std::iter::Iterator for Tokens<'a> {
                 } else if c.is_ascii_alphabetic() || c == '_' {
                     self.identifier(pos)
                 } else {
-                    TokenKind::Error(format!("Unexpected character '{c}'"))
+                    Token::error(pos, format!("Unexpected character '{c}'"))
                 }
             }
         };
-        let token = self.token(pos, token);
         Some(token)
     }
 }
@@ -480,7 +463,12 @@ mod tests {
             fn $name() {
                 use TokenKind::*;
                 let tokens: Vec<_> = Tokens::new($input).collect();
-                let expected = vec![$($s),*];
+
+                let expected: Vec<Token> = (vec![$($s),*])
+                    .into_iter()
+                    .map(|t| Token::new(t.1, t.0, t.2))
+                    .collect();
+
                 assert_eq!(expected, tokens);
             }
         }
@@ -489,81 +477,81 @@ mod tests {
     test_tokens!(
         numbers,
         "1 1.0 1.2e7 2.3e+7 3.3E-06 7_6 8.0e_8",
-        Token::new(0, Number("1")),
-        Token::new(2, Number("1.0")),
-        Token::new(6, Number("1.2e7")),
-        Token::new(12, Number("2.3e+7")),
-        Token::new(19, Number("3.3E-06")),
-        Token::new(27, Number("7_6")),
-        Token::new(31, Number("8.0e_8"))
+        (0, Number, "1"),
+        (2, Number, "1.0"),
+        (6, Number, "1.2e7"),
+        (12, Number, "2.3e+7"),
+        (19, Number, "3.3E-06"),
+        (27, Number, "7_6"),
+        (31, Number, "8.0e_8")
     );
 
     test_tokens!(
         identifiers,
         "asdf x _123 a_23 x3a and or yield async await class else false for from",
-        Token::new(0, Identifier("asdf")),
-        Token::new(5, Identifier("x")),
-        Token::new(7, Identifier("_123")),
-        Token::new(12, Identifier("a_23")),
-        Token::new(17, Identifier("x3a")),
-        Token::new(21, And),
-        Token::new(25, Or),
-        Token::new(28, Yield),
-        Token::new(34, Async),
-        Token::new(40, Await),
-        Token::new(46, Class),
-        Token::new(52, Else),
-        Token::new(57, False),
-        Token::new(63, For),
-        Token::new(67, From)
+        (0, Identifier, "asdf"),
+        (5, Identifier, "x"),
+        (7, Identifier, "_123"),
+        (12, Identifier, "a_23"),
+        (17, Identifier, "x3a"),
+        (21, And, "and"),
+        (25, Or, "or"),
+        (28, Yield, "yield"),
+        (34, Async, "async"),
+        (40, Await, "await"),
+        (46, Class, "class"),
+        (52, Else, "else"),
+        (57, False, "false"),
+        (63, For, "for"),
+        (67, From, "from")
     );
 
     test_tokens!(
         more_keywords,
         "fun if let print return select this true while truewhile",
-        Token::new(0, Fun),
-        Token::new(4, If),
-        Token::new(7, Let),
-        Token::new(11, Print),
-        Token::new(17, Return),
-        Token::new(24, Select),
-        Token::new(31, This),
-        Token::new(36, True),
-        Token::new(41, While),
-        Token::new(47, Identifier("truewhile"))
+        (0, Fun, "fun"),
+        (4, If, "if"),
+        (7, Let, "let"),
+        (11, Print, "print"),
+        (17, Return, "return"),
+        (24, Select, "select"),
+        (31, This, "this"),
+        (36, True, "true"),
+        (41, While, "while"),
+        (47, Identifier, "truewhile")
     );
 
     test_tokens!(
         strings,
         r#"'this is a string that\'s great!\r\n' "foo's" 'bar"s' "#,
-        Token::new(0, String(r#"'this is a string that\'s great!\r\n'"#)),
-        Token::new(38, String(r#""foo's""#)),
-        Token::new(46, String("'bar\"s'"))
+        (0, String, r#"'this is a string that\'s great!\r\n'"#),
+        (38, String, r#""foo's""#),
+        (46, String, "'bar\"s'")
     );
 
     test_tokens!(
         symbols,
         "{ } ( ) [ ] . ! != < <= > >= = == , - + * / ;",
-        Token::new(0, LeftBrace),
-        Token::new(2, RightBrace),
-        Token::new(4, LeftParen),
-        Token::new(6, RightParen),
-        Token::new(8, LeftBracket),
-        Token::new(10, RightBracket),
-        Token::new(12, Dot),
-        Token::new(14, Bang),
-        Token::new(16, BangEqual),
-        Token::new(19, Less),
-        Token::new(21, LessEqual),
-        Token::new(24, Greater),
-        Token::new(26, GreaterEqual),
-        Token::new(29, Equal),
-        Token::new(31, EqualEqual),
-        Token::new(34, Comma),
-        Token::new(36, Minus),
-        Token::new(38, Plus),
-        Token::new(40, Star),
-        Token::new(42, Slash),
-        Token::new(44, Semicolon)
+        (0, LeftBrace, "{"),
+        (2, RightBrace, "}"),
+        (4, LeftParen, "("),
+        (6, RightParen, ")"),
+        (8, LeftBracket, "["),
+        (10, RightBracket, "]"),
+        (12, Dot, "."),
+        (14, Bang, "!"),
+        (16, BangEqual, "!="),
+        (19, Less, "<"),
+        (21, LessEqual, "<="),
+        (24, Greater, ">"),
+        (26, GreaterEqual, ">="),
+        (29, Equal, "="),
+        (31, EqualEqual, "=="),
+        (34, Comma, ","),
+        (36, Minus, "-"),
+        (38, Plus, "+"),
+        (40, Star, "*"),
+        (42, Slash, "/"),
+        (44, Semicolon, ";")
     );
 }