From d2d144a5ecbea7cb93415cb170324c6b333dedeb Mon Sep 17 00:00:00 2001 From: John Doty Date: Wed, 3 Jan 2024 06:18:26 -0800 Subject: [PATCH] [fine] Tokens is not Iterator It was not pulling its weight --- fine/src/parser.rs | 189 ++++++++++++++++++--------------------------- fine/src/tokens.rs | 70 ++++++++--------- 2 files changed, 107 insertions(+), 152 deletions(-) diff --git a/fine/src/parser.rs b/fine/src/parser.rs index a962dcd3..da900c5b 100644 --- a/fine/src/parser.rs +++ b/fine/src/parser.rs @@ -267,7 +267,7 @@ impl<'a> SyntaxTree<'a> { // This is dumb and should be punished, probably. (_, Type::Unreachable) => { - let (line, col) = lines.position(tok.start()); + let (line, col) = lines.position(tok.start); self.errors.push(SyntaxError::new(line, col, format!("cannot apply a unary operator to something that doesn't yield a value"))); Type::Error } @@ -277,7 +277,7 @@ impl<'a> SyntaxTree<'a> { // Missed the whole table, must be an error. (_, arg_type) => { - let (line, col) = lines.position(tok.start()); + let (line, col) = lines.position(tok.start); self.errors.push(SyntaxError::new(line, col, format!("cannot apply unary operator '{tok}' to expression of type '{arg_type}'"))); Type::Error } @@ -305,7 +305,7 @@ impl<'a> SyntaxTree<'a> { // This is dumb and should be punished, probably. (_, _, Type::Unreachable) => { - let (line, col) = lines.position(tok.start()); + let (line, col) = lines.position(tok.start); self.errors.push(SyntaxError::new( line, col, @@ -316,7 +316,7 @@ impl<'a> SyntaxTree<'a> { Type::Error } (_, Type::Unreachable, _) => { - let (line, col) = lines.position(tok.start()); + let (line, col) = lines.position(tok.start); self.errors.push(SyntaxError::new( line, col, @@ -333,7 +333,7 @@ impl<'a> SyntaxTree<'a> { // Missed the whole table, it must be an error. (_, left_type, right_type) => { - let (line, col) = lines.position(tok.start()); + let (line, col) = lines.position(tok.start); self.errors.push(SyntaxError::new(line, col, format!("cannot apply binary operator '{tok}' to expressions of type '{left_type}' (on the left) and '{right_type}' (on the right)"))); Type::Error } @@ -354,8 +354,8 @@ impl<'a> SyntaxTree<'a> { .expr_span(&cond) .expect("If the expression has a type it must have a span"); - let start = lines.position(span.0.start()); - let end = lines.position(span.1.start()); + let start = lines.position(span.0.start); + let end = lines.position(span.1.start); self.errors.push(SyntaxError::new_spanned( start, end, @@ -374,8 +374,8 @@ impl<'a> SyntaxTree<'a> { let span = self .expr_span(&exr) .expect("How did I get this far with a broken parse?"); - let start = lines.position(span.0.start()); - let end = lines.position(span.1.start()); + let start = lines.position(span.0.start); + let end = lines.position(span.1.start); self.errors.push(SyntaxError::new_spanned( start, end, @@ -392,8 +392,8 @@ impl<'a> SyntaxTree<'a> { let span = self .expr_span(&exr) .expect("How did I get this far with a broken parse?"); - let start = lines.position(span.0.start()); - let end = lines.position(span.1.start()); + let start = lines.position(span.0.start); + let end = lines.position(span.1.start); self.errors.push(SyntaxError::new_spanned( start, end, @@ -431,13 +431,8 @@ const UNARY_POWER: u8 = 7; // ! - // const CALL_POWER: u8 = 8; // . () // const PRIMARY_POWER: u8 = 9; -fn token_power<'a>(token: &Option>) -> Option { - let token = match token { - Some(t) => t, - None => return None, - }; - - match token.kind() { +fn token_power<'a>(token: &Token<'a>) -> Option { + match token.kind { TokenKind::Equal => Some(ASSIGNMENT_POWER), TokenKind::Or => Some(OR_POWER), TokenKind::And => Some(AND_POWER), @@ -454,8 +449,8 @@ fn token_power<'a>(token: &Option>) -> Option { pub struct Parser<'a> { tokens: Tokens<'a>, tree: SyntaxTree<'a>, - current: Option>, - previous: Option>, + current: Token<'a>, + previous: Token<'a>, panic_mode: bool, } @@ -465,8 +460,8 @@ impl<'a> Parser<'a> { let mut parser = Parser { tokens: Tokens::new(source), tree: SyntaxTree::new(), - current: None, - previous: None, + current: Token::new(TokenKind::EOF, 0, ""), + previous: Token::new(TokenKind::EOF, 0, ""), panic_mode: false, }; parser.advance(); @@ -475,7 +470,7 @@ impl<'a> Parser<'a> { pub fn parse(mut self) -> (SyntaxTree<'a>, ExprRef, Lines) { let expr = self.expression(); - self.consume(None, "expected end of expression"); + self.consume(TokenKind::EOF, "expected end of expression"); (self.tree, expr, self.tokens.lines()) } @@ -505,30 +500,24 @@ impl<'a> Parser<'a> { fn prefix_expression(&mut self) -> ExprRef { self.trace("prefix"); - let token = self.previous.as_ref(); - match token { - Some(token) => match token.kind() { - TokenKind::Bang => self.unary(), - TokenKind::LeftParen => self.grouping(), - TokenKind::Number => self.number(), - TokenKind::Minus => self.unary(), - TokenKind::String => self.string(), + let token = &self.previous; + match token.kind { + TokenKind::Bang => self.unary(), + TokenKind::LeftParen => self.grouping(), + TokenKind::Number => self.number(), + TokenKind::Minus => self.unary(), + TokenKind::String => self.string(), - TokenKind::True => self - .tree - .add_expr(Expr::Literal(Literal::Bool(true), token.clone())), - TokenKind::False => self - .tree - .add_expr(Expr::Literal(Literal::Bool(false), token.clone())), + TokenKind::True => self + .tree + .add_expr(Expr::Literal(Literal::Bool(true), token.clone())), + TokenKind::False => self + .tree + .add_expr(Expr::Literal(Literal::Bool(false), token.clone())), - TokenKind::If => self.conditional(), + TokenKind::If => self.conditional(), - _ => { - self.error("expected an expression"); - ExprRef::error() - } - }, - None => { + _ => { self.error("expected an expression"); ExprRef::error() } @@ -537,8 +526,7 @@ impl<'a> Parser<'a> { fn infix_expression(&mut self, power: u8, left: ExprRef) -> ExprRef { self.trace("infix"); - let kind = self.previous.as_ref().unwrap().kind(); - match kind { + match self.previous.kind { TokenKind::Plus | TokenKind::Minus | TokenKind::Star @@ -550,7 +538,7 @@ impl<'a> Parser<'a> { } fn number(&mut self) -> ExprRef { - let token = self.previous.as_ref().unwrap(); + let token = &self.previous; // What kind is it? For now let's just ... make it good. let literal = match token.as_str().parse::() { @@ -565,7 +553,7 @@ impl<'a> Parser<'a> { } fn string(&mut self) -> ExprRef { - let token = self.previous.as_ref().unwrap(); + let token = &self.previous; let mut result = String::new(); let mut input = token.as_str().chars(); @@ -590,51 +578,34 @@ impl<'a> Parser<'a> { fn grouping(&mut self) -> ExprRef { let result = self.expression(); - self.consume( - Some(TokenKind::RightParen), - "expected ')' after an expression", - ); + self.consume(TokenKind::RightParen, "expected ')' after an expression"); result } fn conditional(&mut self) -> ExprRef { - let token = self.previous.as_ref().unwrap().clone(); + let token = self.previous.clone(); let condition_expr = self.expression(); - self.consume( - Some(TokenKind::LeftBrace), - "expected '{' to start an 'if' block", - ); + self.consume(TokenKind::LeftBrace, "expected '{' to start an 'if' block"); let then_expr = self.expression(); - self.consume( - Some(TokenKind::RightBrace), - "expected '}' to end an 'if' block", - ); - let else_expr = match &self.current { - Some(token) if token.kind() == TokenKind::Else => { + self.consume(TokenKind::RightBrace, "expected '}' to end an 'if' block"); + let else_expr = if self.current.kind == TokenKind::Else { + self.advance(); + if self.current.kind == TokenKind::If { self.advance(); - match &self.current { - // Allow `else if` without another `{`. - Some(token) if token.kind() == TokenKind::If => { - self.advance(); - Some(self.conditional()) - } - _ => { - self.consume( - Some(TokenKind::LeftBrace), - "expected '{' to start an 'else' block", - ); - let else_expr = self.expression(); - self.consume( - Some(TokenKind::RightBrace), - "Expected '}' to end an 'else' block", - ); - Some(else_expr) - } - } + Some(self.conditional()) + } else { + self.consume( + TokenKind::LeftBrace, + "expected '{' to start an 'else' block", + ); + let else_expr = self.expression(); + self.consume(TokenKind::RightBrace, "Expected '}' to end an 'else' block"); + Some(else_expr) } - _ => None, + } else { + None }; - let tail = self.previous.as_ref().unwrap().clone(); + let tail = self.previous.clone(); self.tree.add_expr(Expr::Conditional( token, condition_expr, @@ -645,8 +616,8 @@ impl<'a> Parser<'a> { } fn unary(&mut self) -> ExprRef { - let token = self.previous.as_ref().unwrap().clone(); - let kind = token.kind(); + let token = self.previous.clone(); + let kind = token.kind; let expr = self.expression_with_power(UNARY_POWER); let op = match kind { TokenKind::Minus => UnaryOp::Negate, @@ -658,8 +629,8 @@ impl<'a> Parser<'a> { } fn binary(&mut self, power: u8, left: ExprRef) -> ExprRef { - let token = self.previous.as_ref().unwrap().clone(); - let op = match token.kind() { + let token = self.previous.clone(); + let op = match token.kind { TokenKind::Plus => BinaryOp::Add, TokenKind::Minus => BinaryOp::Subtract, TokenKind::Star => BinaryOp::Multiply, @@ -673,25 +644,19 @@ impl<'a> Parser<'a> { } fn advance(&mut self) { - self.previous = self.current.take(); - loop { + self.previous = self.current.clone(); + self.current = self.tokens.next(); + while self.current.kind == TokenKind::Error { + self.error_at_current(self.current.to_string()); self.current = self.tokens.next(); - match &self.current { - Some(token) if token.kind() == TokenKind::Error => { - self.error_at_current(token.to_string()) - } - _ => break, - } } } - fn consume(&mut self, kind: Option, error: &str) { - match (&self.current, kind) { - (Some(token), Some(kind)) if token.kind() == kind => self.advance(), - (None, None) => (), - _ => { - self.error_at_current(error); - } + fn consume(&mut self, kind: TokenKind, error: &str) { + if self.current.kind == kind { + self.advance(); + } else { + self.error_at_current(error); } } @@ -709,7 +674,7 @@ impl<'a> Parser<'a> { self.error_at(self.current.clone(), message) } - fn error_at(&mut self, token: Option>, message: T) + fn error_at(&mut self, token: Token<'a>, message: T) where T: Into, { @@ -721,15 +686,13 @@ impl<'a> Parser<'a> { let message: String = message.into(); let (line, column) = self.tokens.token_position(&token); let mut final_message = "Error ".to_string(); - match token { - None => final_message.push_str("at end"), - Some(t) => { - if t.kind() != TokenKind::Error { - final_message.push_str("at '"); - final_message.push_str(t.as_str()); - final_message.push_str("'"); - } - } + + if token.kind == TokenKind::EOF { + final_message.push_str("at end") + } else if token.kind != TokenKind::Error { + final_message.push_str("at '"); + final_message.push_str(token.as_str()); + final_message.push_str("'"); } final_message.push_str(": "); final_message.push_str(&message); diff --git a/fine/src/tokens.rs b/fine/src/tokens.rs index 6c851f28..c4215a7f 100644 --- a/fine/src/tokens.rs +++ b/fine/src/tokens.rs @@ -1,5 +1,8 @@ #[derive(Debug, PartialEq, Eq, Clone, Copy)] pub enum TokenKind { + EOF, + Error, + LeftBrace, RightBrace, LeftBracket, @@ -47,14 +50,12 @@ pub enum TokenKind { Use, While, Yield, - - Error, } #[derive(Debug, PartialEq, Eq, Clone)] pub struct Token<'a> { - kind: TokenKind, - start: usize, + pub kind: TokenKind, + pub start: usize, value: Result<&'a str, String>, } @@ -75,14 +76,6 @@ impl<'a> Token<'a> { } } - pub fn start(&self) -> usize { - self.start - } - - pub fn kind(&self) -> TokenKind { - self.kind - } - pub fn as_str<'b>(&'b self) -> &'a str where 'b: 'a, @@ -102,14 +95,12 @@ impl<'a> std::fmt::Display for Token<'a> { pub struct Lines { newlines: Vec, - eof: usize, } impl Lines { - fn new(eof: usize) -> Self { + fn new() -> Self { Lines { newlines: Vec::new(), - eof, } } @@ -119,16 +110,9 @@ impl Lines { } /// Return the position of the given token as a (line, column) pair. By - /// convention, lines are 1-based and columns are 0-based. Also, in - /// keeping with the iterator-nature of the tokenizer, `None` here - /// indicates end-of-file, and will return the position of the end of the - /// file. - pub fn token_position(&self, token: &Option) -> (usize, usize) { - let start = match token { - Some(t) => t.start, - None => self.eof, - }; - self.position(start) + /// convention, lines are 1-based and columns are 0-based. + pub fn token_position(&self, token: &Token) -> (usize, usize) { + self.position(token.start) } /// Return the position of the given character offset as a (line,column) @@ -162,7 +146,7 @@ impl<'a> Tokens<'a> { source, chars: source.char_indices(), next_char: None, - lines: Lines::new(source.len()), + lines: Lines::new(), }; result.advance(); // Prime the pump result @@ -174,7 +158,7 @@ impl<'a> Tokens<'a> { /// Return the position of the given token as a (line, column) pair. See /// `Lines::token_position` for more information about the range, etc. - pub fn token_position(&self, token: &Option) -> (usize, usize) { + pub fn token_position(&self, token: &Token) -> (usize, usize) { self.lines.token_position(token) } @@ -415,19 +399,15 @@ impl<'a> Tokens<'a> { self.advance(); } } -} -impl<'a> std::iter::Iterator for Tokens<'a> { - type Item = Token<'a>; - - fn next(&mut self) -> Option { + pub fn next(&mut self) -> Token<'a> { self.skip_whitespace(); // TODO: Whitespace preserving/comment preserving let (pos, c) = match self.advance() { Some((p, c)) => (p, c), - None => return None, + None => return self.token(self.source.len(), TokenKind::EOF), }; - let token = match c { + match c { '{' => self.token(pos, TokenKind::LeftBrace), '}' => self.token(pos, TokenKind::RightBrace), '[' => self.token(pos, TokenKind::LeftBracket), @@ -480,8 +460,7 @@ impl<'a> std::iter::Iterator for Tokens<'a> { Token::error(pos, format!("Unexpected character '{c}'")) } } - }; - Some(token) + } } } @@ -490,19 +469,32 @@ mod tests { use super::*; use pretty_assertions::assert_eq; + fn test_tokens_impl(input: &str, expected: Vec) { + let mut result = Vec::new(); + let mut tokens = Tokens::new(input); + let mut is_eof = false; + while !is_eof { + let token = tokens.next(); + is_eof = token.kind == TokenKind::EOF; + result.push(token); + } + + assert_eq!(expected, result); + } + macro_rules! test_tokens { ($name:ident, $input:expr, $($s:expr),+) => { #[test] fn $name() { use TokenKind::*; - let tokens: Vec<_> = Tokens::new($input).collect(); - let expected: Vec = (vec![$($s),*]) + let mut expected: Vec = (vec![$($s),*]) .into_iter() .map(|t| Token::new(t.1, t.0, t.2)) .collect(); + expected.push(Token::new(TokenKind::EOF, $input.len(), "")); - assert_eq!(expected, tokens); + test_tokens_impl($input, expected); } } }