diff --git a/Cargo.lock b/Cargo.lock index 4fd9bdb8..2e15502a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -669,10 +669,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "fine" -version = "0.1.0" - [[package]] name = "flate2" version = "1.0.26" @@ -1543,13 +1539,13 @@ dependencies = [ "bytemuck", "deno_ast", "env_logger", - "fine", "fontdue", "image", "log", "lru", "notify", "oden-js", + "oden-script", "pollster", "sourcemap 7.0.0", "tracy-client", @@ -1576,6 +1572,10 @@ dependencies = [ "walkdir", ] +[[package]] +name = "oden-script" +version = "0.1.0" + [[package]] name = "once_cell" version = "1.18.0" diff --git a/Cargo.toml b/Cargo.toml index b573c760..0b059006 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,13 +13,13 @@ anyhow = "1.0" bytemuck = { version = "1.13", features = ["derive"] } deno_ast = { version = "0.29.3", features = ["transpiling", "typescript"] } env_logger = "0.10" -fine = { path = "fine" } fontdue = "0.7.3" image = { version = "0.24", default-features = false, features = ["png"] } log = "0.4" lru = "0.11.0" notify = "6" oden-js = { path = "oden-js" } +oden-script = { path = "oden-script" } pollster = "0.3" sourcemap = "7.0.0" tracy-client = { version = "0.15.2", default-features = false } diff --git a/fine/src/main.rs b/fine/src/main.rs index d8e6ecab..da0f5d92 100644 --- a/fine/src/main.rs +++ b/fine/src/main.rs @@ -1,37 +1 @@ -use fine::parser::Parser; -use std::env; -use std::fs; - -pub fn process_file(file: &str) { - println!("{file}"); - let source = match fs::read_to_string(file) { - Ok(c) => c, - Err(e) => { - eprintln!("Unable to read file {file}: {e}"); - return; - } - }; - - let (mut tree, expr, lines) = Parser::new(&source).parse(); - if tree.errors.len() > 0 { - for error in tree.errors { - eprintln!("{error}"); - } - return; - } - - let _expr_type = tree.expr_type(&expr, &lines, true); - if tree.errors.len() > 0 { - for error in tree.errors { - eprintln!("{error}"); - } - return; - } -} - -pub fn main() { - let args: Vec = env::args().collect(); - for arg in &args[1..] { - process_file(arg); - } -} +pub fn main() {} diff --git a/fine/src/parser.rs b/fine/src/parser.rs index da900c5b..a962dcd3 100644 --- a/fine/src/parser.rs +++ b/fine/src/parser.rs @@ -267,7 +267,7 @@ impl<'a> SyntaxTree<'a> { // This is dumb and should be punished, probably. (_, Type::Unreachable) => { - let (line, col) = lines.position(tok.start); + let (line, col) = lines.position(tok.start()); self.errors.push(SyntaxError::new(line, col, format!("cannot apply a unary operator to something that doesn't yield a value"))); Type::Error } @@ -277,7 +277,7 @@ impl<'a> SyntaxTree<'a> { // Missed the whole table, must be an error. (_, arg_type) => { - let (line, col) = lines.position(tok.start); + let (line, col) = lines.position(tok.start()); self.errors.push(SyntaxError::new(line, col, format!("cannot apply unary operator '{tok}' to expression of type '{arg_type}'"))); Type::Error } @@ -305,7 +305,7 @@ impl<'a> SyntaxTree<'a> { // This is dumb and should be punished, probably. (_, _, Type::Unreachable) => { - let (line, col) = lines.position(tok.start); + let (line, col) = lines.position(tok.start()); self.errors.push(SyntaxError::new( line, col, @@ -316,7 +316,7 @@ impl<'a> SyntaxTree<'a> { Type::Error } (_, Type::Unreachable, _) => { - let (line, col) = lines.position(tok.start); + let (line, col) = lines.position(tok.start()); self.errors.push(SyntaxError::new( line, col, @@ -333,7 +333,7 @@ impl<'a> SyntaxTree<'a> { // Missed the whole table, it must be an error. (_, left_type, right_type) => { - let (line, col) = lines.position(tok.start); + let (line, col) = lines.position(tok.start()); self.errors.push(SyntaxError::new(line, col, format!("cannot apply binary operator '{tok}' to expressions of type '{left_type}' (on the left) and '{right_type}' (on the right)"))); Type::Error } @@ -354,8 +354,8 @@ impl<'a> SyntaxTree<'a> { .expr_span(&cond) .expect("If the expression has a type it must have a span"); - let start = lines.position(span.0.start); - let end = lines.position(span.1.start); + let start = lines.position(span.0.start()); + let end = lines.position(span.1.start()); self.errors.push(SyntaxError::new_spanned( start, end, @@ -374,8 +374,8 @@ impl<'a> SyntaxTree<'a> { let span = self .expr_span(&exr) .expect("How did I get this far with a broken parse?"); - let start = lines.position(span.0.start); - let end = lines.position(span.1.start); + let start = lines.position(span.0.start()); + let end = lines.position(span.1.start()); self.errors.push(SyntaxError::new_spanned( start, end, @@ -392,8 +392,8 @@ impl<'a> SyntaxTree<'a> { let span = self .expr_span(&exr) .expect("How did I get this far with a broken parse?"); - let start = lines.position(span.0.start); - let end = lines.position(span.1.start); + let start = lines.position(span.0.start()); + let end = lines.position(span.1.start()); self.errors.push(SyntaxError::new_spanned( start, end, @@ -431,8 +431,13 @@ const UNARY_POWER: u8 = 7; // ! - // const CALL_POWER: u8 = 8; // . () // const PRIMARY_POWER: u8 = 9; -fn token_power<'a>(token: &Token<'a>) -> Option { - match token.kind { +fn token_power<'a>(token: &Option>) -> Option { + let token = match token { + Some(t) => t, + None => return None, + }; + + match token.kind() { TokenKind::Equal => Some(ASSIGNMENT_POWER), TokenKind::Or => Some(OR_POWER), TokenKind::And => Some(AND_POWER), @@ -449,8 +454,8 @@ fn token_power<'a>(token: &Token<'a>) -> Option { pub struct Parser<'a> { tokens: Tokens<'a>, tree: SyntaxTree<'a>, - current: Token<'a>, - previous: Token<'a>, + current: Option>, + previous: Option>, panic_mode: bool, } @@ -460,8 +465,8 @@ impl<'a> Parser<'a> { let mut parser = Parser { tokens: Tokens::new(source), tree: SyntaxTree::new(), - current: Token::new(TokenKind::EOF, 0, ""), - previous: Token::new(TokenKind::EOF, 0, ""), + current: None, + previous: None, panic_mode: false, }; parser.advance(); @@ -470,7 +475,7 @@ impl<'a> Parser<'a> { pub fn parse(mut self) -> (SyntaxTree<'a>, ExprRef, Lines) { let expr = self.expression(); - self.consume(TokenKind::EOF, "expected end of expression"); + self.consume(None, "expected end of expression"); (self.tree, expr, self.tokens.lines()) } @@ -500,24 +505,30 @@ impl<'a> Parser<'a> { fn prefix_expression(&mut self) -> ExprRef { self.trace("prefix"); - let token = &self.previous; - match token.kind { - TokenKind::Bang => self.unary(), - TokenKind::LeftParen => self.grouping(), - TokenKind::Number => self.number(), - TokenKind::Minus => self.unary(), - TokenKind::String => self.string(), + let token = self.previous.as_ref(); + match token { + Some(token) => match token.kind() { + TokenKind::Bang => self.unary(), + TokenKind::LeftParen => self.grouping(), + TokenKind::Number => self.number(), + TokenKind::Minus => self.unary(), + TokenKind::String => self.string(), - TokenKind::True => self - .tree - .add_expr(Expr::Literal(Literal::Bool(true), token.clone())), - TokenKind::False => self - .tree - .add_expr(Expr::Literal(Literal::Bool(false), token.clone())), + TokenKind::True => self + .tree + .add_expr(Expr::Literal(Literal::Bool(true), token.clone())), + TokenKind::False => self + .tree + .add_expr(Expr::Literal(Literal::Bool(false), token.clone())), - TokenKind::If => self.conditional(), + TokenKind::If => self.conditional(), - _ => { + _ => { + self.error("expected an expression"); + ExprRef::error() + } + }, + None => { self.error("expected an expression"); ExprRef::error() } @@ -526,7 +537,8 @@ impl<'a> Parser<'a> { fn infix_expression(&mut self, power: u8, left: ExprRef) -> ExprRef { self.trace("infix"); - match self.previous.kind { + let kind = self.previous.as_ref().unwrap().kind(); + match kind { TokenKind::Plus | TokenKind::Minus | TokenKind::Star @@ -538,7 +550,7 @@ impl<'a> Parser<'a> { } fn number(&mut self) -> ExprRef { - let token = &self.previous; + let token = self.previous.as_ref().unwrap(); // What kind is it? For now let's just ... make it good. let literal = match token.as_str().parse::() { @@ -553,7 +565,7 @@ impl<'a> Parser<'a> { } fn string(&mut self) -> ExprRef { - let token = &self.previous; + let token = self.previous.as_ref().unwrap(); let mut result = String::new(); let mut input = token.as_str().chars(); @@ -578,34 +590,51 @@ impl<'a> Parser<'a> { fn grouping(&mut self) -> ExprRef { let result = self.expression(); - self.consume(TokenKind::RightParen, "expected ')' after an expression"); + self.consume( + Some(TokenKind::RightParen), + "expected ')' after an expression", + ); result } fn conditional(&mut self) -> ExprRef { - let token = self.previous.clone(); + let token = self.previous.as_ref().unwrap().clone(); let condition_expr = self.expression(); - self.consume(TokenKind::LeftBrace, "expected '{' to start an 'if' block"); + self.consume( + Some(TokenKind::LeftBrace), + "expected '{' to start an 'if' block", + ); let then_expr = self.expression(); - self.consume(TokenKind::RightBrace, "expected '}' to end an 'if' block"); - let else_expr = if self.current.kind == TokenKind::Else { - self.advance(); - if self.current.kind == TokenKind::If { + self.consume( + Some(TokenKind::RightBrace), + "expected '}' to end an 'if' block", + ); + let else_expr = match &self.current { + Some(token) if token.kind() == TokenKind::Else => { self.advance(); - Some(self.conditional()) - } else { - self.consume( - TokenKind::LeftBrace, - "expected '{' to start an 'else' block", - ); - let else_expr = self.expression(); - self.consume(TokenKind::RightBrace, "Expected '}' to end an 'else' block"); - Some(else_expr) + match &self.current { + // Allow `else if` without another `{`. + Some(token) if token.kind() == TokenKind::If => { + self.advance(); + Some(self.conditional()) + } + _ => { + self.consume( + Some(TokenKind::LeftBrace), + "expected '{' to start an 'else' block", + ); + let else_expr = self.expression(); + self.consume( + Some(TokenKind::RightBrace), + "Expected '}' to end an 'else' block", + ); + Some(else_expr) + } + } } - } else { - None + _ => None, }; - let tail = self.previous.clone(); + let tail = self.previous.as_ref().unwrap().clone(); self.tree.add_expr(Expr::Conditional( token, condition_expr, @@ -616,8 +645,8 @@ impl<'a> Parser<'a> { } fn unary(&mut self) -> ExprRef { - let token = self.previous.clone(); - let kind = token.kind; + let token = self.previous.as_ref().unwrap().clone(); + let kind = token.kind(); let expr = self.expression_with_power(UNARY_POWER); let op = match kind { TokenKind::Minus => UnaryOp::Negate, @@ -629,8 +658,8 @@ impl<'a> Parser<'a> { } fn binary(&mut self, power: u8, left: ExprRef) -> ExprRef { - let token = self.previous.clone(); - let op = match token.kind { + let token = self.previous.as_ref().unwrap().clone(); + let op = match token.kind() { TokenKind::Plus => BinaryOp::Add, TokenKind::Minus => BinaryOp::Subtract, TokenKind::Star => BinaryOp::Multiply, @@ -644,19 +673,25 @@ impl<'a> Parser<'a> { } fn advance(&mut self) { - self.previous = self.current.clone(); - self.current = self.tokens.next(); - while self.current.kind == TokenKind::Error { - self.error_at_current(self.current.to_string()); + self.previous = self.current.take(); + loop { self.current = self.tokens.next(); + match &self.current { + Some(token) if token.kind() == TokenKind::Error => { + self.error_at_current(token.to_string()) + } + _ => break, + } } } - fn consume(&mut self, kind: TokenKind, error: &str) { - if self.current.kind == kind { - self.advance(); - } else { - self.error_at_current(error); + fn consume(&mut self, kind: Option, error: &str) { + match (&self.current, kind) { + (Some(token), Some(kind)) if token.kind() == kind => self.advance(), + (None, None) => (), + _ => { + self.error_at_current(error); + } } } @@ -674,7 +709,7 @@ impl<'a> Parser<'a> { self.error_at(self.current.clone(), message) } - fn error_at(&mut self, token: Token<'a>, message: T) + fn error_at(&mut self, token: Option>, message: T) where T: Into, { @@ -686,13 +721,15 @@ impl<'a> Parser<'a> { let message: String = message.into(); let (line, column) = self.tokens.token_position(&token); let mut final_message = "Error ".to_string(); - - if token.kind == TokenKind::EOF { - final_message.push_str("at end") - } else if token.kind != TokenKind::Error { - final_message.push_str("at '"); - final_message.push_str(token.as_str()); - final_message.push_str("'"); + match token { + None => final_message.push_str("at end"), + Some(t) => { + if t.kind() != TokenKind::Error { + final_message.push_str("at '"); + final_message.push_str(t.as_str()); + final_message.push_str("'"); + } + } } final_message.push_str(": "); final_message.push_str(&message); diff --git a/fine/src/tokens.rs b/fine/src/tokens.rs index c4215a7f..c2bccfb9 100644 --- a/fine/src/tokens.rs +++ b/fine/src/tokens.rs @@ -1,8 +1,5 @@ #[derive(Debug, PartialEq, Eq, Clone, Copy)] pub enum TokenKind { - EOF, - Error, - LeftBrace, RightBrace, LeftBracket, @@ -47,15 +44,16 @@ pub enum TokenKind { Select, This, True, - Use, While, Yield, + + Error, } #[derive(Debug, PartialEq, Eq, Clone)] pub struct Token<'a> { - pub kind: TokenKind, - pub start: usize, + kind: TokenKind, + start: usize, value: Result<&'a str, String>, } @@ -76,6 +74,14 @@ impl<'a> Token<'a> { } } + pub fn start(&self) -> usize { + self.start + } + + pub fn kind(&self) -> TokenKind { + self.kind + } + pub fn as_str<'b>(&'b self) -> &'a str where 'b: 'a, @@ -95,12 +101,14 @@ impl<'a> std::fmt::Display for Token<'a> { pub struct Lines { newlines: Vec, + eof: usize, } impl Lines { - fn new() -> Self { + fn new(eof: usize) -> Self { Lines { newlines: Vec::new(), + eof, } } @@ -110,9 +118,16 @@ impl Lines { } /// Return the position of the given token as a (line, column) pair. By - /// convention, lines are 1-based and columns are 0-based. - pub fn token_position(&self, token: &Token) -> (usize, usize) { - self.position(token.start) + /// convention, lines are 1-based and columns are 0-based. Also, in + /// keeping with the iterator-nature of the tokenizer, `None` here + /// indicates end-of-file, and will return the position of the end of the + /// file. + pub fn token_position(&self, token: &Option) -> (usize, usize) { + let start = match token { + Some(t) => t.start, + None => self.eof, + }; + self.position(start) } /// Return the position of the given character offset as a (line,column) @@ -146,7 +161,7 @@ impl<'a> Tokens<'a> { source, chars: source.char_indices(), next_char: None, - lines: Lines::new(), + lines: Lines::new(source.len()), }; result.advance(); // Prime the pump result @@ -158,7 +173,7 @@ impl<'a> Tokens<'a> { /// Return the position of the given token as a (line, column) pair. See /// `Lines::token_position` for more information about the range, etc. - pub fn token_position(&self, token: &Token) -> (usize, usize) { + pub fn token_position(&self, token: &Option) -> (usize, usize) { self.lines.token_position(token) } @@ -311,11 +326,6 @@ impl<'a> Tokens<'a> { return TokenKind::True; } } - 'u' => { - if ident == "use" { - return TokenKind::Use; - } - } 'w' => { if ident == "while" { return TokenKind::While; @@ -399,15 +409,19 @@ impl<'a> Tokens<'a> { self.advance(); } } +} - pub fn next(&mut self) -> Token<'a> { +impl<'a> std::iter::Iterator for Tokens<'a> { + type Item = Token<'a>; + + fn next(&mut self) -> Option { self.skip_whitespace(); // TODO: Whitespace preserving/comment preserving let (pos, c) = match self.advance() { Some((p, c)) => (p, c), - None => return self.token(self.source.len(), TokenKind::EOF), + None => return None, }; - match c { + let token = match c { '{' => self.token(pos, TokenKind::LeftBrace), '}' => self.token(pos, TokenKind::RightBrace), '[' => self.token(pos, TokenKind::LeftBracket), @@ -460,7 +474,8 @@ impl<'a> Tokens<'a> { Token::error(pos, format!("Unexpected character '{c}'")) } } - } + }; + Some(token) } } @@ -469,32 +484,19 @@ mod tests { use super::*; use pretty_assertions::assert_eq; - fn test_tokens_impl(input: &str, expected: Vec) { - let mut result = Vec::new(); - let mut tokens = Tokens::new(input); - let mut is_eof = false; - while !is_eof { - let token = tokens.next(); - is_eof = token.kind == TokenKind::EOF; - result.push(token); - } - - assert_eq!(expected, result); - } - macro_rules! test_tokens { ($name:ident, $input:expr, $($s:expr),+) => { #[test] fn $name() { use TokenKind::*; + let tokens: Vec<_> = Tokens::new($input).collect(); - let mut expected: Vec = (vec![$($s),*]) + let expected: Vec = (vec![$($s),*]) .into_iter() .map(|t| Token::new(t.1, t.0, t.2)) .collect(); - expected.push(Token::new(TokenKind::EOF, $input.len(), "")); - test_tokens_impl($input, expected); + assert_eq!(expected, tokens); } } } @@ -533,7 +535,7 @@ mod tests { test_tokens!( more_keywords, - "fun if let print return select this true use while truewhile", + "fun if let print return select this true while truewhile", (0, Fun, "fun"), (4, If, "if"), (7, Let, "let"), @@ -542,9 +544,8 @@ mod tests { (24, Select, "select"), (31, This, "this"), (36, True, "true"), - (41, Use, "use"), - (45, While, "while"), - (51, Identifier, "truewhile") + (41, While, "while"), + (47, Identifier, "truewhile") ); test_tokens!(