[fine] Tokens is not Iterator

It was not pulling its weight

Cargo.lock

@@ -669,6 +669,10 @@ dependencies = [
  "windows-sys 0.48.0",
 ]
 
+[[package]]
+name = "fine"
+version = "0.1.0"
+
 [[package]]
 name = "flate2"
 version = "1.0.26"
@@ -1539,13 +1543,13 @@ dependencies = [
  "bytemuck",
  "deno_ast",
  "env_logger",
+ "fine",
  "fontdue",
  "image",
  "log",
  "lru",
  "notify",
  "oden-js",
- "oden-script",
  "pollster",
  "sourcemap 7.0.0",
  "tracy-client",
@@ -1572,10 +1576,6 @@ dependencies = [
  "walkdir",
 ]
 
-[[package]]
-name = "oden-script"
-version = "0.1.0"
-
 [[package]]
 name = "once_cell"
 version = "1.18.0"

Cargo.toml

@@ -13,13 +13,13 @@ anyhow = "1.0"
 bytemuck = { version = "1.13", features = ["derive"] }
 deno_ast = { version = "0.29.3", features = ["transpiling", "typescript"] }
 env_logger = "0.10"
+fine = { path = "fine" }
 fontdue = "0.7.3"
 image = { version = "0.24", default-features = false, features = ["png"] }
 log = "0.4"
 lru = "0.11.0"
 notify = "6"
 oden-js = { path = "oden-js" }
-oden-script = { path = "oden-script" }
 pollster = "0.3"
 sourcemap = "7.0.0"
 tracy-client = { version = "0.15.2", default-features = false }

fine/src/main.rs

@@ -1 +1,37 @@
-pub fn main() {}
+use fine::parser::Parser;
+use std::env;
+use std::fs;
+
+pub fn process_file(file: &str) {
+    println!("{file}");
+    let source = match fs::read_to_string(file) {
+        Ok(c) => c,
+        Err(e) => {
+            eprintln!("Unable to read file {file}: {e}");
+            return;
+        }
+    };
+
+    let (mut tree, expr, lines) = Parser::new(&source).parse();
+    if tree.errors.len() > 0 {
+        for error in tree.errors {
+            eprintln!("{error}");
+        }
+        return;
+    }
+
+    let _expr_type = tree.expr_type(&expr, &lines, true);
+    if tree.errors.len() > 0 {
+        for error in tree.errors {
+            eprintln!("{error}");
+        }
+        return;
+    }
+}
+
+pub fn main() {
+    let args: Vec<String> = env::args().collect();
+    for arg in &args[1..] {
+        process_file(arg);
+    }
+}

fine/src/parser.rs

@@ -267,7 +267,7 @@ impl<'a> SyntaxTree<'a> {
 
                     // This is dumb and should be punished, probably.
                     (_, Type::Unreachable) => {
-                        let (line, col) = lines.position(tok.start());
+                        let (line, col) = lines.position(tok.start);
                         self.errors.push(SyntaxError::new(line, col, format!("cannot apply a unary operator to something that doesn't yield a value")));
                         Type::Error
                     }
@@ -277,7 +277,7 @@ impl<'a> SyntaxTree<'a> {
 
                     // Missed the whole table, must be an error.
                     (_, arg_type) => {
-                        let (line, col) = lines.position(tok.start());
+                        let (line, col) = lines.position(tok.start);
                         self.errors.push(SyntaxError::new(line, col, format!("cannot apply unary operator '{tok}' to expression of type '{arg_type}'")));
                         Type::Error
                     }
@@ -305,7 +305,7 @@ impl<'a> SyntaxTree<'a> {
 
                     // This is dumb and should be punished, probably.
                     (_, _, Type::Unreachable) => {
-                        let (line, col) = lines.position(tok.start());
+                        let (line, col) = lines.position(tok.start);
                         self.errors.push(SyntaxError::new(
                             line,
                             col,
@@ -316,7 +316,7 @@ impl<'a> SyntaxTree<'a> {
                         Type::Error
                     }
                     (_, Type::Unreachable, _) => {
-                        let (line, col) = lines.position(tok.start());
+                        let (line, col) = lines.position(tok.start);
                         self.errors.push(SyntaxError::new(
                             line,
                             col,
@@ -333,7 +333,7 @@ impl<'a> SyntaxTree<'a> {
 
                     // Missed the whole table, it must be an error.
                     (_, left_type, right_type) => {
-                        let (line, col) = lines.position(tok.start());
+                        let (line, col) = lines.position(tok.start);
                         self.errors.push(SyntaxError::new(line, col, format!("cannot apply binary operator '{tok}' to expressions of type '{left_type}' (on the left) and '{right_type}' (on the right)")));
                         Type::Error
                     }
@@ -354,8 +354,8 @@ impl<'a> SyntaxTree<'a> {
                             .expr_span(&cond)
                             .expect("If the expression has a type it must have a span");
 
-                        let start = lines.position(span.0.start());
+                        let start = lines.position(span.0.start);
-                        let end = lines.position(span.1.start());
+                        let end = lines.position(span.1.start);
                         self.errors.push(SyntaxError::new_spanned(
                             start,
                             end,
@@ -374,8 +374,8 @@ impl<'a> SyntaxTree<'a> {
                         let span = self
                             .expr_span(&exr)
                             .expect("How did I get this far with a broken parse?");
-                        let start = lines.position(span.0.start());
+                        let start = lines.position(span.0.start);
-                        let end = lines.position(span.1.start());
+                        let end = lines.position(span.1.start);
                         self.errors.push(SyntaxError::new_spanned(
                                 start,
                                 end,
@@ -392,8 +392,8 @@ impl<'a> SyntaxTree<'a> {
                             let span = self
                                 .expr_span(&exr)
                                 .expect("How did I get this far with a broken parse?");
-                            let start = lines.position(span.0.start());
+                            let start = lines.position(span.0.start);
-                            let end = lines.position(span.1.start());
+                            let end = lines.position(span.1.start);
                             self.errors.push(SyntaxError::new_spanned(
                                 start,
                                 end,
@@ -431,13 +431,8 @@ const UNARY_POWER: u8 = 7; // ! -
 // const CALL_POWER: u8 = 8; // . ()
 // const PRIMARY_POWER: u8 = 9;
 
-fn token_power<'a>(token: &Option<Token<'a>>) -> Option<u8> {
+fn token_power<'a>(token: &Token<'a>) -> Option<u8> {
-    let token = match token {
+    match token.kind {
-        Some(t) => t,
-        None => return None,
-    };
-
-    match token.kind() {
         TokenKind::Equal => Some(ASSIGNMENT_POWER),
         TokenKind::Or => Some(OR_POWER),
         TokenKind::And => Some(AND_POWER),
@@ -454,8 +449,8 @@ fn token_power<'a>(token: &Option<Token<'a>>) -> Option<u8> {
 pub struct Parser<'a> {
     tokens: Tokens<'a>,
     tree: SyntaxTree<'a>,
-    current: Option<Token<'a>>,
+    current: Token<'a>,
-    previous: Option<Token<'a>>,
+    previous: Token<'a>,
 
     panic_mode: bool,
 }
@@ -465,8 +460,8 @@ impl<'a> Parser<'a> {
         let mut parser = Parser {
             tokens: Tokens::new(source),
             tree: SyntaxTree::new(),
-            current: None,
+            current: Token::new(TokenKind::EOF, 0, ""),
-            previous: None,
+            previous: Token::new(TokenKind::EOF, 0, ""),
             panic_mode: false,
         };
         parser.advance();
@@ -475,7 +470,7 @@ impl<'a> Parser<'a> {
 
     pub fn parse(mut self) -> (SyntaxTree<'a>, ExprRef, Lines) {
         let expr = self.expression();
-        self.consume(None, "expected end of expression");
+        self.consume(TokenKind::EOF, "expected end of expression");
         (self.tree, expr, self.tokens.lines())
     }
 
@@ -505,9 +500,8 @@ impl<'a> Parser<'a> {
 
     fn prefix_expression(&mut self) -> ExprRef {
         self.trace("prefix");
-        let token = self.previous.as_ref();
+        let token = &self.previous;
-        match token {
+        match token.kind {
-            Some(token) => match token.kind() {
             TokenKind::Bang => self.unary(),
             TokenKind::LeftParen => self.grouping(),
             TokenKind::Number => self.number(),
@@ -527,18 +521,12 @@ impl<'a> Parser<'a> {
                 self.error("expected an expression");
                 ExprRef::error()
             }
-            },
-            None => {
-                self.error("expected an expression");
-                ExprRef::error()
-            }
         }
     }
 
     fn infix_expression(&mut self, power: u8, left: ExprRef) -> ExprRef {
         self.trace("infix");
-        let kind = self.previous.as_ref().unwrap().kind();
+        match self.previous.kind {
-        match kind {
             TokenKind::Plus
             | TokenKind::Minus
             | TokenKind::Star
@@ -550,7 +538,7 @@ impl<'a> Parser<'a> {
     }
 
     fn number(&mut self) -> ExprRef {
-        let token = self.previous.as_ref().unwrap();
+        let token = &self.previous;
         // What kind is it? For now let's just ... make it good.
 
         let literal = match token.as_str().parse::<f64>() {
@@ -565,7 +553,7 @@ impl<'a> Parser<'a> {
     }
 
     fn string(&mut self) -> ExprRef {
-        let token = self.previous.as_ref().unwrap();
+        let token = &self.previous;
 
         let mut result = String::new();
         let mut input = token.as_str().chars();
@@ -590,51 +578,34 @@ impl<'a> Parser<'a> {
 
     fn grouping(&mut self) -> ExprRef {
         let result = self.expression();
-        self.consume(
+        self.consume(TokenKind::RightParen, "expected ')' after an expression");
-            Some(TokenKind::RightParen),
-            "expected ')' after an expression",
-        );
         result
     }
 
     fn conditional(&mut self) -> ExprRef {
-        let token = self.previous.as_ref().unwrap().clone();
+        let token = self.previous.clone();
         let condition_expr = self.expression();
-        self.consume(
+        self.consume(TokenKind::LeftBrace, "expected '{' to start an 'if' block");
-            Some(TokenKind::LeftBrace),
-            "expected '{' to start an 'if' block",
-        );
         let then_expr = self.expression();
-        self.consume(
+        self.consume(TokenKind::RightBrace, "expected '}' to end an 'if' block");
-            Some(TokenKind::RightBrace),
+        let else_expr = if self.current.kind == TokenKind::Else {
-            "expected '}' to end an 'if' block",
-        );
-        let else_expr = match &self.current {
-            Some(token) if token.kind() == TokenKind::Else => {
             self.advance();
-                match &self.current {
+            if self.current.kind == TokenKind::If {
-                    // Allow `else if` without another `{`.
-                    Some(token) if token.kind() == TokenKind::If => {
                 self.advance();
                 Some(self.conditional())
-                    }
+            } else {
-                    _ => {
                 self.consume(
-                            Some(TokenKind::LeftBrace),
+                    TokenKind::LeftBrace,
                     "expected '{' to start an 'else' block",
                 );
                 let else_expr = self.expression();
-                        self.consume(
+                self.consume(TokenKind::RightBrace, "Expected '}' to end an 'else' block");
-                            Some(TokenKind::RightBrace),
-                            "Expected '}' to end an 'else' block",
-                        );
                 Some(else_expr)
             }
-                }
+        } else {
-            }
+            None
-            _ => None,
         };
-        let tail = self.previous.as_ref().unwrap().clone();
+        let tail = self.previous.clone();
         self.tree.add_expr(Expr::Conditional(
             token,
             condition_expr,
@@ -645,8 +616,8 @@ impl<'a> Parser<'a> {
     }
 
     fn unary(&mut self) -> ExprRef {
-        let token = self.previous.as_ref().unwrap().clone();
+        let token = self.previous.clone();
-        let kind = token.kind();
+        let kind = token.kind;
         let expr = self.expression_with_power(UNARY_POWER);
         let op = match kind {
             TokenKind::Minus => UnaryOp::Negate,
@@ -658,8 +629,8 @@ impl<'a> Parser<'a> {
     }
 
     fn binary(&mut self, power: u8, left: ExprRef) -> ExprRef {
-        let token = self.previous.as_ref().unwrap().clone();
+        let token = self.previous.clone();
-        let op = match token.kind() {
+        let op = match token.kind {
             TokenKind::Plus => BinaryOp::Add,
             TokenKind::Minus => BinaryOp::Subtract,
             TokenKind::Star => BinaryOp::Multiply,
@@ -673,27 +644,21 @@ impl<'a> Parser<'a> {
     }
 
     fn advance(&mut self) {
-        self.previous = self.current.take();
+        self.previous = self.current.clone();
-        loop {
+        self.current = self.tokens.next();
+        while self.current.kind == TokenKind::Error {
+            self.error_at_current(self.current.to_string());
             self.current = self.tokens.next();
-            match &self.current {
-                Some(token) if token.kind() == TokenKind::Error => {
-                    self.error_at_current(token.to_string())
-                }
-                _ => break,
-            }
         }
     }
 
-    fn consume(&mut self, kind: Option<TokenKind>, error: &str) {
+    fn consume(&mut self, kind: TokenKind, error: &str) {
-        match (&self.current, kind) {
+        if self.current.kind == kind {
-            (Some(token), Some(kind)) if token.kind() == kind => self.advance(),
+            self.advance();
-            (None, None) => (),
+        } else {
-            _ => {
             self.error_at_current(error);
         }
     }
-    }
 
     fn error<T>(&mut self, message: T)
     where
@@ -709,7 +674,7 @@ impl<'a> Parser<'a> {
         self.error_at(self.current.clone(), message)
     }
 
-    fn error_at<T>(&mut self, token: Option<Token<'a>>, message: T)
+    fn error_at<T>(&mut self, token: Token<'a>, message: T)
     where
         T: Into<String>,
     {
@@ -721,16 +686,14 @@ impl<'a> Parser<'a> {
         let message: String = message.into();
         let (line, column) = self.tokens.token_position(&token);
         let mut final_message = "Error ".to_string();
-        match token {
+
-            None => final_message.push_str("at end"),
+        if token.kind == TokenKind::EOF {
-            Some(t) => {
+            final_message.push_str("at end")
-                if t.kind() != TokenKind::Error {
+        } else if token.kind != TokenKind::Error {
             final_message.push_str("at '");
-                    final_message.push_str(t.as_str());
+            final_message.push_str(token.as_str());
             final_message.push_str("'");
         }
-            }
-        }
         final_message.push_str(": ");
         final_message.push_str(&message);
 

fine/src/tokens.rs

@@ -1,5 +1,8 @@
 #[derive(Debug, PartialEq, Eq, Clone, Copy)]
 pub enum TokenKind {
+    EOF,
+    Error,
+
     LeftBrace,
     RightBrace,
     LeftBracket,
@@ -44,16 +47,15 @@ pub enum TokenKind {
     Select,
     This,
     True,
+    Use,
     While,
     Yield,
-
-    Error,
 }
 
 #[derive(Debug, PartialEq, Eq, Clone)]
 pub struct Token<'a> {
-    kind: TokenKind,
+    pub kind: TokenKind,
-    start: usize,
+    pub start: usize,
     value: Result<&'a str, String>,
 }
 
@@ -74,14 +76,6 @@ impl<'a> Token<'a> {
         }
     }
 
-    pub fn start(&self) -> usize {
-        self.start
-    }
-
-    pub fn kind(&self) -> TokenKind {
-        self.kind
-    }
-
     pub fn as_str<'b>(&'b self) -> &'a str
     where
         'b: 'a,
@@ -101,14 +95,12 @@ impl<'a> std::fmt::Display for Token<'a> {
 
 pub struct Lines {
     newlines: Vec<usize>,
-    eof: usize,
 }
 
 impl Lines {
-    fn new(eof: usize) -> Self {
+    fn new() -> Self {
         Lines {
             newlines: Vec::new(),
-            eof,
         }
     }
 
@@ -118,16 +110,9 @@ impl Lines {
     }
 
     /// Return the position of the given token as a (line, column) pair. By
-    /// convention, lines are 1-based and columns are 0-based. Also, in
+    /// convention, lines are 1-based and columns are 0-based.
-    /// keeping with the iterator-nature of the tokenizer, `None` here
+    pub fn token_position(&self, token: &Token) -> (usize, usize) {
-    /// indicates end-of-file, and will return the position of the end of the
+        self.position(token.start)
-    /// file.
-    pub fn token_position(&self, token: &Option<Token>) -> (usize, usize) {
-        let start = match token {
-            Some(t) => t.start,
-            None => self.eof,
-        };
-        self.position(start)
     }
 
     /// Return the position of the given character offset as a (line,column)
@@ -161,7 +146,7 @@ impl<'a> Tokens<'a> {
             source,
             chars: source.char_indices(),
             next_char: None,
-            lines: Lines::new(source.len()),
+            lines: Lines::new(),
         };
         result.advance(); // Prime the pump
         result
@@ -173,7 +158,7 @@ impl<'a> Tokens<'a> {
 
     /// Return the position of the given token as a (line, column) pair. See
     /// `Lines::token_position` for more information about the range, etc.
-    pub fn token_position(&self, token: &Option<Token>) -> (usize, usize) {
+    pub fn token_position(&self, token: &Token) -> (usize, usize) {
         self.lines.token_position(token)
     }
 
@@ -326,6 +311,11 @@ impl<'a> Tokens<'a> {
                     return TokenKind::True;
                 }
             }
+            'u' => {
+                if ident == "use" {
+                    return TokenKind::Use;
+                }
+            }
             'w' => {
                 if ident == "while" {
                     return TokenKind::While;
@@ -409,19 +399,15 @@ impl<'a> Tokens<'a> {
             self.advance();
         }
     }
-}
 
-impl<'a> std::iter::Iterator for Tokens<'a> {
+    pub fn next(&mut self) -> Token<'a> {
-    type Item = Token<'a>;
-
-    fn next(&mut self) -> Option<Self::Item> {
         self.skip_whitespace(); // TODO: Whitespace preserving/comment preserving
         let (pos, c) = match self.advance() {
             Some((p, c)) => (p, c),
-            None => return None,
+            None => return self.token(self.source.len(), TokenKind::EOF),
         };
 
-        let token = match c {
+        match c {
             '{' => self.token(pos, TokenKind::LeftBrace),
             '}' => self.token(pos, TokenKind::RightBrace),
             '[' => self.token(pos, TokenKind::LeftBracket),
@@ -474,8 +460,7 @@ impl<'a> std::iter::Iterator for Tokens<'a> {
                     Token::error(pos, format!("Unexpected character '{c}'"))
                 }
             }
-        };
+        }
-        Some(token)
     }
 }
 
@@ -484,19 +469,32 @@ mod tests {
     use super::*;
     use pretty_assertions::assert_eq;
 
+    fn test_tokens_impl(input: &str, expected: Vec<Token>) {
+        let mut result = Vec::new();
+        let mut tokens = Tokens::new(input);
+        let mut is_eof = false;
+        while !is_eof {
+            let token = tokens.next();
+            is_eof = token.kind == TokenKind::EOF;
+            result.push(token);
+        }
+
+        assert_eq!(expected, result);
+    }
+
     macro_rules! test_tokens {
         ($name:ident, $input:expr, $($s:expr),+) => {
             #[test]
             fn $name() {
                 use TokenKind::*;
-                let tokens: Vec<_> = Tokens::new($input).collect();
 
-                let expected: Vec<Token> = (vec![$($s),*])
+                let mut expected: Vec<Token> = (vec![$($s),*])
                     .into_iter()
                     .map(|t| Token::new(t.1, t.0, t.2))
                     .collect();
+                expected.push(Token::new(TokenKind::EOF, $input.len(), ""));
 
-                assert_eq!(expected, tokens);
+                test_tokens_impl($input, expected);
             }
         }
     }
@@ -535,7 +533,7 @@ mod tests {
 
     test_tokens!(
         more_keywords,
-        "fun if let print return select this true while truewhile",
+        "fun if let print return select this true use while truewhile",
         (0, Fun, "fun"),
         (4, If, "if"),
         (7, Let, "let"),
@@ -544,8 +542,9 @@ mod tests {
         (24, Select, "select"),
         (31, This, "this"),
         (36, True, "true"),
-        (41, While, "while"),
+        (41, Use, "use"),
-        (47, Identifier, "truewhile")
+        (45, While, "while"),
+        (51, Identifier, "truewhile")
     );
 
     test_tokens!(