DeCarabas/oden
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

[fine] Tokens is not Iterator

Browse source 
It was not pulling its weight
This commit is contained in:
John Doty 2024-01-03 06:18:26 -08:00
parent c4d2b82968
commit d2d144a5ec
2 changed files with 107 additions and 152 deletions
Download patch file Download diff file Expand all files Collapse all files

189
fine/src/parser.rs
View file

@ -267,7 +267,7 @@ impl<'a> SyntaxTree<'a> {
// This is dumb and should be punished, probably.
(_, Type::Unreachable) => {
let (line, col) = lines.position(tok.start());
let (line, col) = lines.position(tok.start);
self.errors.push(SyntaxError::new(line, col, format!("cannot apply a unary operator to something that doesn't yield a value")));
Type::Error
}
@ -277,7 +277,7 @@ impl<'a> SyntaxTree<'a> {
// Missed the whole table, must be an error.
(_, arg_type) => {
let (line, col) = lines.position(tok.start());
let (line, col) = lines.position(tok.start);
self.errors.push(SyntaxError::new(line, col, format!("cannot apply unary operator '{tok}' to expression of type '{arg_type}'")));
Type::Error
}
@ -305,7 +305,7 @@ impl<'a> SyntaxTree<'a> {
// This is dumb and should be punished, probably.
(_, _, Type::Unreachable) => {
let (line, col) = lines.position(tok.start());
let (line, col) = lines.position(tok.start);
self.errors.push(SyntaxError::new(
line,
col,
@ -316,7 +316,7 @@ impl<'a> SyntaxTree<'a> {
Type::Error
}
(_, Type::Unreachable, _) => {
let (line, col) = lines.position(tok.start());
let (line, col) = lines.position(tok.start);
self.errors.push(SyntaxError::new(
line,
col,
@ -333,7 +333,7 @@ impl<'a> SyntaxTree<'a> {
// Missed the whole table, it must be an error.
(_, left_type, right_type) => {
let (line, col) = lines.position(tok.start());
let (line, col) = lines.position(tok.start);
self.errors.push(SyntaxError::new(line, col, format!("cannot apply binary operator '{tok}' to expressions of type '{left_type}' (on the left) and '{right_type}' (on the right)")));
Type::Error
}
@ -354,8 +354,8 @@ impl<'a> SyntaxTree<'a> {
.expr_span(&cond)
.expect("If the expression has a type it must have a span");
let start = lines.position(span.0.start());
let end = lines.position(span.1.start());
let start = lines.position(span.0.start);
let end = lines.position(span.1.start);
self.errors.push(SyntaxError::new_spanned(
start,
end,
@ -374,8 +374,8 @@ impl<'a> SyntaxTree<'a> {
let span = self
.expr_span(&exr)
.expect("How did I get this far with a broken parse?");
let start = lines.position(span.0.start());
let end = lines.position(span.1.start());
let start = lines.position(span.0.start);
let end = lines.position(span.1.start);
self.errors.push(SyntaxError::new_spanned(
start,
end,
@ -392,8 +392,8 @@ impl<'a> SyntaxTree<'a> {
let span = self
.expr_span(&exr)
.expect("How did I get this far with a broken parse?");
let start = lines.position(span.0.start());
let end = lines.position(span.1.start());
let start = lines.position(span.0.start);
let end = lines.position(span.1.start);
self.errors.push(SyntaxError::new_spanned(
start,
end,
@ -431,13 +431,8 @@ const UNARY_POWER: u8 = 7; // ! -
// const CALL_POWER: u8 = 8; // . ()
// const PRIMARY_POWER: u8 = 9;
fn token_power<'a>(token: &Option<Token<'a>>) -> Option<u8> {
let token = match token {
Some(t) => t,
None => return None,
};
match token.kind() {
fn token_power<'a>(token: &Token<'a>) -> Option<u8> {
match token.kind {
TokenKind::Equal => Some(ASSIGNMENT_POWER),
TokenKind::Or => Some(OR_POWER),
TokenKind::And => Some(AND_POWER),
@ -454,8 +449,8 @@ fn token_power<'a>(token: &Option<Token<'a>>) -> Option<u8> {
pub struct Parser<'a> {
tokens: Tokens<'a>,
tree: SyntaxTree<'a>,
current: Option<Token<'a>>,
previous: Option<Token<'a>>,
current: Token<'a>,
previous: Token<'a>,
panic_mode: bool,
}
@ -465,8 +460,8 @@ impl<'a> Parser<'a> {
let mut parser = Parser {
tokens: Tokens::new(source),
tree: SyntaxTree::new(),
current: None,
previous: None,
current: Token::new(TokenKind::EOF, 0, ""),
previous: Token::new(TokenKind::EOF, 0, ""),
panic_mode: false,
};
parser.advance();
@ -475,7 +470,7 @@ impl<'a> Parser<'a> {
pub fn parse(mut self) -> (SyntaxTree<'a>, ExprRef, Lines) {
let expr = self.expression();
self.consume(None, "expected end of expression");
self.consume(TokenKind::EOF, "expected end of expression");
(self.tree, expr, self.tokens.lines())
}
@ -505,30 +500,24 @@ impl<'a> Parser<'a> {
fn prefix_expression(&mut self) -> ExprRef {
self.trace("prefix");
let token = self.previous.as_ref();
match token {
Some(token) => match token.kind() {
TokenKind::Bang => self.unary(),
TokenKind::LeftParen => self.grouping(),
TokenKind::Number => self.number(),
TokenKind::Minus => self.unary(),
TokenKind::String => self.string(),
let token = &self.previous;
match token.kind {
TokenKind::Bang => self.unary(),
TokenKind::LeftParen => self.grouping(),
TokenKind::Number => self.number(),
TokenKind::Minus => self.unary(),
TokenKind::String => self.string(),
TokenKind::True => self
.tree
.add_expr(Expr::Literal(Literal::Bool(true), token.clone())),
TokenKind::False => self
.tree
.add_expr(Expr::Literal(Literal::Bool(false), token.clone())),
TokenKind::True => self
.tree
.add_expr(Expr::Literal(Literal::Bool(true), token.clone())),
TokenKind::False => self
.tree
.add_expr(Expr::Literal(Literal::Bool(false), token.clone())),
TokenKind::If => self.conditional(),
TokenKind::If => self.conditional(),
_ => {
self.error("expected an expression");
ExprRef::error()
}
},
None => {
_ => {
self.error("expected an expression");
ExprRef::error()
}
@ -537,8 +526,7 @@ impl<'a> Parser<'a> {
fn infix_expression(&mut self, power: u8, left: ExprRef) -> ExprRef {
self.trace("infix");
let kind = self.previous.as_ref().unwrap().kind();
match kind {
match self.previous.kind {
TokenKind::Plus
| TokenKind::Minus
| TokenKind::Star
@ -550,7 +538,7 @@ impl<'a> Parser<'a> {
}
fn number(&mut self) -> ExprRef {
let token = self.previous.as_ref().unwrap();
let token = &self.previous;
// What kind is it? For now let's just ... make it good.
let literal = match token.as_str().parse::<f64>() {
@ -565,7 +553,7 @@ impl<'a> Parser<'a> {
}
fn string(&mut self) -> ExprRef {
let token = self.previous.as_ref().unwrap();
let token = &self.previous;
let mut result = String::new();
let mut input = token.as_str().chars();
@ -590,51 +578,34 @@ impl<'a> Parser<'a> {
fn grouping(&mut self) -> ExprRef {
let result = self.expression();
self.consume(
Some(TokenKind::RightParen),
"expected ')' after an expression",
);
self.consume(TokenKind::RightParen, "expected ')' after an expression");
result
}
fn conditional(&mut self) -> ExprRef {
let token = self.previous.as_ref().unwrap().clone();
let token = self.previous.clone();
let condition_expr = self.expression();
self.consume(
Some(TokenKind::LeftBrace),
"expected '{' to start an 'if' block",
);
self.consume(TokenKind::LeftBrace, "expected '{' to start an 'if' block");
let then_expr = self.expression();
self.consume(
Some(TokenKind::RightBrace),
"expected '}' to end an 'if' block",
);
let else_expr = match &self.current {
Some(token) if token.kind() == TokenKind::Else => {
self.consume(TokenKind::RightBrace, "expected '}' to end an 'if' block");
let else_expr = if self.current.kind == TokenKind::Else {
self.advance();
if self.current.kind == TokenKind::If {
self.advance();
match &self.current {
// Allow `else if` without another `{`.
Some(token) if token.kind() == TokenKind::If => {
self.advance();
Some(self.conditional())
}
_ => {
self.consume(
Some(TokenKind::LeftBrace),
"expected '{' to start an 'else' block",
);
let else_expr = self.expression();
self.consume(
Some(TokenKind::RightBrace),
"Expected '}' to end an 'else' block",
);
Some(else_expr)
}
}
Some(self.conditional())
} else {
self.consume(
TokenKind::LeftBrace,
"expected '{' to start an 'else' block",
);
let else_expr = self.expression();
self.consume(TokenKind::RightBrace, "Expected '}' to end an 'else' block");
Some(else_expr)
}
_ => None,
} else {
None
};
let tail = self.previous.as_ref().unwrap().clone();
let tail = self.previous.clone();
self.tree.add_expr(Expr::Conditional(
token,
condition_expr,
@ -645,8 +616,8 @@ impl<'a> Parser<'a> {
}
fn unary(&mut self) -> ExprRef {
let token = self.previous.as_ref().unwrap().clone();
let kind = token.kind();
let token = self.previous.clone();
let kind = token.kind;
let expr = self.expression_with_power(UNARY_POWER);
let op = match kind {
TokenKind::Minus => UnaryOp::Negate,
@ -658,8 +629,8 @@ impl<'a> Parser<'a> {
}
fn binary(&mut self, power: u8, left: ExprRef) -> ExprRef {
let token = self.previous.as_ref().unwrap().clone();
let op = match token.kind() {
let token = self.previous.clone();
let op = match token.kind {
TokenKind::Plus => BinaryOp::Add,
TokenKind::Minus => BinaryOp::Subtract,
TokenKind::Star => BinaryOp::Multiply,
@ -673,25 +644,19 @@ impl<'a> Parser<'a> {
}
fn advance(&mut self) {
self.previous = self.current.take();
loop {
self.previous = self.current.clone();
self.current = self.tokens.next();
while self.current.kind == TokenKind::Error {
self.error_at_current(self.current.to_string());
self.current = self.tokens.next();
match &self.current {
Some(token) if token.kind() == TokenKind::Error => {
self.error_at_current(token.to_string())
}
_ => break,
}
}
}
fn consume(&mut self, kind: Option<TokenKind>, error: &str) {
match (&self.current, kind) {
(Some(token), Some(kind)) if token.kind() == kind => self.advance(),
(None, None) => (),
_ => {
self.error_at_current(error);
}
fn consume(&mut self, kind: TokenKind, error: &str) {
if self.current.kind == kind {
self.advance();
} else {
self.error_at_current(error);
}
}
@ -709,7 +674,7 @@ impl<'a> Parser<'a> {
self.error_at(self.current.clone(), message)
}
fn error_at<T>(&mut self, token: Option<Token<'a>>, message: T)
fn error_at<T>(&mut self, token: Token<'a>, message: T)
where
T: Into<String>,
{
@ -721,15 +686,13 @@ impl<'a> Parser<'a> {
let message: String = message.into();
let (line, column) = self.tokens.token_position(&token);
let mut final_message = "Error ".to_string();
match token {
None => final_message.push_str("at end"),
Some(t) => {
if t.kind() != TokenKind::Error {
final_message.push_str("at '");
final_message.push_str(t.as_str());
final_message.push_str("'");
}
}
if token.kind == TokenKind::EOF {
final_message.push_str("at end")
} else if token.kind != TokenKind::Error {
final_message.push_str("at '");
final_message.push_str(token.as_str());
final_message.push_str("'");
}
final_message.push_str(": ");
final_message.push_str(&message);

70
fine/src/tokens.rs
View file

@ -1,5 +1,8 @@
#[derive(Debug, PartialEq, Eq, Clone, Copy)]
pub enum TokenKind {
EOF,
Error,
LeftBrace,
RightBrace,
LeftBracket,
@ -47,14 +50,12 @@ pub enum TokenKind {
Use,
While,
Yield,
Error,
}
#[derive(Debug, PartialEq, Eq, Clone)]
pub struct Token<'a> {
kind: TokenKind,
start: usize,
pub kind: TokenKind,
pub start: usize,
value: Result<&'a str, String>,
}
@ -75,14 +76,6 @@ impl<'a> Token<'a> {
}
}
pub fn start(&self) -> usize {
self.start
}
pub fn kind(&self) -> TokenKind {
self.kind
}
pub fn as_str<'b>(&'b self) -> &'a str
where
'b: 'a,
@ -102,14 +95,12 @@ impl<'a> std::fmt::Display for Token<'a> {
pub struct Lines {
newlines: Vec<usize>,
eof: usize,
}
impl Lines {
fn new(eof: usize) -> Self {
fn new() -> Self {
Lines {
newlines: Vec::new(),
eof,
}
}
@ -119,16 +110,9 @@ impl Lines {
}
/// Return the position of the given token as a (line, column) pair. By
/// convention, lines are 1-based and columns are 0-based. Also, in
/// keeping with the iterator-nature of the tokenizer, `None` here
/// indicates end-of-file, and will return the position of the end of the
/// file.
pub fn token_position(&self, token: &Option<Token>) -> (usize, usize) {
let start = match token {
Some(t) => t.start,
None => self.eof,
};
self.position(start)
/// convention, lines are 1-based and columns are 0-based.
pub fn token_position(&self, token: &Token) -> (usize, usize) {
self.position(token.start)
}
/// Return the position of the given character offset as a (line,column)
@ -162,7 +146,7 @@ impl<'a> Tokens<'a> {
source,
chars: source.char_indices(),
next_char: None,
lines: Lines::new(source.len()),
lines: Lines::new(),
};
result.advance(); // Prime the pump
result
@ -174,7 +158,7 @@ impl<'a> Tokens<'a> {
/// Return the position of the given token as a (line, column) pair. See
/// `Lines::token_position` for more information about the range, etc.
pub fn token_position(&self, token: &Option<Token>) -> (usize, usize) {
pub fn token_position(&self, token: &Token) -> (usize, usize) {
self.lines.token_position(token)
}
@ -415,19 +399,15 @@ impl<'a> Tokens<'a> {
self.advance();
}
}
}
impl<'a> std::iter::Iterator for Tokens<'a> {
type Item = Token<'a>;
fn next(&mut self) -> Option<Self::Item> {
pub fn next(&mut self) -> Token<'a> {
self.skip_whitespace(); // TODO: Whitespace preserving/comment preserving
let (pos, c) = match self.advance() {
Some((p, c)) => (p, c),
None => return None,
None => return self.token(self.source.len(), TokenKind::EOF),
};
let token = match c {
match c {
'{' => self.token(pos, TokenKind::LeftBrace),
'}' => self.token(pos, TokenKind::RightBrace),
'[' => self.token(pos, TokenKind::LeftBracket),
@ -480,8 +460,7 @@ impl<'a> std::iter::Iterator for Tokens<'a> {
Token::error(pos, format!("Unexpected character '{c}'"))
}
}
};
Some(token)
}
}
}
@ -490,19 +469,32 @@ mod tests {
use super::*;
use pretty_assertions::assert_eq;
fn test_tokens_impl(input: &str, expected: Vec<Token>) {
let mut result = Vec::new();
let mut tokens = Tokens::new(input);
let mut is_eof = false;
while !is_eof {
let token = tokens.next();
is_eof = token.kind == TokenKind::EOF;
result.push(token);
}
assert_eq!(expected, result);
}
macro_rules! test_tokens {
($name:ident, $input:expr, $($s:expr),+) => {
#[test]
fn $name() {
use TokenKind::*;
let tokens: Vec<_> = Tokens::new($input).collect();
let expected: Vec<Token> = (vec![$($s),*])
let mut expected: Vec<Token> = (vec![$($s),*])
.into_iter()
.map(|t| Token::new(t.1, t.0, t.2))
.collect();
expected.push(Token::new(TokenKind::EOF, $input.len(), ""));
assert_eq!(expected, tokens);
test_tokens_impl($input, expected);
}
}
}