DeCarabas/oden
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

[fine] Dump CSTs and an example zoo

Browse source
This commit is contained in:
John Doty 2024-01-04 13:25:17 -08:00
parent 757db0ba3e
commit 1f6d7ec131
9 changed files with 357 additions and 32 deletions
Download patch file Download diff file Expand all files Collapse all files

29
fine/src/tokens.rs
View file

@ -3,6 +3,9 @@ pub enum TokenKind {
EOF,
Error,
Whitespace,
Comment,
LeftBrace,
RightBrace,
LeftBracket,
@ -390,7 +393,7 @@ impl<'a> Tokens<'a> {
self.next_char.is_none()
}
fn skip_whitespace(&mut self) {
fn whitespace(&mut self, pos: usize) -> Token<'a> {
while let Some((pos, ch)) = self.next_char {
if ch == '\n' {
self.lines.add_line(pos);
@ -399,16 +402,27 @@ impl<'a> Tokens<'a> {
}
self.advance();
}
self.token(pos, TokenKind::Whitespace)
}
fn comment(&mut self, pos: usize) -> Token<'a> {
while let Some((_, ch)) = self.next_char {
if ch == '\n' {
break;
}
self.advance();
}
self.token(pos, TokenKind::Comment)
}
pub fn next(&mut self) -> Token<'a> {
self.skip_whitespace(); // TODO: Whitespace preserving/comment preserving
let (pos, c) = match self.advance() {
Some((p, c)) => (p, c),
None => return self.token(self.source.len(), TokenKind::EOF),
};
match c {
' ' | '\t' | '\r' | '\n' => self.whitespace(pos),
'{' => self.token(pos, TokenKind::LeftBrace),
'}' => self.token(pos, TokenKind::RightBrace),
'[' => self.token(pos, TokenKind::LeftBracket),
@ -427,7 +441,13 @@ impl<'a> Tokens<'a> {
'+' => self.token(pos, TokenKind::Plus),
':' => self.token(pos, TokenKind::Colon),
';' => self.token(pos, TokenKind::Semicolon),
'/' => self.token(pos, TokenKind::Slash),
'/' => {
if self.matches('/') {
self.comment(pos)
} else {
self.token(pos, TokenKind::Slash)
}
}
'*' => self.token(pos, TokenKind::Star),
'!' => {
if self.matches('=') {
@ -484,6 +504,9 @@ mod tests {
while !is_eof {
let token = tokens.next();
is_eof = token.kind == TokenKind::EOF;
if token.kind == TokenKind::Whitespace {
continue;
}
result.push(token);
}