DeCarabas/oden
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Compare commits

base: DeCarabas:9d226b205d2f7930c42ad0cb366a2077c507e7bb
Branches Tags
DeCarabas:main
DeCarabas:debugger
DeCarabas:source-maps
..
compare: DeCarabas:fa53841af9ef1998a0e330dc02c9b64fdb2b65d4
Branches Tags
DeCarabas:main
DeCarabas:debugger
DeCarabas:source-maps

No commits in common. "9d226b205d2f7930c42ad0cb366a2077c507e7bb" and "fa53841af9ef1998a0e330dc02c9b64fdb2b65d4" have entirely different histories.
9d226b205d ... fa53841af9

12 changed files with 118 additions and 365 deletions
Download patch file Download diff file Expand all files Collapse all files

15
fine/build.rs
View file

@ -37,21 +37,6 @@ fn generate_test_for_file(path: PathBuf) -> String {
assertions.push(quote! {
crate::assert_concrete(&_tree, #concrete, #display_path);
});
} else if line == "@compiles-to:" {
let mut compiled = String::new();
while let Some(line) = lines.next() {
let line = match line.strip_prefix("// | ") {
Some(line) => line,
None => break,
};
compiled.push_str(line);
compiled.push_str("\n");
}
assertions.push(quote! {
crate::assert_compiles_to(&_tree, &_lines, #compiled, #display_path);
});
} else if let Some(line) = line.strip_prefix("@type:") {
let (pos, expected) = line
.trim()

116
fine/src/compiler.rs
View file

@ -6,34 +6,8 @@ use crate::{
tokens::TokenKind,
};
macro_rules! compiler_assert_eq {
($compiler:expr, $tr:expr, $ll:expr, $rr:expr, $($t:tt)*) => {{
let left = &$ll;
let right = &$rr;
if left != right {
let semantics = $compiler.semantics;
semantics.dump_compiler_state(Some($tr));
let message = format!($($t)*);
assert_eq!(left, right, "{}", message);
}
}};
}
macro_rules! compiler_assert {
($compiler:expr, $tr:expr, $($t:tt)*) => {{
if !($($t)*) {
let semantics = $compiler.semantics;
semantics.dump_compiler_state(Some($tr));
assert!($($t)*);
}
}};
}
// TODO: If I were cool this would by actual bytecode.
// But I'm not cool.
#[derive(Debug)]
pub enum Instruction {
Panic,
@ -55,7 +29,6 @@ pub enum Instruction {
PushString(usize),
PushTrue,
StoreLocal(usize),
StoreModule(usize),
}
pub enum Export {
@ -79,13 +52,8 @@ impl Module {
init: 0,
}
}
pub fn functions(&self) -> &[Function] {
&self.functions
}
}
// TODO: Debug information.
pub struct Function {
name: String,
instructions: Vec<Instruction>,
@ -95,12 +63,12 @@ pub struct Function {
}
impl Function {
pub fn new(name: &str, args: usize) -> Self {
pub fn new(name: &str) -> Self {
Function {
name: name.to_string(),
instructions: Vec::new(),
strings: Vec::new(),
args,
args: 0,
locals: 0,
}
}
@ -108,29 +76,12 @@ impl Function {
pub fn name(&self) -> &str {
&self.name
}
pub fn args(&self) -> usize {
self.args
}
pub fn locals(&self) -> usize {
self.locals
}
pub fn strings(&self) -> &[String] {
&self.strings
}
pub fn instructions(&self) -> &[Instruction] {
&self.instructions
}
}
struct Compiler<'a> {
semantics: &'a Semantics<'a>,
syntax: &'a SyntaxTree<'a>,
function_bindings: HashMap<String, usize>,
module: Module,
function: Function,
}
@ -162,9 +113,8 @@ pub fn compile(semantics: &Semantics) -> Module {
let mut compiler = Compiler {
semantics,
syntax: semantics.tree(),
function_bindings: HashMap::new(),
module: Module::new(),
function: Function::new("<< module >>", 0),
function: Function::new("<< module >>"),
};
if let Some(t) = semantics.tree().root() {
@ -181,7 +131,7 @@ pub fn compile(semantics: &Semantics) -> Module {
fn file(c: &mut Compiler, t: TreeRef) {
let tree = &c.syntax[t];
compiler_assert_eq!(c, t, tree.kind, TreeKind::File, "must be compiling a file");
assert_eq!(tree.kind, TreeKind::File);
for i in 0..tree.children.len() {
if let Some(t) = tree.nth_tree(i) {
compile_statement(c, t, false);
@ -355,13 +305,10 @@ fn compile_identifier_expression(c: &mut Compiler, t: TreeRef, tree: &Tree) -> O
Instruction::LoadLocal(declaration.index)
}
Location::Argument => {
compiler_assert!(c, t, declaration.index < c.function.args);
assert!(declaration.index < c.function.args);
Instruction::LoadArgument(declaration.index)
}
Location::Module => {
compiler_assert!(c, t, declaration.index < c.module.globals);
Instruction::LoadModule(declaration.index)
}
Location::Module => Instruction::LoadModule(declaration.index),
};
c.push(instruction);
@ -424,22 +371,9 @@ fn compile_let_statement(c: &mut Compiler, t: TreeRef, tree: &Tree, gen_value: b
let environment = c.semantics.environment_of(t);
let declaration = environment.bind(tree.nth_token(1)?)?;
let instruction = match declaration.location {
Location::Local => {
if declaration.index >= c.function.locals {
c.function.locals = declaration.index + 1;
}
Instruction::StoreLocal(declaration.index)
}
Location::Module => {
if declaration.index >= c.module.globals {
c.module.globals = declaration.index + 1;
}
Instruction::StoreModule(declaration.index)
}
_ => panic!("unsuitable location for let declaration"),
};
c.push(instruction);
// NOTE: Because this is a let statement I assume it's local!
assert!(matches!(declaration.location, Location::Local));
c.push(Instruction::StoreLocal(declaration.index));
if gen_value {
c.push(Instruction::PushNothing);
}
@ -447,34 +381,6 @@ fn compile_let_statement(c: &mut Compiler, t: TreeRef, tree: &Tree, gen_value: b
OK
}
fn compile_function_declaration(c: &mut Compiler, tree: &Tree, gen_value: bool) -> CR {
let name = tree.nth_token(1)?;
let block = if tree
.nth_token(3)
.is_some_and(|t| t.kind == TokenKind::Arrow)
{
tree.nth_tree(4)?
} else {
tree.nth_tree(3)?
};
let arg_list = tree.nth_tree(2)?;
let arg_count = c.syntax[arg_list].children.len() - 2;
let mut prev = Function::new(name.as_str(), arg_count);
std::mem::swap(&mut c.function, &mut prev);
c.function_bindings
.insert(c.function.name.clone(), c.module.functions.len());
compile_expression(c, block);
std::mem::swap(&mut c.function, &mut prev);
c.module.functions.push(prev);
if gen_value {
c.push(Instruction::PushNothing);
}
OK
fn compile_function_declaration(_c: &mut Compiler, _tree: &Tree, _gen_value: bool) -> CR {
todo!()
}

28
fine/src/lib.rs
View file

@ -1,32 +1,4 @@
use std::fs;
use parser::parse;
use semantics::{check, Semantics};
pub mod compiler;
pub mod parser;
pub mod semantics;
pub mod tokens;
pub mod vm;
pub fn process_file(file: &str) {
let source = match fs::read_to_string(file) {
Ok(c) => c,
Err(e) => {
eprintln!("Unable to read file {file}: {e}");
return;
}
};
// What am I doing here?
let (tree, lines) = parse(&source);
let semantics = Semantics::new(&tree, &lines);
check(&semantics);
// OK now there might be errors.
let mut errors = semantics.snapshot_errors();
errors.reverse();
for e in errors {
eprintln!("{file}: {}:{}: {}", e.start.0, e.start.1, e.message);
}
}

31
fine/src/main.rs
View file

@ -1,8 +1,37 @@
use fine::parser::parse;
use fine::semantics::Semantics;
use std::env;
use std::fs;
pub fn process_file(file: &str) {
let source = match fs::read_to_string(file) {
Ok(c) => c,
Err(e) => {
eprintln!("Unable to read file {file}: {e}");
return;
}
};
// What am I doing here?
let (tree, lines) = parse(&source);
let semantics = Semantics::new(&tree, &lines);
// This is... probably wrong, I don't know, what am I doing?
for t in tree.trees() {
let _ = semantics.type_of(t);
}
// OK now there might be errors.
let mut errors = semantics.snapshot_errors();
errors.reverse();
for e in errors {
eprintln!("{file}: {}:{}: {}", e.start.0, e.start.1, e.message);
}
}
pub fn main() {
let args: Vec<String> = env::args().collect();
for arg in &args[1..] {
fine::process_file(arg);
process_file(arg);
}
}

23
fine/src/parser.rs
View file

@ -138,6 +138,7 @@ pub enum TreeKind {
BinaryExpression,
IfStatement,
Identifier,
PrintStatement,
}
pub struct Tree<'a> {
@ -555,10 +556,32 @@ fn statement(p: &mut CParser) {
// require a semicolon at the end if it's all by itself.
TokenKind::If => statement_if(p),
TokenKind::Print => statement_print(p),
_ => statement_expression(p),
}
}
fn statement_print(p: &mut CParser) {
assert!(p.at(TokenKind::Print));
let m = p.start();
p.expect(
TokenKind::Print,
"expect 'print' to start a print statement",
);
p.expect(TokenKind::LeftParen, "expect '(' to start a print");
if !p.at(TokenKind::RightParen) {
expression(p);
}
p.expect(TokenKind::RightParen, "expect ')' after a print statement");
if !p.at(TokenKind::RightBrace) {
p.expect(TokenKind::Semicolon, "expect ';' to end a print statement");
}
p.end(m, TreeKind::PrintStatement);
}
fn statement_if(p: &mut CParser) {
assert!(p.at(TokenKind::If));
let m = p.start();

69
fine/src/semantics.rs
View file

@ -401,22 +401,6 @@ impl<'a> Semantics<'a> {
TreeKind::LetStatement => self.environment_of_let(parent, tree),
TreeKind::FunctionDecl => self.environment_of_func(parent, tree),
// TODO: Blocks should introduce a local environment if required.
// Test with a type error in a block statement and a
// binding outside. You will need a new assertion type and
// possibly a compile/run to ensure it works.
//
// let x = 7;
// {
// let x = 23;
// }
// print(x); // 7
//
// {
// let y = 12; // check: `y` is local not global!
// }
// print(y); // error, cannot find 'y'
// TODO: MORE Things that introduce an environment!
_ => parent,
};
@ -440,19 +424,12 @@ impl<'a> Semantics<'a> {
None => Type::Error,
};
let (location, base_index) = match parent.location {
Location::Local => (
Location::Local,
parent.base_index + parent.declarations.len(),
),
Location::Module => (
Location::Module,
parent.base_index + parent.declarations.len(),
),
Location::Argument => (Location::Local, 0),
let base_index = match parent.location {
Location::Local => parent.base_index + parent.declarations.len(),
_ => 0,
};
let mut environment = Environment::new(Some(parent), location, base_index);
let mut environment = Environment::new(Some(parent), Location::Local, base_index);
environment.insert(name, declaration_type);
EnvironmentRef::new(environment)
@ -841,44 +818,6 @@ impl<'a> Semantics<'a> {
}
}
pub fn check(s: &Semantics) {
for t in s.syntax_tree.trees() {
let tree = &s.syntax_tree[t];
match tree.kind {
TreeKind::Error => {} // already reported
TreeKind::File => {}
TreeKind::FunctionDecl => {
let _ = s.environment_of(t);
}
TreeKind::ParamList => {}
TreeKind::Parameter => {}
TreeKind::TypeExpression => {
let _ = s.type_of_type_expr(tree);
}
TreeKind::Block => {
let _ = s.type_of_block(tree);
}
TreeKind::LetStatement => {
let _ = s.environment_of(t);
}
TreeKind::ReturnStatement => {}
TreeKind::ExpressionStatement
| TreeKind::LiteralExpression
| TreeKind::GroupingExpression
| TreeKind::UnaryExpression
| TreeKind::ConditionalExpression
| TreeKind::CallExpression
| TreeKind::BinaryExpression => {
let _ = s.type_of(t);
}
TreeKind::ArgumentList => {}
TreeKind::Argument => {}
TreeKind::IfStatement => {}
TreeKind::Identifier => {}
}
}
}
#[cfg(test)]
mod tests {
use super::*;

21
fine/src/tokens.rs
View file

@ -49,6 +49,7 @@ pub enum TokenKind {
Import,
Let,
Or,
Print,
Return,
Select,
This,
@ -303,6 +304,11 @@ impl<'a> Tokens<'a> {
return TokenKind::Or;
}
}
'p' => {
if ident == "print" {
return TokenKind::Print;
}
}
'r' => {
if ident == "return" {
return TokenKind::Return;
@ -569,17 +575,18 @@ mod tests {
test_tokens!(
more_keywords,
"fun if import let return select this true while truewhile",
"fun if import let print return select this true while truewhile",
(0, Fun, "fun"),
(4, If, "if"),
(7, Import, "import"),
(14, Let, "let"),
(18, Return, "return"),
(25, Select, "select"),
(32, This, "this"),
(37, True, "true"),
(42, While, "while"),
(48, Identifier, "truewhile")
(18, Print, "print"),
(24, Return, "return"),
(31, Select, "select"),
(38, This, "this"),
(43, True, "true"),
(48, While, "while"),
(54, Identifier, "truewhile")
);
test_tokens!(

12
fine/src/vm.rs
View file

@ -1,12 +0,0 @@
// use crate::compiler::{Function, Module};
// // TODO: VM state structure
// // TODO: Runtime module vs compiled module
// struct StackFrame<'a> {
// function: &'a Function,
// }
// pub fn eval(module: &Module) {
// }

94
fine/tests/example_tests.rs
View file

@ -1,32 +1,30 @@
use fine::compiler::{compile, Function, Module};
use fine::parser::SyntaxTree;
use fine::semantics::{Semantics, Type};
use fine::tokens::Lines;
use pretty_assertions::assert_eq;
use std::fmt::Write as _;
fn rebase_section(source_path: &str, section: &str, value: &str) {
fn rebase_concrete(source_path: &str, dump: &str) {
let contents = std::fs::read_to_string(source_path)
.expect(&format!("unable to read input file {}", source_path));
let mut result = String::new();
let mut lines = contents.lines();
// Search for the section.
let mut found_section = false;
let marker = format!("// @{section}:");
// Search for the "concrete:" section.
let mut found_concrete_section = false;
while let Some(line) = lines.next() {
result.push_str(line);
result.push_str("\n");
if line == marker {
found_section = true;
if line == "// @concrete:" {
found_concrete_section = true;
break;
}
}
if !found_section {
if !found_concrete_section {
panic!(
"unable to locate the {section} section in {source_path}. Is there a line that starts with '// @{section}:'?"
"unable to locate the concrete section in {}. Is there a line that starts with '// concrete:'?",
source_path
);
}
@ -41,7 +39,7 @@ fn rebase_section(source_path: &str, section: &str, value: &str) {
// OK we're out of concrete syntax tree; copy in the
// new CST. (We do this inline so we don't lose
// `line`.)
for expected_line in value.lines() {
for expected_line in dump.lines() {
result.push_str("// | ");
result.push_str(expected_line);
result.push_str("\n");
@ -72,24 +70,18 @@ fn rebase_section(source_path: &str, section: &str, value: &str) {
std::fs::write(source_path, result).expect("unable to write the new file!");
}
fn should_rebase() -> bool {
fn assert_concrete(tree: &SyntaxTree, expected: &str, source_path: &str) {
let dump = tree.dump(false);
let rebase = std::env::var("FINE_TEST_REBASE")
.unwrap_or(String::new())
.to_lowercase();
match rebase.as_str() {
"1" | "true" | "yes" | "y" => true,
_ => false,
}
}
fn assert_concrete(tree: &SyntaxTree, expected: &str, source_path: &str) {
let dump = tree.dump(false);
if dump != expected {
if should_rebase() {
rebase_section(source_path, "concrete", &dump)
} else {
assert_eq!(expected, dump, "concrete syntax trees did not match (set FINE_TEST_REBASE=1 to auto-rebase if the diff is expected)")
"1" | "true" | "yes" | "y" => {
if dump != expected {
rebase_concrete(source_path, &dump)
}
}
_ => assert_eq!(expected, dump, "concrete syntax trees did not match (set FINE_TEST_REBASE=1 to auto-rebase if the diff is expected)"),
}
}
@ -189,58 +181,4 @@ fn assert_type_error_at(
);
}
fn dump_function(out: &mut String, function: &Function) -> std::fmt::Result {
writeln!(
out,
"function {} ({} args, {} locals):",
function.name(),
function.args(),
function.locals()
)?;
let strings = function.strings();
writeln!(out, " strings ({}):", strings.len())?;
for (i, s) in strings.iter().enumerate() {
writeln!(out, " {}: {}", i, s)?; // TODO: ESCAPE
}
let code = function.instructions();
writeln!(out, " code ({}):", code.len())?;
for (i, inst) in code.iter().enumerate() {
writeln!(out, " {}: {:?}", i, inst)?;
}
Ok(())
}
fn dump_module(out: &mut String, module: &Module) -> std::fmt::Result {
for function in module.functions() {
dump_function(out, function)?;
}
Ok(())
}
fn assert_compiles_to(tree: &SyntaxTree, lines: &Lines, expected: &str, source_path: &str) {
let semantics = Semantics::new(tree, lines);
let module = compile(&semantics);
let mut actual = String::new();
dump_module(&mut actual, &module).expect("no dumping?");
if expected != actual {
if should_rebase() {
rebase_section(source_path, "compiles-to", &actual)
} else {
semantic_assert_eq!(
&semantics,
None,
expected,
actual,
"did not compile as expected (set FINE_TEST_REBASE=1 to auto-rebase if the diff is expected)"
)
}
}
}
include!(concat!(env!("OUT_DIR"), "/generated_tests.rs"));

22
fine/tests/expression/argument.fine
View file

@ -1,8 +1,3 @@
fun foo(x: f64) {
x + 7
}
// @type: 20 f64
// @concrete:
// | File
// | FunctionDecl
@ -27,14 +22,9 @@ fun foo(x: f64) {
// | Number:'"7"'
// | RightBrace:'"}"'
// |
// @compiles-to:
// | function foo (1 args, 0 locals):
// | strings (0):
// | code (3):
// | 0: LoadArgument(0)
// | 1: PushFloat(7.0)
// | 2: FloatAdd
// | function << module >> (0 args, 0 locals):
// | strings (0):
// | code (0):
// |
fun foo(x: f64) {
x + 7
}
// @type: 613 f64

23
fine/tests/expression/arithmetic.fine
View file

@ -1,6 +1,3 @@
1 * 2 + -3 * 4;
// @type: 6 f64
// @concrete:
// | File
// | ExpressionStatement
@ -21,19 +18,7 @@
// | LiteralExpression
// | Number:'"4"'
// | Semicolon:'";"'
// |
// @compiles-to:
// | function << module >> (0 args, 0 locals):
// | strings (0):
// | code (10):
// | 0: PushFloat(1.0)
// | 1: PushFloat(2.0)
// | 2: FloatMultiply
// | 3: PushFloat(3.0)
// | 4: PushFloat(-1.0)
// | 5: FloatMultiply
// | 6: PushFloat(4.0)
// | 7: FloatMultiply
// | 8: FloatAdd
// | 9: Discard
// |
//
1 * 2 + -3 * 4;
// @type: 532 f64

29
fine/tests/expression/variable.fine
View file

@ -1,8 +1,3 @@
let x = 23;
let y = x * 2;
y;
// @type: 27 f64
// @concrete:
// | File
// | LetStatement
@ -23,21 +18,17 @@ y;
// | LiteralExpression
// | Number:'"2"'
// | Semicolon:'";"'
// | ExpressionStatement
// | PrintStatement
// | Print:'"print"'
// | LeftParen:'"("'
// | Identifier
// | Identifier:'"y"'
// | RightParen:'")"'
// | Semicolon:'";"'
// |
// @compiles-to:
// | function << module >> (0 args, 0 locals):
// | strings (0):
// | code (8):
// | 0: PushFloat(23.0)
// | 1: StoreModule(0)
// | 2: LoadModule(0)
// | 3: PushFloat(2.0)
// | 4: FloatMultiply
// | 5: StoreModule(1)
// | 6: LoadModule(1)
// | 7: Discard
// |
let x = 23;
let y = x * 2;
print(y);
// @type: 667 f64