DeCarabas/oden
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

[fine] Tokens are by reference, ephemera

Browse source 
It's another jump, perhaps, but smaller arrays, and now we can track
ephemera efficiently without bloating child trees. (We could also
put ephemera inline with the child trees but then nth_token would be
unwieldy, and it would lower our data density.)
This commit is contained in:
John Doty 2024-04-06 17:39:33 -07:00
parent a5173be680
commit 0b0b5d72d0
3 changed files with 298 additions and 105 deletions
Download patch file Download diff file Expand all files Collapse all files

100
fine/src/compiler.rs
View file

@ -2,11 +2,11 @@ use std::collections::HashMap;
use std::rc::Rc;
use crate::{
parser::{Child, SyntaxTree, Tree, TreeKind, TreeRef},
parser::{Child, SyntaxTree, TokenRef, Tree, TreeKind, TreeRef},
semantics::{
string_constant_to_string, Declaration, Location, ModuleId, Origin, Semantics, Type,
},
tokens::TokenKind,
tokens::{Token, TokenKind},
};
pub const EXTERN_BUILTIN_NOOP: usize = 0;
@ -163,6 +163,42 @@ struct Compiler<'a> {
function: Function,
}
impl<'a> std::ops::Index<TreeRef> for Compiler<'a> {
type Output = Tree;
#[inline]
fn index(&self, index: TreeRef) -> &Self::Output {
&self.syntax[index]
}
}
impl<'a> std::ops::Index<&TreeRef> for Compiler<'a> {
type Output = Tree;
#[inline]
fn index(&self, index: &TreeRef) -> &Self::Output {
&self.syntax[index]
}
}
impl<'a> std::ops::Index<TokenRef> for Compiler<'a> {
type Output = Token;
#[inline]
fn index(&self, index: TokenRef) -> &Self::Output {
&self.syntax[index]
}
}
impl<'a> std::ops::Index<&TokenRef> for Compiler<'a> {
type Output = Token;
#[inline]
fn index(&self, index: &TokenRef) -> &Self::Output {
&self.syntax[index]
}
}
impl<'a> Compiler<'a> {
fn add_string(&mut self, result: String) -> usize {
let index = self.function.strings.len();
@ -261,7 +297,7 @@ fn function_from_function_decl(
tree: &Tree,
) -> Result<Function, &'static str> {
// TODO: If this is a method the name should be different.
let name = tree.nth_token(1).ok_or("no id")?.as_str(source);
let name = syntax[tree.nth_token(1).ok_or("no id")?].as_str(source);
let param_list = tree
.child_tree_of_kind(syntax, TreeKind::ParamList)
@ -271,8 +307,12 @@ fn function_from_function_decl(
Ok(Function::new(name, param_count))
}
fn function_from_class_decl(source: &str, tree: &Tree) -> Result<Function, &'static str> {
let name = tree.nth_token(1).ok_or("no name")?.as_str(source);
fn function_from_class_decl(
source: &str,
syntax: &SyntaxTree,
tree: &Tree,
) -> Result<Function, &'static str> {
let name = syntax[tree.nth_token(1).ok_or("no name")?].as_str(source);
// TODO: I think this is incorrect!
let field_count = tree.children.len() - 2;
@ -306,7 +346,7 @@ pub fn compile_module(semantics: &Semantics) -> Rc<CompiledModule> {
let tree = &semantics.tree()[t];
let function = match tree.kind {
TreeKind::FunctionDecl => function_from_function_decl(&source, &syntax_tree, tree),
TreeKind::ClassDecl => function_from_class_decl(&source, tree),
TreeKind::ClassDecl => function_from_class_decl(&source, &syntax_tree, tree),
_ => Err("don't know how to make a function of this"),
};
@ -335,7 +375,7 @@ pub fn compile_module(semantics: &Semantics) -> Rc<CompiledModule> {
}
fn file(c: &mut Compiler, t: TreeRef) {
let tree = &c.syntax[t];
let tree = &c[t];
compiler_assert_eq!(c, t, tree.kind, TreeKind::File, "must be compiling a file");
let children: Vec<_> = tree.child_trees().collect();
@ -381,7 +421,7 @@ fn compile_expression(c: &mut Compiler, t: TreeRef) {
}
fn compile_literal(c: &mut Compiler, t: TreeRef, tr: &Tree) -> CR {
let tok = tr.nth_token(0).ok_or("no token")?;
let tok = &c[tr.nth_token(0).ok_or("no token")?];
match c.semantics.type_of(t) {
Type::F64 => c.push(Instruction::PushFloat(
tok.as_str(c.source).parse().unwrap(),
@ -410,7 +450,7 @@ fn compile_grouping(c: &mut Compiler, t: &Tree) -> CR {
fn compile_unary_operator(c: &mut Compiler, t: TreeRef, tr: &Tree) -> CR {
compile_expression(c, tr.nth_tree(1).ok_or("no arg")?);
let tok = tr.nth_token(0).ok_or("no op")?;
let tok = &c[tr.nth_token(0).ok_or("no op")?];
match tok.kind {
TokenKind::Minus => {
c.push(Instruction::PushFloat(-1.0));
@ -464,7 +504,7 @@ where
}
fn compile_binary_expression(c: &mut Compiler, t: TreeRef, tr: &Tree) -> CR {
let op = tr.nth_token(1).ok_or("no op")?;
let op = &c[tr.nth_token(1).ok_or("no op")?];
match op.kind {
TokenKind::Plus => compile_simple_binary_expression(c, tr, |c, t| match t {
Type::F64 => Instruction::FloatAdd,
@ -583,7 +623,7 @@ fn compile_binary_expression(c: &mut Compiler, t: TreeRef, tr: &Tree) -> CR {
c.push(Instruction::Dup);
let lvalue = tr.nth_tree(0).ok_or("no lvalue")?;
let ltree = &c.syntax[lvalue];
let ltree = &c[lvalue];
#[allow(unused_assignments)]
let mut environment = None;
@ -591,7 +631,7 @@ fn compile_binary_expression(c: &mut Compiler, t: TreeRef, tr: &Tree) -> CR {
let declaration = match ltree.kind {
// TODO: Assign to list access
TreeKind::Identifier => {
let id = ltree.nth_token(0).ok_or("no id")?.as_str(&c.source);
let id = c[ltree.nth_token(0).ok_or("no id")?].as_str(&c.source);
environment = Some(c.semantics.environment_of(lvalue));
environment
.as_ref()
@ -600,7 +640,7 @@ fn compile_binary_expression(c: &mut Compiler, t: TreeRef, tr: &Tree) -> CR {
.ok_or("cannot bind destination")?
}
TreeKind::MemberAccess => {
let id = ltree.nth_token(2).ok_or("no member")?.as_str(&c.source);
let id = c[ltree.nth_token(2).ok_or("no member")?].as_str(&c.source);
let t = ltree.nth_tree(0).ok_or("no lhs exp")?;
let typ = c.semantics.type_of(t);
@ -655,7 +695,7 @@ fn compile_binary_expression(c: &mut Compiler, t: TreeRef, tr: &Tree) -> CR {
}
fn compile_identifier_expression(c: &mut Compiler, t: TreeRef, tree: &Tree) -> CR {
let ident = tree.nth_token(0).ok_or("no ident")?.as_str(&c.source);
let ident = c[tree.nth_token(0).ok_or("no ident")?].as_str(&c.source);
let environment = c.semantics.environment_of(t);
let declaration = environment.bind(ident).ok_or("not found")?;
@ -758,15 +798,15 @@ fn compile_pattern(c: &mut Compiler, t: TreeRef) -> CR {
let type_expr = tree.child_tree_of_kind(&c.syntax, TreeKind::TypeExpression);
let and_index = tree.children.iter().position(|c| match c {
Child::Token(t) => t.kind == TokenKind::And,
let and_index = tree.children.iter().position(|child| match child {
Child::Token(t) => c[t].kind == TokenKind::And,
_ => false,
});
// If you have a binding, dup and store now, it is in scope.
if let Some(binding) = tree.child_tree_of_kind(&c.syntax, TreeKind::VariableBinding) {
if let Some(variable) = binding.nth_token(0) {
let id = variable.as_str(&c.source);
let id = c[variable].as_str(&c.source);
let environment = c.semantics.environment_of(t);
let Some(declaration) = environment.bind(id) else {
ice!(c, t, "cannot bind pattern variable `{id}`");
@ -845,7 +885,7 @@ fn compile_type_expr_eq(c: &mut Compiler, t: TreeRef) {
}
fn compile_type_identifier_eq(c: &mut Compiler, t: TreeRef, tree: &Tree) -> CR {
let identifier = tree.nth_token(0).ok_or("no id")?.as_str(&c.source);
let identifier = c[tree.nth_token(0).ok_or("no id")?].as_str(&c.source);
match identifier {
"f64" => {
c.push(Instruction::IsFloat);
@ -971,7 +1011,7 @@ fn compile_new_object_expression(c: &mut Compiler, t: TreeRef, tree: &Tree) -> C
let mut field_bindings = HashMap::new();
for field in field_list.children_of_kind(&c.syntax, TreeKind::FieldValue) {
let f = &c.syntax[field];
let name = f.nth_token(0).ok_or("no field name")?;
let name = &c.syntax[f.nth_token(0).ok_or("no field name")?];
field_bindings.insert(name.as_str(&c.source), field);
}
@ -990,10 +1030,8 @@ fn compile_new_object_expression(c: &mut Compiler, t: TreeRef, tree: &Tree) -> C
let type_reference = tree
.child_tree_of_kind(&c.syntax, TreeKind::TypeIdentifier)
.ok_or("no type ref")?;
let identifier = type_reference
.nth_token(0)
.ok_or("no type id")?
.as_str(&c.source);
let identifier = type_reference.nth_token(0).ok_or("no type id")?;
let identifier = c[identifier].as_str(&c.source);
let environment = c.semantics.environment_of(t);
let declaration = environment.bind(identifier).ok_or("cannot bind type")?;
match declaration.location {
@ -1008,7 +1046,7 @@ fn compile_new_object_expression(c: &mut Compiler, t: TreeRef, tree: &Tree) -> C
fn compile_field_value(c: &mut Compiler, t: TreeRef, tree: &Tree) -> CR {
if let Some(colon) = tree.nth_token(1) {
if colon.kind == TokenKind::Colon {
if c[colon].kind == TokenKind::Colon {
compile_expression(c, tree.nth_tree(2).ok_or("no val")?);
return OK;
}
@ -1016,7 +1054,7 @@ fn compile_field_value(c: &mut Compiler, t: TreeRef, tree: &Tree) -> CR {
// Form 2: { x, ... }
let environment = c.semantics.environment_of(t);
let id = tree.nth_token(0).ok_or("no id")?.as_str(&c.source);
let id = c[tree.nth_token(0).ok_or("no id")?].as_str(&c.source);
let declaration = environment.bind(id).ok_or("cannot bind")?;
compile_load_declaration(c, t, declaration)
@ -1030,7 +1068,7 @@ fn compile_member_access(c: &mut Compiler, t: TreeRef, tree: &Tree) -> CR {
compile_expression(c, lhs);
let typ = c.semantics.type_of(lhs);
let ident = tree.nth_token(2).ok_or("no ident")?.as_str(&c.source);
let ident = c[tree.nth_token(2).ok_or("no ident")?].as_str(&c.source);
let environment = c.semantics.member_environment(t, &typ);
let declaration = environment.bind(ident).ok_or("cannot bind")?;
@ -1114,7 +1152,7 @@ fn compile_expression_statement(c: &mut Compiler, tree: &Tree, gen_value: bool)
if tree
.nth_token(1)
.is_some_and(|t| t.kind == TokenKind::Semicolon)
.is_some_and(|t| c[t].kind == TokenKind::Semicolon)
{
c.push(Instruction::Discard);
if gen_value {
@ -1134,7 +1172,7 @@ fn compile_let_statement(c: &mut Compiler, t: TreeRef, tree: &Tree, gen_value: b
compile_expression(c, tree.nth_tree(3).ok_or("no val")?);
let environment = c.semantics.environment_of(t);
let declaration = environment
.bind(tree.nth_token(1).ok_or("no id")?.as_str(&c.source))
.bind(c[tree.nth_token(1).ok_or("no id")?].as_str(&c.source))
.ok_or("cannot bind")?;
// TODO: ASSERT LOCAL DECLARATION?
@ -1198,7 +1236,7 @@ fn compile_function(c: &mut Compiler, t: TreeRef) -> CR {
c.push(Instruction::LoadArgument(count - 1 - i));
}
let name = tree.nth_token(1).ok_or("no name")?.as_str(&c.source);
let name = c[tree.nth_token(1).ok_or("no name")?].as_str(&c.source);
let name_index = c.add_string(name.to_string());
c.push(Instruction::PushString(name_index));
c.push(Instruction::PushInt(t.index().try_into().unwrap()));
@ -1250,8 +1288,8 @@ fn compile_return_statement(c: &mut Compiler, tree: &Tree) -> CR {
fn compile_for_statement(c: &mut Compiler, tree: &Tree, gen_value: bool) -> CR {
// Figure out the variable.
let vt = tree.nth_tree(1).ok_or("no var")?;
let var = &c.syntax[vt];
let id = var.nth_token(0).ok_or("no id")?.as_str(&c.source);
let var = &c[vt];
let id = c[var.nth_token(0).ok_or("no id")?].as_str(&c.source);
let body = tree.nth_tree(4).ok_or("no body")?;
let env = c.semantics.environment_of(body);