[fine] Oh no a runtime and module loading and stuff

Lots of test work to use the new mechanism. I'm not sure I like it.

fine/build.rs

@@ -57,7 +57,7 @@ fn generate_test_for_file(path: PathBuf) -> String {
             }
 
             assertions.push(quote! {
-                crate::assert_concrete(source.clone(), _tree.clone(), #concrete, #display_path);
+                crate::assert_concrete(source.clone(), #concrete, #display_path);
             });
         } else if line == "@compiles-to:" {
             let mut compiled = String::new();
@@ -72,7 +72,7 @@ fn generate_test_for_file(path: PathBuf) -> String {
             }
 
             assertions.push(quote! {
-                crate::assert_compiles_to(source.clone(), _tree.clone(), _lines.clone(), #compiled, #display_path);
+                crate::assert_compiles_to(_module.clone(), #compiled, #display_path);
             });
         } else if let Some(line) = line.strip_prefix("@type:") {
             let (pos, expected) = line
@@ -85,7 +85,7 @@ fn generate_test_for_file(path: PathBuf) -> String {
                 .expect(&format!("Unable to parse position '{pos}'"));
             let expected = expected.trim();
             assertions.push(quote! {
-                crate::assert_type_at(source.clone(), _tree.clone(), _lines.clone(), #pos, #expected, #display_path);
+                crate::assert_type_at(_module.clone(), #pos, #expected, #display_path);
             });
         } else if let Some(line) = line.strip_prefix("@type-error:") {
             let (pos, expected) = line
@@ -98,21 +98,21 @@ fn generate_test_for_file(path: PathBuf) -> String {
                 .expect(&format!("Unable to parse position '{pos}'"));
             let expected = expected.trim();
             assertions.push(quote! {
-                crate::assert_type_error_at(source.clone(), _tree.clone(), _lines.clone(), #pos, #expected, #display_path);
+                crate::assert_type_error_at(_module.clone(), &_errors, #pos, #expected, #display_path);
             });
         } else if line == "@no-errors" {
             assertions.push(quote! {
-                crate::assert_no_errors(source.clone(), _tree.clone(), _lines.clone());
+                crate::assert_no_errors(_module.clone(), &_errors);
             });
         } else if let Some(line) = line.strip_prefix("@eval:") {
             let expected = line.trim();
             assertions.push(quote! {
-                crate::assert_eval_ok(source.clone(), _tree.clone(), _lines.clone(), #expected);
+                crate::assert_eval_ok(_module.clone(), #expected);
             });
         } else if let Some(line) = line.strip_prefix("@check-error:") {
             let expected = line.trim();
             assertions.push(quote! {
-                crate::assert_check_error(source.clone(), _tree.clone(), _lines.clone(), #expected);
+                crate::assert_check_error(_module.clone(), &_errors, #expected);
             });
         } else if line == "@expect-errors:" {
             let mut errors = Vec::new();
@@ -127,7 +127,7 @@ fn generate_test_for_file(path: PathBuf) -> String {
 
             let errors = ExpectedErrors(errors);
             assertions.push(quote! {
-                crate::assert_errors(source.clone(), _tree.clone(), _lines.clone(), #errors);
+                crate::assert_errors(_module.clone(), &_errors, #errors);
             });
         } else if line.starts_with("@") {
             panic!("Test file {display_path} has unknown directive: {line}");
@@ -139,7 +139,9 @@ fn generate_test_for_file(path: PathBuf) -> String {
         #disabled
         fn #name() {
             let source : std::rc::Rc<str> = #contents.into();
-            let (_tree, _lines) = fine::parser::parse(&source);
+            let mut runtime = crate::test_runtime(source.clone());
+            let (_errors, _module) = runtime.load_module("__test__").unwrap();
+
             #(#assertions)*
         }
     };

fine/src/compiler.rs

@@ -237,7 +237,7 @@ macro_rules! inst_panic {
 //     ($compiler:expr, $tr:expr, $($t:tt)*) => {{}};
 // }
 
-pub fn compile(semantics: Rc<Semantics>) -> Rc<Module> {
+pub fn compile(semantics: &Semantics) -> Rc<Module> {
     let source = semantics.source();
     let syntax_tree = semantics.tree();
 
@@ -998,6 +998,8 @@ fn compile_list_constructor_element(c: &mut Compiler, tree: &Tree) -> CR {
 fn compile_statement(c: &mut Compiler, t: TreeRef, gen_value: bool) {
     let tree = &c.semantics.tree()[t];
     let cr = match tree.kind {
+        TreeKind::Error => None,
+
         TreeKind::Block => compile_block_statement(c, t, gen_value),
         TreeKind::ClassDecl => compile_class_declaration(c, t, tree, gen_value),
         TreeKind::ExpressionStatement => compile_expression_statement(c, tree, gen_value),

fine/src/lib.rs

@@ -1,8 +1,8 @@
-use std::{fs, rc::Rc};
+use std::{collections::HashMap, fs, rc::Rc};
 
 use compiler::compile;
 use parser::parse;
-use semantics::{check, Semantics};
+use semantics::{check, Error, Semantics};
 use vm::{eval, Context};
 
 pub mod compiler;
@@ -11,38 +11,139 @@ pub mod semantics;
 pub mod tokens;
 pub mod vm;
 
-// struct SourceModule {
+pub enum ModuleSource {
-//     semantics: Rc<Semantics>,
+    SourceText(String),
-// }
+}
 
-// impl SourceModule {
+#[derive(Debug)]
-//     pub fn new(source: &str) -> Self {
+pub enum ModuleLoadError {
-//         let source: Rc<str> = source.into();
+    IO(std::io::Error),
-//         let (syntax, lines) = parse(&source);
+}
-//         let semantics = Rc::new(Semantics::new(source, syntax, lines));
-//         SourceModule { semantics }
-//     }
-// }
 
-// struct Environment {}
+pub trait ModuleLoader {
+    fn normalize_module_name(&self, name: String) -> String;
+    fn load_module(&self, name: &String) -> Result<ModuleSource, ModuleLoadError>;
+}
+
+pub struct StandardModuleLoader {}
+
+impl ModuleLoader for StandardModuleLoader {
+    fn normalize_module_name(&self, name: String) -> String {
+        match std::fs::canonicalize(&name) {
+            Ok(p) => match p.into_os_string().into_string() {
+                Ok(s) => s,
+                Err(_) => name,
+            },
+            Err(_) => name,
+        }
+    }
+
+    fn load_module(&self, name: &String) -> Result<ModuleSource, ModuleLoadError> {
+        match fs::read_to_string(name) {
+            Ok(c) => Ok(ModuleSource::SourceText(c)),
+            Err(e) => Err(ModuleLoadError::IO(e)),
+        }
+    }
+}
+
+pub struct Module {
+    module: Rc<compiler::Module>,
+    semantics: Rc<Semantics>,
+}
+
+impl Module {
+    pub fn semantics(&self) -> Rc<Semantics> {
+        self.semantics.clone()
+    }
+
+    pub fn compiled(&self) -> Rc<compiler::Module> {
+        self.module.clone()
+    }
+}
+
+pub struct Runtime {
+    modules: HashMap<String, Rc<Module>>,
+    loader: Box<dyn ModuleLoader>,
+}
+
+impl Runtime {
+    pub fn new(loader: Box<dyn ModuleLoader>) -> Self {
+        Runtime {
+            modules: HashMap::new(),
+            loader,
+        }
+    }
+
+    pub fn load_module(&mut self, name: &str) -> Result<(Vec<Error>, Rc<Module>), ModuleLoadError> {
+        let mut init_pending = HashMap::new();
+        let mut names = Vec::new();
+        let name = self.loader.normalize_module_name(name.to_string());
+        names.push(name.clone());
+
+        while let Some(name) = names.pop() {
+            if self.modules.contains_key(&name) {
+                continue;
+            }
+            if !init_pending.contains_key(&name) {
+                let loaded = self.loader.load_module(&name)?;
+                match loaded {
+                    ModuleSource::SourceText(source) => {
+                        let source: Rc<str> = source.into();
+                        let (tree, lines) = parse(&source);
+                        let semantics = Rc::new(Semantics::new(source, tree, lines));
+
+                        let mut normalized = Vec::new();
+                        for import in semantics.imports() {
+                            let import = self.loader.normalize_module_name(import);
+                            names.push(import.clone());
+                            normalized.push(import);
+                        }
+
+                        init_pending.insert(name, (normalized, semantics));
+                    }
+                }
+            }
+        }
+
+        for (_, (imports, semantics)) in init_pending.iter() {
+            let mut import_table = HashMap::new();
+            for import in imports.iter() {
+                let target = if let Some(module) = self.modules.get(&*import) {
+                    Rc::downgrade(&module.semantics)
+                } else {
+                    Rc::downgrade(&init_pending.get(&*import).unwrap().1)
+                };
+                import_table.insert(import.clone(), target);
+            }
+            semantics.set_imports(import_table);
+        }
+
+        let mut errors = Vec::new();
+        for (name, (_, semantics)) in init_pending.into_iter() {
+            check(&semantics);
+            errors.append(&mut semantics.snapshot_errors());
+            let module = compile(&semantics);
+            self.modules
+                .insert(name, Rc::new(Module { semantics, module }));
+        }
+
+        let result = self.modules.get(&name).unwrap().clone();
+        Ok((errors, result))
+    }
+}
 
 pub fn process_file(file: &str) {
-    let source = match fs::read_to_string(file) {
+    let mut runtime = Runtime::new(Box::new(StandardModuleLoader {}));
-        Ok(c) => c,
+
-        Err(e) => {
+    let (errors, module) = match runtime.load_module(file) {
-            eprintln!("Unable to read file {file}: {e}");
+        Ok(r) => r,
+        Err(_) => {
+            eprintln!("Error loading module");
             return;
         }
     };
 
-    // What am I doing here?
-    let source: Rc<str> = source.into();
-    let (tree, lines) = parse(&source);
-    let semantics = Rc::new(Semantics::new(source, tree, lines));
-    check(&semantics);
-
     // OK now there might be errors.
-    let errors = semantics.snapshot_errors();
     if errors.len() > 0 {
         for e in errors {
             eprintln!("{file}: {}:{}: {}", e.start.0, e.start.1, e.message);
@@ -50,7 +151,8 @@ pub fn process_file(file: &str) {
         return;
     }
 
-    let module = compile(semantics);
+    // shrug
+    let module = module.module.clone();
     let main_function = module.functions[module.init].clone();
 
     let mut context = Context::new(module.clone());

fine/src/semantics.rs

@@ -3,7 +3,12 @@ use crate::{
     tokens::{Lines, Token, TokenKind},
     vm::StackValue,
 };
-use std::{cell::RefCell, collections::HashMap, fmt, rc::Rc};
+use std::{
+    cell::{OnceCell, RefCell},
+    collections::HashMap,
+    fmt,
+    rc::{Rc, Weak},
+};
 
 // TODO: Unused variables?
 // TODO: Underscore for discard?
@@ -610,6 +615,8 @@ pub struct Semantics {
     syntax_tree: Rc<SyntaxTree>,
     lines: Rc<Lines>,
 
+    import_map: OnceCell<HashMap<String, Weak<Semantics>>>,
+
     // Instead of physical parents, this is the set of *logical* parents.
     // This is what is used for binding.
     logical_parents: Vec<Option<TreeRef>>,
@@ -635,6 +642,7 @@ impl Semantics {
             source,
             syntax_tree: tree.clone(),
             lines,
+            import_map: OnceCell::new(),
             logical_parents,
             errors: RefCell::new(vec![]),
             types: RefCell::new(vec![Incremental::None; tree.len()]),
@@ -653,6 +661,10 @@ impl Semantics {
         semantics
     }
 
+    pub fn set_imports(&self, imports: HashMap<String, Weak<Semantics>>) {
+        self.import_map.set(imports).expect("imports already set");
+    }
+
     pub fn source(&self) -> Rc<str> {
         self.source.clone()
     }
@@ -665,6 +677,10 @@ impl Semantics {
         self.lines.clone()
     }
 
+    pub fn imports(&self) -> Vec<String> {
+        vec![]
+    }
+
     pub fn snapshot_errors(&self) -> Vec<Error> {
         let mut result = (*self.errors.borrow()).clone();
         result.sort_by(|a, b| match a.start.0.cmp(&b.start.0) {

fine/tests/example_tests.rs

@@ -1,8 +1,8 @@
-use fine::compiler::{compile, Function, Module};
+use fine::compiler::{Function, Module};
-use fine::parser::SyntaxTree;
+use fine::semantics::{Error, Type};
-use fine::semantics::{check, Error, Semantics, Type};
-use fine::tokens::Lines;
 use fine::vm::{eval_export_fn, Context};
+use fine::{ModuleLoadError, ModuleLoader, ModuleSource, Runtime, StandardModuleLoader};
+
 use pretty_assertions::assert_eq;
 use std::fmt::Write as _;
 use std::rc::Rc;
@@ -84,7 +84,8 @@ fn should_rebase() -> bool {
     }
 }
 
-fn assert_concrete(source: Rc<str>, tree: Rc<SyntaxTree>, expected: &str, source_path: &str) {
+fn assert_concrete(source: Rc<str>, expected: &str, source_path: &str) {
+    let (tree, _) = fine::parser::parse(&source);
     let dump = tree.dump(&source, false);
     if dump != expected {
         if should_rebase() {
@@ -128,15 +129,46 @@ macro_rules! semantic_assert_eq {
     }};
 }
 
-fn assert_type_at(
+struct TestLoader {
     source: Rc<str>,
-    tree: Rc<SyntaxTree>,
+    base: StandardModuleLoader,
-    lines: Rc<Lines>,
+}
-    pos: usize,
+
-    expected: &str,
+impl TestLoader {
-    _source_path: &str,
+    fn new(source: Rc<str>) -> Box<Self> {
-) {
+        Box::new(TestLoader {
-    let semantics = Semantics::new(source, tree.clone(), lines);
+            source,
+            base: StandardModuleLoader {},
+        })
+    }
+}
+
+impl ModuleLoader for TestLoader {
+    fn normalize_module_name(&self, name: String) -> String {
+        if name == "__test__" {
+            name
+        } else {
+            self.base.normalize_module_name(name)
+        }
+    }
+
+    fn load_module(&self, name: &String) -> Result<ModuleSource, ModuleLoadError> {
+        if name == "__test__" {
+            Ok(ModuleSource::SourceText(self.source.to_string()))
+        } else {
+            self.base.load_module(name)
+        }
+    }
+}
+
+fn test_runtime(source: Rc<str>) -> Runtime {
+    Runtime::new(TestLoader::new(source))
+}
+
+fn assert_type_at(module: Rc<fine::Module>, pos: usize, expected: &str, _source_path: &str) {
+    let semantics = module.semantics();
+    let tree = semantics.tree();
+
     let tree_ref = match tree.find_tree_at(pos) {
         Some(t) => t,
         None => semantic_panic!(
@@ -158,14 +190,15 @@ fn assert_type_at(
 }
 
 fn assert_type_error_at(
-    source: Rc<str>,
+    module: Rc<fine::Module>,
-    tree: Rc<SyntaxTree>,
+    errors: &[Error],
-    lines: Rc<Lines>,
     pos: usize,
     expected: &str,
     _source_path: &str,
 ) {
-    let semantics = Semantics::new(source, tree.clone(), lines);
+    let semantics = module.semantics();
+    let tree = semantics.tree();
+
     let tree_ref = match tree.find_tree_at(pos) {
         Some(t) => t,
         None => semantic_panic!(
@@ -184,7 +217,6 @@ fn assert_type_error_at(
         tree[tree_ref].kind
     );
 
-    let errors = semantics.snapshot_errors();
     semantic_assert!(
         &semantics,
         Some(tree_ref),
@@ -225,15 +257,9 @@ fn dump_module(out: &mut String, module: &Module) -> std::fmt::Result {
     Ok(())
 }
 
-fn assert_compiles_to(
+fn assert_compiles_to(module: Rc<fine::Module>, expected: &str, source_path: &str) {
-    source: Rc<str>,
+    let semantics = module.semantics();
-    tree: Rc<SyntaxTree>,
+    let module = module.compiled();
-    lines: Rc<Lines>,
-    expected: &str,
-    source_path: &str,
-) {
-    let semantics = Rc::new(Semantics::new(source, tree, lines));
-    let module = compile(semantics.clone());
 
     let mut actual = String::new();
     dump_module(&mut actual, &module).expect("no dumping?");
@@ -253,12 +279,10 @@ fn assert_compiles_to(
     }
 }
 
-fn assert_no_errors(source: Rc<str>, tree: Rc<SyntaxTree>, lines: Rc<Lines>) {
+fn assert_no_errors(module: Rc<fine::Module>, errors: &[Error]) {
-    let semantics = Semantics::new(source, tree, lines);
+    let semantics = module.semantics();
-    check(&semantics);
 
-    let expected_errors: Vec<Error> = Vec::new();
+    let expected_errors: &[Error] = &[];
-    let errors = semantics.snapshot_errors();
     semantic_assert_eq!(
         &semantics,
         None,
@@ -268,10 +292,10 @@ fn assert_no_errors(source: Rc<str>, tree: Rc<SyntaxTree>, lines: Rc<Lines>) {
     );
 }
 
-fn assert_eval_ok(source: Rc<str>, tree: Rc<SyntaxTree>, lines: Rc<Lines>, expected: &str) {
+fn assert_eval_ok(module: Rc<fine::Module>, expected: &str) {
-    let semantics = Rc::new(Semantics::new(source, tree, lines));
+    let semantics = module.semantics();
+    let module = module.compiled();
 
-    let module = compile(semantics.clone());
     let mut context = Context::new(module.clone());
     context.init().expect("Unable to initialize module");
 
@@ -310,20 +334,10 @@ fn assert_eval_ok(source: Rc<str>, tree: Rc<SyntaxTree>, lines: Rc<Lines>, expec
     }
 }
 
-fn assert_errors(
+fn assert_errors(module: Rc<fine::Module>, errors: &[Error], expected_errors: Vec<&str>) {
-    source: Rc<str>,
+    let semantics = module.semantics();
-    tree: Rc<SyntaxTree>,
-    lines: Rc<Lines>,
-    expected_errors: Vec<&str>,
-) {
-    let semantics = Semantics::new(source, tree, lines);
-    check(&semantics);
 
-    let errors: Vec<String> = semantics
+    let errors: Vec<String> = errors.iter().map(|e| format!("{}", e)).collect();
-        .snapshot_errors()
-        .iter()
-        .map(|e| format!("{}", e))
-        .collect();
 
     semantic_assert_eq!(
         &semantics,
@@ -334,11 +348,9 @@ fn assert_errors(
     );
 }
 
-fn assert_check_error(source: Rc<str>, tree: Rc<SyntaxTree>, lines: Rc<Lines>, expected: &str) {
+fn assert_check_error(module: Rc<fine::Module>, errors: &[Error], expected: &str) {
-    let semantics = Semantics::new(source, tree, lines);
+    let semantics = module.semantics();
-    check(&semantics);
 
-    let errors = semantics.snapshot_errors();
     semantic_assert!(
         &semantics,
         None,

fine/tests/expression/alternates.fine

@@ -141,5 +141,5 @@ fun test() -> f64 {
   // like the above.
 }
 
-/// @ignore WIP
+// @ignore never finished compiling `match`
 // @no-errors