[quickjs][oden][oden-js] Source maps

The worst support but it should cost very little if nobody is using
them (psst we're using them)

Cargo.lock

@@ -504,6 +504,16 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "41b319d1b62ffbd002e057f36bebd1f42b9f97927c9577461d855f3513c4289f"
 
+[[package]]
+name = "debugid"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d"
+dependencies = [
+ "serde",
+ "uuid",
+]
+
 [[package]]
 name = "deno_ast"
 version = "0.29.3"
@@ -1536,6 +1546,7 @@ dependencies = [
  "notify",
  "oden-js",
  "pollster",
+ "sourcemap 7.0.0",
  "tracy-client",
  "wgpu",
  "winit",
@@ -2107,6 +2118,22 @@ dependencies = [
  "url",
 ]
 
+[[package]]
+name = "sourcemap"
+version = "7.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dbecc42a2b6131acc3bf9a25c9fe4161dba438eb52131bba83c5d781b5b70be3"
+dependencies = [
+ "data-encoding",
+ "debugid",
+ "if_chain",
+ "rustc_version",
+ "serde",
+ "serde_json",
+ "unicode-id",
+ "url",
+]
+
 [[package]]
 name = "spirv"
 version = "0.2.0+1.5.4"
@@ -2218,7 +2245,7 @@ dependencies = [
  "rustc-hash",
  "serde",
  "siphasher",
- "sourcemap",
+ "sourcemap 6.2.3",
  "string_cache",
  "swc_atoms",
  "swc_eq_ignore_macros",
@@ -2281,7 +2308,7 @@ dependencies = [
  "once_cell",
  "rustc-hash",
  "serde",
- "sourcemap",
+ "sourcemap 6.2.3",
  "swc_atoms",
  "swc_common",
  "swc_ecma_ast",
@@ -2819,6 +2846,12 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "uuid"
+version = "1.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d"
+
 [[package]]
 name = "valuable"
 version = "0.1.0"

Cargo.toml

@@ -20,6 +20,7 @@ lru = "0.11.0"
 notify = "6"
 oden-js = { path = "oden-js" }
 pollster = "0.3"
+sourcemap = "7.0.0"
 tracy-client = { version = "0.15.2", default-features = false }
 wgpu = "0.17"
 winit = "0.28"

oden-js-sys/quickjs/quickjs.c

@@ -306,6 +306,8 @@ struct JSRuntime {
     uint32_t operator_count;
 #endif
     void *user_opaque;
+    JSMapSourceFunc *map_source_func;
+    void *map_source_opaque;
 };
 
 struct JSClass {
@@ -6518,17 +6520,29 @@ static void build_backtrace(JSContext *ctx, JSValueConst error_obj,
             b = p->u.func.function_bytecode;
             backtrace_barrier = b->backtrace_barrier;
             if (b->has_debug) {
-                /* TODO: SOURCEMAP: Indirect through the source map here if
-                   there is one. */
                 line_num1 = find_line_num(ctx, b,
                                           sf->cur_pc - b->byte_code_buf - 1);
-                atom_str = JS_AtomToCString(ctx, b->debug.filename);
+
+                JSAtom file = b->debug.filename;
+                if (ctx->rt->map_source_func) {
+                    ctx->rt->map_source_func(
+                        ctx,
+                        ctx->rt->map_source_opaque,
+                        file, line_num1,
+                        &file, &line_num1);
+                }
+
+                atom_str = JS_AtomToCString(ctx, file);
                 dbuf_printf(&dbuf, " (%s",
                             atom_str ? atom_str : "<null>");
                 JS_FreeCString(ctx, atom_str);
                 if (line_num1 != -1)
                     dbuf_printf(&dbuf, ":%d", line_num1);
                 dbuf_putc(&dbuf, ')');
+
+                if (ctx->rt->map_source_func) {
+                    JS_FreeAtomRT(ctx->rt, file);
+                }
             }
         } else {
             dbuf_printf(&dbuf, " (native)");
@@ -54067,3 +54081,8 @@ void JS_AddIntrinsicTypedArrays(JSContext *ctx)
     JS_AddIntrinsicAtomics(ctx);
 #endif
 }
+
+void JS_SetSourceMapFunc(JSRuntime *rt, JSMapSourceFunc *map_func, void *opaque) {
+    rt->map_source_func = map_func;
+    rt->map_source_opaque = opaque;
+}

oden-js-sys/quickjs/quickjs.h

@@ -1044,6 +1044,12 @@ int JS_CountModuleExport(JSContext *ctx, const JSModuleDef *m);
 JSAtom JS_GetModuleExportName(JSContext *ctx, const JSModuleDef *m, int idx);
 JSValueConst JS_GetModuleExportValue(JSContext *ctx, const JSModuleDef *m, int idx);
 
+/* Source map support */
+
+typedef void JSMapSourceFunc(JSContext *ctx, void *opaque, JSAtom file, int line, JSAtom *pfile, int *pline);
+
+void JS_SetSourceMapFunc(JSRuntime *rt, JSMapSourceFunc *map_func, void *opaque);
+
 #undef js_unlikely
 #undef js_force_inline
 

oden-js/src/atom.rs

@@ -2,6 +2,7 @@ use crate::{ContextRef, Runtime, ValueResult};
 use oden_js_sys as sys;
 use std::ops::Deref;
 
+#[derive(Eq, PartialEq, Debug, Hash)]
 pub struct AtomRef {
     pub(crate) atom: sys::JSAtom,
 }
@@ -39,6 +40,11 @@ impl Atom {
             rt: Runtime::from_raw(unsafe { sys::JS_GetRuntime(ctx.ctx) }),
         }
     }
+
+    pub(crate) fn consume(atom: Atom) -> u32 {
+        let atom = std::mem::ManuallyDrop::new(atom);
+        atom.atom.atom
+    }
 }
 
 impl Deref for Atom {
@@ -56,3 +62,17 @@ impl Drop for Atom {
         }
     }
 }
+
+impl PartialEq for Atom {
+    fn eq(&self, other: &Self) -> bool {
+        self.atom == other.atom
+    }
+}
+
+impl Eq for Atom {}
+
+impl std::hash::Hash for Atom {
+    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+        self.atom.hash(state)
+    }
+}

oden-js/src/module/loader.rs

@@ -1,6 +1,5 @@
 use super::ModuleRef;
-use crate::{throw_error, ContextRef, Error, Result};
+use crate::{Atom, AtomRef, ContextRef, Error, Result};
-use oden_js_sys as sys;
 use std::path::Path;
 
 pub enum ModuleSource<'a> {
@@ -9,7 +8,22 @@ pub enum ModuleSource<'a> {
 }
 
 pub trait ModuleLoader {
-    fn load(&self, context: &ContextRef, name: &str) -> Result<ModuleSource>;
+    fn load<'a>(&mut self, context: &'a ContextRef, name: &str) -> Result<ModuleSource<'a>>;
+
+    // Return true if this loader also supports source maps, otherwise false.
+    // NOTE: This must return `true` if you want to have a non-trivial
+    //       implementation of `map_source`; if this returns `false` (the
+    //       default) then your object will never even receive a map_source
+    //       call.
+    fn support_source_map(&self) -> bool {
+        false
+    }
+
+    // Map a given file/line pair to the source file/line pair. This is only
+    // called if `support_source_map` returns true.
+    fn map_source(&self, context: &ContextRef, file: &AtomRef, line: i32) -> Result<(Atom, i32)> {
+        Ok((file.dup(context), line))
+    }
 }
 
 pub struct DefaultModuleLoader {}
@@ -21,7 +35,7 @@ impl DefaultModuleLoader {
 }
 
 impl ModuleLoader for DefaultModuleLoader {
-    fn load(&self, _context: &ContextRef, name: &str) -> Result<ModuleSource> {
+    fn load<'a>(&mut self, _context: &'a ContextRef, name: &str) -> Result<ModuleSource<'a>> {
         // Attempt to open the file.
         let path = Path::new(name);
         match std::fs::read_to_string(path) {
@@ -30,24 +44,3 @@ impl ModuleLoader for DefaultModuleLoader {
         }
     }
 }
-
-pub(crate) fn load_module(
-    context: &ContextRef,
-    name: &str,
-    loader: &Box<dyn ModuleLoader>,
-) -> *mut sys::JSModuleDef {
-    match loader.load(context, name) {
-        Ok(ModuleSource::Native(native)) => native.module,
-        Ok(ModuleSource::JavaScript(js)) => match context.eval_module(&js, name) {
-            Ok(v) => v.module,
-            Err(e) => {
-                throw_error(context, e);
-                return std::ptr::null_mut();
-            }
-        },
-        Err(e) => {
-            throw_error(context, e);
-            return std::ptr::null_mut();
-        }
-    }
-}

oden-js/src/module/mod.rs

@@ -8,7 +8,7 @@ pub mod native;
 
 #[derive(Debug, Clone)]
 pub struct ModuleRef {
-    module: *mut sys::JSModuleDef,
+    pub(crate) module: *mut sys::JSModuleDef,
 }
 
 impl ModuleRef {

oden-js/src/runtime.rs

@@ -1,8 +1,8 @@
 use crate::{
-    module::loader::{load_module, DefaultModuleLoader, ModuleLoader},
+    module::loader::{DefaultModuleLoader, ModuleLoader, ModuleSource},
     promise::{PromiseEvent, PromiseHandle},
-    ContextRef, DefaultRejectedPromiseTracker, Promise, RejectedPromiseTracker, Result, Value,
+    throw_error, Atom, AtomRef, ContextRef, DefaultRejectedPromiseTracker, Promise,
-    ValueRef,
+    RejectedPromiseTracker, Result, Value, ValueRef,
 };
 use oden_js_sys as sys;
 use std::cell::{RefCell, RefMut};
@@ -42,7 +42,7 @@ impl Drop for PromiseEntry {
 
 struct PrivateState {
     refs: u64,
-    loader: Arc<Box<dyn ModuleLoader>>,
+    loader: Arc<RefCell<Box<dyn ModuleLoader>>>,
     promise_send: Sender<(PromiseHandle, PromiseEvent)>,
     promise_recv: Receiver<(PromiseHandle, PromiseEvent)>,
     promise_table: HashMap<PromiseHandle, PromiseEntry>, // !
@@ -54,7 +54,7 @@ impl PrivateState {
         let (send, recv) = channel();
         Box::new(RefCell::new(PrivateState {
             refs: 1,
-            loader: Arc::new(Box::new(DefaultModuleLoader::new())),
+            loader: Arc::new(RefCell::new(Box::new(DefaultModuleLoader::new()))),
             promise_send: send,
             promise_recv: recv,
             promise_table: HashMap::new(),
@@ -90,7 +90,64 @@ impl PrivateState {
                 .clone()
         };
         let context = ContextRef::from_raw(ctx);
-        load_module(&context, path, &loader)
+
+        let result = {
+            let mut loader = loader.borrow_mut();
+            loader.load(&context, path)
+        };
+
+        match result {
+            Ok(ModuleSource::Native(native)) => native.module,
+            Ok(ModuleSource::JavaScript(js)) => match context.eval_module(&js, path) {
+                Ok(v) => v.module,
+                Err(e) => {
+                    throw_error(&context, e);
+                    std::ptr::null_mut()
+                }
+            },
+            Err(e) => {
+                throw_error(&context, e);
+                std::ptr::null_mut()
+            }
+        }
+    }
+
+    unsafe extern "C" fn source_mapper(
+        ctx: *mut sys::JSContext,
+        opaque: *mut std::os::raw::c_void,
+        file: u32,
+        line: i32,
+        pfile: *mut u32,
+        pline: *mut i32,
+    ) {
+        let ctx = ContextRef::from_raw(ctx);
+        let file = AtomRef::from_raw(file, &ctx);
+
+        let loader = unsafe {
+            let ptr = opaque as *const RefCell<PrivateState>;
+            ptr.as_ref()
+                .expect("We already know this runtime is one of ours!")
+                .borrow()
+                .loader
+                .clone()
+        };
+
+        let result = {
+            let loader = loader.borrow();
+            loader.map_source(&ctx, &file, line)
+        };
+
+        match result {
+            Ok((file, line)) => {
+                *pfile = Atom::consume(file);
+                *pline = line;
+            }
+            Err(e) => {
+                throw_error(&ctx, e);
+                *pfile = file.atom;
+                *pline = line;
+            }
+        }
     }
 
     unsafe extern "C" fn promise_rejection_tracker(
@@ -171,7 +228,17 @@ impl Runtime {
         T: ModuleLoader + 'static,
     {
         let mut state = unsafe { PrivateState::from_rt_mut(self.rt) };
-        state.loader = Arc::new(Box::new(loader));
+        if loader.support_source_map() {
+            unsafe {
+                let opaque = sys::JS_GetRuntimeOpaque(self.rt);
+                sys::JS_SetSourceMapFunc(self.rt, Some(PrivateState::source_mapper), opaque);
+            }
+        } else {
+            unsafe {
+                sys::JS_SetSourceMapFunc(self.rt, None, std::ptr::null_mut());
+            }
+        }
+        state.loader = Arc::new(RefCell::new(Box::new(loader)));
     }
 
     /// Set a tracker to be notified whenever a promise is rejected. By

src/script.rs

@@ -1,11 +1,12 @@
 use notify::{RecommendedWatcher, RecursiveMode, Watcher};
 use oden_js::{
     module::loader::{ModuleLoader, ModuleSource},
-    Context, ContextRef, RejectedPromiseTracker, Result, Runtime, Value,
+    Atom, Context, ContextRef, RejectedPromiseTracker, Result, Runtime, Value,
 };
+use sourcemap::SourceMap;
+use std::collections::HashMap;
 use std::ffi::OsStr;
 use std::sync::mpsc::{channel, Receiver, Sender};
-use std::sync::{Arc, Mutex};
 use std::time::Instant;
 use tracy_client::span;
 use winit::event::*;
@@ -21,36 +22,66 @@ mod typescript;
 use typescript::transpile_to_javascript;
 
 struct Loader {
-    watcher: Arc<Mutex<RecommendedWatcher>>,
+    watcher: RecommendedWatcher,
+    source_map: HashMap<Atom, SourceMap>,
 }
 
 impl Loader {
     pub fn new(reload_trigger: Sender<()>) -> Loader {
-        let watcher = Arc::new(Mutex::new(
+        Loader {
-            notify::recommended_watcher(move |_| {
+            watcher: notify::recommended_watcher(move |_| {
                 let _ = reload_trigger.send(());
             })
             .expect("Unable to create watcher"),
-        ));
+            source_map: HashMap::new(),
-        Loader { watcher }
+        }
     }
 }
 
 impl ModuleLoader for Loader {
-    fn load(&self, _context: &ContextRef, name: &str) -> Result<ModuleSource> {
+    fn load<'a>(&mut self, context: &'a ContextRef, name: &str) -> Result<ModuleSource<'a>> {
         eprintln!("Loading {name}...");
         let path = io::resolve_path(name, &["ts"])?;
         let contents = std::fs::read_to_string(&path)?;
         let contents = if path.extension().and_then(OsStr::to_str) == Some("ts") {
-            transpile_to_javascript(name, contents)?
+            let (c, sm) = transpile_to_javascript(name, contents)?;
+            let a = context.new_atom(name)?;
+            self.source_map.insert(a, sm);
+            c
         } else {
             contents
         };
 
-        let mut watcher = self.watcher.lock().unwrap();
+        let _ = self.watcher.watch(&path, RecursiveMode::NonRecursive);
-        let _ = watcher.watch(&path, RecursiveMode::NonRecursive);
         Ok(ModuleSource::JavaScript(contents))
     }
+
+    fn support_source_map(&self) -> bool {
+        true
+    }
+
+    fn map_source(
+        &self,
+        context: &ContextRef,
+        file: &oden_js::AtomRef,
+        line: i32,
+    ) -> Result<(Atom, i32)> {
+        let file = file.dup(&context);
+
+        // Yuck.
+        if let Ok(line) = line.try_into() {
+            if let Some(sm) = self.source_map.get(&file) {
+                if let Some(token) = sm.lookup_token(line, 0) {
+                    if let Ok(src_line) = token.get_src_line().try_into() {
+                        return Ok((file, src_line));
+                    }
+                }
+            }
+        }
+
+        eprintln!("Not mapped");
+        Ok((file, line))
+    }
 }
 
 struct RejectedPromiseHandler {

src/script/typescript.rs

@@ -1,7 +1,8 @@
 use deno_ast::{parse_module, MediaType, ParseParams, SourceTextInfo};
 use oden_js::{Error, Result};
+use sourcemap::SourceMap;
 
-pub fn transpile_to_javascript(path: &str, input: String) -> Result<String> {
+pub fn transpile_to_javascript(path: &str, input: String) -> Result<(String, SourceMap)> {
     let text_info = SourceTextInfo::new(input.into());
     let parsed_source = parse_module(ParseParams {
         specifier: path.to_string(),
@@ -25,9 +26,18 @@ pub fn transpile_to_javascript(path: &str, input: String) -> Result<String> {
         )
     })?;
 
-    let options: deno_ast::EmitOptions = Default::default();
+    let options = deno_ast::EmitOptions {
+        inline_source_map: false,
+        source_map: true,
+        ..Default::default()
+    };
     let transpiled = parsed_source
         .transpile(&options)
         .map_err(|e| Error::ParseError(path.into(), e.to_string()))?;
-    return Ok(transpiled.text);
+
+    let source_map = transpiled.source_map.unwrap();
+    let bytes = source_map.into_bytes();
+    let source_map = SourceMap::from_slice(&bytes).expect("unable to parse source map");
+
+    return Ok((transpiled.text, source_map));
 }