DeCarabas/oden
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

[quickjs][oden][oden-js] Source maps

Browse source 
The worst support but it should cost very little if nobody is using
them (psst we're using them)
This commit is contained in:
John Doty 2023-09-20 09:31:57 -07:00
parent eb9fed759a
commit c96a1a4979
10 changed files with 554 additions and 374 deletions
Download patch file Download diff file Expand all files Collapse all files

37
Cargo.lock generated
View file

@ -504,6 +504,16 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41b319d1b62ffbd002e057f36bebd1f42b9f97927c9577461d855f3513c4289f"
[[package]]
name = "debugid"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d"
dependencies = [
"serde",
"uuid",
]
[[package]]
name = "deno_ast"
version = "0.29.3"
@ -1536,6 +1546,7 @@ dependencies = [
"notify",
"oden-js",
"pollster",
"sourcemap 7.0.0",
"tracy-client",
"wgpu",
"winit",
@ -2107,6 +2118,22 @@ dependencies = [
"url",
]
[[package]]
name = "sourcemap"
version = "7.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dbecc42a2b6131acc3bf9a25c9fe4161dba438eb52131bba83c5d781b5b70be3"
dependencies = [
"data-encoding",
"debugid",
"if_chain",
"rustc_version",
"serde",
"serde_json",
"unicode-id",
"url",
]
[[package]]
name = "spirv"
version = "0.2.0+1.5.4"
@ -2218,7 +2245,7 @@ dependencies = [
"rustc-hash",
"serde",
"siphasher",
"sourcemap",
"sourcemap 6.2.3",
"string_cache",
"swc_atoms",
"swc_eq_ignore_macros",
@ -2281,7 +2308,7 @@ dependencies = [
"once_cell",
"rustc-hash",
"serde",
"sourcemap",
"sourcemap 6.2.3",
"swc_atoms",
"swc_common",
"swc_ecma_ast",
@ -2819,6 +2846,12 @@ dependencies = [
"serde",
]
[[package]]
name = "uuid"
version = "1.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d"
[[package]]
name = "valuable"
version = "0.1.0"

1
Cargo.toml
View file

@ -20,6 +20,7 @@ lru = "0.11.0"
notify = "6"
oden-js = { path = "oden-js" }
pollster = "0.3"
sourcemap = "7.0.0"
tracy-client = { version = "0.15.2", default-features = false }
wgpu = "0.17"
winit = "0.28"

25
oden-js-sys/quickjs/quickjs.c
View file

@ -306,6 +306,8 @@ struct JSRuntime {
uint32_t operator_count;
#endif
void *user_opaque;
JSMapSourceFunc *map_source_func;
void *map_source_opaque;
};
struct JSClass {
@ -6518,17 +6520,29 @@ static void build_backtrace(JSContext *ctx, JSValueConst error_obj,
b = p->u.func.function_bytecode;
backtrace_barrier = b->backtrace_barrier;
if (b->has_debug) {
/* TODO: SOURCEMAP: Indirect through the source map here if
there is one. */
line_num1 = find_line_num(ctx, b,
sf->cur_pc - b->byte_code_buf - 1);
atom_str = JS_AtomToCString(ctx, b->debug.filename);
JSAtom file = b->debug.filename;
if (ctx->rt->map_source_func) {
ctx->rt->map_source_func(
ctx,
ctx->rt->map_source_opaque,
file, line_num1,
&file, &line_num1);
}
atom_str = JS_AtomToCString(ctx, file);
dbuf_printf(&dbuf, " (%s",
atom_str ? atom_str : "<null>");
JS_FreeCString(ctx, atom_str);
if (line_num1 != -1)
dbuf_printf(&dbuf, ":%d", line_num1);
dbuf_putc(&dbuf, ')');
if (ctx->rt->map_source_func) {
JS_FreeAtomRT(ctx->rt, file);
}
}
} else {
dbuf_printf(&dbuf, " (native)");
@ -54067,3 +54081,8 @@ void JS_AddIntrinsicTypedArrays(JSContext *ctx)
JS_AddIntrinsicAtomics(ctx);
#endif
}
void JS_SetSourceMapFunc(JSRuntime *rt, JSMapSourceFunc *map_func, void *opaque) {
rt->map_source_func = map_func;
rt->map_source_opaque = opaque;
}

6
oden-js-sys/quickjs/quickjs.h
View file

@ -1044,6 +1044,12 @@ int JS_CountModuleExport(JSContext *ctx, const JSModuleDef *m);
JSAtom JS_GetModuleExportName(JSContext *ctx, const JSModuleDef *m, int idx);
JSValueConst JS_GetModuleExportValue(JSContext *ctx, const JSModuleDef *m, int idx);
/* Source map support */
typedef void JSMapSourceFunc(JSContext *ctx, void *opaque, JSAtom file, int line, JSAtom *pfile, int *pline);
void JS_SetSourceMapFunc(JSRuntime *rt, JSMapSourceFunc *map_func, void *opaque);
#undef js_unlikely
#undef js_force_inline

20
oden-js/src/atom.rs
View file

@ -2,6 +2,7 @@ use crate::{ContextRef, Runtime, ValueResult};
use oden_js_sys as sys;
use std::ops::Deref;
#[derive(Eq, PartialEq, Debug, Hash)]
pub struct AtomRef {
pub(crate) atom: sys::JSAtom,
}
@ -39,6 +40,11 @@ impl Atom {
rt: Runtime::from_raw(unsafe { sys::JS_GetRuntime(ctx.ctx) }),
}
}
pub(crate) fn consume(atom: Atom) -> u32 {
let atom = std::mem::ManuallyDrop::new(atom);
atom.atom.atom
}
}
impl Deref for Atom {
@ -56,3 +62,17 @@ impl Drop for Atom {
}
}
}
impl PartialEq for Atom {
fn eq(&self, other: &Self) -> bool {
self.atom == other.atom
}
}
impl Eq for Atom {}
impl std::hash::Hash for Atom {
fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
self.atom.hash(state)
}
}

43
oden-js/src/module/loader.rs
View file

@ -1,6 +1,5 @@
use super::ModuleRef;
use crate::{throw_error, ContextRef, Error, Result};
use oden_js_sys as sys;
use crate::{Atom, AtomRef, ContextRef, Error, Result};
use std::path::Path;
pub enum ModuleSource<'a> {
@ -9,7 +8,22 @@ pub enum ModuleSource<'a> {
}
pub trait ModuleLoader {
fn load(&self, context: &ContextRef, name: &str) -> Result<ModuleSource>;
fn load<'a>(&mut self, context: &'a ContextRef, name: &str) -> Result<ModuleSource<'a>>;
// Return true if this loader also supports source maps, otherwise false.
// NOTE: This must return `true` if you want to have a non-trivial
// implementation of `map_source`; if this returns `false` (the
// default) then your object will never even receive a map_source
// call.
fn support_source_map(&self) -> bool {
false
}
// Map a given file/line pair to the source file/line pair. This is only
// called if `support_source_map` returns true.
fn map_source(&self, context: &ContextRef, file: &AtomRef, line: i32) -> Result<(Atom, i32)> {
Ok((file.dup(context), line))
}
}
pub struct DefaultModuleLoader {}
@ -21,7 +35,7 @@ impl DefaultModuleLoader {
}
impl ModuleLoader for DefaultModuleLoader {
fn load(&self, _context: &ContextRef, name: &str) -> Result<ModuleSource> {
fn load<'a>(&mut self, _context: &'a ContextRef, name: &str) -> Result<ModuleSource<'a>> {
// Attempt to open the file.
let path = Path::new(name);
match std::fs::read_to_string(path) {
@ -30,24 +44,3 @@ impl ModuleLoader for DefaultModuleLoader {
}
}
}
pub(crate) fn load_module(
context: &ContextRef,
name: &str,
loader: &Box<dyn ModuleLoader>,
) -> *mut sys::JSModuleDef {
match loader.load(context, name) {
Ok(ModuleSource::Native(native)) => native.module,
Ok(ModuleSource::JavaScript(js)) => match context.eval_module(&js, name) {
Ok(v) => v.module,
Err(e) => {
throw_error(context, e);
return std::ptr::null_mut();
}
},
Err(e) => {
throw_error(context, e);
return std::ptr::null_mut();
}
}
}

2
oden-js/src/module/mod.rs
View file

@ -8,7 +8,7 @@ pub mod native;
#[derive(Debug, Clone)]
pub struct ModuleRef {
module: *mut sys::JSModuleDef,
pub(crate) module: *mut sys::JSModuleDef,
}
impl ModuleRef {

81
oden-js/src/runtime.rs
View file

@ -1,8 +1,8 @@
use crate::{
module::loader::{load_module, DefaultModuleLoader, ModuleLoader},
module::loader::{DefaultModuleLoader, ModuleLoader, ModuleSource},
promise::{PromiseEvent, PromiseHandle},
ContextRef, DefaultRejectedPromiseTracker, Promise, RejectedPromiseTracker, Result, Value,
ValueRef,
throw_error, Atom, AtomRef, ContextRef, DefaultRejectedPromiseTracker, Promise,
RejectedPromiseTracker, Result, Value, ValueRef,
};
use oden_js_sys as sys;
use std::cell::{RefCell, RefMut};
@ -42,7 +42,7 @@ impl Drop for PromiseEntry {
struct PrivateState {
refs: u64,
loader: Arc<Box<dyn ModuleLoader>>,
loader: Arc<RefCell<Box<dyn ModuleLoader>>>,
promise_send: Sender<(PromiseHandle, PromiseEvent)>,
promise_recv: Receiver<(PromiseHandle, PromiseEvent)>,
promise_table: HashMap<PromiseHandle, PromiseEntry>, // !
@ -54,7 +54,7 @@ impl PrivateState {
let (send, recv) = channel();
Box::new(RefCell::new(PrivateState {
refs: 1,
loader: Arc::new(Box::new(DefaultModuleLoader::new())),
loader: Arc::new(RefCell::new(Box::new(DefaultModuleLoader::new()))),
promise_send: send,
promise_recv: recv,
promise_table: HashMap::new(),
@ -90,7 +90,64 @@ impl PrivateState {
.clone()
};
let context = ContextRef::from_raw(ctx);
load_module(&context, path, &loader)
let result = {
let mut loader = loader.borrow_mut();
loader.load(&context, path)
};
match result {
Ok(ModuleSource::Native(native)) => native.module,
Ok(ModuleSource::JavaScript(js)) => match context.eval_module(&js, path) {
Ok(v) => v.module,
Err(e) => {
throw_error(&context, e);
std::ptr::null_mut()
}
},
Err(e) => {
throw_error(&context, e);
std::ptr::null_mut()
}
}
}
unsafe extern "C" fn source_mapper(
ctx: *mut sys::JSContext,
opaque: *mut std::os::raw::c_void,
file: u32,
line: i32,
pfile: *mut u32,
pline: *mut i32,
) {
let ctx = ContextRef::from_raw(ctx);
let file = AtomRef::from_raw(file, &ctx);
let loader = unsafe {
let ptr = opaque as *const RefCell<PrivateState>;
ptr.as_ref()
.expect("We already know this runtime is one of ours!")
.borrow()
.loader
.clone()
};
let result = {
let loader = loader.borrow();
loader.map_source(&ctx, &file, line)
};
match result {
Ok((file, line)) => {
*pfile = Atom::consume(file);
*pline = line;
}
Err(e) => {
throw_error(&ctx, e);
*pfile = file.atom;
*pline = line;
}
}
}
unsafe extern "C" fn promise_rejection_tracker(
@ -171,7 +228,17 @@ impl Runtime {
T: ModuleLoader + 'static,
{
let mut state = unsafe { PrivateState::from_rt_mut(self.rt) };
state.loader = Arc::new(Box::new(loader));
if loader.support_source_map() {
unsafe {
let opaque = sys::JS_GetRuntimeOpaque(self.rt);
sys::JS_SetSourceMapFunc(self.rt, Some(PrivateState::source_mapper), opaque);
}
} else {
unsafe {
sys::JS_SetSourceMapFunc(self.rt, None, std::ptr::null_mut());
}
}
state.loader = Arc::new(RefCell::new(Box::new(loader)));
}
/// Set a tracker to be notified whenever a promise is rejected. By

53
src/script.rs
View file

@ -1,11 +1,12 @@
use notify::{RecommendedWatcher, RecursiveMode, Watcher};
use oden_js::{
module::loader::{ModuleLoader, ModuleSource},
Context, ContextRef, RejectedPromiseTracker, Result, Runtime, Value,
Atom, Context, ContextRef, RejectedPromiseTracker, Result, Runtime, Value,
};
use sourcemap::SourceMap;
use std::collections::HashMap;
use std::ffi::OsStr;
use std::sync::mpsc::{channel, Receiver, Sender};
use std::sync::{Arc, Mutex};
use std::time::Instant;
use tracy_client::span;
use winit::event::*;
@ -21,36 +22,66 @@ mod typescript;
use typescript::transpile_to_javascript;
struct Loader {
watcher: Arc<Mutex<RecommendedWatcher>>,
watcher: RecommendedWatcher,
source_map: HashMap<Atom, SourceMap>,
}
impl Loader {
pub fn new(reload_trigger: Sender<()>) -> Loader {
let watcher = Arc::new(Mutex::new(
notify::recommended_watcher(move |_| {
Loader {
watcher: notify::recommended_watcher(move |_| {
let _ = reload_trigger.send(());
})
.expect("Unable to create watcher"),
));
Loader { watcher }
source_map: HashMap::new(),
}
}
}
impl ModuleLoader for Loader {
fn load(&self, _context: &ContextRef, name: &str) -> Result<ModuleSource> {
fn load<'a>(&mut self, context: &'a ContextRef, name: &str) -> Result<ModuleSource<'a>> {
eprintln!("Loading {name}...");
let path = io::resolve_path(name, &["ts"])?;
let contents = std::fs::read_to_string(&path)?;
let contents = if path.extension().and_then(OsStr::to_str) == Some("ts") {
transpile_to_javascript(name, contents)?
let (c, sm) = transpile_to_javascript(name, contents)?;
let a = context.new_atom(name)?;
self.source_map.insert(a, sm);
c
} else {
contents
};
let mut watcher = self.watcher.lock().unwrap();
let _ = watcher.watch(&path, RecursiveMode::NonRecursive);
let _ = self.watcher.watch(&path, RecursiveMode::NonRecursive);
Ok(ModuleSource::JavaScript(contents))
}
fn support_source_map(&self) -> bool {
true
}
fn map_source(
&self,
context: &ContextRef,
file: &oden_js::AtomRef,
line: i32,
) -> Result<(Atom, i32)> {
let file = file.dup(&context);
// Yuck.
if let Ok(line) = line.try_into() {
if let Some(sm) = self.source_map.get(&file) {
if let Some(token) = sm.lookup_token(line, 0) {
if let Ok(src_line) = token.get_src_line().try_into() {
return Ok((file, src_line));
}
}
}
}
eprintln!("Not mapped");
Ok((file, line))
}
}
struct RejectedPromiseHandler {

16
src/script/typescript.rs
View file

@ -1,7 +1,8 @@
use deno_ast::{parse_module, MediaType, ParseParams, SourceTextInfo};
use oden_js::{Error, Result};
use sourcemap::SourceMap;
pub fn transpile_to_javascript(path: &str, input: String) -> Result<String> {
pub fn transpile_to_javascript(path: &str, input: String) -> Result<(String, SourceMap)> {
let text_info = SourceTextInfo::new(input.into());
let parsed_source = parse_module(ParseParams {
specifier: path.to_string(),
@ -25,9 +26,18 @@ pub fn transpile_to_javascript(path: &str, input: String) -> Result<String> {
)
})?;
let options: deno_ast::EmitOptions = Default::default();
let options = deno_ast::EmitOptions {
inline_source_map: false,
source_map: true,
..Default::default()
};
let transpiled = parsed_source
.transpile(&options)
.map_err(|e| Error::ParseError(path.into(), e.to_string()))?;
return Ok(transpiled.text);
let source_map = transpiled.source_map.unwrap();
let bytes = source_map.into_bytes();
let source_map = SourceMap::from_slice(&bytes).expect("unable to parse source map");
return Ok((transpiled.text, source_map));
}