Compare commits

...

2 commits

Author SHA1 Message Date
eb9fed759a [oden] Explain myself. 2023-09-16 22:56:27 -07:00
c02eb25873 [oden] Use deno_ast instead of raw swc for typescript
OK, look, I hate SWC and I really don't want to use it if I can help
it, but the other options are worse.
2023-09-16 22:56:21 -07:00
4 changed files with 216 additions and 286 deletions

273
Cargo.lock generated
View file

@ -61,7 +61,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f"
dependencies = [
"cfg-if",
"getrandom",
"once_cell",
"version_check",
]
@ -499,6 +498,59 @@ version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308"
[[package]]
name = "data-url"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41b319d1b62ffbd002e057f36bebd1f42b9f97927c9577461d855f3513c4289f"
[[package]]
name = "deno_ast"
version = "0.29.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "577ec3850834c2578eb44afa9250f9a807f8497664e6e2aaae19cea0aac2fe3b"
dependencies = [
"anyhow",
"base64",
"deno_media_type",
"dprint-swc-ext",
"serde",
"swc_atoms",
"swc_common",
"swc_config",
"swc_config_macro",
"swc_ecma_ast",
"swc_ecma_codegen",
"swc_ecma_codegen_macros",
"swc_ecma_loader",
"swc_ecma_parser",
"swc_ecma_transforms_base",
"swc_ecma_transforms_classes",
"swc_ecma_transforms_macros",
"swc_ecma_transforms_proposal",
"swc_ecma_transforms_react",
"swc_ecma_transforms_typescript",
"swc_ecma_utils",
"swc_ecma_visit",
"swc_eq_ignore_macros",
"swc_macros_common",
"swc_visit",
"swc_visit_macros",
"text_lines",
"url",
]
[[package]]
name = "deno_media_type"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a798670c20308e5770cc0775de821424ff9e85665b602928509c8c70430b3ee0"
dependencies = [
"data-url",
"serde",
"url",
]
[[package]]
name = "digest"
version = "0.10.7"
@ -530,6 +582,22 @@ version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ea835d29036a4087793836fa931b08837ad5e957da9e23886b29586fb9b6650"
[[package]]
name = "dprint-swc-ext"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a0a2492465344a58a37ae119de59e81fe5a2885f2711c7b5048ef0dfa14ce42"
dependencies = [
"bumpalo",
"num-bigint",
"rustc-hash",
"swc_atoms",
"swc_common",
"swc_ecma_ast",
"swc_ecma_parser",
"text_lines",
]
[[package]]
name = "either"
version = "1.8.1"
@ -1044,79 +1112,6 @@ version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
[[package]]
name = "lexical"
version = "6.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c7aefb36fd43fef7003334742cbf77b243fcd36418a1d1bdd480d613a67968f6"
dependencies = [
"lexical-core",
]
[[package]]
name = "lexical-core"
version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2cde5de06e8d4c2faabc400238f9ae1c74d5412d03a7bd067645ccbc47070e46"
dependencies = [
"lexical-parse-float",
"lexical-parse-integer",
"lexical-util",
"lexical-write-float",
"lexical-write-integer",
]
[[package]]
name = "lexical-parse-float"
version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "683b3a5ebd0130b8fb52ba0bdc718cc56815b6a097e28ae5a6997d0ad17dc05f"
dependencies = [
"lexical-parse-integer",
"lexical-util",
"static_assertions",
]
[[package]]
name = "lexical-parse-integer"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d0994485ed0c312f6d965766754ea177d07f9c00c9b82a5ee62ed5b47945ee9"
dependencies = [
"lexical-util",
"static_assertions",
]
[[package]]
name = "lexical-util"
version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5255b9ff16ff898710eb9eb63cb39248ea8a5bb036bea8085b1a767ff6c4e3fc"
dependencies = [
"static_assertions",
]
[[package]]
name = "lexical-write-float"
version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "accabaa1c4581f05a3923d1b4cfd124c329352288b7b9da09e766b0668116862"
dependencies = [
"lexical-util",
"lexical-write-integer",
"static_assertions",
]
[[package]]
name = "lexical-write-integer"
version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e1b6f3d1f4422866b68192d62f77bc5c700bee84f3069f2469d7bc8c77852446"
dependencies = [
"lexical-util",
"static_assertions",
]
[[package]]
name = "libc"
version = "0.2.144"
@ -1207,9 +1202,9 @@ dependencies = [
[[package]]
name = "memchr"
version = "2.5.0"
version = "2.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c"
[[package]]
name = "memmap2"
@ -1532,6 +1527,7 @@ version = "0.1.0"
dependencies = [
"anyhow",
"bytemuck",
"deno_ast",
"env_logger",
"fontdue",
"image",
@ -1540,12 +1536,6 @@ dependencies = [
"notify",
"oden-js",
"pollster",
"swc_common",
"swc_ecma_codegen",
"swc_ecma_parser",
"swc_ecma_transforms_base",
"swc_ecma_transforms_typescript",
"swc_ecma_visit",
"tracy-client",
"wgpu",
"winit",
@ -1629,6 +1619,12 @@ version = "1.0.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c"
[[package]]
name = "pathdiff"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd"
[[package]]
name = "peeking_take_while"
version = "0.1.2"
@ -2193,9 +2189,9 @@ dependencies = [
[[package]]
name = "swc_atoms"
version = "0.5.6"
version = "0.5.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93d0307dc4bfd107d49c7528350c372758cfca94fb503629b9a056e6a1572860"
checksum = "9f54563d7dcba626d4acfe14ed12def7ecc28e004debe3ecd2c3ee07cc47e449"
dependencies = [
"once_cell",
"rustc-hash",
@ -2207,11 +2203,10 @@ dependencies = [
[[package]]
name = "swc_common"
version = "0.31.16"
version = "0.32.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c6414bd4e553f5638961d39b07075ffd37a3d63176829592f4a5900260d94ca1"
checksum = "39cb7fcd56655c8ae7dcf2344f0be6cbff4d9c7cb401fe3ec8e56e1de8dfe582"
dependencies = [
"ahash 0.8.3",
"ast_node",
"better_scoped_tls",
"cfg-if",
@ -2223,6 +2218,7 @@ dependencies = [
"rustc-hash",
"serde",
"siphasher",
"sourcemap",
"string_cache",
"swc_atoms",
"swc_eq_ignore_macros",
@ -2259,14 +2255,15 @@ dependencies = [
[[package]]
name = "swc_ecma_ast"
version = "0.106.6"
version = "0.109.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ebf4d6804b1da4146c4c0359d129e3dd43568d321f69d7953d9abbca4ded76ba"
checksum = "7bc2286cedd688a68f214faa1c19bb5cceab7c9c54d0cbe3273e4c1704e38f69"
dependencies = [
"bitflags 2.3.2",
"is-macro",
"num-bigint",
"scoped-tls",
"serde",
"string_enum",
"swc_atoms",
"swc_common",
@ -2275,9 +2272,9 @@ dependencies = [
[[package]]
name = "swc_ecma_codegen"
version = "0.141.10"
version = "0.144.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac23ce426fca1f664579a8c4b9b42e47e2cb387bdce55c7fe1121c00dad009ea"
checksum = "8e62ba2c0ed1f119fc1a76542d007f1b2c12854d54dea15f5491363227debe11"
dependencies = [
"memchr",
"num-bigint",
@ -2306,14 +2303,27 @@ dependencies = [
]
[[package]]
name = "swc_ecma_parser"
version = "0.136.7"
name = "swc_ecma_loader"
version = "0.44.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "411341a6433540222c2a16043934f4398bba7e816059fd814f602e44731ad8c1"
checksum = "e7d7c322462657ae27ac090a2c89f7e456c94416284a2f5ecf66c43a6a3c19d1"
dependencies = [
"anyhow",
"pathdiff",
"serde",
"swc_common",
"tracing",
]
[[package]]
name = "swc_ecma_parser"
version = "0.139.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3eab46cb863bc5cd61535464e07e5b74d5f792fa26a27b9f6fd4c8daca9903b7"
dependencies = [
"either",
"lexical",
"num-bigint",
"num-traits",
"serde",
"smallvec",
"smartstring",
@ -2327,9 +2337,9 @@ dependencies = [
[[package]]
name = "swc_ecma_transforms_base"
version = "0.129.13"
version = "0.132.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "589da1447e30a46498a99b25d52602fb367fff7e6774fa83cae71829ac935415"
checksum = "01ffd4a8149052bfc1ec1832fcbe04f317846ce635a49ec438df33b06db27d26"
dependencies = [
"better_scoped_tls",
"bitflags 2.3.2",
@ -2349,10 +2359,24 @@ dependencies = [
]
[[package]]
name = "swc_ecma_transforms_macros"
version = "0.5.2"
name = "swc_ecma_transforms_classes"
version = "0.121.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f59c4b6ed5d78d3ad9fc7c6f8ab4f85bba99573d31d9a2c0a712077a6b45efd2"
checksum = "f4b7fee0e2c6f12456d2aefb2418f2f26529b995945d493e1dce35a5a22584fc"
dependencies = [
"swc_atoms",
"swc_common",
"swc_ecma_ast",
"swc_ecma_transforms_base",
"swc_ecma_utils",
"swc_ecma_visit",
]
[[package]]
name = "swc_ecma_transforms_macros"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8188eab297da773836ef5cf2af03ee5cca7a563e1be4b146f8141452c28cc690"
dependencies = [
"pmutil",
"proc-macro2",
@ -2362,12 +2386,31 @@ dependencies = [
]
[[package]]
name = "swc_ecma_transforms_react"
version = "0.175.17"
name = "swc_ecma_transforms_proposal"
version = "0.166.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b58f795939b4412d3ba94a5299dc6d64834ef0e272098872399637e8ec795833"
checksum = "122fd9a69f464694edefbf9c59106b3c15e5cc8cb8575a97836e4fb79018e98f"
dependencies = [
"either",
"rustc-hash",
"serde",
"smallvec",
"swc_atoms",
"swc_common",
"swc_ecma_ast",
"swc_ecma_transforms_base",
"swc_ecma_transforms_classes",
"swc_ecma_transforms_macros",
"swc_ecma_utils",
"swc_ecma_visit",
]
[[package]]
name = "swc_ecma_transforms_react"
version = "0.178.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "675b5c755b0448268830e85e59429095d3423c0ce4a850b209c6f0eeab069f63"
dependencies = [
"ahash 0.8.3",
"base64",
"dashmap",
"indexmap",
@ -2388,9 +2431,9 @@ dependencies = [
[[package]]
name = "swc_ecma_transforms_typescript"
version = "0.179.18"
version = "0.182.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "491ac322e85eee53c33b972cb73e5b86327f293ecb5ec6a1bc8b2f5f052bd7a3"
checksum = "4eba97b1ea71739fcf278aedad4677a3cacb52288a3f3566191b70d16a889de6"
dependencies = [
"serde",
"swc_atoms",
@ -2404,9 +2447,9 @@ dependencies = [
[[package]]
name = "swc_ecma_utils"
version = "0.119.9"
version = "0.122.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22e881888b8d5f2c869dd026bb367e0c789385f4d2b39bd0321fead742b1f995"
checksum = "11006a3398ffd4693c4d3b0a1b1a5030edbdc04228159f5301120a6178144708"
dependencies = [
"indexmap",
"num_cpus",
@ -2422,9 +2465,9 @@ dependencies = [
[[package]]
name = "swc_ecma_visit"
version = "0.92.5"
version = "0.95.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f61da6cac0ec3b7e62d367cfbd9e38e078a4601271891ad94f0dac5ff69f839"
checksum = "0f628ec196e76e67892441e14eef2e423a738543d32bffdabfeec20c29582117"
dependencies = [
"num-bigint",
"swc_atoms",
@ -2513,6 +2556,15 @@ dependencies = [
"winapi-util",
]
[[package]]
name = "text_lines"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7fd5828de7deaa782e1dd713006ae96b3bee32d3279b79eb67ecf8072c059bcf"
dependencies = [
"serde",
]
[[package]]
name = "thiserror"
version = "1.0.40"
@ -2764,6 +2816,7 @@ dependencies = [
"form_urlencoded",
"idna",
"percent-encoding",
"serde",
]
[[package]]

View file

@ -11,6 +11,7 @@ tracing = ["tracy-client/enable"]
[dependencies]
anyhow = "1.0"
bytemuck = { version = "1.13", features = ["derive"] }
deno_ast = { version = "0.29.3", features = ["transpiling", "typescript"] }
env_logger = "0.10"
fontdue = "0.7.3"
image = { version = "0.24", default-features = false, features = ["png"] }
@ -19,12 +20,6 @@ lru = "0.11.0"
notify = "6"
oden-js = { path = "oden-js" }
pollster = "0.3"
swc_common = "0.31.16"
swc_ecma_codegen = "0.141.10"
swc_ecma_parser = "0.136.7"
swc_ecma_transforms_base = "0.129.13"
swc_ecma_transforms_typescript = "0.179.18"
swc_ecma_visit = "0.92.5"
tracy-client = { version = "0.15.2", default-features = false }
wgpu = "0.17"
winit = "0.28"

View file

@ -26,8 +26,8 @@ I'm using QuickJS, lightly patched, with my own bindings.
#### QuickJS
The easy alternative to QuickJS is to use Deno Core, which binds V8.
Unfortunately, Deno Core has kind of a wild architecture, where it goes through flatbuf for all its communications, and keeps the JavaScript engine at arm's length.
The easy alternative to QuickJS is to use Deno Core, which binds V8.
Unfortunately, Deno Core has kind of a wild architecture, where it goes through flatbuf for all its communications, and keeps the JavaScript engine at arm's length.
It doesn't expose things like "make a new native class."
And while it feels "possible" to use (see https://deno.com/blog/roll-your-own-javascript-runtime-pt2), it was not quite as easy to use for what I wanted to do as you might like.
@ -44,3 +44,25 @@ See above about being abandoned: we have small patches in place to e.g. expose i
#### My Own Bindings
The other bindings are maddeningly incomplete, and don't expose features I want to expose out of QuickJS.
### TypeScript
I'm using `deno_ast` to parse typescript at load time and "transpile" to javascript before loading.
`deno_ast` is built on swc, which I really really don't like very much, but my other options are:
1. Actually use the typescript compiler.
This is a neat idea but involves loading the compiler into QuickJS (which requires some patches to tsc since qjs is not node).
Also the typescript compiler is SLOW (especially when run in QJS) and finicky.
But this would be a full-fidelity option for sure.
2. Use tree-sitter to parse the typescript and strip the types out myself.
The tree-sitter parser is fast and gives me everything with full fidelity.
This *also* has the nice side-effect of letting me edit the source file in place, so source maps aren't a problem.
The downside is that finding every part of TS that has to be cut out to make JS is time consuming and error prone and I don't want to do it, particularly.
3. Parse Typescript myself.
I can write a parser, sure, doesn't bother me, but the syntax for typescript is not formally documented anywhere.
And it has the same problem as #2 when it comes time to emit JS.
So for now, I stick with `deno_ast`.
It does what I need it to do.

View file

@ -1,173 +1,33 @@
use deno_ast::{parse_module, MediaType, ParseParams, SourceTextInfo};
use oden_js::{Error, Result};
use std::cell::RefCell;
use std::rc::Rc;
use swc_common::{
self, comments::SingleThreadedComments, errors::Diagnostic, sync::Lrc, FileName, Globals, Mark,
SourceMap, GLOBALS,
};
use swc_ecma_codegen::{text_writer::JsWriter, Emitter};
use swc_ecma_parser::{lexer::Lexer, Parser, StringInput, Syntax};
use swc_ecma_transforms_base::{fixer::fixer, hygiene::hygiene, resolver};
use swc_ecma_transforms_typescript::strip;
use swc_ecma_visit::{swc_ecma_ast::EsVersion, FoldWith};
struct DiagnosticCollector {
cell: Rc<RefCell<Vec<Diagnostic>>>,
}
impl DiagnosticCollector {
pub fn into_handler(self) -> swc_common::errors::Handler {
swc_common::errors::Handler::with_emitter(true, false, Box::new(self))
}
}
impl swc_common::errors::Emitter for DiagnosticCollector {
fn emit(&mut self, db: &swc_common::errors::DiagnosticBuilder<'_>) {
use std::ops::Deref;
self.cell.borrow_mut().push(db.deref().clone());
}
}
fn is_fatal_swc_diagnostic(diagnostic: &Diagnostic) -> bool {
use swc_common::errors::Level;
match diagnostic.level {
Level::Bug
| Level::Cancelled
| Level::FailureNote
| Level::Fatal
| Level::PhaseFatal
| Level::Error => true,
Level::Help | Level::Note | Level::Warning => false,
}
}
fn format_swc_diagnostic(source_map: &SourceMap, diagnostic: &Diagnostic) -> String {
if let Some(span) = &diagnostic.span.primary_span() {
let file_name = source_map.span_to_filename(*span);
let loc = source_map.lookup_char_pos(span.lo);
format!(
"{} at {}:{}:{}",
diagnostic.message(),
file_name,
loc.line,
loc.col_display + 1,
)
} else {
diagnostic.message()
}
}
fn diagnostics_to_parse_error<'a>(
name: &str,
source_map: &SourceMap,
diagnostics: impl Iterator<Item = &'a Diagnostic>,
) -> Error {
Error::ParseError(
name.into(),
diagnostics
.map(|d| format_swc_diagnostic(source_map, d))
.collect::<Vec<_>>()
.join("\n\n"),
)
}
fn ensure_no_fatal_swc_diagnostics<'a>(
name: &str,
source_map: &SourceMap,
diagnostics: impl Iterator<Item = &'a Diagnostic>,
) -> Result<()> {
let fatal_diagnostics = diagnostics
.filter(|d| is_fatal_swc_diagnostic(d))
.collect::<Vec<_>>();
if !fatal_diagnostics.is_empty() {
Err(diagnostics_to_parse_error(
name,
source_map,
fatal_diagnostics.into_iter(),
))
} else {
Ok(())
}
}
pub fn transpile_to_javascript(path: &str, input: String) -> Result<String> {
// NOTE: This was taken almost verbatim from
// https://github.com/swc-project/swc/blob/main/crates/swc_ecma_transforms_typescript/examples/ts_to_js.rs
// This appears to be similar to what deno_ast does, but this has
// the advantage of actually compiling. :P
//
// NOTE: Really need to figure out how to get this to generate a source
// map for the transpilation or something.
let cm: Lrc<SourceMap> = Default::default();
let diagnostics_cell: Rc<RefCell<Vec<Diagnostic>>> = Default::default();
let handler = DiagnosticCollector {
cell: diagnostics_cell.clone(),
}
.into_handler();
let fm = cm.new_source_file(FileName::Custom(path.into()), input.into());
let comments = SingleThreadedComments::default();
let lexer = Lexer::new(
Syntax::Typescript(Default::default()),
EsVersion::Es2020,
StringInput::from(&*fm),
Some(&comments),
);
let mut parser = Parser::new_from(lexer);
for e in parser.take_errors() {
e.into_diagnostic(&handler).emit();
}
let module = parser
.parse_module()
.map_err(|e| e.into_diagnostic(&handler))
.map_err(|mut e| {
e.emit();
let diagnostics = diagnostics_cell.borrow();
diagnostics_to_parse_error(path, &cm, diagnostics.iter())
})?;
let globals = Globals::default();
GLOBALS.set(&globals, || {
let unresolved_mark = Mark::new();
let top_level_mark = Mark::new();
// Optionally transforms decorators here before the resolver pass
// as it might produce runtime declarations.
// Conduct identifier scope analysis
let module = module.fold_with(&mut resolver(unresolved_mark, top_level_mark, true));
// Remove typescript types
let module = module.fold_with(&mut strip(top_level_mark));
// Fix up any identifiers with the same name, but different contexts
let module = module.fold_with(&mut hygiene());
// Ensure that we have enough parenthesis.
let module = module.fold_with(&mut fixer(Some(&comments)));
let mut buf = vec![];
{
let mut emitter = Emitter {
cfg: swc_ecma_codegen::Config {
minify: false,
..Default::default()
},
cm: cm.clone(),
comments: Some(&comments),
wr: JsWriter::new(cm.clone(), "\n", &mut buf, None),
};
emitter.emit_module(&module).unwrap();
}
let diagnostics = diagnostics_cell.borrow();
ensure_no_fatal_swc_diagnostics(path, &cm, diagnostics.iter())?;
Ok(String::from_utf8(buf).expect("non-utf8?"))
let text_info = SourceTextInfo::new(input.into());
let parsed_source = parse_module(ParseParams {
specifier: path.to_string(),
media_type: MediaType::TypeScript,
text_info,
capture_tokens: true,
maybe_syntax: None,
scope_analysis: false,
})
.map_err(|e| {
let position = e.display_position();
Error::ParseError(
path.into(),
format!(
"{} at {}:{}:{}",
e.message(),
path,
position.line_number,
position.column_number
),
)
})?;
let options: deno_ast::EmitOptions = Default::default();
let transpiled = parsed_source
.transpile(&options)
.map_err(|e| Error::ParseError(path.into(), e.to_string()))?;
return Ok(transpiled.text);
}