diff --git a/Cargo.lock b/Cargo.lock index b62386bc..b0f0a3ff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -61,6 +61,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" dependencies = [ "cfg-if", + "getrandom", "once_cell", "version_check", ] @@ -498,59 +499,6 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308" -[[package]] -name = "data-url" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41b319d1b62ffbd002e057f36bebd1f42b9f97927c9577461d855f3513c4289f" - -[[package]] -name = "deno_ast" -version = "0.29.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "577ec3850834c2578eb44afa9250f9a807f8497664e6e2aaae19cea0aac2fe3b" -dependencies = [ - "anyhow", - "base64", - "deno_media_type", - "dprint-swc-ext", - "serde", - "swc_atoms", - "swc_common", - "swc_config", - "swc_config_macro", - "swc_ecma_ast", - "swc_ecma_codegen", - "swc_ecma_codegen_macros", - "swc_ecma_loader", - "swc_ecma_parser", - "swc_ecma_transforms_base", - "swc_ecma_transforms_classes", - "swc_ecma_transforms_macros", - "swc_ecma_transforms_proposal", - "swc_ecma_transforms_react", - "swc_ecma_transforms_typescript", - "swc_ecma_utils", - "swc_ecma_visit", - "swc_eq_ignore_macros", - "swc_macros_common", - "swc_visit", - "swc_visit_macros", - "text_lines", - "url", -] - -[[package]] -name = "deno_media_type" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a798670c20308e5770cc0775de821424ff9e85665b602928509c8c70430b3ee0" -dependencies = [ - "data-url", - "serde", - "url", -] - [[package]] name = "digest" version = "0.10.7" @@ -582,22 +530,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ea835d29036a4087793836fa931b08837ad5e957da9e23886b29586fb9b6650" -[[package]] -name = "dprint-swc-ext" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a0a2492465344a58a37ae119de59e81fe5a2885f2711c7b5048ef0dfa14ce42" -dependencies = [ - "bumpalo", - "num-bigint", - "rustc-hash", - "swc_atoms", - "swc_common", - "swc_ecma_ast", - "swc_ecma_parser", - "text_lines", -] - [[package]] name = "either" version = "1.8.1" @@ -1112,6 +1044,79 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" +[[package]] +name = "lexical" +version = "6.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7aefb36fd43fef7003334742cbf77b243fcd36418a1d1bdd480d613a67968f6" +dependencies = [ + "lexical-core", +] + +[[package]] +name = "lexical-core" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cde5de06e8d4c2faabc400238f9ae1c74d5412d03a7bd067645ccbc47070e46" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683b3a5ebd0130b8fb52ba0bdc718cc56815b6a097e28ae5a6997d0ad17dc05f" +dependencies = [ + "lexical-parse-integer", + "lexical-util", + "static_assertions", +] + +[[package]] +name = "lexical-parse-integer" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d0994485ed0c312f6d965766754ea177d07f9c00c9b82a5ee62ed5b47945ee9" +dependencies = [ + "lexical-util", + "static_assertions", +] + +[[package]] +name = "lexical-util" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5255b9ff16ff898710eb9eb63cb39248ea8a5bb036bea8085b1a767ff6c4e3fc" +dependencies = [ + "static_assertions", +] + +[[package]] +name = "lexical-write-float" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accabaa1c4581f05a3923d1b4cfd124c329352288b7b9da09e766b0668116862" +dependencies = [ + "lexical-util", + "lexical-write-integer", + "static_assertions", +] + +[[package]] +name = "lexical-write-integer" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1b6f3d1f4422866b68192d62f77bc5c700bee84f3069f2469d7bc8c77852446" +dependencies = [ + "lexical-util", + "static_assertions", +] + [[package]] name = "libc" version = "0.2.144" @@ -1202,9 +1207,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.6.3" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "memmap2" @@ -1527,7 +1532,6 @@ version = "0.1.0" dependencies = [ "anyhow", "bytemuck", - "deno_ast", "env_logger", "fontdue", "image", @@ -1536,6 +1540,12 @@ dependencies = [ "notify", "oden-js", "pollster", + "swc_common", + "swc_ecma_codegen", + "swc_ecma_parser", + "swc_ecma_transforms_base", + "swc_ecma_transforms_typescript", + "swc_ecma_visit", "tracy-client", "wgpu", "winit", @@ -1619,12 +1629,6 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" -[[package]] -name = "pathdiff" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd" - [[package]] name = "peeking_take_while" version = "0.1.2" @@ -2189,9 +2193,9 @@ dependencies = [ [[package]] name = "swc_atoms" -version = "0.5.9" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f54563d7dcba626d4acfe14ed12def7ecc28e004debe3ecd2c3ee07cc47e449" +checksum = "93d0307dc4bfd107d49c7528350c372758cfca94fb503629b9a056e6a1572860" dependencies = [ "once_cell", "rustc-hash", @@ -2203,10 +2207,11 @@ dependencies = [ [[package]] name = "swc_common" -version = "0.32.0" +version = "0.31.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39cb7fcd56655c8ae7dcf2344f0be6cbff4d9c7cb401fe3ec8e56e1de8dfe582" +checksum = "c6414bd4e553f5638961d39b07075ffd37a3d63176829592f4a5900260d94ca1" dependencies = [ + "ahash 0.8.3", "ast_node", "better_scoped_tls", "cfg-if", @@ -2218,7 +2223,6 @@ dependencies = [ "rustc-hash", "serde", "siphasher", - "sourcemap", "string_cache", "swc_atoms", "swc_eq_ignore_macros", @@ -2255,15 +2259,14 @@ dependencies = [ [[package]] name = "swc_ecma_ast" -version = "0.109.0" +version = "0.106.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bc2286cedd688a68f214faa1c19bb5cceab7c9c54d0cbe3273e4c1704e38f69" +checksum = "ebf4d6804b1da4146c4c0359d129e3dd43568d321f69d7953d9abbca4ded76ba" dependencies = [ "bitflags 2.3.2", "is-macro", "num-bigint", "scoped-tls", - "serde", "string_enum", "swc_atoms", "swc_common", @@ -2272,9 +2275,9 @@ dependencies = [ [[package]] name = "swc_ecma_codegen" -version = "0.144.1" +version = "0.141.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e62ba2c0ed1f119fc1a76542d007f1b2c12854d54dea15f5491363227debe11" +checksum = "ac23ce426fca1f664579a8c4b9b42e47e2cb387bdce55c7fe1121c00dad009ea" dependencies = [ "memchr", "num-bigint", @@ -2302,28 +2305,15 @@ dependencies = [ "syn 2.0.18", ] -[[package]] -name = "swc_ecma_loader" -version = "0.44.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7d7c322462657ae27ac090a2c89f7e456c94416284a2f5ecf66c43a6a3c19d1" -dependencies = [ - "anyhow", - "pathdiff", - "serde", - "swc_common", - "tracing", -] - [[package]] name = "swc_ecma_parser" -version = "0.139.0" +version = "0.136.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eab46cb863bc5cd61535464e07e5b74d5f792fa26a27b9f6fd4c8daca9903b7" +checksum = "411341a6433540222c2a16043934f4398bba7e816059fd814f602e44731ad8c1" dependencies = [ "either", + "lexical", "num-bigint", - "num-traits", "serde", "smallvec", "smartstring", @@ -2337,9 +2327,9 @@ dependencies = [ [[package]] name = "swc_ecma_transforms_base" -version = "0.132.2" +version = "0.129.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01ffd4a8149052bfc1ec1832fcbe04f317846ce635a49ec438df33b06db27d26" +checksum = "589da1447e30a46498a99b25d52602fb367fff7e6774fa83cae71829ac935415" dependencies = [ "better_scoped_tls", "bitflags 2.3.2", @@ -2358,25 +2348,11 @@ dependencies = [ "tracing", ] -[[package]] -name = "swc_ecma_transforms_classes" -version = "0.121.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4b7fee0e2c6f12456d2aefb2418f2f26529b995945d493e1dce35a5a22584fc" -dependencies = [ - "swc_atoms", - "swc_common", - "swc_ecma_ast", - "swc_ecma_transforms_base", - "swc_ecma_utils", - "swc_ecma_visit", -] - [[package]] name = "swc_ecma_transforms_macros" -version = "0.5.3" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8188eab297da773836ef5cf2af03ee5cca7a563e1be4b146f8141452c28cc690" +checksum = "f59c4b6ed5d78d3ad9fc7c6f8ab4f85bba99573d31d9a2c0a712077a6b45efd2" dependencies = [ "pmutil", "proc-macro2", @@ -2385,32 +2361,13 @@ dependencies = [ "syn 2.0.18", ] -[[package]] -name = "swc_ecma_transforms_proposal" -version = "0.166.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "122fd9a69f464694edefbf9c59106b3c15e5cc8cb8575a97836e4fb79018e98f" -dependencies = [ - "either", - "rustc-hash", - "serde", - "smallvec", - "swc_atoms", - "swc_common", - "swc_ecma_ast", - "swc_ecma_transforms_base", - "swc_ecma_transforms_classes", - "swc_ecma_transforms_macros", - "swc_ecma_utils", - "swc_ecma_visit", -] - [[package]] name = "swc_ecma_transforms_react" -version = "0.178.3" +version = "0.175.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "675b5c755b0448268830e85e59429095d3423c0ce4a850b209c6f0eeab069f63" +checksum = "b58f795939b4412d3ba94a5299dc6d64834ef0e272098872399637e8ec795833" dependencies = [ + "ahash 0.8.3", "base64", "dashmap", "indexmap", @@ -2431,9 +2388,9 @@ dependencies = [ [[package]] name = "swc_ecma_transforms_typescript" -version = "0.182.3" +version = "0.179.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eba97b1ea71739fcf278aedad4677a3cacb52288a3f3566191b70d16a889de6" +checksum = "491ac322e85eee53c33b972cb73e5b86327f293ecb5ec6a1bc8b2f5f052bd7a3" dependencies = [ "serde", "swc_atoms", @@ -2447,9 +2404,9 @@ dependencies = [ [[package]] name = "swc_ecma_utils" -version = "0.122.0" +version = "0.119.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11006a3398ffd4693c4d3b0a1b1a5030edbdc04228159f5301120a6178144708" +checksum = "22e881888b8d5f2c869dd026bb367e0c789385f4d2b39bd0321fead742b1f995" dependencies = [ "indexmap", "num_cpus", @@ -2465,9 +2422,9 @@ dependencies = [ [[package]] name = "swc_ecma_visit" -version = "0.95.0" +version = "0.92.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f628ec196e76e67892441e14eef2e423a738543d32bffdabfeec20c29582117" +checksum = "0f61da6cac0ec3b7e62d367cfbd9e38e078a4601271891ad94f0dac5ff69f839" dependencies = [ "num-bigint", "swc_atoms", @@ -2556,15 +2513,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "text_lines" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fd5828de7deaa782e1dd713006ae96b3bee32d3279b79eb67ecf8072c059bcf" -dependencies = [ - "serde", -] - [[package]] name = "thiserror" version = "1.0.40" @@ -2816,7 +2764,6 @@ dependencies = [ "form_urlencoded", "idna", "percent-encoding", - "serde", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 7d8b2f08..9c3d0ff3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,6 @@ tracing = ["tracy-client/enable"] [dependencies] anyhow = "1.0" bytemuck = { version = "1.13", features = ["derive"] } -deno_ast = { version = "0.29.3", features = ["transpiling", "typescript"] } env_logger = "0.10" fontdue = "0.7.3" image = { version = "0.24", default-features = false, features = ["png"] } @@ -20,6 +19,12 @@ lru = "0.11.0" notify = "6" oden-js = { path = "oden-js" } pollster = "0.3" +swc_common = "0.31.16" +swc_ecma_codegen = "0.141.10" +swc_ecma_parser = "0.136.7" +swc_ecma_transforms_base = "0.129.13" +swc_ecma_transforms_typescript = "0.179.18" +swc_ecma_visit = "0.92.5" tracy-client = { version = "0.15.2", default-features = false } wgpu = "0.17" winit = "0.28" diff --git a/README.md b/README.md index a647ece4..9099406b 100644 --- a/README.md +++ b/README.md @@ -26,8 +26,8 @@ I'm using QuickJS, lightly patched, with my own bindings. #### QuickJS -The easy alternative to QuickJS is to use Deno Core, which binds V8. -Unfortunately, Deno Core has kind of a wild architecture, where it goes through flatbuf for all its communications, and keeps the JavaScript engine at arm's length. +The easy alternative to QuickJS is to use Deno Core, which binds V8. +Unfortunately, Deno Core has kind of a wild architecture, where it goes through flatbuf for all its communications, and keeps the JavaScript engine at arm's length. It doesn't expose things like "make a new native class." And while it feels "possible" to use (see https://deno.com/blog/roll-your-own-javascript-runtime-pt2), it was not quite as easy to use for what I wanted to do as you might like. @@ -44,25 +44,3 @@ See above about being abandoned: we have small patches in place to e.g. expose i #### My Own Bindings The other bindings are maddeningly incomplete, and don't expose features I want to expose out of QuickJS. - -### TypeScript - -I'm using `deno_ast` to parse typescript at load time and "transpile" to javascript before loading. -`deno_ast` is built on swc, which I really really don't like very much, but my other options are: - -1. Actually use the typescript compiler. - This is a neat idea but involves loading the compiler into QuickJS (which requires some patches to tsc since qjs is not node). - Also the typescript compiler is SLOW (especially when run in QJS) and finicky. - But this would be a full-fidelity option for sure. - -2. Use tree-sitter to parse the typescript and strip the types out myself. - The tree-sitter parser is fast and gives me everything with full fidelity. - This *also* has the nice side-effect of letting me edit the source file in place, so source maps aren't a problem. - The downside is that finding every part of TS that has to be cut out to make JS is time consuming and error prone and I don't want to do it, particularly. - -3. Parse Typescript myself. - I can write a parser, sure, doesn't bother me, but the syntax for typescript is not formally documented anywhere. - And it has the same problem as #2 when it comes time to emit JS. - -So for now, I stick with `deno_ast`. -It does what I need it to do. diff --git a/src/script/typescript.rs b/src/script/typescript.rs index 229da5d5..9e94bb35 100644 --- a/src/script/typescript.rs +++ b/src/script/typescript.rs @@ -1,33 +1,173 @@ -use deno_ast::{parse_module, MediaType, ParseParams, SourceTextInfo}; use oden_js::{Error, Result}; +use std::cell::RefCell; +use std::rc::Rc; + +use swc_common::{ + self, comments::SingleThreadedComments, errors::Diagnostic, sync::Lrc, FileName, Globals, Mark, + SourceMap, GLOBALS, +}; +use swc_ecma_codegen::{text_writer::JsWriter, Emitter}; +use swc_ecma_parser::{lexer::Lexer, Parser, StringInput, Syntax}; +use swc_ecma_transforms_base::{fixer::fixer, hygiene::hygiene, resolver}; +use swc_ecma_transforms_typescript::strip; +use swc_ecma_visit::{swc_ecma_ast::EsVersion, FoldWith}; + +struct DiagnosticCollector { + cell: Rc>>, +} + +impl DiagnosticCollector { + pub fn into_handler(self) -> swc_common::errors::Handler { + swc_common::errors::Handler::with_emitter(true, false, Box::new(self)) + } +} + +impl swc_common::errors::Emitter for DiagnosticCollector { + fn emit(&mut self, db: &swc_common::errors::DiagnosticBuilder<'_>) { + use std::ops::Deref; + self.cell.borrow_mut().push(db.deref().clone()); + } +} + +fn is_fatal_swc_diagnostic(diagnostic: &Diagnostic) -> bool { + use swc_common::errors::Level; + match diagnostic.level { + Level::Bug + | Level::Cancelled + | Level::FailureNote + | Level::Fatal + | Level::PhaseFatal + | Level::Error => true, + Level::Help | Level::Note | Level::Warning => false, + } +} + +fn format_swc_diagnostic(source_map: &SourceMap, diagnostic: &Diagnostic) -> String { + if let Some(span) = &diagnostic.span.primary_span() { + let file_name = source_map.span_to_filename(*span); + let loc = source_map.lookup_char_pos(span.lo); + format!( + "{} at {}:{}:{}", + diagnostic.message(), + file_name, + loc.line, + loc.col_display + 1, + ) + } else { + diagnostic.message() + } +} + +fn diagnostics_to_parse_error<'a>( + name: &str, + source_map: &SourceMap, + diagnostics: impl Iterator, +) -> Error { + Error::ParseError( + name.into(), + diagnostics + .map(|d| format_swc_diagnostic(source_map, d)) + .collect::>() + .join("\n\n"), + ) +} + +fn ensure_no_fatal_swc_diagnostics<'a>( + name: &str, + source_map: &SourceMap, + diagnostics: impl Iterator, +) -> Result<()> { + let fatal_diagnostics = diagnostics + .filter(|d| is_fatal_swc_diagnostic(d)) + .collect::>(); + if !fatal_diagnostics.is_empty() { + Err(diagnostics_to_parse_error( + name, + source_map, + fatal_diagnostics.into_iter(), + )) + } else { + Ok(()) + } +} pub fn transpile_to_javascript(path: &str, input: String) -> Result { - let text_info = SourceTextInfo::new(input.into()); - let parsed_source = parse_module(ParseParams { - specifier: path.to_string(), - media_type: MediaType::TypeScript, - text_info, - capture_tokens: true, - maybe_syntax: None, - scope_analysis: false, - }) - .map_err(|e| { - let position = e.display_position(); - Error::ParseError( - path.into(), - format!( - "{} at {}:{}:{}", - e.message(), - path, - position.line_number, - position.column_number - ), - ) - })?; + // NOTE: This was taken almost verbatim from + // https://github.com/swc-project/swc/blob/main/crates/swc_ecma_transforms_typescript/examples/ts_to_js.rs + // This appears to be similar to what deno_ast does, but this has + // the advantage of actually compiling. :P + // + // NOTE: Really need to figure out how to get this to generate a source + // map for the transpilation or something. + let cm: Lrc = Default::default(); + let diagnostics_cell: Rc>> = Default::default(); + let handler = DiagnosticCollector { + cell: diagnostics_cell.clone(), + } + .into_handler(); - let options: deno_ast::EmitOptions = Default::default(); - let transpiled = parsed_source - .transpile(&options) - .map_err(|e| Error::ParseError(path.into(), e.to_string()))?; - return Ok(transpiled.text); + let fm = cm.new_source_file(FileName::Custom(path.into()), input.into()); + let comments = SingleThreadedComments::default(); + + let lexer = Lexer::new( + Syntax::Typescript(Default::default()), + EsVersion::Es2020, + StringInput::from(&*fm), + Some(&comments), + ); + + let mut parser = Parser::new_from(lexer); + + for e in parser.take_errors() { + e.into_diagnostic(&handler).emit(); + } + + let module = parser + .parse_module() + .map_err(|e| e.into_diagnostic(&handler)) + .map_err(|mut e| { + e.emit(); + let diagnostics = diagnostics_cell.borrow(); + diagnostics_to_parse_error(path, &cm, diagnostics.iter()) + })?; + + let globals = Globals::default(); + GLOBALS.set(&globals, || { + let unresolved_mark = Mark::new(); + let top_level_mark = Mark::new(); + + // Optionally transforms decorators here before the resolver pass + // as it might produce runtime declarations. + + // Conduct identifier scope analysis + let module = module.fold_with(&mut resolver(unresolved_mark, top_level_mark, true)); + + // Remove typescript types + let module = module.fold_with(&mut strip(top_level_mark)); + + // Fix up any identifiers with the same name, but different contexts + let module = module.fold_with(&mut hygiene()); + + // Ensure that we have enough parenthesis. + let module = module.fold_with(&mut fixer(Some(&comments))); + + let mut buf = vec![]; + { + let mut emitter = Emitter { + cfg: swc_ecma_codegen::Config { + minify: false, + ..Default::default() + }, + cm: cm.clone(), + comments: Some(&comments), + wr: JsWriter::new(cm.clone(), "\n", &mut buf, None), + }; + + emitter.emit_module(&module).unwrap(); + } + + let diagnostics = diagnostics_cell.borrow(); + ensure_no_fatal_swc_diagnostics(path, &cm, diagnostics.iter())?; + Ok(String::from_utf8(buf).expect("non-utf8?")) + }) }