diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0e2ddbf4..68642d1f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -107,6 +107,13 @@ jobs: run: pnpm test working-directory: crates/astro_napi + - name: Build astro2tsx binding + run: pnpm --filter ./crates/astro2tsx run build-dev + + - name: Test astro2tsx binding + run: pnpm test + working-directory: crates/astro2tsx + - name: Build compiler package run: pnpm run build:compiler diff --git a/Cargo.toml b/Cargo.toml index 25b15391..97fc7cae 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,6 +33,17 @@ oxc_syntax = { git = "https://github.com/withastro/oxc", rev = "8bb526fc0c20beb4 oxc_transformer = { git = "https://github.com/withastro/oxc", rev = "8bb526fc0c20beb4649b223d3ac39851505caa5a" } oxc_sourcemap = "6.0.1" +# Biome dependencies via git, all pinned to the same fork revision +biome_diagnostics = { git = "https://github.com/Princesseuh/biome", rev = "e0a22c5f5b6fd09abe5e93ec0b5416f0ffbb73cb" } +biome_html_parser = { git = "https://github.com/Princesseuh/biome", rev = "e0a22c5f5b6fd09abe5e93ec0b5416f0ffbb73cb" } +biome_html_syntax = { git = "https://github.com/Princesseuh/biome", rev = "e0a22c5f5b6fd09abe5e93ec0b5416f0ffbb73cb" } +biome_js_parser = { git = "https://github.com/Princesseuh/biome", rev = "e0a22c5f5b6fd09abe5e93ec0b5416f0ffbb73cb" } +biome_parser = { git = "https://github.com/Princesseuh/biome", rev = "e0a22c5f5b6fd09abe5e93ec0b5416f0ffbb73cb" } +biome_js_syntax = { git = "https://github.com/Princesseuh/biome", rev = "e0a22c5f5b6fd09abe5e93ec0b5416f0ffbb73cb" } +biome_languages = { git = "https://github.com/Princesseuh/biome", rev = "e0a22c5f5b6fd09abe5e93ec0b5416f0ffbb73cb" } +biome_rowan = { git = "https://github.com/Princesseuh/biome", rev = "e0a22c5f5b6fd09abe5e93ec0b5416f0ffbb73cb" } +biome_string_case = { git = "https://github.com/Princesseuh/biome", rev = "e0a22c5f5b6fd09abe5e93ec0b5416f0ffbb73cb" } + # NAPI napi = { version = "3", features = ["tokio_rt"] } napi-build = "2" diff --git a/biome.jsonc b/biome.jsonc index 5b819f73..f16b0d56 100644 --- a/biome.jsonc +++ b/biome.jsonc @@ -11,7 +11,10 @@ "!crates/astro_napi/*.cjs", "!crates/astro_napi/*.mjs", "!crates/astro_napi/index.d.ts", - "!crates/astro_napi/npm" + "!crates/astro_napi/npm", + "!crates/astro2tsx/*.js", + "!crates/astro2tsx/index.d.ts", + "!crates/astro2tsx/npm" ] }, "vcs": { diff --git a/crates/astro2tsx/Cargo.toml b/crates/astro2tsx/Cargo.toml new file mode 100644 index 00000000..aff89d0f --- /dev/null +++ b/crates/astro2tsx/Cargo.toml @@ -0,0 +1,44 @@ +[package] +name = "astro2tsx" +version = "0.1.0" +authors.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false +description = "Converts .astro files to .tsx for tsserver intellisense, using Biome's HTML CST" + +[lib] +crate-type = ["cdylib", "lib"] +test = true +doctest = false + +[dependencies] +biome_diagnostics = { workspace = true } +biome_html_parser = { workspace = true } +biome_html_syntax = { workspace = true } +biome_js_parser = { workspace = true } +biome_parser = { workspace = true } +biome_js_syntax = { workspace = true } +biome_languages = { workspace = true } +biome_rowan = { workspace = true } +biome_string_case = { workspace = true } + +napi = { workspace = true } +napi-derive = { workspace = true } +oxc_sourcemap = { workspace = true } +oxc_syntax = { workspace = true } + +[dev-dependencies] +divan = { workspace = true } +insta = { workspace = true, features = ["glob", "serde"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" + +[build-dependencies] +napi-build = { workspace = true } + +[[bench]] +name = "convert" +harness = false diff --git a/crates/astro2tsx/__test__/astro2tsx.test.ts b/crates/astro2tsx/__test__/astro2tsx.test.ts new file mode 100644 index 00000000..f0166be9 --- /dev/null +++ b/crates/astro2tsx/__test__/astro2tsx.test.ts @@ -0,0 +1,209 @@ +import { readdirSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { strict as assert } from 'node:assert'; +import ts from 'typescript'; +import { test } from 'node:test'; +import { convertToTsx } from '../index.js'; + +test('emits the TSX prefix and a Fragment-wrapped body', () => { + const result = convertToTsx('
Hi
'; + const map = JSON.parse(convertToTsx(input).map!); + assert.equal(map.version, 3); + assert.deepEqual(map.sources, ['input.astro']); + assert.deepEqual(map.sourcesContent, [input]); + assert.deepEqual(map.names, []); + assert.ok(map.mappings.length > 0); +}); + +test('names the source after the filename option', () => { + const map = JSON.parse(convertToTsx('', { filename: 'Index.astro' }).map!); + assert.deepEqual(map.sources, ['Index.astro']); +}); + +test('appends the inline source map comment by default', () => { + const { code, map } = convertToTsx('Hi
'); + const marker = '\n//# sourceMappingURL=data:application/json;charset=utf-8;base64,'; + assert.ok(code.includes(marker)); + const blob = code.slice(code.indexOf(marker) + marker.length); + assert.deepEqual(JSON.parse(Buffer.from(blob, 'base64').toString('utf8')), JSON.parse(map!)); +}); + +test("sourcemap: 'external' leaves the code without the comment", () => { + const inline = convertToTsx('Hi
'); + const external = convertToTsx('Hi
', { sourcemap: 'external' }); + assert.doesNotMatch(external.code, /sourceMappingURL/); + assert.ok(inline.code.startsWith(external.code)); +}); + +test('every clean-parse fixture emits syntactically valid TSX', async () => { + // Frontmatter is user JS emitted verbatim, so a fixture whose frontmatter is + // itself invalid TS legitimately produces invalid output. + const invalidUserCode = new Set(['props_generic_invalid']); + + const dir = join(import.meta.dirname, '../tests/fixtures'); + let checked = 0; + for (const file of readdirSync(dir)) { + if (!file.endsWith('.astro')) continue; + const name = file.slice(0, -'.astro'.length); + if (invalidUserCode.has(name)) continue; + + let source = readFileSync(join(dir, file), 'utf8'); + while (source.startsWith('// @config ')) { + source = source.slice(source.indexOf('\n') + 1); + } + + const result = convertToTsx(source, { filename: `${name}.astro` }); + if (result.hasParseErrors) continue; + + const sourceFile = ts.createSourceFile( + `${name}.tsx`, + result.code, + ts.ScriptTarget.Latest, + false, + ts.ScriptKind.TSX, + ); + const diagnostics = ( + sourceFile as unknown as { parseDiagnostics: { messageText: unknown; start: number }[] } + ).parseDiagnostics; + assert.deepEqual( + diagnostics.map((d) => `${name}: ${JSON.stringify(d.messageText)} at ${d.start}`), + [], + `invalid TSX emitted for ${name}:\n${result.code}`, + ); + checked++; + } + assert.ok(checked > 50, `expected to check most fixtures, checked ${checked}`); +}); + +test('offsets are UTF-16 code units, not bytes', () => { + const source = '---\nconst \u{1f984} = 1;\n---\n'; + const result = convertToTsx(source, { sourcemap: 'external' }); + + // The unicorn is four bytes but two UTF-16 units, so a byte offset would overshoot. + const style = result.styles[0]; + assert.equal(source.slice(style.position.start, style.position.end), '.a{color:red}'); + assert.equal( + source.slice(result.frontmatterSource.start, result.frontmatterSource.end).at(-1), + '-', + ); + + const { generatedOffsets, sourceOffsets, lengths } = result; + for (let i = 0; i < generatedOffsets.length; i++) { + assert.equal( + result.code.slice(generatedOffsets[i], generatedOffsets[i] + lengths[i]), + source.slice(sourceOffsets[i], sourceOffsets[i] + lengths[i]), + `run ${i} is not verbatim`, + ); + } +}); + +test('reports frontmatter status and positioned diagnostics', () => { + assert.equal(convertToTsx('---\nlet x = 1;\n---\n').frontmatterStatus, 'closed'); + assert.equal(convertToTsx('---\nlet x = 1;\n').frontmatterStatus, 'open'); + assert.equal(convertToTsx('').frontmatterStatus, 'doesnt-exist'); + + const broken = convertToTsx('{x}
'; + const skipped = convertToTsx(source, { sourcemap: false }); + assert.equal(skipped.map, undefined); + assert.doesNotMatch(skipped.code, /sourceMappingURL/); + + // Opting out must not change the code or the offsets editors navigate by. + const external = convertToTsx(source, { sourcemap: 'external' }); + assert.equal(skipped.code, external.code); + assert.deepEqual(Array.from(skipped.generatedOffsets), Array.from(external.generatedOffsets)); + assert.deepEqual(Array.from(skipped.lengths), Array.from(external.lengths)); + assert.ok(external.map); +}); diff --git a/crates/astro2tsx/benches/convert.rs b/crates/astro2tsx/benches/convert.rs new file mode 100644 index 00000000..6702fba3 --- /dev/null +++ b/crates/astro2tsx/benches/convert.rs @@ -0,0 +1,114 @@ +use astro2tsx::{ConvertOptions, SourceMapMode, convert_to_tsx}; +use divan::counter::BytesCount; + +fn main() { + divan::main(); +} + +fn options() -> ConvertOptions { + ConvertOptions { + filename: Some("Component.astro".to_string()), + sourcemap: SourceMapMode::External, + } +} + +fn bench_convert(bencher: divan::Bencher<'_, '_>, source: &str) { + bencher + .counter(BytesCount::of_str(source)) + .bench_local(|| convert_to_tsx(divan::black_box(source), options())); +} + +mod components { + use super::*; + + #[divan::bench] + fn favicon(bencher: divan::Bencher<'_, '_>) { + bench_convert(bencher, include_str!("fixtures/Favicon.astro")); + } + + #[divan::bench] + fn pill_link(bencher: divan::Bencher<'_, '_>) { + bench_convert(bencher, include_str!("fixtures/PillLink.astro")); + } + + #[divan::bench] + fn social_links(bencher: divan::Bencher<'_, '_>) { + bench_convert(bencher, include_str!("fixtures/SocialLinks.astro")); + } + + #[divan::bench] + fn header_drop_down(bencher: divan::Bencher<'_, '_>) { + bench_convert(bencher, include_str!("fixtures/HeaderDropDown.astro")); + } + + #[divan::bench] + fn seo(bencher: divan::Bencher<'_, '_>) { + bench_convert(bencher, include_str!("fixtures/SEO.astro")); + } + + #[divan::bench] + fn expression_heavy(bencher: divan::Bencher<'_, '_>) { + bench_convert(bencher, include_str!("fixtures/ExpressionHeavy.astro")); + } +} + +/// Repeats the section so the input grows without its shape changing. +fn build_page(sections: usize) -> String { + let fixture = include_str!("fixtures/ExpressionHeavy.astro"); + let (frontmatter, body) = fixture + .rsplit_once("---\n") + .expect("fixture has a frontmatter fence"); + let mut source = String::with_capacity(frontmatter.len() + 4 + body.len() * sections); + source.push_str(frontmatter); + source.push_str("---\n"); + for _ in 0..sections { + source.push_str(body); + } + source +} + +#[divan::bench(args = [8, 64, 256])] +fn large_page(bencher: divan::Bencher<'_, '_>, sections: usize) { + let source = build_page(sections); + bencher + .counter(BytesCount::of_str(&source)) + .bench_local(|| convert_to_tsx(divan::black_box(&source), options())); +} + +mod phases { + use super::*; + + #[divan::bench] + fn convert_only(bencher: divan::Bencher<'_, '_>) { + let source = build_page(64); + bencher + .counter(BytesCount::of_str(&source)) + .bench_local(|| convert_to_tsx(divan::black_box(&source), options())); + } + + #[divan::bench] + fn sourcemap_encode(bencher: divan::Bencher<'_, '_>) { + let source = build_page(64); + let result = convert_to_tsx(&source, options()); + bencher + .counter(BytesCount::of_str(&result.code)) + .bench_local(|| result.source_map(divan::black_box(&source), "Component.astro")); + } + + /// Inline is the napi binding's default, so this is the production path. + #[divan::bench] + fn convert_with_inline_map(bencher: divan::Bencher<'_, '_>) { + let source = build_page(64); + bencher + .counter(BytesCount::of_str(&source)) + .bench_local(|| { + convert_to_tsx( + divan::black_box(&source), + ConvertOptions { + filename: Some("Component.astro".to_string()), + sourcemap: SourceMapMode::Inline, + }, + ) + }); + } +} diff --git a/crates/astro2tsx/benches/fixtures/ExpressionHeavy.astro b/crates/astro2tsx/benches/fixtures/ExpressionHeavy.astro new file mode 100644 index 00000000..36402415 --- /dev/null +++ b/crates/astro2tsx/benches/fixtures/ExpressionHeavy.astro @@ -0,0 +1,45 @@ +--- +interface Item { + href: string; + title: string; + tags: string[]; + draft?: boolean; +} + +interface Props { + items: Item[]; + heading?: string; +} + +const { items, heading = 'Latest posts' } = Astro.props; +const published = items.filter((item) => !item.draft); +--- + +Nothing here yet.
} +x
diff --git a/crates/astro2tsx/tests/fixtures/frontmatter_top_level_return.snap b/crates/astro2tsx/tests/fixtures/frontmatter_top_level_return.snap new file mode 100644 index 00000000..209e0cf2 --- /dev/null +++ b/crates/astro2tsx/tests/fixtures/frontmatter_top_level_return.snap @@ -0,0 +1,33 @@ +--- +source: crates/astro2tsx/tests/snapshots.rs +info: + has_parse_errors: false + frontmatter_status: Closed + frontmatter_source: + start: 0 + end: 51 + frontmatter: + start: 30 + end: 76 + body: + start: 90 + end: 101 + scripts: [] + styles: [] +input_file: crates/astro2tsx/tests/fixtures/frontmatter_top_level_return.astro +--- +/* @jsxImportSource astro */ + + +if (cond) { + throw Astro.redirect('/x'); +} + +{};x
+ +x
x
foobar
diff --git a/crates/astro2tsx/tests/fixtures/multibyte.snap b/crates/astro2tsx/tests/fixtures/multibyte.snap new file mode 100644 index 00000000..ea2e64f5 --- /dev/null +++ b/crates/astro2tsx/tests/fixtures/multibyte.snap @@ -0,0 +1,27 @@ +--- +source: crates/astro2tsx/tests/snapshots.rs +info: + has_parse_errors: false + frontmatter_status: DoesntExist + frontmatter_source: + start: 0 + end: 0 + frontmatter: + start: 30 + end: 30 + body: + start: 41 + end: 68 + scripts: [] + styles: [] +input_file: crates/astro2tsx/tests/fixtures/multibyte.astro +--- +/* @jsxImportSource astro */ + +foobar
+ +🦄 {π}
diff --git a/crates/astro2tsx/tests/fixtures/multibyte_astral.snap b/crates/astro2tsx/tests/fixtures/multibyte_astral.snap new file mode 100644 index 00000000..e7c55cbe --- /dev/null +++ b/crates/astro2tsx/tests/fixtures/multibyte_astral.snap @@ -0,0 +1,31 @@ +--- +source: crates/astro2tsx/tests/snapshots.rs +info: + has_parse_errors: false + frontmatter_status: Closed + frontmatter_source: + start: 0 + end: 27 + frontmatter: + start: 30 + end: 52 + body: + start: 66 + end: 85 + scripts: [] + styles: [] +input_file: crates/astro2tsx/tests/fixtures/multibyte_astral.astro +--- +/* @jsxImportSource astro */ + + +const π = Math.PI; + +{};🦄 {π}
+ +body
+body
+

after
"); + assert!( + raw.contains("lost") && raw.contains("after"), + "is:raw content should stay inline:\n{raw}" + ); + + let style = convert_to_tsx( + "", ConvertOptions::default()); + let range = tag.styles[0].range; + assert_eq!(range.start, range.end); + assert!(tag.code[..range.start as usize].ends_with("")); +} + +#[test] +fn props_binding_needs_a_local_name() { + for (input, has_props) in [ + ("---\nimport Foo from './Props';\nFoo;\n---\n", false), + ( + "---\nimport { Props as Other } from './t';\n---\n", + false, + ), + ("---\nexport { Props } from './t';\n---\n", false), + ( + "---\n// mentions Props only in a comment\n---\n", + false, + ), + ( + "---\nimport { Other as Props } from './t';\n---\n", + true, + ), + ("---\nimport type { Props } from './t';\n---\n", true), + ("---\nimport Props from './t';\n---\n", true), + ( + "---\nexport interface Props { a: string }\n---\n", + true, + ), + ] { + let actual = convert(input); + assert_eq!( + actual.contains("_props: Props"), + has_props, + "wrong Props detection for {input:?}:\n{actual}" + ); + } +} + +#[test] +fn frontmatter_is_terminated_even_when_a_comment_ends_with_a_semicolon() { + let actual = convert("---\nconst x = foo\n// note;\n---\n"); + assert!( + actual.contains("{};