diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 409809d4e..2dd3d5475 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -141,7 +141,11 @@ jobs: run: pnpm test packages/cli/test/native-output.test.ts - name: Helper diagnostics and object/link parity if: matrix.shard == 1 - run: pnpm test packages/compiler/test/native-codegen-integration.test.ts + run: >- + pnpm test + packages/compiler/test/native-codegen-integration.test.ts + packages/cli/test/native-link-info.test.ts + tests/harness/native-object-example.test.ts - name: LLVM-tier helper object differential (${{ matrix.shard }}/3) env: SCRIPTC_LLVM_HELPER_ONLY: "1" @@ -167,6 +171,12 @@ jobs: "$PREFIX/node_modules/.bin/scriptc" build tests/corpus/001-hello.ts \ --emit=obj -o "$RUNNER_TEMP/installed.o" file "$RUNNER_TEMP/installed.o" | grep 'Mach-O 64-bit object arm64' + "$PREFIX/node_modules/.bin/scriptc" build tests/corpus/001-hello.ts \ + --print=native-link-info -o "$RUNNER_TEMP/installed-link.o" \ + > "$RUNNER_TEMP/installed-link.json" + node examples/native-object/link.mjs cc \ + "$RUNNER_TEMP/installed-link.json" "$RUNNER_TEMP/installed-program" + test "$("$RUNNER_TEMP/installed-program")" = 'hello world' # Exercises the supported Windows GNU target and the built CLI end to end: # TS7 must open its synthetic project, ambient files must resolve across diff --git a/README.md b/README.md index c237e1c2e..1219ce3ce 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,13 @@ assembly/object emission is rejected until the helper's AddressSanitizer pipeline matches the executable path. +External object consumption is experimental. Use +`--print=native-link-info` to emit the object and print a versioned JSON recipe +containing its target, `main` entry, exact `@scriptc/runtime` source pack, +required system libraries, FFI inputs, and ABI marker. The recipe never uses +hidden scriptc cache paths. See [`examples/native-object`](./examples/native-object) +for C-driver and direct Apple-linker builds. + ## Use Node APIs Supported Node APIs compile to the native runtime. For example, `server.ts`: diff --git a/docs/src/app/cli/page.mdx b/docs/src/app/cli/page.mdx index 03d984bc2..0dfbcdf3e 100644 --- a/docs/src/app/cli/page.mdx +++ b/docs/src/app/cli/page.mdx @@ -52,6 +52,13 @@ outputs use the matching native helper installed with scriptc and do not invoke an external compiler, archiver, linker, or SDK. The object is a relocatable program object with undefined scr_* runtime symbols and a required scr_runtime_abi_v1 marker, not a standalone library. +External consumption is experimental and requires the exact runtime version +reported by --print=native-link-info. That option still writes the +object, performs no link, and prints a versioned JSON recipe with the target, +main entry, installed source runtime pack, FFI inputs, and system +libraries. It never reports private scriptc cache paths. See +Native Program Objects for complete C-driver and +direct-linker examples. --emit=exe is the default and retains the existing executable behavior. @@ -83,6 +90,9 @@ Prebuilds the release runtime objects and native TLS/dynamic-engine archives aga
--emit <ir|c|llvm|asm|obj|exe>
Select the invocation's one primary artifact. ir, c, and llvm need only Node. asm and obj use the bundled LLVM helper on macOS 15+ arm64 and emit artifacts targeting macOS 14.0. exe is the default.
+
--print <native-link-info>
+
Build an object (equivalent to --emit=obj) and print its machine-readable external link recipe as JSON instead of printing the artifact path. The document names the exact installed source runtime pack and all link inputs, but does not invoke a linker.
+
--dynamic
Embed the dynamic engine (~620KB) so npm dependencies and any-typed code can run. Static stays the default — without this flag, dynamic-tier sites are per-site compile errors. See npm Dependencies.
@@ -182,6 +192,12 @@ An explicit --backend llvm pins the LLVM backend and fails with dia Bundled scriptc LLVM helper Not used + + External link of --emit=obj with the reported source runtime pack + Not used by the artifact + C compiler required for runtime sources + macOS linker and SDK required + --emit=exe Required to run scriptc diff --git a/docs/src/app/how-it-works/page.mdx b/docs/src/app/how-it-works/page.mdx index a38e4a0c5..6e000acce 100644 --- a/docs/src/app/how-it-works/page.mdx +++ b/docs/src/app/how-it-works/page.mdx @@ -14,6 +14,13 @@ TypeScript ──tsc: parse + typecheck──▶ lowering ──▶ typed IR ─ 3. **Backends** — `--emit=c` writes readable C and stops; `--emit=llvm` writes textual LLVM IR and stops. Neither source-output command discovers or invokes a native toolchain. On macOS 15+ arm64, `--emit=asm|obj` sends LLVM IR to a version-matched out-of-process helper linked to LLVM 22; it needs no clang or linker and emits macOS 14-targeted artifacts. Executable builds default to LLVM and can fall back to C on a native program outside the LLVM tier (one stderr note; `--backend llvm` pins it and fails with a diagnostic instead). The production wasm32-wasi target never falls back. 4. **Link** — the runtime is a C library of link-gated feature units: binaries pay only for what they use. A hello-world links nothing but libSystem; a regex-using program links the regex engine; an `http` server links the net stack. +Program objects define main and leave their selected +scr_* runtime functions undefined. The +scr_runtime_abi_v1 reference is a strong link-time compatibility +check. --print=native-link-info exposes the exact source runtime +pack and link ordering for external builds; that object ABI is currently +experimental and exact-runtime-version compatible, not semver-stable. + Inspect any stage yourself: ```console diff --git a/docs/src/app/native-objects/layout.tsx b/docs/src/app/native-objects/layout.tsx new file mode 100644 index 000000000..3aa7dc660 --- /dev/null +++ b/docs/src/app/native-objects/layout.tsx @@ -0,0 +1,7 @@ +import { pageMetadata } from "@/lib/page-metadata"; + +export const metadata = pageMetadata("native-objects"); + +export default function Layout({ children }: { children: React.ReactNode }) { + return children; +} diff --git a/docs/src/app/native-objects/page.mdx b/docs/src/app/native-objects/page.mdx new file mode 100644 index 000000000..979bfa9e2 --- /dev/null +++ b/docs/src/app/native-objects/page.mdx @@ -0,0 +1,84 @@ +# Native Program Objects + +`scriptc build --emit=obj` produces one relocatable macOS arm64 program +object without invoking clang, a linker, or an SDK. The object defines +`main`; it is intended to become the program in an external native link. It +is not a host-callable library—use `scriptc build --lib --profile ...` for +that interface. + +## ABI and runtime contract + +The external object ABI is **experimental**. Its `scr_*` function and data +surface may change before 1.0, so consumers must use the exact +`@scriptc/runtime` version reported by the same compiler installation. This +is stricter than semver compatibility. + +The object intentionally leaves its selected runtime symbols undefined. It +also holds a strong reference to `scr_runtime_abi_v1`, which the matching +runtime defines. Linking an object against a runtime with another ABI marker +fails at link time with the missing versioned symbol; it cannot become a +latent runtime incompatibility. + +## Machine-readable link information + +Add `--print=native-link-info` to emit the object and print a JSON document +instead of the ordinary path line: + +```console +$ scriptc build main.ts --print=native-link-info -o app.o > link-info.json +``` + +The `scriptc.native-link-info.v1` document reports: + +- target triple, object format, architecture, minimum OS, and relocation model; +- the `main` entry and versioned runtime ABI marker; +- the matching installed `@scriptc/runtime` source-pack root and exact source + sets, include paths, defines, and compile flags selected by the program; +- ordered program, FFI, runtime, and vendor inputs; and +- required system libraries and frameworks. + +Paths inside each source set are relative to `runtime_pack.root`. FFI library +paths are the manifest-resolved absolute inputs. No path points into scriptc's +private build cache. The source pack requires a C compiler; the final link +requires the macOS SDK and linker. Precompiled runtime packs are not shipped +yet. + +## C compiler as linker driver + +The repository's `examples/native-object` directory is a runnable example +with a TypeScript program, a C FFI function, and a small consumer for the JSON +recipe: + +```console +$ cd examples/native-object +$ clang -target arm64-apple-macosx14.0.0 -O2 -c native.c -o native.o +$ scriptc build main.ts --ffi ffi.json --print=native-link-info -o app.o > link-info.json +$ node link.mjs cc link-info.json app-cc +$ ./app-cc +42 +``` + +The script compiles each reported source set and gives clang only the link +inputs and system libraries from the document. `--emit=obj` itself remains +clang-free; this compiler invocation belongs to the external runtime build. + +## Native Apple linker + +The same example can invoke Apple `ld` directly after compiling the reported +runtime source sets: + +```console +$ node link.mjs ld link-info.json app-ld +$ ./app-ld +42 +``` + +This lane asks `xcrun` for the selected macOS SDK and linker, then supplies +the target's minimum OS, every ordered object/archive input, and each reported +system library. It demonstrates the code-generation boundary precisely: +scriptc owns `app.o`; an external toolchain owns runtime compilation and the +platform link. + +Outbound FFI declarations retain the same C ABI in clang-compiled LLVM and +helper-produced object paths. Scalar widths, string/byte pointer-plus-length +pairs, and callback signatures follow the [Native FFI](/ffi) manifest. diff --git a/docs/src/app/quickstart/page.mdx b/docs/src/app/quickstart/page.mdx index 9aeff097e..b69c6a28e 100644 --- a/docs/src/app/quickstart/page.mdx +++ b/docs/src/app/quickstart/page.mdx @@ -90,5 +90,6 @@ The package's JS is embedded into the binary at build time — the executable ne ## Next steps - [CLI Reference](/cli) — every command and flag, including `--emit`, `--backend llvm`, and `--sanitize`. +- [Native Program Objects](/native-objects) — consume `app.o` from an external C or linker build. - [Platform Support](/platforms) — cross-compiling to Linux and Windows with zig. - [Limitations](/limitations) — what doesn't compile yet. diff --git a/docs/src/lib/docs-navigation.ts b/docs/src/lib/docs-navigation.ts index c97a05e17..7a5077217 100644 --- a/docs/src/lib/docs-navigation.ts +++ b/docs/src/lib/docs-navigation.ts @@ -22,6 +22,7 @@ export const navSections: NavSection[] = [ { name: "Coverage Reports", href: "/coverage" }, { name: "npm Dependencies", href: "/dependencies" }, { name: "Native FFI", href: "/ffi" }, + { name: "Native Program Objects", href: "/native-objects" }, { name: "Platform Support", href: "/platforms" }, ], }, diff --git a/docs/src/lib/page-titles.ts b/docs/src/lib/page-titles.ts index 3d662cf26..55771a959 100644 --- a/docs/src/lib/page-titles.ts +++ b/docs/src/lib/page-titles.ts @@ -6,6 +6,7 @@ export const PAGE_TITLES: Record = { coverage: "Coverage Reports", dependencies: "npm Dependencies", ffi: "Native FFI", + "native-objects": "Native Program Objects", platforms: "Platform Support", "how-it-works": "How It Works", limitations: "Limitations", diff --git a/examples/native-object/README.md b/examples/native-object/README.md new file mode 100644 index 000000000..57fb7fbb5 --- /dev/null +++ b/examples/native-object/README.md @@ -0,0 +1,27 @@ +# External program object + +This macOS arm64 example links a scriptc program object, a small C FFI +implementation, and the exact installed source runtime pack. It uses no +scriptc cache path. + +```console +$ clang -target arm64-apple-macosx14.0.0 -O2 -c native.c -o native.o +$ scriptc build main.ts --ffi ffi.json --print=native-link-info -o app.o > link-info.json +$ node link.mjs cc link-info.json app-cc +$ ./app-cc +42 +$ node link.mjs ld link-info.json app-ld +$ ./app-ld +42 +``` + +`cc` uses the C compiler as a linker driver. `ld` compiles the same reported +runtime sources and invokes the Apple linker directly with the selected SDK. +The object defines `main`; it is a complete program object, not a library to +load into another process. Use `scriptc build --lib --profile ...` for a +host-callable static library. + +The external object ABI is experimental. Always consume the runtime pack at +the exact `runtime_pack.version` reported by the same scriptc installation. +The object requires `scr_runtime_abi_v1`, so a mismatched runtime fails during +the link instead of starting with an incompatible ABI. diff --git a/examples/native-object/ffi.json b/examples/native-object/ffi.json new file mode 100644 index 000000000..bf3e8fecd --- /dev/null +++ b/examples/native-object/ffi.json @@ -0,0 +1,13 @@ +{ + "ffi_format": 1, + "functions": [ + { + "name": "nativeDouble", + "symbol": "native_double", + "params": ["f64"], + "returns": "f64" + } + ], + "libraries": ["./native.o"], + "system_libraries": [] +} diff --git a/examples/native-object/link.mjs b/examples/native-object/link.mjs new file mode 100644 index 000000000..cba5962e7 --- /dev/null +++ b/examples/native-object/link.mjs @@ -0,0 +1,107 @@ +#!/usr/bin/env node +import { spawnSync } from "node:child_process"; +import { mkdirSync, readFileSync, rmSync } from "node:fs"; +import { basename, dirname, join, resolve } from "node:path"; + +const [mode, infoArg, outputArg] = process.argv.slice(2); +if ((mode !== "cc" && mode !== "ld") || !infoArg || !outputArg) { + console.error("usage: node link.mjs "); + process.exitCode = 2; +} else { + const info = JSON.parse(readFileSync(infoArg, "utf8")); + if (info.schema !== "scriptc.native-link-info.v1") { + throw new Error(`unsupported native link info schema: ${info.schema}`); + } + const runtimePackage = JSON.parse( + readFileSync(join(info.runtime_pack.root, "package.json"), "utf8"), + ); + if ( + runtimePackage.name !== info.runtime_pack.package || + runtimePackage.version !== info.runtime_pack.version + ) { + throw new Error( + `runtime pack identity mismatch: expected ${info.runtime_pack.package}@${info.runtime_pack.version} ` + + `at ${info.runtime_pack.root}, found ${runtimePackage.name ?? ""}@${runtimePackage.version ?? ""}`, + ); + } + const run = (command, args, options = {}) => { + const result = spawnSync(command, args, { stdio: "inherit", ...options }); + if (result.error) throw result.error; + if (result.status !== 0) throw new Error(`${command} exited ${result.status}`); + }; + const capture = (command, args) => { + const result = spawnSync(command, args, { encoding: "utf8" }); + if (result.error) throw result.error; + if (result.status !== 0) throw new Error(result.stderr || `${command} exited ${result.status}`); + return result.stdout.trim(); + }; + + const infoPath = resolve(infoArg); + const output = resolve(outputArg); + const buildDir = join(dirname(infoPath), `.native-link-${mode}`); + rmSync(buildDir, { recursive: true, force: true }); + mkdirSync(buildDir, { recursive: true }); + const runtimeObjects = []; + const vendorArchives = []; + for (const set of info.runtime_pack.source_sets) { + const setDir = join(buildDir, set.name); + mkdirSync(setDir, { recursive: true }); + const objects = []; + for (const source of set.sources) { + const object = join(setDir, `${source.replace(/[^A-Za-z0-9]+/g, "_")}.o`); + run("clang", [ + ...set.c_flags, + ...set.defines.map((define) => `-D${define}`), + ...set.include_directories.flatMap((path) => [ + "-I", join(info.runtime_pack.root, path), + ]), + "-c", join(info.runtime_pack.root, source), "-o", object, + ]); + objects.push(object); + } + if (set.output === "objects") { + runtimeObjects.push(...objects); + } else { + const archive = join(buildDir, basename(set.suggested_output)); + run("ar", ["rcs", archive, ...objects]); + vendorArchives.push(archive); + } + } + + const inputs = [ + info.program.object, + ...info.ffi.libraries, + ...runtimeObjects, + ...vendorArchives, + ]; + const libraries = info.link.system_libraries.map((name) => `-l${name}`); + const frameworks = info.link.frameworks.flatMap((name) => ["-framework", name]); + if (mode === "cc") { + // Darwin compiler drivers add libSystem themselves. It remains explicit + // in the document because a direct ld invocation must name it. + const driverLibraries = info.link.system_libraries + .filter((name) => name !== "System") + .map((name) => `-l${name}`); + run("clang", [ + ...info.link.driver_flags, + ...inputs, + ...driverLibraries, + ...frameworks, + "-o", output, + ]); + } else { + const sdk = capture("xcrun", ["--sdk", "macosx", "--show-sdk-path"]); + const sdkVersion = capture("xcrun", ["--sdk", "macosx", "--show-sdk-version"]); + const linker = capture("xcrun", ["--sdk", "macosx", "--find", "ld"]); + run(linker, [ + "-arch", info.target.architecture, + "-platform_version", "macos", info.target.minimum_os, sdkVersion, + "-syslibroot", sdk, + ...(info.link.driver_flags.includes("-Wl,-dead_strip") ? ["-dead_strip"] : []), + ...inputs, + ...libraries, + ...frameworks, + "-o", output, + ]); + } +} diff --git a/examples/native-object/main.ts b/examples/native-object/main.ts new file mode 100644 index 000000000..3e8ccab70 --- /dev/null +++ b/examples/native-object/main.ts @@ -0,0 +1,3 @@ +declare function nativeDouble(value: number): number; + +console.log(nativeDouble(21)); diff --git a/examples/native-object/native.c b/examples/native-object/native.c new file mode 100644 index 000000000..f17de5033 --- /dev/null +++ b/examples/native-object/native.c @@ -0,0 +1,3 @@ +double native_double(double value) { + return value * 2.0; +} diff --git a/packages/cli/README.md b/packages/cli/README.md index 0da61a06d..3e5b87b35 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -39,6 +39,9 @@ emission currently requires a macOS 15+ arm64 host and emits objects targeting macOS 14.0 (`arm64-apple-macosx14.0.0`). Objects retain undefined `scr_*` runtime references plus the `scr_runtime_abi_v1` compatibility marker; they are not library archives. +External consumption is experimental and requires the exact matching runtime. +`scriptc build app.ts --print=native-link-info -o app.o` prints the versioned +JSON target/runtime/link recipe without performing a link. `--emit=asm|obj --sanitize` is rejected until ASan pipeline parity is available. diff --git a/packages/cli/src/bootstrap.ts b/packages/cli/src/bootstrap.ts index 833af553a..a0bb1e43f 100644 --- a/packages/cli/src/bootstrap.ts +++ b/packages/cli/src/bootstrap.ts @@ -56,6 +56,7 @@ async function tryFastPath(): Promise { if ( (command !== "build" && command !== "run") || inputArg === undefined || (values.emit !== undefined && values.emit !== "exe") || + values.print !== undefined || values["emit-ir"] || values.lib || values["from-c"] || values["provenance-sources"] || (values["external-types"] ?? []).length > 0 diff --git a/packages/cli/src/main.ts b/packages/cli/src/main.ts index 39395f9a9..46dbfb88b 100644 --- a/packages/cli/src/main.ts +++ b/packages/cli/src/main.ts @@ -70,7 +70,7 @@ async function main(): Promise { const [command, inputArg] = positionals; if (command === "cache") { if (inputArg !== "warm") fail(`unknown cache command "${inputArg ?? ""}" (supported: warm)\n\n${USAGE}`); - if (values.lib || values.dynamic || values.backend !== undefined || values.emit !== undefined || values["from-c"] || values.ffi !== undefined || values.profile !== undefined || (values["npm-static"] ?? []).length > 0 || values["provenance-sources"] || externalTypeArgs.length > 0 || values.out !== undefined || values["emit-ir"] || !values["keep-c"]) { + if (values.lib || values.dynamic || values.backend !== undefined || values.emit !== undefined || values.print !== undefined || values["from-c"] || values.ffi !== undefined || values.profile !== undefined || (values["npm-static"] ?? []).length > 0 || values["provenance-sources"] || externalTypeArgs.length > 0 || values.out !== undefined || values["emit-ir"] || !values["keep-c"]) { fail(`scriptc cache warm takes only native optimization/sanitizer options and profile names\n\n${USAGE}`); } const optimization = values.optimization; @@ -118,9 +118,9 @@ async function main(): Promise { if (inputArg) { fail("scriptc build --lib takes no input positional: the profile names the entry module"); } - if (values.dynamic || values.backend !== undefined || values.emit !== undefined || values.optimization !== undefined || values.ffi !== undefined || (values["npm-static"] ?? []).length > 0 || externalTypeArgs.length > 0) { + if (values.dynamic || values.backend !== undefined || values.emit !== undefined || values.print !== undefined || values.optimization !== undefined || values.ffi !== undefined || (values["npm-static"] ?? []).length > 0 || externalTypeArgs.length > 0) { fail( - "scriptc build --lib takes no --dynamic/--backend/--emit/--optimization/--npm-static/--ffi/--external-types: the profile pins the emission and optimization, npm imports are judged automatically, outbound FFI belongs to executable builds, and external type mappings belong to coverage", + "scriptc build --lib takes no --dynamic/--backend/--emit/--print/--optimization/--npm-static/--ffi/--external-types: the profile pins the emission and optimization, npm imports are judged automatically, outbound FFI belongs to executable builds, and external type mappings belong to coverage", ); } const profilePath = resolve(profileArg); @@ -154,6 +154,16 @@ async function main(): Promise { if (command === "coverage" && values.emit !== undefined) { fail(`--emit is a build/run option\n\n${USAGE}`); } + if (values.print !== undefined && values.print !== "native-link-info") { + fail(`unknown print kind "${values.print}" (supported: native-link-info)\n\n${USAGE}`); + } + const printNativeLinkInfo = values.print === "native-link-info"; + if (printNativeLinkInfo && command !== "build") { + fail(`--print=native-link-info is a build option\n\n${USAGE}`); + } + if (printNativeLinkInfo && values.emit !== undefined && values.emit !== "obj") { + fail(`--print=native-link-info requires --emit=obj\n\n${USAGE}`); + } if (externalTypeArgs.length > 0 && command !== "coverage") { fail(`--external-types is a coverage-only option\n\n${USAGE}`); } @@ -193,7 +203,9 @@ async function main(): Promise { const output = command === "coverage" ? null : resolveOutputOptions(command, { - ...(values.emit === undefined ? {} : { emit: values.emit }), + ...(values.emit === undefined && !printNativeLinkInfo + ? {} + : { emit: values.emit ?? "obj" }), emitIr: values["emit-ir"], ...(values.backend === undefined ? {} : { backend: values.backend }), fromC: values["from-c"], @@ -247,6 +259,7 @@ async function main(): Promise { if (output === null || !output.ok) throw new Error("internal output-option state"); const { outDir, outPath, defaultOutputPath } = selectOutputPaths(input, output.cliOutputKind, values.out); + let nativeLinkInfo: object | undefined; const build = async (): Promise => { if (values["from-c"]) { if (ffiProfilePath !== undefined) { @@ -273,6 +286,7 @@ async function main(): Promise { ...(optimization !== undefined ? { optimization } : {}), ...(npmStatic !== undefined ? { npmStatic } : {}), ...(ffiProfilePath !== undefined ? { ffiProfilePath } : {}), + ...(printNativeLinkInfo ? { nativeLinkInfo: true } : {}), }); if (!result.ok) { const color = process.stderr.isTTY ?? false; @@ -291,6 +305,8 @@ async function main(): Promise { process.stderr.write(`scriptc: backend c (llvm refused: ${result.artifact.llvmRefusal})\n`); } if (!values["keep-c"]) rmSync(result.artifact.translationUnitPath, { force: true }); + } else if (result.artifact.kind === "obj") { + nativeLinkInfo = result.artifact.nativeLinkInfo; } return result.artifact.path; }; @@ -323,7 +339,14 @@ async function main(): Promise { }); }); } - process.stdout.write(`${binary}\n`); + if (printNativeLinkInfo) { + if (nativeLinkInfo === undefined) throw new Error("internal native-link-info state"); + // Keep stdout pure JSON for tooling; the ordinary artifact path is in + // program.object inside the document. + process.stdout.write(`${JSON.stringify(nativeLinkInfo, null, 2)}\n`); + } else { + process.stdout.write(`${binary}\n`); + } return 0; } diff --git a/packages/cli/src/usage.ts b/packages/cli/src/usage.ts index e68ff03d7..8fa1ad047 100644 --- a/packages/cli/src/usage.ts +++ b/packages/cli/src/usage.ts @@ -20,6 +20,8 @@ Options: -o, --out primary output path (default: .scriptc/) --emit primary output: ir, c, llvm, asm, obj, or exe (default: exe). asm/obj currently support macOS 15+ arm64 + --print print machine-readable metadata instead of the output path + (native-link-info implies --emit=obj and never links) --backend code generator. llvm is the default and the output that ships; c emits readable C for inspecting what the compiler produced, and program behavior is identical @@ -69,6 +71,7 @@ Options: export const CLI_OPTIONS = { out: { type: "string", short: "o" }, emit: { type: "string" }, + print: { type: "string" }, backend: { type: "string" }, optimization: { type: "string" }, "from-c": { type: "boolean", default: false }, diff --git a/packages/cli/test/native-link-info.test.ts b/packages/cli/test/native-link-info.test.ts new file mode 100644 index 000000000..aef407372 --- /dev/null +++ b/packages/cli/test/native-link-info.test.ts @@ -0,0 +1,152 @@ +import { execFile } from "node:child_process"; +import { createRequire } from "node:module"; +import { chmod, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { release as osRelease, tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { promisify } from "node:util"; +import { afterEach, describe, expect, test } from "vitest"; +import type { NativeLinkInfo } from "@scriptc/compiler"; + +const execFileAsync = promisify(execFile); +const require = createRequire(import.meta.url); +const repoRoot = join(import.meta.dirname, "../../.."); +const cliEntry = join(repoRoot, "packages/cli/src/main.ts"); +const tsxLoader = join(dirname(require.resolve("tsx/package.json")), "dist/loader.mjs"); +const supported = process.platform === "darwin" && process.arch === "arm64" && + Number.parseInt(osRelease().split(".", 1)[0] ?? "", 10) >= 24; +const dirs: string[] = []; + +afterEach(async () => { + await Promise.all(dirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))); +}); + +function cli(args: string[]) { + return execFileAsync(process.execPath, ["--import", tsxLoader, cliEntry, ...args], { + maxBuffer: 16 * 1024 * 1024, + }); +} + +async function fixture() { + const dir = await mkdtemp(join(tmpdir(), "scriptc-native-link-info-")); + dirs.push(dir); + const entry = join(dir, "main.ts"); + await writeFile(entry, 'console.log("external link");\n'); + return { dir, entry, object: join(dir, "app.o") }; +} + +test("print option validation is explicit", async () => { + const { entry } = await fixture(); + await expect(cli(["build", entry, "--print=wat"])) + .rejects.toMatchObject({ stderr: expect.stringContaining("unknown print kind") }); + await expect(cli(["build", entry, "--print=native-link-info", "--emit=llvm"])) + .rejects.toMatchObject({ stderr: expect.stringContaining("requires --emit=obj") }); + await expect(cli(["run", entry, "--print=native-link-info"])) + .rejects.toMatchObject({ stderr: expect.stringContaining("is a build option") }); +}); + +describe.runIf(supported)("macOS arm64 native link info", () => { + test("prints a stable cache-independent source-pack recipe", async () => { + const { entry, object } = await fixture(); + const { stdout, stderr } = await cli([ + "build", entry, "--print=native-link-info", "-o", object, + ]); + expect(stderr).toBe(""); + const info = JSON.parse(stdout) as NativeLinkInfo; + expect(info).toMatchObject({ + schema: "scriptc.native-link-info.v1", + format: 1, + object_abi: { stability: "experimental", compatibility: "exact-runtime-version" }, + target: { + name: "macos-arm64", + llvm_triple: "arm64-apple-macosx14.0.0", + object_format: "macho", + minimum_os: "14.0", + }, + program: { object, entry_symbol: "main" }, + runtime_abi: { version: 1, marker: "scr_runtime_abi_v1" }, + runtime_pack: { + kind: "source", + package: "@scriptc/runtime", + path_base: "runtime_pack.root", + }, + link: { system_libraries: ["System"], frameworks: [] }, + }); + expect(info.runtime_pack.root).not.toContain("node_modules/.cache"); + expect(info.link.input_order.join("\n")).not.toContain("node_modules/.cache"); + const runtime = info.runtime_pack.source_sets.find((set) => set.name === "runtime"); + expect(runtime?.sources).toEqual(expect.arrayContaining([ + "src/scr_console.c", + "src/scr_async.c", + "src/scr_cycle.c", + ])); + await expect(readFile(object)).resolves.toBeInstanceOf(Buffer); + }); + + test("printing link info emits an object without invoking external tools", async () => { + const { dir, entry, object } = await fixture(); + const traps = join(dir, "traps"); + await execFileAsync("mkdir", [traps]); + const trapLog = join(dir, "trap.log"); + for (const tool of ["clang", "cc", "gcc", "zig", "ar", "ld", "xcrun"]) { + const path = join(traps, tool); + await writeFile(path, `#!/bin/sh\nprintf '${tool}\\n' >> '${trapLog}'\nexit 97\n`); + await chmod(path, 0o755); + } + const { stdout, stderr } = await execFileAsync( + process.execPath, + ["--import", tsxLoader, cliEntry, "build", entry, "--print=native-link-info", "-o", object], + { + env: { + ...process.env, + PATH: `${traps}:${process.env["PATH"] ?? ""}`, + SCRIPTC_CACHE_DIR: join(dir, "cache"), + }, + maxBuffer: 16 * 1024 * 1024, + }, + ); + expect(stderr).toBe(""); + expect((JSON.parse(stdout) as NativeLinkInfo).program.object).toBe(object); + await expect(readFile(trapLog)).rejects.toMatchObject({ code: "ENOENT" }); + }); + + test("an out-of-tree C-driver build consumes the reported object and runtime pack", async () => { + const { dir, entry, object } = await fixture(); + const { stdout } = await cli([ + "build", entry, "--print=native-link-info", "-o", object, + ]); + const info = JSON.parse(stdout) as NativeLinkInfo; + const runtime = info.runtime_pack.source_sets.find((set) => set.name === "runtime")!; + const executable = join(dir, "app"); + await execFileAsync("clang", [ + ...runtime.c_flags, + ...runtime.defines.map((define) => `-D${define}`), + ...runtime.include_directories.flatMap((path) => ["-I", join(info.runtime_pack.root, path)]), + ...runtime.sources.map((path) => join(info.runtime_pack.root, path)), + object, + ...info.link.system_libraries.map((name) => `-l${name}`), + "-o", executable, + ]); + await expect(execFileAsync(executable, [], { encoding: "utf8" })) + .resolves.toMatchObject({ stdout: "external link\n", stderr: "" }); + }, 30_000); + + test("a mismatched runtime ABI fails at link time through the versioned marker", async () => { + const { dir, entry, object } = await fixture(); + await cli(["build", entry, "--emit=obj", "-o", object]); + const stub = join(dir, "wrong-runtime.c"); + await writeFile(stub, [ + "void scr_runtime_abi_v2(void) {}", + "void scr_console_log(void) {}", + "void scr_init(void) {}", + "void scr_lib_init(void) {}", + "void scr_str_release(void) {}", + "void scr_str_retain_v(void) {}", + "void scr_error_vts(void) {}", + "", + ].join("\n")); + const error = await execFileAsync("clang", [ + "-target", "arm64-apple-macosx14.0.0", object, stub, "-o", join(dir, "bad"), + ]).then(() => null, (failure: { stderr?: string }) => failure); + expect(error?.stderr).toContain("scr_runtime_abi_v1"); + }); +}); diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index 4b802cb07..36fbcfad8 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -82,6 +82,7 @@ import type { import { CAUGHT, ffiCallbackType, isFfiContextParam, isRefCounted, isUnitType, moduleEmbedsBuiltin, moduleEmbedsCompressedNpm, moduleUsesDynInvoke, moduleUsesFetch, moduleUsesFsWatch, moduleUsesHttpServer, moduleUsesNet, moduleUsesNodeTest, moduleUsesProcessEvents, moduleUsesStream, moduleUsesTls, moduleUsesTlsCa, NPM_COMPRESS_MIN, POINTER_KINDS, RUNTIME_EMITTER_CLASS, RUNTIME_ERROR_CLASSES, RUNTIME_STREAM_CLASSES, VOID } from "../../ir/ir.js"; import { matchIntegerBytesForLoop } from "../../ir/integer-loops.js"; import { allocateFfiCallbackAdapters, hasForeignFfiCallback, hasRetainedFfiCallback, type FfiCallbackAdapter } from "../ffi-callbacks.js"; +import { RUNTIME_ABI_MARKER } from "../runtime-abi.js"; import { computeMayThrow } from "../c/may-throw.js"; import { mangleArgPack, mangleAsyncSpawn, mangleClassObj, mangleFnClosure, mangleFunction, mangleGenDrop, mangleGenSpawn, mangleGlobal, mangleLocal, mangleRecordStruct, mangleTrampoline, mangleWrapper } from "../mangle.js"; import { BlockBuilder } from "./blocks.js"; @@ -1101,7 +1102,7 @@ class LlEmitter { `declare void @scr_init()`, `declare void @scr_lib_init(i32, ptr)`, ...(this.runtimeAbiMarker && this.mod.lib === undefined - ? [`declare void @scr_runtime_abi_v1()`] + ? [`declare void @${RUNTIME_ABI_MARKER}()`] : []), ); for (const d of this.decls) out.push(d); @@ -1280,7 +1281,7 @@ class LlEmitter { out.push( `define i32 @${this.wasi ? "__main_argc_argv" : "main"}(i32 %argc, ptr %argv) ${FN_ATTRS} {`, `entry:`, - ...(this.runtimeAbiMarker ? [` call void @scr_runtime_abi_v1()`] : []), + ...(this.runtimeAbiMarker ? [` call void @${RUNTIME_ABI_MARKER}()`] : []), ` call void @scr_init()`, ...stamps, // Event-surface programs (signal/exit listeners) fill the loop's diff --git a/packages/compiler/src/backend/native-link-info.test.ts b/packages/compiler/src/backend/native-link-info.test.ts new file mode 100644 index 000000000..7f7456e7b --- /dev/null +++ b/packages/compiler/src/backend/native-link-info.test.ts @@ -0,0 +1,123 @@ +import { mkdtemp, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, test } from "vitest"; +import { loadFfiProfile } from "../ffi/ffi-manifest.js"; +import { createNativeLinkInfo, type NativeLinkFeatures } from "./native-link-info.js"; +import { MACOS_ARM64_TARGET } from "./targets.js"; + +const BASE: NativeLinkFeatures = { + dynamic: false, + regex: false, + copying: false, + textDecoderLegacy: false, + fileHandle: false, + fetch: false, + netIsland: false, + zlib: false, + assert: false, + inspect: false, + dynInvoke: false, + dc: false, + dynAsync: false, + events: false, + emitter: false, + symbol: false, + searchParams: false, + qs: false, + parseArgs: false, + stream: false, + net: false, + http: false, + http2: false, + dgram: false, + watch: false, + foreignFfi: false, + nodeTest: false, + tls: false, + tlsCa: false, +}; + +describe("native link info recipes", () => { + test("feature source sets reproduce runtime and vendor gates", async () => { + const info = await createNativeLinkInfo({ + programObject: "/out/app.o", + target: MACOS_ARM64_TARGET, + features: { + ...BASE, + dynamic: true, + regex: true, + inspect: true, + zlib: true, + tls: true, + }, + ffi: null, + }); + const runtime = info.runtime_pack.source_sets.find((set) => set.name === "runtime")!; + expect(runtime.sources).toEqual(expect.arrayContaining([ + "src/scr_regex.c", + "src/scr_assert.c", + "src/scr_inspect.c", + "src/scr_zlib.c", + "src/scr_zlib_island.c", + "src/scr_tls.c", + "src/scr_tls_ca.c", + "src/scr_island.c", + "src/scr_web.c", + "src/scr_inspect_island.c", + ])); + expect(info.runtime_pack.source_sets.map((set) => set.name)).toEqual([ + "runtime", "quickjs", "mbedtls", + ]); + expect(info.runtime_pack.source_sets.find((set) => set.name === "mbedtls")!.sources.length) + .toBeGreaterThan(100); + expect(info.link.system_libraries).toEqual(["System", "z", "m"]); + }); + + test("dev object recipes keep runtime optimization in lockstep", async () => { + const info = await createNativeLinkInfo({ + programObject: "/out/app.o", + target: MACOS_ARM64_TARGET, + features: BASE, + ffi: null, + optimization: "dev", + }); + expect(info.runtime_pack.source_sets[0]?.c_flags).toContain("-O0"); + expect(info.runtime_pack.source_sets[0]?.c_flags).not.toContain("-O2"); + }); + + test("FFI symbols and resolved inputs remain ordered before the runtime", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-native-link-info-unit-")); + const library = join(dir, "native.o"); + const manifest = join(dir, "ffi.json"); + await writeFile(library, "fixture"); + await writeFile(manifest, JSON.stringify({ + ffi_format: 1, + functions: [{ + name: "nativeScale", + symbol: "native_scale", + params: ["f64"], + returns: "f64", + }], + libraries: ["./native.o"], + system_libraries: ["sqlite3"], + })); + const loaded = loadFfiProfile(manifest); + if (!loaded.ok) throw new Error(loaded.diagnostics[0]?.message); + const info = await createNativeLinkInfo({ + programObject: "/out/app.o", + target: MACOS_ARM64_TARGET, + features: BASE, + ffi: loaded.profile, + }); + expect(info.ffi).toEqual({ + format: 1, + symbols: ["native_scale"], + libraries: [library], + }); + expect(info.link.input_order.slice(0, 3)).toEqual([ + "/out/app.o", library, "runtime/*.o", + ]); + expect(info.link.system_libraries).toEqual(["sqlite3", "System"]); + }); +}); diff --git a/packages/compiler/src/backend/native-link-info.ts b/packages/compiler/src/backend/native-link-info.ts new file mode 100644 index 000000000..8e9597620 --- /dev/null +++ b/packages/compiler/src/backend/native-link-info.ts @@ -0,0 +1,330 @@ +import { readFile, readdir } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import type { FfiProfile } from "../ffi/ffi-manifest.js"; +import { compilerReleaseVersion } from "../library/sidecar.js"; +import { EXECUTABLE_RUNTIME_SOURCES, runtimeSrcDir } from "./native-toolchain.js"; +import { + EXTERNAL_OBJECT_ABI_STABILITY, + RUNTIME_ABI_MARKER, + RUNTIME_ABI_VERSION, +} from "./runtime-abi.js"; +import type { NativeTargetSpec } from "./targets.js"; +import { LRE_SOURCES, QJS_ENGINE_SOURCES } from "./vendor-archives.js"; + +export interface NativeLinkFeatures { + dynamic: boolean; + regex: boolean; + copying: boolean; + textDecoderLegacy: boolean; + fileHandle: boolean; + fetch: boolean; + netIsland: boolean; + zlib: boolean; + assert: boolean; + inspect: boolean; + dynInvoke: boolean; + dc: boolean; + dynAsync: boolean; + events: boolean; + emitter: boolean; + symbol: boolean; + searchParams: boolean; + qs: boolean; + parseArgs: boolean; + stream: boolean; + net: boolean; + http: boolean; + http2: boolean; + dgram: boolean; + watch: boolean; + foreignFfi: boolean; + nodeTest: boolean; + tls: boolean; + tlsCa: boolean; +} + +export interface NativeSourceSet { + name: "runtime" | "libregexp" | "quickjs" | "mbedtls"; + output: "objects" | "archive"; + suggested_output: string; + sources: string[]; + include_directories: string[]; + defines: string[]; + c_flags: string[]; +} + +export interface NativeLinkInfo { + schema: "scriptc.native-link-info.v1"; + format: 1; + compiler_version: string; + object_abi: { + stability: "experimental"; + compatibility: "exact-runtime-version"; + }; + target: { + name: NativeTargetSpec["name"]; + llvm_triple: NativeTargetSpec["llvmTriple"]; + architecture: "arm64"; + object_format: NativeTargetSpec["objectFormat"]; + minimum_os: NativeTargetSpec["minimumOs"]; + relocation_model: NativeTargetSpec["relocationModel"]; + }; + program: { + object: string; + entry_symbol: "main"; + undefined_runtime_symbol_prefix: "scr_"; + }; + runtime_abi: { + version: typeof RUNTIME_ABI_VERSION; + marker: typeof RUNTIME_ABI_MARKER; + }; + runtime_pack: { + kind: "source"; + package: "@scriptc/runtime"; + version: string; + root: string; + path_base: "runtime_pack.root"; + source_sets: NativeSourceSet[]; + }; + ffi: { + format: number | null; + symbols: string[]; + libraries: string[]; + }; + link: { + input_order: string[]; + driver_flags: string[]; + system_libraries: string[]; + frameworks: string[]; + }; +} + +function unique(values: readonly T[]): T[] { + return [...new Set(values)]; +} + +function packagePath(...parts: string[]): string { + return parts.join("/"); +} + +function runtimeSourceRecipe( + features: NativeLinkFeatures, + target: NativeTargetSpec, + env: NodeJS.ProcessEnv, + optimization: "release" | "dev", +): { sets: NativeSourceSet[]; systemLibraries: string[]; driverFlags: string[] } { + const dynamic = features.dynamic; + const curlFetch = dynamic && features.fetch && env["SCRIPTC_FETCH_CURL"] === "1"; + const nativeFetch = features.fetch && !curlFetch; + const netIsland = dynamic && (features.netIsland || nativeFetch); + const net = features.net || nativeFetch || netIsland; + const http = features.http || nativeFetch || netIsland; + const tls = features.tls || nativeFetch || netIsland; + const tlsCa = features.tlsCa || tls; + const runtimeSources = [ + ...EXECUTABLE_RUNTIME_SOURCES, + ...(features.copying ? ["scr_copying.c"] : []), + ...(features.fileHandle ? ["scr_file_handle.c"] : []), + ...(features.regex ? ["scr_regex.c"] : []), + ...(features.assert || features.regex || features.symbol ? ["scr_assert.c"] : []), + ...(features.inspect ? ["scr_inspect.c"] : []), + ...(features.dynInvoke || nativeFetch ? ["scr_dyn_invoke.c"] : []), + ...(features.dc ? ["scr_dc.c"] : []), + ...(features.dynAsync || features.dynInvoke || features.dc || nativeFetch + ? ["scr_async_dyn.c"] + : []), + ...(features.zlib ? ["scr_zlib.c"] : []), + ...(features.zlib && dynamic ? ["scr_zlib_island.c"] : []), + ...(features.events ? ["scr_events.c", "scr_readline.c"] : []), + ...(features.emitter ? ["scr_events_emitter.c"] : []), + ...(features.emitter || net ? ["scr_dyn_handle.c"] : []), + ...(features.symbol ? ["scr_symbol.c"] : []), + ...(features.searchParams ? ["scr_url_params.c"] : []), + ...(features.qs ? ["scr_qs.c"] : []), + ...(features.parseArgs ? ["scr_util.c"] : []), + ...(features.stream ? ["scr_stream.c"] : []), + ...(net || features.dgram + ? ["scr_loop_kqueue.c", "scr_loop_epoll.c", "scr_loop_wsapoll.c"] + : []), + ...(net ? ["scr_net.c"] : []), + ...(http ? ["scr_http.c"] : []), + ...(features.http2 ? ["scr_http2.c"] : []), + ...(features.dgram ? ["scr_dgram.c"] : []), + ...(features.watch ? ["scr_watch.c"] : []), + ...(features.foreignFfi ? ["scr_ffi_queue.c"] : []), + ...(features.nodeTest ? ["scr_test.c"] : []), + ...(tlsCa ? ["scr_tls_ca.c"] : []), + ...(tls ? ["scr_tls.c"] : []), + ...(nativeFetch ? ["scr_fetch.c"] : []), + ...(dynamic + ? [ + "scr_island.c", + "scr_web.c", + ...(features.inspect ? ["scr_inspect_island.c"] : []), + ...(netIsland ? ["scr_net_island.c"] : []), + ...(curlFetch ? ["scr_fetch_curl.c"] : []), + ] + : []), + ]; + const commonTargetFlags = ["-target", target.llvmTriple]; + const sets: NativeSourceSet[] = [{ + name: "runtime", + output: "objects", + suggested_output: "runtime/*.o", + sources: unique(runtimeSources).map((source) => packagePath("src", source)), + include_directories: unique([ + "src", + ...(features.regex || dynamic ? ["vendor/quickjs-ng"] : []), + ...(tls ? ["vendor/mbedtls/include"] : []), + ...(curlFetch ? ["vendor/curl/include"] : []), + ]), + defines: unique([ + ...(features.textDecoderLegacy ? ["SCR_TEXT_DECODER_LEGACY"] : []), + ...(dynamic ? ["SCR_DYNAMIC"] : []), + ]), + c_flags: [ + "-std=c11", ...commonTargetFlags, "-pthread", + optimization === "dev" ? "-O0" : "-O2", + "-fno-math-errno", "-fno-strict-aliasing", "-Wno-deprecated-declarations", + ], + }]; + if (features.regex && !dynamic) { + sets.push({ + name: "libregexp", + output: "archive", + suggested_output: "libscriptc-regexp.a", + sources: LRE_SOURCES.map((source) => packagePath("vendor", "quickjs-ng", source)), + include_directories: ["vendor/quickjs-ng"], + defines: [], + c_flags: ["-std=c11", ...commonTargetFlags, "-Os"], + }); + } + if (dynamic) { + sets.push({ + name: "quickjs", + output: "archive", + suggested_output: "libscriptc-quickjs.a", + sources: QJS_ENGINE_SOURCES.map((source) => packagePath("vendor", "quickjs-ng", source)), + include_directories: ["vendor/quickjs-ng"], + defines: ["QUICKJS_NG_BUILD", "_GNU_SOURCE", "NDEBUG"], + c_flags: [ + "-std=gnu11", ...commonTargetFlags, "-fvisibility=hidden", + "-funsigned-char", "-Os", + ], + }); + } + // mbedTLS is the one source set whose membership is intentionally the + // package's complete library/*.c set. The package version and ABI marker + // pin those bytes as one exact runtime pack. + if (tls) { + sets.push({ + name: "mbedtls", + output: "archive", + suggested_output: "libscriptc-mbedtls.a", + sources: [], + include_directories: ["vendor/mbedtls/include", "vendor/mbedtls/library"], + defines: [], + c_flags: ["-std=c11", ...commonTargetFlags, "-Os"], + }); + } + const systemLibraries = unique([ + "System", + ...((features.zlib || nativeFetch) ? ["z"] : []), + ...(dynamic ? ["m"] : []), + ...(curlFetch ? ["curl"] : []), + ]); + return { + sets, + systemLibraries, + driverFlags: [ + ...commonTargetFlags, + "-pthread", + ...(dynamic ? ["-Wl,-dead_strip"] : []), + ], + }; +} + +export async function createNativeLinkInfo(options: { + programObject: string; + target: NativeTargetSpec; + features: NativeLinkFeatures; + ffi: FfiProfile | null; + optimization?: "release" | "dev"; + env?: NodeJS.ProcessEnv; +}): Promise { + const root = dirname(runtimeSrcDir()); + const runtimeVersion = (JSON.parse( + await readFile(join(root, "package.json"), "utf8"), + ) as { version: string }).version; + const recipe = runtimeSourceRecipe( + options.features, + options.target, + options.env ?? process.env, + options.optimization ?? "release", + ); + const mbedtls = recipe.sets.find((set) => set.name === "mbedtls"); + if (mbedtls !== undefined) { + mbedtls.sources = (await readdir(join(root, "vendor", "mbedtls", "library"))) + .filter((name) => !name.startsWith(".") && name.endsWith(".c")) + .sort() + .map((name) => packagePath("vendor", "mbedtls", "library", name)); + } + const ffiLibraries = options.ffi?.libraries ?? []; + const vendorInputs = recipe.sets + .filter((set) => set.output === "archive") + .map((set) => set.suggested_output); + return { + schema: "scriptc.native-link-info.v1", + format: 1, + compiler_version: compilerReleaseVersion(), + object_abi: { + stability: EXTERNAL_OBJECT_ABI_STABILITY, + compatibility: "exact-runtime-version", + }, + target: { + name: options.target.name, + llvm_triple: options.target.llvmTriple, + architecture: "arm64", + object_format: options.target.objectFormat, + minimum_os: options.target.minimumOs, + relocation_model: options.target.relocationModel, + }, + program: { + object: options.programObject, + entry_symbol: "main", + undefined_runtime_symbol_prefix: "scr_", + }, + runtime_abi: { + version: RUNTIME_ABI_VERSION, + marker: RUNTIME_ABI_MARKER, + }, + runtime_pack: { + kind: "source", + package: "@scriptc/runtime", + version: runtimeVersion, + root, + path_base: "runtime_pack.root", + source_sets: recipe.sets, + }, + ffi: { + format: options.ffi?.ffiFormat ?? null, + symbols: options.ffi?.functions.map((fn) => fn.symbol) ?? [], + libraries: [...ffiLibraries], + }, + link: { + input_order: [ + options.programObject, + ...ffiLibraries, + "runtime/*.o", + ...vendorInputs, + "system_libraries", + ], + driver_flags: recipe.driverFlags, + system_libraries: unique([ + ...(options.ffi?.systemLibraries ?? []), + ...recipe.systemLibraries, + ]), + frameworks: [], + }, + }; +} diff --git a/packages/compiler/src/backend/native-toolchain.ts b/packages/compiler/src/backend/native-toolchain.ts index 235cb6b43..d66d2db7e 100644 --- a/packages/compiler/src/backend/native-toolchain.ts +++ b/packages/compiler/src/backend/native-toolchain.ts @@ -79,7 +79,7 @@ function stableTestMemo( return pending; } -const RUNTIME_SOURCES = ["scr_number.c", "scr_string.c", "scr_array.c", "scr_bytes.c", "scr_bytes_io.c", "scr_map.c", "scr_closure.c", "scr_ffi.c", "scr_object.c", "scr_union.c", "scr_exception.c", "scr_error.c", "scr_console.c", "scr_lib.c", "scr_path.c", "scr_url.c", "scr_json.c", "scr_async.c", "scr_child.c", "scr_cycle.c"]; +export const EXECUTABLE_RUNTIME_SOURCES = ["scr_number.c", "scr_string.c", "scr_array.c", "scr_bytes.c", "scr_bytes_io.c", "scr_map.c", "scr_closure.c", "scr_ffi.c", "scr_object.c", "scr_union.c", "scr_exception.c", "scr_error.c", "scr_console.c", "scr_lib.c", "scr_path.c", "scr_url.c", "scr_json.c", "scr_async.c", "scr_child.c", "scr_cycle.c"] as const; /** Environment variables consumed by clang, its linker/subtools, or the * platform SDK selection. They are implicit command-line inputs: changing one @@ -921,7 +921,7 @@ export function cacheTargetIdentity( /** The library base: the executable lane's unconditional sources minus the * fiber/loop and child-process units, plus the library-mode TU. */ const LIB_RUNTIME_SOURCES = [ - ...RUNTIME_SOURCES.filter( + ...EXECUTABLE_RUNTIME_SOURCES.filter( (f) => f !== "scr_async.c" && f !== "scr_child.c" && f !== "scr_ffi.c", ), "scr_library.c", @@ -2535,8 +2535,8 @@ async function nativeSourceFiles( * libregexp, mbedTLS, zlib, curl, and Ryū translation units. */ export async function implicitDependencyProbeIncludes(rtDir: string): Promise { const vendor = join(rtDir, "..", "vendor"); - const quickjsSources = new Set([...QJS_ENGINE_SOURCES, ...LRE_SOURCES]); - const zlibSources = new Set(ZLIB_SOURCES); + const quickjsSources = new Set([...QJS_ENGINE_SOURCES, ...LRE_SOURCES]); + const zlibSources = new Set(ZLIB_SOURCES); const fileGroups = await Promise.all([ nativeSourceFiles(rtDir, false), nativeSourceFiles(join(vendor, "ryu"), false), @@ -3954,8 +3954,8 @@ async function compileCInternal( ); } const runtimeSources = targetPlatform(driver) === "wasi" - ? RUNTIME_SOURCES.filter((source) => source !== "scr_child.c") - : RUNTIME_SOURCES; + ? EXECUTABLE_RUNTIME_SOURCES.filter((source) => source !== "scr_child.c") + : EXECUTABLE_RUNTIME_SOURCES; // scr_async.c submits callback-style filesystem work to a native worker. // POSIX drivers need the thread compile/link mode; win32 uses CreateThread. const threadArgs = targetPlatform(driver) === "win32" || targetPlatform(driver) === "wasi" diff --git a/packages/compiler/src/backend/runtime-abi.ts b/packages/compiler/src/backend/runtime-abi.ts new file mode 100644 index 000000000..73191971d --- /dev/null +++ b/packages/compiler/src/backend/runtime-abi.ts @@ -0,0 +1,11 @@ +/** External program-object ABI identity. The spelling is intentionally part + * of the link contract: a program object keeps this symbol undefined and the + * matching runtime defines it, so incompatible manual links fail before the + * program can start. */ +export const RUNTIME_ABI_VERSION = 1 as const; +export const RUNTIME_ABI_MARKER = "scr_runtime_abi_v1" as const; + +/** Object consumption is useful today, but the complete scr_* surface may + * still change before 1.0. The versioned marker prevents accidental mixing; + * it does not promise semver stability yet. */ +export const EXTERNAL_OBJECT_ABI_STABILITY = "experimental" as const; diff --git a/packages/compiler/src/backend/vendor-archives.ts b/packages/compiler/src/backend/vendor-archives.ts index 8c20609bb..9bcfadbf0 100644 --- a/packages/compiler/src/backend/vendor-archives.ts +++ b/packages/compiler/src/backend/vendor-archives.ts @@ -13,6 +13,11 @@ const execFileAsync = promisify(execFile); export const QJS_COMMIT = "3c8f3d68953955950074c41c6e4d999562ae82a7"; export const MBEDTLS_VERSION = "3.6.7"; export const ZLIB_VERSION = "1.3.1"; +/** Exact translation-unit membership shared by cache builds and external + * source-pack recipes. */ +export const QJS_ENGINE_SOURCES = ["dtoa.c", "libregexp.c", "libunicode.c", "quickjs.c"] as const; +export const LRE_SOURCES = ["libregexp.c", "libunicode.c"] as const; +export const ZLIB_SOURCES = ["adler32.c", "compress.c", "crc32.c", "deflate.c", "infback.c", "inffast.c", "inflate.c", "inftrees.c", "trees.c", "uncompr.c", "zutil.c"] as const; export interface VendorArchiveContext { runtimeSrcDir(): string; @@ -222,7 +227,6 @@ export function createVendorArchives(context: VendorArchiveContext) { /** The qjs library target's source list (CMakeLists.txt `qjs_sources` with * QJS_BUILD_LIBC off — cutils is header-only): the cross-build recipe * compiles exactly these. */ - const QJS_ENGINE_SOURCES = ["dtoa.c", "libregexp.c", "libunicode.c", "quickjs.c"]; /** The engine archive for one target, per-TU compiler invocations plus ar. * The flags mirror what the former CMake @@ -280,7 +284,6 @@ export function createVendorArchives(context: VendorArchiveContext) { * libregexp and its unicode tables (cutils is header-only). Deliberately * NOT the engine — a regex-using static binary links ~110KB of matcher, * never the ~620KB island. */ - const LRE_SOURCES = ["libregexp.c", "libunicode.c"]; function lreObjectPaths( sanitize: boolean, @@ -341,7 +344,6 @@ export function createVendorArchives(context: VendorArchiveContext) { * except the gzFile file-I/O units (gz*.c — nothing in scr_zlib.c * references the gzFile API, and those TUs alone want unistd/io headers). * Host builds never touch this list — they link the system libz. */ - const ZLIB_SOURCES = ["adler32.c", "compress.c", "crc32.c", "deflate.c", "infback.c", "inffast.c", "inflate.c", "inftrees.c", "trees.c", "uncompr.c", "zutil.c"]; function zlibObjectPaths( sanitize: boolean, diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index 558404dd5..3356cd324 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -7,7 +7,8 @@ import { emitCModule } from "./backend/c/c-emitter.js"; import { emitLlvmModule, LlvmUnsupportedError } from "./backend/llvm/emitter.js"; import { emitNativeArtifact, NativeCodegenError } from "./backend/native-codegen.js"; import { privateSiblingPath } from "./backend/build-cache.js"; -import { nativeCodegenTargetRefusal } from "./backend/targets.js"; +import { nativeCodegenTarget, nativeCodegenTargetRefusal } from "./backend/targets.js"; +import { createNativeLinkInfo, type NativeLinkInfo } from "./backend/native-link-info.js"; import { splitLlvmLibraryProgram, splitLlvmProgram } from "./backend/llvm/split.js"; import { rebaseLibrarySourceComments, replaceLibraryIdentity, stripLibraryIdentity, stripLibrarySourceComments } from "./backend/library-identity-markers.js"; import { checkerPanicDiag, ffiNativeBuildDiag, libAsyncExportDiag, libAsyncSurfaceDiag, libExportUnresolvedDiag, libGenericExportDiag, libIntBoundaryDiag, libNpmIneligibleDiag, libSidecarDiag, libUnmappableSignatureDiag, iceDiag, isCheckerPanic, LIB_INBOUND_BYTES_TRAP_CODE, LIB_RUNTIME_TRAP_CODES, nativeCodegenDiag, type ScrDiagnostic } from "./diagnostics/diagnostic.js"; @@ -51,6 +52,13 @@ import { compilerImplementationIdentity } from "./library/compiler-self-identity export const VERSION = "0.0.1"; +export { + EXTERNAL_OBJECT_ABI_STABILITY, + RUNTIME_ABI_MARKER, + RUNTIME_ABI_VERSION, +} from "./backend/runtime-abi.js"; +export type { NativeLinkInfo, NativeLinkFeatures } from "./backend/native-link-info.js"; + export { InternalCompilerError } from "./errors.js"; export { compileC, @@ -205,6 +213,9 @@ export interface CompileBaseOptions { * lower to direct C ABI calls. Source outputs retain those declarations; * archive/system-library inputs join only an executable link. */ ffiProfilePath?: string; + /** Attach the machine-readable external link recipe to an object result. + * Valid only with outputKind "obj"; it never invokes a linker. */ + nativeLinkInfo?: boolean; } /** Executable compile options. This remains the compatibility type for the @@ -237,7 +248,7 @@ export type CompileArtifact = | { kind: "c"; path: string } | { kind: "llvm"; path: string } | { kind: "asm"; path: string } - | { kind: "obj"; path: string } + | { kind: "obj"; path: string; nativeLinkInfo?: NativeLinkInfo } | { kind: "exe"; path: string; @@ -1149,6 +1160,17 @@ async function compileTracked( ): Promise { entryPath = resolve(entryPath); const outputKind = opts.outputKind ?? "exe"; + if (opts.nativeLinkInfo === true && outputKind !== "obj") { + return { + ok: false, + diagnostics: [nativeCodegenDiag( + "SC3002", + "native link info is available only for object output", + entryPath, + )], + sourceTexts: new Map(), + }; + } if (opts.nativeProgramObject === true && (outputKind !== "exe" || opts.backend !== "llvm")) { return { @@ -1531,6 +1553,31 @@ async function compileTracked( } } await removeStaleSourceArtifacts([opts.outPath]); + if (outputKind === "obj" && opts.nativeLinkInfo === true) { + const target = nativeCodegenTarget(); + if (target === null) { + throw new InternalCompilerError("native object emitted without a native target"); + } + return { + ok: true, + artifact: { + kind: "obj", + path: opts.outPath, + nativeLinkInfo: await createNativeLinkInfo({ + programObject: opts.outPath, + target, + features: executableNativeFeatures( + lowered.module, + "llvm", + opts.dynamic ?? false, + opts.optimization ?? "release", + ), + ffi, + optimization: opts.optimization ?? "release", + }), + }, + }; + } return { ok: true, artifact: { kind: outputKind, path: opts.outPath } }; } diff --git a/packages/compiler/test/llvm-runtime-abi.test.ts b/packages/compiler/test/llvm-runtime-abi.test.ts index 21a92a218..3e1d0e9c4 100644 --- a/packages/compiler/test/llvm-runtime-abi.test.ts +++ b/packages/compiler/test/llvm-runtime-abi.test.ts @@ -36,6 +36,7 @@ import { promisify } from "node:util"; import { describe, expect, test } from "vitest"; import { emitLlvmModule } from "../src/backend/llvm/emitter.js"; import { resolveCc, runtimeSrcDir } from "../src/backend/native-toolchain.js"; +import { RUNTIME_ABI_MARKER } from "../src/backend/runtime-abi.js"; import type { IrModule } from "../src/ir/ir.js"; import { compile } from "../src/index.js"; @@ -243,11 +244,11 @@ describe("LLVM backend declares match scr_runtime.h prototypes", () => { }], }; const llvm = emitLlvmModule(mod, { runtimeAbiMarker: true }); - expect(llvm).toContain("declare void @scr_runtime_abi_v1()"); - expect(llvm).toContain("call void @scr_runtime_abi_v1()"); - expect(await readFile(headerPath, "utf8")).toContain("void scr_runtime_abi_v1(void);"); + expect(llvm).toContain(`declare void @${RUNTIME_ABI_MARKER}()`); + expect(llvm).toContain(`call void @${RUNTIME_ABI_MARKER}()`); + expect(await readFile(headerPath, "utf8")).toContain(`void ${RUNTIME_ABI_MARKER}(void);`); expect(await readFile(join(repoRoot, "packages/runtime/src/scr_console.c"), "utf8")) - .toContain("void scr_runtime_abi_v1(void) {}"); + .toContain(`void ${RUNTIME_ABI_MARKER}(void) {}`); }); test("ScrBytes structural type matches the C runtime layout", async () => { const outDir = await mkdtemp(join(tmpdir(), "scriptc-llvm-layout-")); diff --git a/packages/compiler/test/native-codegen-integration.test.ts b/packages/compiler/test/native-codegen-integration.test.ts index 0971c44d9..c083f3873 100644 --- a/packages/compiler/test/native-codegen-integration.test.ts +++ b/packages/compiler/test/native-codegen-integration.test.ts @@ -165,6 +165,7 @@ describe.runIf(supported)("LLVM native helper integration", () => { .toEqual(await symbols(clangObject, ["-u"])); expect(await symbols(helperObject, ["-gU"])) .toEqual(await symbols(clangObject, ["-gU"])); + expect(await symbols(helperObject, ["-gU"])).toEqual(["0000000000000000 T _main"]); expect(await symbols(helperObject, ["-u"])).toContain("_scr_runtime_abi_v1"); const clangExe = join(dir, "clang-program"); @@ -268,4 +269,55 @@ describe.runIf(supported)("LLVM native helper integration", () => { expect(undefinedSymbols).toContain("_sf_callback_mix"); expect(undefinedSymbols).toContain("_scr_runtime_abi_v1"); }); + + test("helper and clang object paths retain the same outbound FFI C ABI", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-helper-ffi-parity-")); + dirs.push(dir); + const entry = join(dir, "main.ts"); + const profile = join(dir, "ffi.json"); + await writeFile(entry, [ + "declare function nativeMix(value: number, text: string, bytes: Uint8Array): number;", + "console.log(nativeMix(2, 'ok', new Uint8Array([1, 2])));", + "", + ].join("\n")); + await writeFile(profile, JSON.stringify({ + ffi_format: 1, + functions: [{ + name: "nativeMix", + symbol: "native_mix", + params: ["f64", "string", "bytes"], + returns: "f64", + }], + libraries: [], + system_libraries: [], + })); + const object = join(dir, "helper.o"); + const result = await compile(entry, { + outDir: dir, + outPath: object, + outputKind: "obj", + ffiProfilePath: profile, + }); + if (!result.ok) throw new Error(result.diagnostics.map((d) => d.message).join("\n")); + const llvm = join(dir, "main.ll"); + const sourceResult = await compile(entry, { + outDir: dir, + outPath: llvm, + outputKind: "llvm", + ffiProfilePath: profile, + }); + if (!sourceResult.ok) throw new Error(sourceResult.diagnostics.map((d) => d.message).join("\n")); + const clangObject = join(dir, "clang.o"); + await execFileAsync("clang", [ + "-O2", "-Wno-override-module", "-target", MACOS_ARM64_TARGET.llvmTriple, + "-c", llvm, "-o", clangObject, + ]); + const nativeMixSignature = (await readFile(llvm, "utf8")) + .split("\n").find((line) => line.includes("@native_mix(")); + expect(nativeMixSignature).toBe("declare double @native_mix(double, ptr, i64, ptr, i64)"); + for (const candidate of [object, clangObject]) { + const undefinedSymbols = (await execFileAsync("nm", ["-u", candidate], { encoding: "utf8" })).stdout; + expect(undefinedSymbols).toContain("_native_mix"); + } + }); }); diff --git a/tests/harness/native-object-example.test.ts b/tests/harness/native-object-example.test.ts new file mode 100644 index 000000000..a46bac651 --- /dev/null +++ b/tests/harness/native-object-example.test.ts @@ -0,0 +1,94 @@ +import { execFileSync } from "node:child_process"; +import { + chmodSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { release as osRelease, tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, describe, expect, test } from "vitest"; + +const supported = process.platform === "darwin" && process.arch === "arm64" && + Number.parseInt(osRelease().split(".", 1)[0] ?? "", 10) >= 24; +const repoRoot = join(import.meta.dirname, "../.."); +const fixture = join(repoRoot, "examples/native-object"); +const scratch = mkdtempSync(join(tmpdir(), "scriptc-native-object-example-")); + +afterAll(() => rmSync(scratch, { recursive: true, force: true })); + +describe.runIf(supported)("external native object example", () => { + test("rejects a stale runtime pack before invoking the toolchain", () => { + const runtimeRoot = join(scratch, "stale-runtime"); + const traps = join(scratch, "stale-traps"); + const trapLog = join(scratch, "stale-toolchain.log"); + const linkInfo = join(scratch, "stale-link-info.json"); + mkdirSync(runtimeRoot, { recursive: true }); + mkdirSync(traps, { recursive: true }); + writeFileSync(join(runtimeRoot, "package.json"), JSON.stringify({ + name: "@scriptc/runtime", + version: "2.0.0", + })); + const clang = join(traps, "clang"); + writeFileSync(clang, `#!/bin/sh\nprintf invoked > '${trapLog}'\nexit 97\n`); + chmodSync(clang, 0o755); + writeFileSync(linkInfo, JSON.stringify({ + schema: "scriptc.native-link-info.v1", + runtime_pack: { + package: "@scriptc/runtime", + version: "1.0.0", + root: runtimeRoot, + source_sets: [], + }, + program: { object: join(scratch, "missing.o") }, + ffi: { libraries: [] }, + link: { driver_flags: [], system_libraries: [], frameworks: [] }, + })); + + let failure: { stderr?: Buffer } | null = null; + try { + execFileSync(process.execPath, [ + join(fixture, "link.mjs"), "cc", linkInfo, join(scratch, "stale-program"), + ], { + env: { ...process.env, PATH: `${traps}:${process.env["PATH"] ?? ""}` }, + stdio: "pipe", + }); + } catch (error) { + failure = error as { stderr?: Buffer }; + } + expect(failure?.stderr?.toString("utf8")).toContain( + "runtime pack identity mismatch: expected @scriptc/runtime@1.0.0", + ); + expect(existsSync(trapLog)).toBe(false); + }); + + test("links and runs through both the C driver and Apple ld", () => { + const nativeObject = join(scratch, "native.o"); + const programObject = join(scratch, "app.o"); + const manifest = join(scratch, "ffi.json"); + const linkInfo = join(scratch, "link-info.json"); + const raw = JSON.parse(readFileSync(join(fixture, "ffi.json"), "utf8")) as { libraries: string[] }; + raw.libraries = [nativeObject]; + writeFileSync(manifest, JSON.stringify(raw)); + execFileSync("clang", [ + "-target", "arm64-apple-macosx14.0.0", "-O2", "-c", + join(fixture, "native.c"), "-o", nativeObject, + ]); + const json = execFileSync("node", [ + join(repoRoot, "packages/cli/dist/main.js"), + "build", join(fixture, "main.ts"), + "--ffi", manifest, + "--print=native-link-info", "-o", programObject, + ], { encoding: "utf8" }); + writeFileSync(linkInfo, json); + + for (const mode of ["cc", "ld"] as const) { + const executable = join(scratch, `app-${mode}`); + execFileSync("node", [join(fixture, "link.mjs"), mode, linkInfo, executable]); + expect(execFileSync(executable, [], { encoding: "utf8" })).toBe("42\n"); + } + }, 60_000); +});