diff --git a/.github/workflows/clang-tidy.yml b/.github/workflows/clang-tidy.yml index 41096e7..ede211d 100644 --- a/.github/workflows/clang-tidy.yml +++ b/.github/workflows/clang-tidy.yml @@ -48,7 +48,7 @@ jobs: - name: Install dependencies run: | sudo apt-get update - sudo apt-get install -y ninja-build libgmp-dev \ + sudo apt-get install -y ninja-build libgmp-dev libgc-dev \ "clang-${LLVM_VERSION}" "clang-tidy-${LLVM_VERSION}" \ "clang-tools-${LLVM_VERSION}" "llvm-${LLVM_VERSION}-dev" \ "llvm-${LLVM_VERSION}-tools" diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index 791b674..97f62c7 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -36,7 +36,7 @@ jobs: - name: Install build dependencies run: | sudo apt-get update - sudo apt-get install -y ninja-build libgmp-dev lcov gcc-14 g++-14 \ + sudo apt-get install -y ninja-build libgmp-dev libgc-dev lcov gcc-14 g++-14 \ "llvm-${LLVM_VERSION}-dev" "llvm-${LLVM_VERSION}-tools" python3-pip echo "/usr/lib/llvm-${LLVM_VERSION}/bin" >> "$GITHUB_PATH" diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 64be54f..1584cd9 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -45,7 +45,7 @@ jobs: - name: Install build dependencies run: | sudo apt-get update - sudo apt-get install -y ninja-build libgmp-dev "clang-${LLVM_VERSION}" \ + sudo apt-get install -y ninja-build libgmp-dev libgc-dev "clang-${LLVM_VERSION}" \ "llvm-${LLVM_VERSION}-dev" "llvm-${LLVM_VERSION}-tools" - name: Initialize CodeQL diff --git a/.github/workflows/scan-build.yml b/.github/workflows/scan-build.yml index adcc8e4..0767459 100644 --- a/.github/workflows/scan-build.yml +++ b/.github/workflows/scan-build.yml @@ -42,7 +42,7 @@ jobs: - name: Install dependencies run: | sudo apt-get update - sudo apt-get install -y ninja-build libgmp-dev "clang-${LLVM_VERSION}" \ + sudo apt-get install -y ninja-build libgmp-dev libgc-dev "clang-${LLVM_VERSION}" \ "clang-tools-${LLVM_VERSION}" "llvm-${LLVM_VERSION}-dev" "llvm-${LLVM_VERSION}-tools" echo "/usr/lib/llvm-${LLVM_VERSION}/bin" >> "$GITHUB_PATH" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c92fc47..e63984e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,7 +35,7 @@ jobs: - name: Install build dependencies run: | sudo apt-get update - sudo apt-get install -y ninja-build libgmp-dev \ + sudo apt-get install -y ninja-build libgmp-dev libgc-dev \ "llvm-${LLVM_VERSION}-dev" "llvm-${LLVM_VERSION}-tools" python3-pip if [ "${{ matrix.compiler }}" = "clang" ]; then sudo apt-get install -y "clang-${LLVM_VERSION}" @@ -83,7 +83,7 @@ jobs: - name: Install build dependencies run: | sudo apt-get update - sudo apt-get install -y ninja-build libgmp-dev \ + sudo apt-get install -y ninja-build libgmp-dev libgc-dev \ "clang-${LLVM_VERSION}" "llvm-${LLVM_VERSION}-dev" \ "llvm-${LLVM_VERSION}-tools" python3-pip echo "/usr/lib/llvm-${LLVM_VERSION}/bin" >> "$GITHUB_PATH" diff --git a/CMakeLists.txt b/CMakeLists.txt index d5172d8..ccf191d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -94,6 +94,12 @@ endif() message(STATUS "Found GMP library") +# Configure the Boehm-Demers-Weiser GC (M2 value model). Shipped as pkg-config +# `bdw-gc`; exposes the imported target PkgConfig::BDWGC. +find_package(PkgConfig REQUIRED) +pkg_check_modules(BDWGC REQUIRED IMPORTED_TARGET bdw-gc) +message(STATUS "Found Boehm GC ${BDWGC_VERSION}") + # Configure LLVM find_package(LLVM REQUIRED CONFIG) message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}") diff --git a/ROADMAP.md b/ROADMAP.md new file mode 100644 index 0000000..5e41571 --- /dev/null +++ b/ROADMAP.md @@ -0,0 +1,298 @@ +# NORA Roadmap — From FEP Interpreter to a Compiled `#lang racket/base` Hello-World + +## Current state + +`norac` parses **one** self-contained linklet (a Fully-Expanded-Program subset) and interprets it with a CEK/CESK machine (explicit typed `Frame` stack in a heap `std::vector`). The constraints that dominate everything below, verified against the tree: + +- **No proper tail calls.** `applyProcedure` unconditionally `Kont.emplace_back(Frame::Call)` (`src/Interpreter.cpp:510`); a `Call` frame is popped only when its activation returns a value (`:375`). Tail loops grow `Kont` by one frame per iteration — O(depth) heap where Racket guarantees O(1). The expander is deeply/mutually tail-recursive (`for-loop` 563×, `loop` 172×), so this is disqualifying. The machine is *already* a trampoline, which makes the fix a local one (reuse the `Call` frame) rather than an architecture change. +- **Value-copy model, no shared heap, no GC, no identity.** Values are `std::unique_ptr` deep-**cloned** on every bind/lookup (`Environment::envLookup` returns `clone()`). `ast::List` is a flat `SmallVector`, not cons cells (so `cons` is O(n), list-building O(n²)). Symbols are not interned (`AST.h:212` FIXME); `eq?`/`eqv?`/`equal?` are stubbed by a name-comparing `valueEq`. All integers are GMP `mpz_class` (RAII) — no fixnums, flonums, rationals, or complex. Cycle teardown is a manual `AllScopes` hack, not a collector. +- **6 primitives total** (`+ - *`, `current-continuation-marks`, `continuation-mark-set-first`, `continuation-mark-set->list`). Verified: no `box`/`cons`/`eq?`/`car`/`cdr` exist anywhere in `src/`. The expander needs **544 distinct kernel primitives** (measured by the R3 spike — `tools/freevars.py` → [`docs/expander-primitive-surface.md`](docs/expander-primitive-surface.md)); *running* a `racket/base` program needs a second, larger wave beyond that (see M16). +- **Linklets are parsed but not linked.** `visit(Linklet)` builds one `GlobalEnv` from body `define-values` and keeps the last form's value; imports are rejected by the parser, exports are unused. No instance objects, no import/export wiring, no `#%linklet` runtime API. Free identifiers resolve **lazily at eval time**, not eagerly at link time. +- **NIR is an empty, partly-broken opt-in scaffold** (`src/include/nir/`, `src/mlir/`): one `UnimplementedOp`, a stale duplicate `Ops.td` with the obsolete `list` API, class-name mismatches, never linked into the interpreter. Rewrite, don't extend. +- ~88–90 integration `.rkt` tests (lit/FileCheck) + 1 Catch2 unit file. **No Racket oracle infrastructure exists yet.** + +The expander artifact (`expander/expander.rktl`, 8.6 MB) is a single `(linklet () (<71 exports>) )` with **empty imports** — every kernel primitive is a **free variable** the host must supply — **~2875 internal `define-values`** (~2000 at top level), and **71 exports** (`boot`, `eval`, `expand`, `compile`, `read`, `datum->syntax`, `make-namespace`, `namespace-require`, `dynamic-require`, …). Its body tail *is* the boot sequence. It also consumes the linklet runtime itself as free-variable primitives (`compile-linklet`, `instantiate-linklet`, `make-instance`, `primitive-table`, …) — the expander is a linklet that compiles and runs linklets by calling back into the host. + +## Target + +A **standalone AOT executable** `./hello`, produced from `hello.rkt` (`#lang racket/base`), that prints `hello world`: + +``` +hello.rkt (#lang racket/base) + --[NORA's reader + #lang/module-reader protocol]--> (real read-syntax over a file port) + --[NORA's own interpreter runs expander.rktl]--> (self-hosted expander, phase-1 macros execute) +expanded FEP / linklet-bundle graph (+ racket/base graph deps) + --[NORA backend: FEP -> NIR (MLIR) -> LLVM IR]--> (Racket-level opts in NIR) +object files + libnora_rt (Boehm GC + primitives + printer + main()) + --[link]--> ./hello -> prints "hello world" +``` + +## Issue tracking + +Every milestone and spike below is a GitHub issue (label `roadmap`), cross-linked by dependency. Browse: + +- **Spikes:** [R1 #88](https://github.com/pmatos/nora/issues/88) · [R3 #89](https://github.com/pmatos/nora/issues/89) · [R5 #90](https://github.com/pmatos/nora/issues/90) +- **Track A (interpreter → self-hosted expander):** [M0 #91](https://github.com/pmatos/nora/issues/91) · [M0-N #92](https://github.com/pmatos/nora/issues/92) · [M1 #93](https://github.com/pmatos/nora/issues/93) · [M2 #94](https://github.com/pmatos/nora/issues/94) · [M3 #95](https://github.com/pmatos/nora/issues/95) · [M4 #96](https://github.com/pmatos/nora/issues/96) · [M5 #97](https://github.com/pmatos/nora/issues/97) · [M6 #98](https://github.com/pmatos/nora/issues/98) · [M7 #99](https://github.com/pmatos/nora/issues/99) · [M8 #100](https://github.com/pmatos/nora/issues/100) · [M9 #101](https://github.com/pmatos/nora/issues/101) · [M10 #102](https://github.com/pmatos/nora/issues/102) · [M11 #103](https://github.com/pmatos/nora/issues/103) · [M12 #104](https://github.com/pmatos/nora/issues/104) · [M13 #105](https://github.com/pmatos/nora/issues/105) · [M14 #106](https://github.com/pmatos/nora/issues/106) · [M15 #107](https://github.com/pmatos/nora/issues/107) · [M16 #108](https://github.com/pmatos/nora/issues/108) · [M17 #109](https://github.com/pmatos/nora/issues/109) +- **Track B (NIR/LLVM AOT backend):** [B0 #110](https://github.com/pmatos/nora/issues/110) · [B1 #111](https://github.com/pmatos/nora/issues/111) · [B2 #112](https://github.com/pmatos/nora/issues/112) · [B3 #113](https://github.com/pmatos/nora/issues/113) · [B4 #114](https://github.com/pmatos/nora/issues/114) · [B5 #115](https://github.com/pmatos/nora/issues/115) + +## Toolchain version lock (governs everything below) + +All version-sensitive artifacts **must** derive from **one pinned Racket commit**, recorded as a single controlled fact (`ORACLE_RACKET_COMMIT`): + +- `expander/expander.rktl` (regenerated via the `raco` demodularizer), +- the differential oracle (M0), +- the precompiled/embedded `racket/base` `.zo` graph (M16), +- the fasl v2/v3 wire format the deserializer targets (M13), +- the `primitive-table` name→module grouping the expander expects (M8). + +A mismatch in any one silently invalidates **every** M10+ differential test. Regenerating `expander.rktl` + the `racket/base` `.zo` graph from that pinned build is a **scheduled build-infra task** (M0 deliverable) — the `raco` demodularizer toolchain is part of the budget, not free. + +## Strategy — two interleaved tracks over one shared substrate + +Build **one shared prerequisite substrate** first — the *spine* — then run two tracks that share a runtime and converge at the end: + +- **Spine (critical path, blocks both tracks):** a single **value model + garbage-collected heap + object identity + proper tail calls + closure representation**, whose representation ABI is **co-designed once** and used by both the interpreter and compiled code. This is the pivot; getting it wrong forces rewriting both tracks. +- **Track A — interpreter → self-hosted expander:** grow the primitive surface (6 → 544), add the literal/value types the expander's data needs, build the printer, build linklet instances + the `#%linklet`/`primitive-table` runtime API, instantiate `expander.rktl`, prove phase-1 macro execution, add the fasl/`.zo` loader, the reader/`#lang` protocol, and the `racket/base` graph, culminating in an *interpreted* `#lang racket/base` hello. This is the long pole; staff it heaviest. +- **Track B — NIR/LLVM AOT backend:** rewrite the NIR dialect for real, prove the whole toolchain on a trivial FEP→NIR→LLVM→`./out` walking skeleton (needs **no** GC, can start day one), then build the runtime lib on the frozen M2 ABI **sharing Track A's primitive library**, cover all FEP forms, and link a linklet-bundle graph into an executable. + +**Honest parallelism.** Only **B0 and B1 run genuinely ahead** of Track A — a trivial `(+ 1 2)`/`if` → `./out prints 42` needs no GC, no identity, no primitives. Everything past B1 interleaves with Track A: **B2 shares the M4–M7 primitive library and printer, B4 depends on M8's instance model.** Racket-as-oracle (differential testing, never as the expander) still decouples *test authoring* — B2–B4 can be exercised on Racket-produced `racket/base` FEP fixtures — but their *acceptance* ("compile and run a `racket/base` fixture") is gated on the shared primitives existing. The claim "Track B races far ahead in parallel" is false and is not planned around. + +Legend: **[S]** spine · **[A]** Track A · **[B]** Track B. Effort: S=small, M=medium, L=large, XL=extra-large. Coarse person-time in each milestone header is *individual* duration, not additive across the serial chain — see "Effort realism". + +--- + +## Milestone ladder + +### M0 — Oracle + differential harness + version lock **[S] · ~2–3 wk** · [#91](https://github.com/pmatos/nora/issues/91) +- **Goal:** Make Racket usable as a differential oracle; lock behavior and provenance. +- **Deliverables:** `test/oracle/` harness with a `%racket` lit substitution that runs an expression through the **pinned** `racket` and `norac`, normalizes printed output, and diffs; gated on `NORA_HAVE_RACKET` (auto-skips green when absent). Record `ORACLE_RACKET_COMMIT`; a build-infra task that **regenerates `expander.rktl` (via the `raco` demodularizer) and the `racket/base` `.zo` graph from that commit**. New opt-in CI `oracle` job; new CMake `lsan` preset. **The gensym/alpha-normalizer is scaffolded here but scoped to M-effort and lands for real just before M10** (see M0-N). +- **Depends-on:** — +- **Acceptance:** `ctest` runs a handful of `.rkt` files through both engines and passes; harness skips green with no Racket; `expander.rktl` and the `.zo` graph both carry a recorded provenance stamp matching `ORACLE_RACKET_COMMIT`. +- **Risk:** Unpinned oracle → flaky diffs; a provenance mismatch invalidates M10+. + +### M0-N — Global-consistent alpha/gensym normalizer **[S→M] · ~3–4 wk** · [#92](https://github.com/pmatos/nora/issues/92) +- **Goal:** A canonicalizer that can diff fully-expanded programs. Not "small": it must handle lifts, `let-values` temporaries, module-path-index encodings, and scope-annotated identifiers. +- **Deliverables:** a normalizer that renames **globally and consistently** across the whole expanded program (not local alpha-renaming), tolerant of a shifted gensym counter (every generated name renumbered coherently). A fallback "observational-equivalence" acceptance mode for cases text-normalization cannot reconcile. +- **Depends-on:** M0. +- **Acceptance:** normalizes non-trivial *real expander output* (not toy cases) to a stable canonical form; two runs of the same input over the pinned Racket normalize identically. +- **Risk:** See R7 — hash-iteration order and gensym-counter parity can defeat any normalizer; this milestone gates M10 and must be proven on real output before M10 is declared reachable. + +### M1 — Proper tail calls in the interpreter **[S] · ~3–4 wk** · [#93](https://github.com/pmatos/nora/issues/93) · **✅ DONE** (`1d7b11b`) +- **Goal:** O(1)-space tail calls in the CEK machine. +- **Deliverables (as built):** no static AST marking was needed — a dynamic approach fits this trampoline. `continueStep(Frame::Seq)` pops the sequence frame **before** its final expression (non-`begin0`), mirroring `IfBranch`; `applyProcedure` **reuses** the enclosing `Frame::Call` when it is on top of `Kont` (move in the new callee, `clear()` its marks) instead of `emplace_back`. Added `Interpreter::getPeakKont()` and a minimal `zero?` predicate (full numeric predicates are M4). +- **Depends-on:** M0. (Implemented ahead of M0; tests are self-contained Catch2 + one lit test.) +- **Acceptance (met):** built test-first — peak `Kont` is identical at depth 100 vs 100000 and `< 16` (O(1)); a tail loop returns the right value; non-tail recursion still grows the continuation; mutual `ev`/`od` recursion is bounded; `test/integration/tailcall.rkt` runs end to end. **25/25 tests pass under debug, asan, and ubsan.** +- **Risk (retired):** the "top is `Call`" reuse condition is exact once Seq/if/let-body frames pop before their tail sub-expression; verified clean under asan/ubsan. + +### M2 — Shared value model + GC + closure representation (the pivot) **[XL] · ~3–5 mo** · [#94](https://github.com/pmatos/nora/issues/94) +- **Goal:** Replace clone/value-copy with a shared, identity-preserving, GC-managed heap, and fix closure capture. **This is the single ABI decision point for both tracks — freeze it here.** +- **Deliverables:** + - a tagged 64-bit `nr_value` word (low-bit/3-bit tagging, Chez/CS-style: fixnum immediate, char/`#f`/`#t`/`null`/`void`/`eof` immediates, 8-byte-aligned heap pointers); a uniform `ObjHeader` (type tag + GC bits + len/meta). + - **Boehm–Demers–Weiser conservative GC (`libgc`) for both interpreter and compiled code — committed, not "first."** A conservative collector scans the native stack, which is exactly what the compiled `tailcc` path needs. **Moving/precise GC (stack maps/statepoints/shadow stack) is explicitly out of scope** for hello and filed as a future XL that would itself change the ABI; there is **no** "reserved forwarding slot" pretense. + - GMP routed through GC: an explicit task to **migrate off `mpz_class` (RAII) to GC-routed raw `mpz_t`**, using Boehm atomic allocation for pointer-free limbs, so bignum `nr_value`s need no finalizers. Own test. + - `Environment` returns **shared** references, not clones. **Closure representation designed here: a flat closure = header + code-ptr + vector of captured cells**, aligned byte-for-byte with B2's compiled closure layout, so a lambda capturing over the ~2000-entry top level is O(captured), not O(env). No naive env chaining/copying. + - a minimal `cons`/`eq?`/`box`/`unbox`/`set-box!` quintet pulled in here (they are the natural first identity/mutation smoke tests). `eq?`/`eqv?`/`equal?` cycle-safe (seen-set). Interned symbols. `clone()` and the `AllScopes` teardown deleted. +- **Depends-on:** M1; spike **R1** first. +- **Acceptance:** a **C++-level `nr_value` unit test** proves pointer identity and mutation without surface syntax; `(let ([p (cons 1 2)]) (eq? p p))` → `#t`; `(let ([b (box 0)]) (set-box! b 5) (unbox b))` → `5`; a tail loop allocating ≫ heap-size of garbage completes (GC demonstrably collects, RSS bounded); all ~90 tests + asan/lsan green. +- **Risk:** Broad refactor of `AST.h`/`ASTRuntime.h`/`Environment`/`Runtime.cpp`. Migrate one type at a time behind the `ValueNode` interface with golden tests green at each step. **Do this before growing the primitive table** — every primitive signature depends on it. + +### M3 — Literal reader coverage + literal value types + bounded parsing **[L] · ~1–1.5 mo** · [#95](https://github.com/pmatos/nora/issues/95) +- **Goal:** Parse the whole 8.6 MB expander artifact and construct its data literals. +- **Deliverables:** lexer/parser for flonums, exact rationals (`1/2`), byte strings (`#"…"`), `#hash(...)`/`#hasheq(...)` reader syntax, and acceptance of `define-syntaxes` (58×, must parse in this flattened artifact); value types `Flonum` (double, boxed), `Rational` (GMP `mpq`), `ByteString`, immutable `Hash`, plus fixnum/bignum promotion on `nr_value` (fixnum fast-path via `__builtin_*_overflow`, mpz fallback). **Bounded/iterative parsing (or a raised parser stack) for deeply-nested data**, since the recursive-descent parser will otherwise blow the C++ stack on this file. +- **Depends-on:** M2. +- **Acceptance:** `norac` parses `expander.rktl` end-to-end without a lex/parse error **and without stack overflow** (parse-only smoke test on the real artifact + a pathological-depth fixture); differential tests for each literal kind. +- **Risk:** Today `0.0`/`1/2`/`7/8` lex as **identifiers** — the file will not even parse until this lands; deep nesting is an independent failure mode. + +### M4 — Core datatypes, equality & the numeric tower **[L] · ~1.5–2 mo** · [#96](https://github.com/pmatos/nora/issues/96) +- **Goal:** The pure-value layer the expander builds at load time. +- **Deliverables:** pairs/lists (`car cdr cons null? pair? list length list* append apply map …`), mutable pairs, mutable/immutable strings (real code-unit buffers — drop the "lexeme-includes-quotes" hack) and chars (immediate codepoint), bytes, vectors (+ `unsafe-vector*-ref/-set!/-length`), boxes, `void`/`eof`; `values`/`call-with-values`; interned symbols + gensym; fixnum/`unsafe-fx*` ops + generic-int subset (`add1 sub1 zero? exact-integer? fixnum? number? quotient expt`). Completes the **GMP raw-`mpz_t`** migration for the bignum arithmetic paths. +- **Depends-on:** M2 (M3 in parallel). +- **Acceptance:** each of the top-50 datatype/numeric prims passes a `norac`-vs-`racket` differential test. +- **Risk:** `unsafe-*` ops are a bit-layout contract — they must assume the tag with no dispatch; co-designed with M2. + +### M5 — Structs, properties, hash tables (iteration-order-matched) **[L] · ~2–3 mo** · [#97](https://github.com/pmatos/nora/issues/97) +- **Goal:** The heaviest expander object surface — everything in the expander is a struct or a hash. +- **Deliverables:** struct system (`make-struct-type` 156–190×, `make-struct-field-accessor` **506×**, `make-struct-field-mutator`, `make-struct-type-property`, `struct-copy` 136×, `struct->vector`, `current-inspector`) with property values (`prop:authentic` 92×, `prop:equal+hash`, `prop:procedure`, `prop:custom-write`, …); prefab structs (global prefab table); `eq`/`eqv`/`equal` hashes — mutable (`hash-set!` 119×) and immutable HAMT (`hash-set` 167×, structural sharing) — plus the `hash-iterate-*` protocol (incl. `unsafe-immutable-hash-iterate-*`) and `#hash`/`#hasheq` literals. +- **Depends-on:** M4. +- **Acceptance:** differential tests for struct construction/accessor/mutator/property/`struct-copy` and hash CRUD+iteration. **Additionally: the `eq`/`equal`-hash iteration order either matches Racket CS or is proven unused for code-emission ordering** (see R7) — this is a gating acceptance criterion, not a footnote. +- **Risk:** Load-bearing #1 after pairs; a subtly wrong `make-struct-type`/`hash-ref` corrupts the expander invisibly. Hash iteration order silently threatens *every* downstream expand-equivalence test. + +### M6 — Printer + errors, exceptions, parameters & control **[L] · ~1.5–2 mo** · [#98](https://github.com/pmatos/nora/issues/98) +- **Goal:** The value printer plus the control/diagnostic layer the expander touches on any non-trivial path. **The printer moves here (before/with the error system), because error text is formatted through it.** +- **Deliverables:** a real `write`/`display`/`print` subsystem (cycle handling, `prop:custom-write`) — the offending-value formatter that `raise-argument-error` and friends depend on; `raise` / `error` (115×) / `raise-argument-error` (**489×**) / `raise-arguments-error`, the `exn`/`exn:fail` struct hierarchy, `with-handlers`→`call-with-exception-handler`; parameters built on continuation marks (`make-parameter`, `parameterization-key`, `extend-parameterization`, `parameterization?`); `dynamic-wind`, `call-with-continuation-prompt`, `abort-current-continuation` for the eval loop (the marks 3-op family already exists). +- **Depends-on:** M5. +- **Acceptance:** differential tests on normalized raised-error message text (which *requires* the printer) and handler control flow. +- **Risk:** You cannot match Racket's error text without the value printer — hence it is a hard dependency of this milestone, and it is also an explicit B2 deliverable. + +### M7 — Full kernel (expander) primitive surface, data-driven **[XL, incremental] · ~2.5–3.5 mo** · [#99](https://github.com/pmatos/nora/issues/99) +- **Goal:** Implement the *complete* finite set of primitives the **expander** references (a distinct, smaller set than the `racket/base` runtime surface in M16). +- **Deliverables:** the remaining prims by frequency (of the 544 measured by R3) — strings/bytes/chars + `format` (127×), the `letrec`/varref mechanism (`unsafe-undefined` **729×** sentinel + read-guard; `variable-reference-from-unsafe?` **637×** ⇒ always `#f` = safe path; `variable-reference->instance`, `variable-reference-constant?`, `variable-reference?`), srcloc structs + `syntax-source/line/column/position/span`, weak collections, places/atomic as **no-ops or a single global box** (`unsafe-make-place-local`/`-ref`/`-set!` = one global box per key; `unsafe-start/end-atomic` = no-ops), in-memory string/bytes ports + a stdout port so `display`/`format` print. (The real filesystem reader is M14/M15, not here.) +- **First task = spike R3:** instrument the interpreter to log **every unbound free identifier** hit while instantiating `expander.rktl` → an exact finite worklist; then batch-implement + differential-test. +- **Depends-on:** M4, M5, M6. +- **Acceptance:** the unbound-id log is **empty** after instantiating `expander.rktl` (meaningful only under M8's eager link-time binding); each prim batch has differential tests. +- **Risk:** The long-tail *fidelity* (not mere presence) is the danger; the logging instrumentation turns "mysterious expander crash" into a burn-down list. + +### M8 — Linklet instances + `#%linklet` runtime API + eager resolution **[L] · ~1.5–2 mo** · [#100](https://github.com/pmatos/nora/issues/100) +- **Goal:** Real linklet instantiation/linking, the runtime API the expander calls back into, and **eager link-time free-variable resolution**. +- **Deliverables:** an `Instance` type = named table of shared, mutable, GC'd variable **cells**; generalize `visit(Linklet)` to bind grouped imports to import-instances' cells and produce an instance (not a last-value); the `#%linklet` primitives NORA implements natively (`compile-linklet` = capture the parsed FEP AST + compiled marker; `instantiate-linklet` = reuse NORA's own linklet path; `eval-linklet`, `recompile-linklet`, `make-instance`, `instance-variable-value`/`-set!`, `instance-variable-names`, `linklet?`); `primitive-table : symbol → hash`, `primitive-lookup`, `declare-primitive-module!`; primitives grouped into pseudo-modules (`#%kernel`, `#%paramz`, `#%unsafe`, `#%flfxnum`; `#%foreign`/`#%network`/`#%place`/`#%futures` stubbed). **Design deliverable: instantiation eagerly binds every free reference to the primitive instance's cells** (as real linklets do), so M7/M9's "empty unbound-id log" is a complete proof, not a coverage artifact of executed paths. +- **Depends-on:** M7. +- **Acceptance:** a 2-linklet program (one exports, one imports a value) evaluates correctly; `(primitive-table '#%kernel)` returns a hash whose `car`/`cons` entries work; a linklet with an unbound free var **fails at instantiation**, not lazily at first use. +- **Risk:** Cross-instance references are mutable boxed cells shared by pointer — exactly what value-copy could not express; depends entirely on M2. + +### M9 — Expander instantiates + `(boot)` **[L] · ~1–2 mo** · [#101](https://github.com/pmatos/nora/issues/101) +- **Goal:** Run all ~2875 `define-values` and the boot tail to completion without an unbound/arity error. +- **Deliverables:** driver that instantiates the expander linklet into an instance exposing the **71 exports**, runs the boot tail (`namespace-init!`, `declare-reexporting-module!` loop for `#%kernel`/`#%paramz`/… and `#%linklet`/`#%boot`, `current-namespace ← ns`, `dynamic-require '#%kernel`), then fetches and calls the `boot` export. Replaces `main`'s "print last form" with an instantiate-then-boot driver; `runtime-instances` registration. +- **Depends-on:** M8; spike **R2** (measure cost). +- **Acceptance (fast CI smoke):** `expander.rktl` instantiates — all ~2875 closures built via flat capture, boot runs — with an **empty** unbound-id log (eager binding) and no arity error. +- **Risk:** Even with TCO+GC+flat closures, instantiating 2875 defines may be too slow/heavy (RSS/time). **Measure via R2 before committing to a full `expand` run;** if minutes/GBs, prioritize interpreter perf (persistent structures, allocation, closure layout — the exact thing M2's flat-closure decision de-risks). + +### M10 — Self-hosted expand of a `#%kernel` expression **[M] · ~3–5 wk** · [#102](https://github.com/pmatos/nora/issues/102) +- **Goal:** Call the expander's own `expand` on trivial input (self-hosted, no Racket oracle inside). `#%kernel` has **no macros**, so this exercises expansion without executing compile-time code. +- **Deliverables:** driver path `make-namespace` → set `current-namespace` → `namespace-require ''#%kernel` → build syntax via `datum->syntax` for `(module m '#%kernel (#%module-begin (display "hi")))` → `expand` → `syntax->datum`. No source loading, no user reader. +- **Depends-on:** M9, **M0-N (normalizer proven on real output)**. +- **Acceptance (expand-equivalence oracle):** NORA's `expand` output = `racket -e '(expand …)'` after **global-consistent** normalization. +- **Risk:** Hygienic gensym suffixes and hash-iteration order differ (R7); diff only after normalization, and fall back to observational equivalence where text cannot reconcile. + +### M11 — Compile + eval a `#%kernel` module; bundle/directory model **[M–L] · ~4–6 wk** · [#103](https://github.com/pmatos/nora/issues/103) +- **Goal:** End-to-end interpreted compile→instantiate of an expanded `#%kernel` module, modeling the **linklet-bundle/directory** structure explicitly. +- **Deliverables:** `compile` (expanded syntax → a **`linklet-directory` / `linklet-bundle`**, keys `decl`, `data`, `stx`, phase `0`/`1` bodies — *not* a monolithic single linklet) → phase-ordered instantiation (decl before body; syntax-literal/data linklets before phase-0 body) → `eval`/instantiate in the namespace → observe `"hi"` on stdout. **As soon as this exists, route NORA-produced FEP through the B-track compiler as a fixture** (see R8) so the two dialects integrate early, not at B5. +- **Depends-on:** M10. +- **Acceptance:** program output `"hi"` == Racket; exercises `compile-linklet`/`instantiate-linklet` and the bundle/directory instantiation order end to end. +- **Risk:** First real exercise of the compiled-linklet round-trip and the bundle structure in the runtime API. + +### M12 — Phase-1 transformer execution (one-macro module) **[M–L] · ~1–1.5 mo** · [#104](https://github.com/pmatos/nora/issues/104) +- **Goal:** Prove that NORA can **compile and instantiate a macro transformer at phase 1 and run it during expansion** — the single hardest mechanism, isolated *before* the full `racket/base` macro graph hits it. +- **Deliverables:** a tiny hand-written language exporting one macro (e.g. a `swap!`/`my-let` `define-syntax`); self-host-expand a module using it, which forces the expander to compile the transformer linklet, instantiate it at phase 1, and invoke it. Fed via `datum->syntax` (no reader/fs yet). +- **Depends-on:** M11. +- **Acceptance:** the one-macro module expands to the same normalized FEP as Racket; the transformer demonstrably executed inside NORA (traced instantiate at phase 1). +- **Risk:** This is the M10→"real hello" cliff made explicit. Everything in `racket/base` runs through this path ~thousands of times; discovering a phase-1 bug here is far cheaper than at M17. + +### M13 — fasl v2/v3 + `#~` compiled reader + bundle deserializer **[L] · ~1.5–2 mo** · [#105](https://github.com/pmatos/nora/issues/105) +- **Goal:** Load a machine-independent `.zo` `racket/base` graph. This is a large, finicky, **version-locked** format implementation, not "a bundle deserializer." +- **Deliverables:** the fasl v2/v3 reader (graph refs, srcloc/`prefab`/`path`/interned-symbol encodings), the `#~` compiled reader, `read-accept-compiled`, and the `linklet-directory`/`linklet-bundle` structure. A loaded `.zo` is fed through NORA's **own** `compile-linklet`/`instantiate-linklet` path (loading does not bypass the compiler). +- **Depends-on:** M11 (bundle model), **version-locked to `ORACLE_RACKET_COMMIT`**. +- **Acceptance:** a Racket-produced `.zo` bundle deserializes and its linklets instantiate under NORA, matching the oracle for a small module. +- **Risk:** Format drift vs. the pinned build silently corrupts everything; the fasl graph/`prefab`/`path` encodings are the finicky part. + +### M14 — Minimal file I/O + embedded `racket/base` module table **[M–L] · ~1–1.5 mo** · [#106](https://github.com/pmatos/nora/issues/106) +- **Goal:** Give the goal a real front door without a full OS layer. **Decision (committed now):** *embed the precompiled `racket/base` graph as an in-memory module table so no collection-path search is needed for the graph*, and provide only the minimal filesystem surface needed to read the **user's** `hello.rkt`. +- **Deliverables:** a real `path` datatype + `path->string`/`build-path`/`simplify-path` (minimal), `file-exists?`, `open-input-file` + UTF-8-decoding file input ports (for `hello.rkt`); an in-memory module registry preloaded with the M13 `racket/base` bundle graph, so `standard-module-name-resolver` resolves `racket/base` against the embedded table rather than the filesystem/collections. +- **Depends-on:** M13. +- **Acceptance:** `hello.rkt` reads as a UTF-8 char port; `(require racket/base)` resolves to the embedded graph with **zero** filesystem access. +- **Risk:** This decision changes M16's shape; making it *now* avoids building a collection-path search algorithm we do not need for hello. + +### M15 — Reader + `#lang` / module-reader protocol **[L] · ~1.5–2 mo** · [#107](https://github.com/pmatos/nora/issues/107) +- **Goal:** Real `read-syntax` for `#lang racket/base` over a file port. Reading `#lang` is the **language-loading protocol**, not `read` on s-expressions. +- **Deliverables:** the reader pipeline — read the `#lang` line → `read-language` → load the **reader module** (`syntax/module-reader`, itself macro-heavy, expanded/instantiated through M12's phase-1 path) → `read-syntax` producing syntax objects with **source locations** from the char port. `read` on datums, readtables to the extent `racket/base`'s reader needs. (`read-accept-compiled`/`#~` already landed in M13.) +- **Depends-on:** M14 (file ports), M16 partial or M13 (the `racket/base` reader module must be loadable). **The "`norac hello.rkt` reads" step lives here — it must not hide behind `datum->syntax`.** +- **Acceptance:** `read-syntax` on `hello.rkt` produces srcloc-bearing syntax whose `syntax->datum` matches Racket's, and whose `#lang` line correctly selects `racket/base`'s reader. +- **Risk:** The reader module is itself a program that must expand/instantiate under NORA — another exercise of M12's machinery. + +### M16 — Module loader/resolver + `racket/base` graph + second primitive burn-down **[XL] · ~2.5–3.5 mo** · [#108](https://github.com/pmatos/nora/issues/108) +- **Goal:** Instantiate the entire `racket/base` linklet graph and implement the **second, larger** primitive wave it needs at *runtime* (distinct from the expander surface in M7). +- **Deliverables:** module registry + **phase-ordered instantiation** (a module required `for-syntax` is instantiated at phase 1 before its requirer expands); `standard-module-name-resolver` + `current-load/use-compiled` wired against the M14 embedded table. **A second R3-style logging burn-down over `racket/base` *instantiation* (not expander instantiation)** — this surfaces primitives the expander never touches: the full keyword-application protocol (`make-keyword-procedure`, `keyword-apply`), the `for` runtime, more of the numeric tower (flonums, `bitwise-*`, `arithmetic-shift`), and the full print system. This wave is **comparable in size to M4–M7, not a footnote**, and is a first-class dependency of the goal. +- **Depends-on:** M12 (phase-1 macros execute), M13 (`.zo` loader), M14 (embedded table), M15 (reader). +- **Acceptance:** the `racket/base` graph instantiates with an **empty** second-wave unbound-id log; `(module m racket/base 42)` (read via M15 or fed via `datum->syntax`) self-host-expands and matches the oracle; `racket/base`'s macros execute through M12's phase-1 path. +- **Risk:** `racket/base` is a macro-heavy graph of dozens of modules — far heavier than `#%kernel`. Precompiled `.zo` avoids re-expanding it; **source-expansion of `racket/base` is the hardest possible bootstrap and is deferred** as a self-hosting-purity stretch goal. + +### M17 — Interpreted `#lang racket/base` hello (Track A checkpoint) **[L/XL] · ~1–1.5 mo** · [#109](https://github.com/pmatos/nora/issues/109) +- **Goal:** End-to-end self-hosted **interpretation** of a real `#lang racket/base` program. This is the integration crucible: it runs the full read → phase-1-macro-expand → compile → instantiate path over the entire `racket/base` graph. +- **Deliverables:** `norac hello.rkt` **reads** (M15), self-host-expands (NORA runs `expander.rktl`, `racket/base`'s transformers execute at phase 1 via M12), compiles to a bundle, and evals `(display "hello world")`. +- **Depends-on:** M16. +- **Acceptance:** `norac hello.rkt` stdout == `racket hello.rkt` stdout. **Track A complete; strongly-recommended checkpoint feeding B5.** +- **Risk:** Re-labeled **L/XL** (not M): it is the first time the entire compile+instantiate path runs over the entire `racket/base` macro graph. M12's one-macro proof de-risks it but does not eliminate the integration surface. + +--- + +### B0 — Real NIR dialect skeleton (representation-agnostic) **[M] · ~1–1.5 mo** *(can start day one)* · [#110](https://github.com/pmatos/nora/issues/110) +- **Goal:** Replace the broken scaffold with a real, round-trippable dialect — **without baking the value representation before M2 freezes it.** +- **Deliverables:** proper `NIR_Dialect` + **representation-agnostic** ops (`nir.const/quote`, `nir.if`+`nir.truthy`, `nir.app`/`nir.tailapp`, `nir.primcall`, `nir.lambda`/`nir.case_lambda`, `nir.linklet`/`nir.defvar`, `nir.box_new/ref/set`, `nir.let_values`/`nir.values`, `nir.wcm`) with verifiers and `CallOpInterface`/`SymbolOpInterface`/`RegionBranchOpInterface`; fix the duplicate/stale `.td`, class-name mismatch, and CMake wiring; a `noract` tool and `norac --emit=nir`. **Concrete value/unboxing types (`!nir.fixnum`, `!nir.box`, `!nir.closure`, interned `!nir.symbol`, struct descriptors) and unboxing bridges are deferred until after the M2 ABI freeze** (via R1), so lowering assumptions are not committed before the tag layout is decided. +- **Depends-on:** — for the ops; the concrete-type layer depends on R1/M2. +- **Acceptance:** `nir-opt` round-trips a `nir.constant`/`nir.return` `.mlir` (lit test); backend `mlir`-preset CI job (ccache, one compiler/config) builds. +- **Risk:** MLIR scaffolding is heavy before the first executable; keep MLIR out of the default matrix and out of the runtime/binary. + +### B1 — Walking skeleton: FEP → NIR → LLVM → object → `./out` prints 42 **[M] · ~1–1.5 mo** *(highest-value early de-risk)* · [#111](https://github.com/pmatos/nora/issues/111) +- **Goal:** Prove the *entire* toolchain on a trivial sublanguage (int literals, `+`, `if`) with a **throwaway** minimal value model — **no GC, no identity, no primitives.** +- **Deliverables:** FEP→NIR lowering (tail position computed here) + NIR→LLVM `TypeConverter`/ConversionPatterns → LLVM IR → `.o` via `addPassesToEmitFile`; a tiny runtime `main()`; a link step; `norac --emit=exe`. +- **Depends-on:** B0 (parallel with M1/M2). +- **Acceptance:** `norac --emit=exe tiny.rkt && ./out` prints the integer; == Racket oracle. +- **Risk:** Validates MLIR build + lowering + linking + runtime `main()` before broadening — classic end-to-end-first. **This is the only genuinely oracle-decoupled backend milestone.** + +### B2 — Runtime library + ABI (`libnora_rt`), sharing Track A's primitives **[L] · ~2–3 mo** · [#112](https://github.com/pmatos/nora/issues/112) +- **Goal:** The shared runtime built on the **frozen M2 value representation**, reusing the M4–M7 primitive library and the M6 printer — **not a second heap, not a second primitive set.** +- **Deliverables:** static `libnora_rt` = Boehm GC + `nr_value` allocation/constructors + **the kernel primitive library written once and shared with the interpreter** + **the value printer (shared with M6)** + symbol intern table + multiple-values buffer/ABI + arity/error helpers + `main()`/boot. Uniform code signature `nr_value code(nr_value env, i64 argc, nr_value* argv)`; closure = header + code-ptr + flat free-vars (byte-identical to M2's flat closure); MV = return-register fast path + thread-local values buffer. +- **Depends-on:** **M2 ABI freeze (hard gate)**, **M4–M7 (shared primitives + printer)**, B1. +- **Acceptance:** compiled `(let ([b (box 0)]) (set-box! b 5) (unbox b))` prints `5`; a compiled program that formats a value via the printer matches the oracle; runtime asan/lsan clean; triangulates `racket == norac-interp == norac-compiled`. +- **Risk:** Do **not** let A and B invent two heaps or two primitive sets — B2 consumes the M2 ABI and the Track-A primitive/printer library verbatim. + +### B3 — Full FEP → NIR coverage **[L] · ~2–3 mo** · [#113](https://github.com/pmatos/nora/issues/113) +- **Goal:** Compile every FEP form the interpreter supports. +- **Deliverables:** closure conversion (`make_closure` + top-level `code`), general `nir.app` + arity check, `case-lambda`, `values`/`let-values`, `letrec`, `set!`/boxes, structs (`make-struct-type` + props), vectors/hashes/strings, `#%variable-reference`, `with-continuation-mark` + mark queries, `error`/`raise`; **proper tail calls** via `tailcc` + `musttail` on the uniform signature (trampoline fallback behind a flag); prompts + one-shot escape continuations via the runtime. Racket-level opts in NIR (primcall inlining, fixnum unboxing, known-call devirtualization, dead-export elimination). **Full multi-shot `call/cc` deferred** — confirmed unnecessary for hello via R6. +- **Depends-on:** B2; spike **R6**; commit the continuation/mark ABI **early** (before this milestone) and share it with the interpreter. +- **Acceptance:** every interpreter-supported FEP form has a compile-and-run differential test; deep self/mutual tail loops run in O(1) native stack; **NORA-produced `#%kernel` FEP (from M11) compiles and runs**, not just oracle FEP. +- **Risk:** `musttail` target-portability (trampoline fallback); continuations-on-native-stack is the deepest backend problem — scoped to one-shot/prompts for hello. + +### B4 — Linklet-bundle graph → NIR modules + runtime link/boot **[L] · ~1.5–2.5 mo** · [#114](https://github.com/pmatos/nora/issues/114) +- **Goal:** Compile and link a multi-linklet **bundle/directory** graph into one executable. +- **Deliverables:** each linklet → an `instantiate(imports…) -> instance` function with top-level vars as heap boxes; **import resolution via runtime instance objects (not raw linker symbols)** — needed for mutable top-levels, re-instantiation, `dynamic-require`; boot instantiates the primitive instance + linklets in **dependency + phase order** (mirroring M11's bundle/directory instantiation exactly); wire the embedded `racket/base` graph deps. +- **Depends-on:** B3, **M8 (instance model), M11 (bundle/directory structure)**. +- **Acceptance:** a compiled multi-linklet hello graph links into one executable and runs; interpreted and compiled linking agree on instance semantics. +- **Risk:** Must mirror M8/M11 instance + bundle semantics exactly so interpreted and compiled linking agree. + +### B5 — `./hello` from `#lang racket/base` (FINAL GOAL) **[L, convergence] · ~1–2 mo** · [#115](https://github.com/pmatos/nora/issues/115) +- **Goal:** The standalone compiled executable. +- **Deliverables:** `hello.rkt` → NORA **reads + self-host-expands** (Track A: M15 + M17 path) → expanded linklet-bundle graph (+ embedded `racket/base` graph from M16) → NORA backend compiles (NIR→LLVM) → object files → link `libnora_rt` → `./hello`. +- **Depends-on:** M17 (self-hosted expander produces the FEP + `racket/base` graph) and B4 (compiler links a bundle graph). +- **Acceptance:** `./hello` prints `hello world`; stdout == `racket hello.rkt`. **Goal pipeline complete.** +- **Risk:** The convergence point; both tracks must land. Because M11 already routed NORA-FEP through the compiler (R8), the two dialects are integrated well before here. Continuations confirmed out of scope (R6). + +--- + +## Critical path & honest parallelism + +**Strict chain to the first `./hello`:** + +``` +M0 → M0-N → M1 → M2(freeze ABv) + │ + Track A (long pole, staff heaviest): + M3 → M4 → M5 → M6 → M7 → M8 → M9 → M10 → M11 → M12 → M13 → M14 → M15 → M16 → M17 + │ │ + Track B: └── B0 → B1 (run ahead) │ + B2 ⟵ needs M2 ABI + M4–M7 + M6 printer │ + B3 ⟵ needs B2 │ + B4 ⟵ needs B3 + M8 + M11 (bundle model) │ + ▼ + (M17) + (B4) ───────────► B5 +``` + +- **Shared spine `M0 → M0-N → M1 → M2` blocks everything meaningful.** M2 is the fork and the ABI freeze; both tracks and the shared primitive/printer library consume it. One heap, one primitive set. +- **Only B0 and B1 run genuinely ahead.** B2 onward interleave with Track A: **B2 depends on M4–M7 + M6, B4 depends on M8 + M11.** Track B's *completion* is gated mid-Track-A. Staffing math: the two tracks are **not** fully parallelizable — plan for it. +- **The `racket/base` runtime primitive wave (M16) is a second cost center comparable to M4–M7**, on the critical path, and must not be conflated with the expander surface. +- **NORA-produced FEP flows through the compiler from M11 on** (R8), so B5 is a link-up, not a first meeting of two dialects. + +## Top risks & early spikes (ordered by "hurts most if discovered late") + +1. **R1 — Value model & GC ABI (highest; before M2 and before B0's concrete types).** Prototype tagged `nr_value` + Boehm heap + `eq?` identity + one mutable type + **flat closure capture**; prove a garbage-generating tail loop actually collects, and that the *same* representation plugs into both a trivial interpreter path **and** a trivial compiled program. **Commit Boehm-conservative for both interpreter and compiled code here** (moving/precise GC is out of scope, filed as a future XL). Getting this wrong rewrites both tracks. +2. **R7 — Hash-iteration-order & gensym-counter parity (threatens *every* M10→B5 differential test).** `eq-hash-code` is allocation-derived, so NORA's HAMT/`eq`-hash iteration order will differ from Racket CS; if the expander ever iterates a hash to emit code order, no alpha-normalizer can reconcile the result. Prove early that M5's hash iteration either matches Racket or is unused for output ordering; design M0-N for **global** consistent renaming; budget for an "observationally equivalent" acceptance fallback. +3. **R5 — MLIR/toolchain integration (parallel with M1/M2).** Ship **B1** (FEP→NIR→LLVM→`./out prints 42`) as early as possible to validate the MLIR build, lowering, linking, and runtime `main()` on one trivial case. Highest-value early Track-B de-risk. +4. **R3 — Primitive-surface long tail (two passes).** Instrument the interpreter to **log every unbound free identifier** — first while instantiating `expander.rktl` (first task in M7), then again while instantiating the `racket/base` graph (first task in M16). Converts two ~hundreds-of-prims unknowns into finite burn-down checklists; fidelity is caught by the eval-equivalence oracle. **Eager link-time binding (M8) is required for the log to be complete.** +5. **R2 — Expander cost (at M9).** Measure instantiate time + RSS of merely building all ~2875 closures **before** committing to a full `expand` run. If minutes/GBs, prioritize interpreter perf; the M2 flat-closure decision is the primary lever. +6. **R4 — `racket/base` provenance + fasl (before M13/M16).** Committed: **precompiled/embedded `.zo` graph** (fasl v2/v3 + `#~` + `read-accept-compiled`, version-locked to `ORACLE_RACKET_COMMIT`); source-expansion of `racket/base` deferred. Confirm the fasl `prefab`/`path`/graph encodings and the bundle/directory phase order early. +7. **R6 — Continuations in compiled code (before B3).** Confirm `#lang racket/base` hello does **not** exercise full multi-shot `call/cc` (near-certain), scoping compiled continuations to prompts + one-shot escapes. Commit the continuation/mark ABI early and share it with the interpreter. +8. **R8 — Two-dialect integration (at M11, not B5).** As soon as interpreted `#%kernel` compile exists, route **NORA-produced** FEP through the B-track compiler as a fixture, so shape/ordering differences between NORA-FEP and Racket-FEP surface early — not at the final milestone. +9. **R0 — TCO spike (M1).** Small: prototype `Call`-frame reuse + early frame-pop on a deep loop; assert bounded `Kont`. + +## Effort realism + +The S/M/L/XL scale saturates at the top; the honest picture is a **from-scratch Racket-CS bootstrap** where the incumbent (Chez) supplied GC, the numeric tower, structs, TCO, and continuations for free. NORA starts from a value-copy tree-walker with 6 primitives. + +- **Cost centers (each multi-month, serial on the critical path):** **M2** (value model + GC + closures), **M5** (structs + hashes), **M7** (expander primitives), **M12** (phase-1 macro execution), **M16** (`racket/base` graph + second primitive wave), **M17** (integration crucible). +- The per-milestone person-time in each header is *individual* duration. **In strict series (this plan's own critical path), the interpreted hello (M17) is realistically 1–3 person-years before the backend is complete;** "months" applies to individual milestones, never the whole chain. Parallelism buys back only B0/B1 and modest overlap, because B2+ share Track A's substrate. +- Guardrails throughout: keep all ~90 integration tests green at every step (FileCheck catches any printed-form regression immediately), triangulate `racket == interp == compiled`, keep MLIR out of the default build, and use Racket purely as a differential oracle — never as the expander. + +## Immediate next 3 actions (spikes) + +1. **R1 — value-model/GC ABI spike (throwaway branch).** ([#88](https://github.com/pmatos/nora/issues/88)) **DONE** (commit `4e52ba6`): [`spike/r1-value-model/`](spike/r1-value-model/) proves a tree-walking interpreter and compiled C++ compute identical results over one shared heap/ABI, a flat closure applies identically in both, and Boehm churns **2.98 GiB of garbage through a 0.1 MiB live heap** (~24,000×, RSS 4 MiB; clean under `-O2` and ASan). The frozen tag/immediate/`ObjHeader`/flat-closure layout and the Boehm-conservative commitment (moving GC out of scope) are recorded in [`docs/value-model-abi.md`](docs/value-model-abi.md) — the reference M2 and B0/B2 consume. +2. **R3 (static pass) — unbound-identifier enumerator on today's tree.** ([#89](https://github.com/pmatos/nora/issues/89)) **DONE:** `tools/freevars.py` (scope-aware walker, robust to the M3 lexing gap) reports **544 distinct free identifiers** (22,173 refs) over 2,003 body forms; ranked, categorized worklist in [`docs/expander-primitive-surface.md`](docs/expander-primitive-surface.md). Seeds M4–M7. +3. **R5 — MLIR toolchain spike toward B1.** ([#90](https://github.com/pmatos/nora/issues/90)) **DONE & VERIFIED** under MLIR 22 (commits `a17adcb`, `891e0d7`): the broken scaffold is rewritten into a minimal representation-agnostic `nir.constant`/`nir.return` dialect + `nir-opt` driver; `cmake --preset mlir` builds it and `nir-opt test/mlir/nir-roundtrip.mlir | nir-opt | FileCheck` **passes**. The MLIR build (tablegen → dialect → nir-opt → parse/verify/print) is de-risked end to end; the default `norac` build stays green and MLIR remains opt-in. Unblocks B0/B1. See [`src/mlir/README.md`](src/mlir/README.md). \ No newline at end of file diff --git a/docs/expander-primitive-surface.md b/docs/expander-primitive-surface.md new file mode 100644 index 0000000..67af50e --- /dev/null +++ b/docs/expander-primitive-surface.md @@ -0,0 +1,588 @@ +# Expander primitive surface (R3-static) + +Auto-generated by `tools/freevars.py` from `expander/expander.rktl`. This is the +**exact set of free (primitive) identifiers** the flattened expander linklet +references and that the host (NORA) must supply. It is the worklist that seeds +milestones M4-M7 (interpreter primitive surface) and, later, the second wave +in M16 for `racket/base`'s *runtime* surface. + +## Summary + +- **544 distinct free identifiers** (22173 references) +- Method: a scope-aware s-expression walk that tracks `lambda`/`case-lambda`/`let-values`/`letrec-values`/`define-values` binders and skips quoted data. Robust to the M3 lexing gap (the production C++ parser cannot yet ingest the artifact). +- Correction to the roadmap's earlier estimate: the real surface is **544**, not ~350-370. + +## Category breakdown (heuristic) + +| Category | Distinct prims | +|---|---:| +| other | 127 | +| numeric tower | 60 | +| hash tables | 44 | +| errors/control/srcloc | 43 | +| pairs/lists/core | 42 | +| unsafe-* fastpaths | 37 | +| ports/IO/read | 34 | +| strings | 26 | +| paths/filesystem | 25 | +| structs/props/inspectors | 23 | +| bytes | 20 | +| vectors | 14 | +| boxes/cells | 13 | +| chars | 9 | +| regexp | 9 | +| varref/place/atomic | 5 | +| symbols | 5 | +| parameters | 5 | +| keywords | 2 | +| impersonators | 1 | +| **TOTAL** | **544** | + +## Full ranked list (count — identifier) + +``` + 2376 values + 2067 void + 1264 not + 817 pair? + 729 unsafe-undefined + 679 car + 630 variable-reference-from-unsafe? + 619 cons + 593 cdr + 592 eq? + 585 null + 566 list + 505 make-struct-field-accessor + 487 raise-argument-error + 396 hash-ref + 390 unsafe-car + 387 unsafe-cdr + 317 equal? + 284 null? + 279 eqv? + 243 unsafe-fx< + 192 symbol? + 187 unsafe-fx+ + 171 + + 167 hash-set + 152 make-struct-type + 148 add1 + 127 format + 119 hash-set! + 119 current-inspector + 111 fx+ + 107 hash-iterate-next + 107 hash-iterate-first + 103 string-append + 95 unsafe-place-local-ref + 95 error + 92 < + 92 prop:authentic + 91 unsafe-vector-ref + 90 vector-ref + 89 unsafe-vector-set! + 86 parameterization-key + 83 char=? + 83 unbox + 82 list? + 80 string? + 80 = + 79 hash-iterate-key+value + 79 list* + 78 length + 75 - + 74 make-hasheq + 71 append + 64 hash? + 64 raise-arguments-error + 63 cadr + 61 char? + 60 procedure-arity-includes? + 60 box + 59 procedure? + 58 apply + 55 hash-count + 53 continuation-mark-set-first + 49 string-ref + 49 make-struct-field-mutator + 48 vector? + 47 zero? + 44 sub1 + 43 > + 42 vector*-ref + 41 extend-parameterization + 41 unsafe-immutable-hash-iterate-first + 41 unsafe-immutable-hash-iterate-next + 38 path? + 38 bytes-length + 38 char->integer + 38 make-vector + 38 unsafe-fx= + 38 unsafe-vector-length + 37 build-path + 37 eof-object? + 37 make-parameter + 36 fx= + 34 hasheq + 33 string-length + 33 unsafe-make-place-local + 32 list-ref + 32 current-code-inspector + 29 exact-nonnegative-integer? + 29 unsafe-place-local-set! + 29 read-char-or-special + 28 unsafe-immutable-hash-iterate-key + 28 string->symbol + 27 <= + 27 set-box! + 25 bytes? + 25 vector-length + 25 symbol->string + 24 hash-iterate-key + 24 cddr + 23 make-struct-type-property + 23 vector + 23 write-string + 23 box? + 22 make-hasheqv + 22 peek-char-or-special + 21 inspector-superior? + 20 immutable? + 19 exact-integer? + 19 * + 19 fixnum? + 18 subbytes + 18 current-continuation-marks + 18 object-name + 18 fx- + 18 port-next-location + 17 split-path + 17 char<=? + 17 substring + 16 / + 16 unsafe-fx- + 16 >= + 16 number? + 15 raise + 15 bytes-append + 15 regexp-match? + 15 memq + 15 symbol-interned? + 15 prefab-struct-key + 15 boolean? + 15 write-bytes + 14 gensym + 14 system-type + 14 path->complete-path + 14 vector-copy! + 14 hash-remove + 14 positive? + 14 weak-box-value + 14 struct->vector + 14 fx> + 13 path-for-some-system? + 13 complete-path? + 13 vector-set! + 13 hasheqv + 13 max + 13 path->string + 13 unsafe-immutable-hash-iterate-key+value + 13 inspector? + 12 bytes->path + 12 string->bytes/utf-8 + 12 integer? + 12 make-string + 12 fxlshift + 11 caddr + 11 call-with-values + 11 hash-eq? + 11 make-hash + 11 prop:custom-write + 11 unsafe-vector*-set! + 11 read-bytes + 11 expt + 11 find-system-path + 10 string->path + 10 unsafe-fx<= + 10 make-weak-hasheq + 10 integer->char + 10 make-weak-box + 10 fprintf + 10 display + 10 bytes->string/utf-8 + 10 srcloc-source + 10 fx< + 10 list->vector + 9 break-enabled-key + 9 relative-path? + 9 file-exists? + 9 directory-exists? + 9 ceiling + 9 input-port? + 9 unquoted-printing-string + 9 hash-clear! + 9 keyword? + 9 current-input-port + 8 caar + 8 path->bytes + 8 raise-argument-error* + 8 error-print-width + 8 number->string + 8 hash-eqv? + 8 make-weak-hash + 8 hash-iterate-value + 8 current-directory + 8 fxior + 8 arithmetic-shift + 8 version + 8 simplify-path + 7 symbol=? + 6 cdar + 6 system-path-convention-type + 6 bytes=? + 6 read-case-sensitive + 6 vector->immutable-vector + 6 mpair? + 6 mcar + 6 read-byte + 6 quotient + 6 hash + 6 negative? + 6 box-cas! + 6 unbox* + 6 hash-remove! + 6 srcloc + 6 current-load-relative-directory + 6 integer->integer-bytes + 5 call-with-continuation-prompt + 5 string + 5 path-convention-type + 5 raise-arguments-error* + 5 fxvector? + 5 flvector? + 5 hash-equal-always? + 5 log-message + 5 current-logger + 5 srcloc->string + 5 string->uninterned-symbol + 5 raise-result-error + 5 error-print-source-location + 5 srcloc-span + 5 srcloc-position + 5 srcloc-column + 5 srcloc-line + 5 unsafe-fx>= + 5 bytes + 5 file-position + 5 exn:fail? + 5 peek-byte + 5 default-continuation-prompt-tag + 4 abort-current-continuation + 4 string->bytes/locale + 4 regexp-replace* + 4 bytes-ref + 4 bytes->path-element + 4 read-accept-bar-quote + 4 unsafe-fxrshift + 4 real? + 4 mcdr + 4 hash-equal? + 4 dynamic-wind + 4 log-level? + 4 regexp-replace + 4 make-prefab-struct + 4 box-immutable + 4 regexp? + 4 prop:sealed + 4 fxand + 4 datum-intern-literal + 4 call-with-continuation-barrier + 4 hash-copy + 4 explode-path + 4 hash-for-each + 4 flonum? + 4 void? + 4 integer-bytes->integer + 4 string=? + 4 char-whitespace? + 4 integer-length + 4 string-foldcase + 4 char-alphabetic? + 4 port-read-handler + 3 make-continuation-prompt-tag + 3 raise-mismatch-error + 3 path-element->bytes + 3 byte-regexp + 3 current-environment-variables + 3 environment-variables-ref + 3 struct-type? + 3 unsafe-bytes-ref + 3 keyword->string + 3 make-ephemeron-hasheq + 3 logger-name + 3 prop:equal+hash + 3 make-ephemeron + 3 ephemeron-value + 3 vector-immutable + 3 set-box*! + 3 vector*-set! + 3 current-thread + 3 make-inspector + 3 error-syntax->string-handler + 3 write-byte + 3 get-output-bytes + 3 real->single-flonum + 3 open-output-bytes + 3 make-fxvector + 3 make-flvector + 3 string->unreadable-symbol + 3 make-rectangular + 3 eof + 3 set-mcdr! + 3 bitwise-ior + 3 unsafe-vector*-length + 3 call-with-escape-continuation + 3 current-compile-realm + 3 load-on-demand-enabled + 3 min + 3 read-on-demand-source + 3 filesystem-change-evt + 3 string-copy! + 3 error-module-path->string-handler + 3 peek-bytes + 2 parameterization? + 2 make-thread-cell + 2 exception-handler-key + 2 absolute-path? + 2 bytes->string/locale + 2 cleanse-path + 2 regexp-match + 2 fxvector-length + 2 flvector-length + 2 unsafe-flvector-ref + 2 unsafe-fxvector-ref + 2 make-hashalw + 2 hashalw + 2 open-input-file + 2 close-input-port + 2 hash-keys-subset? + 2 current-memory-use + 2 current-gc-milliseconds + 2 current-inexact-monotonic-milliseconds + 2 list->string + 2 regexp-match-positions + 2 cdddr + 2 prefab-key? + 2 primitive-table + 2 always-evt + 2 sync/timeout + 2 exn:fail:contract? + 2 error-syntax->name-handler + 2 prefab-struct-type-key+field-count + 2 denominator + 2 numerator + 2 real->floating-point-bytes + 2 prefab-key->struct-type + 2 unsafe-fxvector-set! + 2 unsafe-flvector-set! + 2 byte-pregexp + 2 regexp + 2 pregexp + 2 string->keyword + 2 floating-point-bytes->real + 2 mcons + 2 fx>= + 2 string-set! + 2 vector->list + 2 bytesfl + 2 inexact->exact + 2 exact->inexact + 2 string| + 2 char-numeric? + 2 code| + 2 flush-output + 2 continuation-prompt-available? + 2 caadr + 2 current-read-interaction + 1 check-for-break + 1 bytes-convert + 1 bytes-open-converter + 1 string-locale-downcase + 1 andmap + 1 resolve-path + 1 prop:impersonator-of + 1 prop:checked-procedure + 1 prop:procedure + 1 current-print + 1 for-each + 1 raise-range-error* + 1 exn:fail:contract + 1 exact? + 1 error-value->string-handler + 1 arity-at-least-value + 1 arity-at-least? + 1 procedure-arity + 1 hash-map + 1 make-ephemeron-hasheqv + 1 make-ephemeron-hashalw + 1 make-ephemeron-hash + 1 hash-ephemeron? + 1 make-weak-hasheqv + 1 make-weak-hashalw + 1 hash-weak? + 1 with-input-from-file + 1 unsafe-make-uninterruptible-lock + 1 unsafe-uninterruptible-lock-acquire + 1 unsafe-uninterruptible-lock-release + 1 modulo + 1 string->list + 1 current-plumber + 1 plumber-add-flush! + 1 cadddr + 1 unsafe-string-length + 1 equal-hash-code + 1 eq-hash-code + 1 unsafe-root-continuation-prompt-tag + 1 print-syntax-width + 1 unsafe-struct*-cas! + 1 hash-ref-key + 1 primitive->compiled-position + 1 compiled-position->primitive + 1 primitive-in-category? + 1 primitive-lookup + 1 linklet? + 1 compile-linklet + 1 recompile-linklet + 1 eval-linklet + 1 instantiate-linklet + 1 linklet-import-variables + 1 linklet-export-variables + 1 linklet-add-target-machine-info + 1 linklet-summarize-target-machine-info + 1 instance? + 1 make-instance + 1 instance-name + 1 instance-data + 1 instance-variable-names + 1 instance-variable-value + 1 instance-set-variable-value! + 1 instance-unset-variable! + 1 instance-describe-variable! + 1 linklet-virtual-machine-bytes + 1 linklet-cross-machine-type + 1 write-linklet-bundle-hash + 1 read-linklet-bundle-hash + 1 variable-reference? + 1 variable-reference->instance + 1 variable-reference-constant? + 1 sync + 1 semaphore-post + 1 semaphore-peek-evt + 1 make-semaphore + 1 never-evt + 1 prop:exn:srclocs + 1 struct:exn:fail + 1 symbol->immutable-string + 1 string-append-immutable + 1 current-write-relative-directory + 1 byte-pregexp? + 1 byte-regexp? + 1 pregexp? + 1 unsafe-fxvector-length + 1 unsafe-flvector-length + 1 symbol-unreadable? + 1 imag-part + 1 real-part + 1 complex? + 1 rational? + 1 single-flonum? + 1 output-port? + 1 bytes->immutable-bytes + 1 string->immutable-string + 1 vector*-length + 1 vector-cas! + 1 write + 1 datum->syntax + 1 syntax-property + 1 syntax-span + 1 syntax-position + 1 syntax-column + 1 syntax-line + 1 syntax-source + 1 syntax-e + 1 syntax? + 1 make-bytes + 1 memv + 1 path->directory-path + 1 raise-result-arity-error + 1 eval-jit-enabled + 1 caddar + 1 cadar + 1 open-input-bytes + 1 file-size + 1 filesystem-change-evt-cancel + 1 exn:fail:filesystem? + 1 filesystem-change-evt-ready? + 1 read-char + 1 exn:fail:read:non-char + 1 exn:fail:read:eof + 1 exn-continuation-marks + 1 char-utf-8-length + 1 port-counts-lines? + 1 peek-char + 1 fx* + 1 string->bytes/latin-1 + 1 fxmin + 1 string->number + 1 log + 1 make-polar + 1 real->extfl + 1 extflonum-available? + 1 single-flonum-available? + 1 fxrshift + 1 flvector-set! + 1 fxvector-set! + 1 exn:fail:out-of-memory + 1 placeholder-set! + 1 make-placeholder + 1 make-immutable-hashalw + 1 make-hashalw-placeholder + 1 make-immutable-hasheqv + 1 make-hasheqv-placeholder + 1 make-immutable-hasheq + 1 make-hasheq-placeholder + 1 make-immutable-hash + 1 make-hash-placeholder + 1 byte? + 1 make-reader-graph + 1 current-output-port + 1 current-error-port + 1 struct:exn:fail:filesystem + 1 error-message->adjusted-string + 1 prop:evt + 1 system-library-subpath + 1 file-or-directory-modify-seconds + 1 port-count-lines! + 1 expand-user-path + 1 cadadr + 1 reparameterize +``` diff --git a/docs/value-model-abi.md b/docs/value-model-abi.md new file mode 100644 index 0000000..94f591a --- /dev/null +++ b/docs/value-model-abi.md @@ -0,0 +1,118 @@ +# NORA value-model ABI (frozen by spike R1) + +This is the shared runtime representation of Racket values in NORA: one tagged +word, one heap-object header, one closure layout, and one garbage collector, +used by **both** the tree-walking interpreter (milestone M2) and statically +compiled code (B0/B2). Freezing it once is the whole point of spike R1 — the +interpreter and the compiler must agree byte-for-byte or they cannot share a +heap, and changing it later forces rewriting both tracks. + +The design below is validated by the runnable spike in +[`spike/r1-value-model/`](../spike/r1-value-model/) (issue #88). Numbers quoted +here are what the spike prints on x86-64 with Boehm GC 8.2. + +## 1. `nr_value`: a tagged 64-bit word + +``` + bit 0 == 1 fixnum value = (int64)w >> 1 (63-bit) + low 3 bits == 0b000 (w != 0) heap pointer an 8-byte-aligned ObjHeader* + low 3 bits == 0b010 singleton subtype = w >> 3 + low 3 bits == 0b110 character codepoint = w >> 3 + low 3 bits == 0b100 reserved +``` + +Rationale: + +- **Fixnums are odd** (`(v << 1) | 1`). Arithmetic (`+`, `-`, `<`) works on the + raw word with a single untag/retag and an `__builtin_*_overflow` check; the + overflow path promotes to a bignum (M4 — the spike asserts instead). +- **Heap pointers keep tag `000`**, so an `ObjHeader*` *is* its own `nr_value` + with no masking on dereference — the common operation stays free. Every heap + object is 8-byte aligned (Boehm returns ≥16-byte-aligned granules). +- **Singletons and chars are even and non-zero-low-3**, so they never collide + with odd fixnums or `000` pointers. + +Singleton subtypes (`(subtype << 3) | 0b010`): `#f`, `#t`, `'()`, `void`, `eof`, +`unsafe-undefined` (the 729×-referenced sentinel), and a letrec `uninit` hole. +Racket truthiness is `w != #f` — everything else, including `0` and `'()`, is +truthy. + +Reserved tag `100` is left for a future need (e.g. an immediate flonum on 64-bit +if we ever want NaN-free unboxed doubles); nothing depends on it today. + +## 2. Heap objects: uniform `ObjHeader` + +Every heap object starts with an 8-byte header so payloads stay 8-aligned: + +```c +struct ObjHeader { uint32_t type; uint32_t meta; }; // type = NrObjType, meta = len/arity/flags +``` + +Spike object sizes: `NrPair` 24 B (`header + car + cdr`), `NrBox` 16 B, +`NrClosure` 24 B + captures, `NrSymbol` 24 B. The header is deliberately fat +enough (`meta`) to carry a length/arity without a second word; a future precise +GC would use spare header bits for mark/forward state (see §5). + +## 3. Closures: flat capture, one code signature + +```c +typedef nr_value (*nr_code)(nr_value self, int64_t argc, const nr_value *argv); +struct NrClosure { ObjHeader h; nr_code code; uint32_t nfree, pad; nr_value free[]; }; +``` + +A closure is `header + code-pointer + inline captured cells`. Capture is +**flat**: a lambda closing over the ~2000-entry linklet top level captures only +the variables it uses, so building it is O(captured), not O(env) — the property +that de-risks instantiating the expander's thousands of closures (M9/R2). + +`nr_code`'s signature is the exact one B2's compiled code uses: the callee +receives its own closure as `self` and reads captures from `free[]`. So the same +`nrt_apply(clos, argc, argv)` entry point dispatches an interpreter-built +closure and a compiler-emitted one identically — proven in the spike, where a +closure capturing `10` applied to `5` yields `15` through both paths. + +## 4. Garbage collection: Boehm conservative — committed for both tracks + +NORA uses the **Boehm-Demers-Weiser conservative collector** (`libgc`) for the +interpreter **and** compiled code, sharing one heap. A conservative collector +scans the native C/C++ stack and registers, which is exactly what both the CEK +machine's registers (`Kont`/`Env`/`Val`) and compiled `tailcc` frames need — no +stack maps, no shadow stack, no write barriers to bring up first. + +Spike evidence: a loop allocating a box and a pair per iteration for 50 M +iterations **churns 2.98 GiB through a 0.1 MiB live heap (≈24,000×)** with peak +RSS 4 MiB — Boehm reclaims the per-iteration garbage and the heap stays bounded. +The interpreter and compiled paths behave identically. + +**Out of scope for the hello-world goal (filed, not planned):** a moving/precise +GC (LLVM statepoints / shadow stack). It would change this ABI (object headers, +interior-pointer rules, `musttail` interaction) and is an XL effort of its own. +There is **no** "reserved forwarding slot" pretending to future-proof the ABI — +if precise GC is ever adopted it is an explicit ABI break. + +## 5. Known conservative-GC caveats (accepted) + +- **False retention from fixnum bit patterns.** An odd fixnum word can, by + coincidence, look like an interior pointer into a live object, briefly keeping + garbage alive. This is a correctness-preserving imprecision inherent to + conservative GC; the spike shows it does not stop the heap from staying + bounded. It is one motivation for the (out-of-scope) precise-GC future. +- **GMP.** Bignums must route their limb storage through GC-atomic allocation + (pointer-free) and drop `mpz_class` RAII for GC-managed raw `mpz_t`, so bignum + `nr_value`s need no finalizers (M2/M4 task). +- **Non-GC containers.** Anything holding the only reference to a live object + from malloc'd memory the GC does not scan (e.g. a `std::unordered_map` intern + table) must either be a registered root or hold *uncollectable* objects. The + spike interns symbols with `GC_MALLOC_UNCOLLECTABLE` (symbols are permanent). + +## 6. What M2 / B0 / B2 must consume verbatim + +1. The tag scheme and immediate encodings in §1 (`nrt.h` is the reference). +2. The `ObjHeader` shape and 8-alignment in §2. +3. The flat-closure layout and `nr_code` signature in §3 — interpreter closures + and compiled closures are the same objects. +4. Boehm conservative GC as the single heap for both tracks (§4). + +The spike's `nrt.h`/`nrt.cpp` are the throwaway proof; M2 promotes this ABI into +the real `src/` value model, and B2 links the same primitive/runtime library +into compiled output. diff --git a/docs/value-model-gc-migration.md b/docs/value-model-gc-migration.md new file mode 100644 index 0000000..ab00bf1 --- /dev/null +++ b/docs/value-model-gc-migration.md @@ -0,0 +1,241 @@ +All load-bearing facts verified against the tree: `Environment::Env` is `std::map>` living inside `Scope` (a `make_shared` malloc block); `AllScopes` at `Interpreter.h:197` with teardown at `Interpreter.cpp:31-42`; `newScope` uses `make_shared` (`Interpreter.cpp:44`); `Box`/`Pair` are `shared_ptr` over `unique_ptr`; `Closure` holds `unique_ptr`; `Integer::~Integer()` at `AST.cpp:178`; `Symbol` uses `shared_ptr Uninterned` + static `InternTable`; both test mains use `CATCH_CONFIG_MAIN`; `bdw-gc` 8.2.12 is present via pkg-config; the ABI's `nr_value` immediate layer is reusable but `nrt.h`'s `ObjHeader`-based object accessors are not. Here is the final plan. + +--- + +# NORA M2 — Value model + Boehm-GC migration (final plan) + +## 0. Purpose and hard promise + +Migrate the interpreter's runtime values from a deep-copying, RAII, `shared_ptr`/`unique_ptr` object graph to a shared, Boehm-GC'd value model, converging structurally on the frozen R1 `nr_value` ABI. The whole test suite — both Catch2 unit mains (`test_interpreter`, `test_parse`) and the `lit`/FileCheck integration corpus — stays green under `debug`, `asan`, and `ubsan` at **every** slice, and **no slice ever leaves a GC cell embedding a destructor-bearing member, a live pointer into malloc-heap memory Boehm cannot scan, or a collectable value reachable only through an unscanned root.** + +The honest shape of that promise: the *immediate* leaves (bool/char/fixnum/…) and the machine-plumbing refactors are genuinely small independent slices. The *mutually-recursive heap-object set* (`Box`, `Pair`, `List`, `Vector`, `Values`, `ContinuationMarkSet`, `Closure`) plus `Scope`/`Environment` form one strongly-connected component with **no valid one-at-a-time topological order** — a not-yet-migrated container can always hold an already-migrated element and vice-versa. We keep those slices individually green not by pretending an order exists, but by erecting explicit **transition scaffolding** (a legacy pin table + a GC keep-alive root) that makes every cross-tier reference safe, then demolishing it in the scope cutover. Several slices are behavior-preserving *characterization* refactors, not red→green; they are labeled as such and each carries an explicit characterization assertion. + +--- + +## 1. Representation decision — (B): GC-backed `ValueNode`, staged toward the R1 `nr_value` ABI + +Adopt **(B)**. Keep the `ValueNode` single-inheritance hierarchy and its virtual `accept`/`getKind` dispatch, but (i) allocate every runtime value with `GC_MALLOC` via placement-new, (ii) reduce every leaf/cell to pointer-free scalars and `nr_value` words — no `unique_ptr`/`shared_ptr`/`mpz_class`/`SmallString`/`SmallVector`/`std::map` members, (iii) make `clone()` an identity/no-op so shared references replace deep copies, and (iv) delete `AllScopes` and the teardown. + +**Why not (A) (one-shot flatten to POD tagged `nr_value` words).** The value classes are welded to three subsystems that (A) would rewrite simultaneously in a single non-green cutover: the 27-way `ASTVisitor`/`dyn_cast` dispatch (`Runtime.cpp`, `valueEq`); the parser, where the *same* classes are AST literals *and* runtime values (`QuotedExpr` wraps a `ValueNode`, `Integer` is both a literal and the number type); and the tagged-immediate encoding. (B) changes **allocation and lifetime while keeping types and public API identical**, which is what keeps each slice green. + +**Why the vtable is safe under Boehm.** The vptr is written by placement-new and points into `.rodata`; it is never a GC hazard and is never traced as a heap pointer. Boehm skips destructors, which is safe **iff** every member of a GC cell is trivially destructible or itself GC-managed — the invariant the ladder enforces. `ASTNode`'s base subobject (`const Kind` + `SMRange`, i.e. two `const char*`) is already trivially destructible, so a skipped virtual dtor on a POD-of-words leaf is genuinely safe. + +**Convergence on the frozen ABI — and the crucial caveat.** Single inheritance keeps `(void*)this == (void*)base`, so the eventual flatten to `nr_value` is mechanical. **However, throughout all of M2 the GC cells remain polymorphic C++ objects: the vptr sits at offset 0, *not* an `ObjHeader{u32 type; u32 meta}`.** Reading `nr_obj(w)->type` on an M2 cell would read the low 32 bits of a vtable pointer — garbage. Therefore: + +- M2 reuses **only the immediate/tag layer** of the frozen runtime: `nr_value`, `nr_fixnum`/`nr_fixnum_val`, `nr_bool`/`nr_truthy`, `nr_char`/`nr_char_val`, the `NR_*` singleton immediates, and the tag predicates. Heap pointers stay at tag `000` (a base pointer is its own `nr_value`). +- M2 **does not** use `nrt.h`'s object entry points (`nrt_cons`/`nrt_car`/`nrt_box`/`nrt_unbox`, `NrPair`/`NrBox`/`NrClosure`). Those `assert(nr_has_type(...))` on an `ObjHeader.type` that M2 cells do not have. They are dropped from the promoted header (or gated behind a `NORA_FLAT_ABI` macro) so nobody wires `nrt_unbox` onto a polymorphic cell and reads the vptr as a type tag. Object construction/access stays C++-RTTI (`getKind`/`dyn_cast`) for the whole milestone. +- The **flatten** — mapping the vptr slot onto `ObjHeader`, `getKind`→`NrObjType`, and swapping virtual dispatch for a `type`-`switch` — is a later, ABI/perf-driven step (post-M2, due only when compiled code (B2) shares the heap), **not** a GC-correctness requirement. `docs/value-model-abi.md` §6's "M2 consumes the ABI verbatim" is corrected to: *"M2 lands GC + shared references over the existing polymorphic hierarchy, reusing the `nr_value` immediate encoding; a later slice flattens the object layout to `ObjHeader`/`nr_value` for B2 heap-sharing."* + +--- + +## 2. Invariants every slice preserves + +**GC-cell ⇒ POD-of-words.** The instant a type is `GC_MALLOC`'d, its members may be only pointer-free scalars, raw GC pointers, and `nr_value` words. Two failure modes drive everything: + +- **(L) Leak** — a skipped destructor never frees owned malloc heap (mpz limbs, `shared_ptr` control blocks, tree nodes). LSan does **not** catch this (Boehm's heap is invisible to it); the churn capstones (§5) do. +- **(D) Cross-heap dangle** — Boehm never scans malloc memory. A collectable GC value reachable *only* through malloc storage (a `unique_ptr`/`SmallVector` backing store, a `make_shared` `Scope`, an unscanned `std::map` node) is collected mid-evaluation → use-after-free. + +**Two distinct rooting mechanisms, never conflated:** + +1. **GC scanning** roots *migrated* (GC-cell) values. It works **only where the word physically lives is scanned by Boehm** — the C stack, or a container whose backing store is `gc_allocator`'d *and* whose header is itself reachable from a scanned location. +2. **A manual strong reference** roots *legacy* (malloc `ValueNode`) values regardless of where a referring word sits, because it keeps the object alive by identity, not by location. + +**`Interpreter` stays stack-resident.** Its registers (`Val`, the `Kont` header, `Result`, `Env`) are scanned only because the object lives on the scanned C stack. A `make_unique()` would move them to unscanned malloc and void the in-flight-value guarantee. A comment/`static_assert`-style note is added at its definition; if heap allocation is ever needed it must derive from `gc` (`gc_cpp.h`) or be registered as a root. + +**Ordering law.** Immediates (never allocate) and interned symbols (`GC_MALLOC_UNCOLLECTABLE`, permanent roots) can be introduced freely. The **first *collectable* value** (String) may only appear once every place it can be parked across an allocation is covered by one of the two rooting mechanisms — which is exactly why the transition scaffolding (§3) is erected in the same slice. + +--- + +## 3. The transition scaffolding (the migration vehicle) + +A stack-only handle plus two temporary roots. Everything here is scaffolding that exists only for Phases 3–5 and is **deleted in S17/S18**. + +```cpp +struct Value { // lives ONLY on the C++ stack / Kont / env / GC-cell slots — a bare word + nr_value W; // immediate | GC pointer to a migrated cell | legacy-index immediate +}; +``` + +`Value::W` is one of: +- an **immediate** (`nr_fixnum`, `nr_bool`, `nr_char`, `NR_NULL`/`NR_VOID`/`NR_EOF`/…); +- a **migrated** heap value: a GC pointer (tag `000`) to a polymorphic GC cell; +- a **legacy** value: a reserved-tag immediate (`NR_TAG_LEGACY`, the `0b100` slot the ABI marks "reserved") carrying an index into the legacy pin table. Because it is a non-pointer word, Boehm ignores it, so it is safe in *any* storage — a register, the malloc env map, or a migrated GC cell's slot. + +**The two scaffolding roots (erected in S10, demolished in S17):** + +- **Legacy pin table** — `std::deque>`, an `Interpreter` member (a plain RAII container). It owns every still-malloc `ValueNode` and destroys each **exactly once** at interpreter teardown. This is the *manual strong reference* of §2: a legacy value referenced from anywhere — including a GC cell's word slot — stays alive and is freed once. It does **not** need to be GC-scanned. +- **GC keep-alive root** — `std::vector>`, an `Interpreter` member. Its header is on the scanned stack, so Boehm traces its buffer. Whenever a **migrated** (GC-pointer) word is written into storage Boehm cannot scan — a still-legacy container's slot, or the still-malloc env map — that word is appended here. This is the *GC scanning* mechanism of §2, extended to reach malloc-resident words. It holds no destructors and is reclaimed wholesale (its cells too) once the scaffolding is gone. + +Together these dissolve the SCC's two hazard directions: a **migrated cell holding a legacy element** is safe (legacy-index word + pin table), and a **legacy container (or malloc env map) holding a migrated element** is safe (keep-alive root). This is what lets each container flip to a GC cell as its own individually-green slice with no valid topological order and no `intoCell`-to-word bridge for unmigrated types (which cannot exist). + +**Cost, and why it is acceptable.** The scaffolding over-retains: legacy values and any migrated value that ever entered malloc storage live until teardown, so memory is O(work) during Phases 3–5. That is correct (no UAF, no double-free, no LSan leak — everything is freed once at teardown), and it does not pollute the capstones, which are deliberately written over already-migrated, non-parked value garbage (§5). The final slices delete the scaffolding, at which point memory is bounded by the GC. + +In Phases 1–2, before any GC cell can hold a `Value`, the legacy alternative is simply materialized at the boundary (a fresh RAII `ValueNode` *view* owned by the still-legacy container, or a `shared_ptr` inside the stack-resident `Value`); the pin-index encoding is switched on in S10 when the first GC cell that can hold an arbitrary `Value` approaches. + +--- + +## 4. Slice ladder + +Legend: **RED** = a genuine failing test drives the slice; **CHAR** = behavior-preserving refactor guarded by an explicit characterization assertion. + +### Phase 0 — Collector up; scan the roots that are already reachable + +**S0 — libgc linked, `GC_INIT`, heap hooks. (RED — link failure.)** *Detailed in §7.* +Promote `spike/r1-value-model/nrt.{h,cpp}` → `src/nora_rt.{h,cpp}`, **stripping/gating the `ObjHeader`-based object accessors** (§1) and **removing `GC_set_all_interior_pointers(1)`** from `nrt_init`. Wire `PkgConfig::BDWGC` into `src/CMakeLists.txt` and both test exes; `GC_INIT()` first in `main`; convert both test mains to `CATCH_CONFIG_RUNNER`; expose `getGCHeapSize()`/`getGCTotalBytes()`. RED: `nr_*`/`GC_*` unresolved. Depends-on: —. + +**S1 — `gc_allocator` the containers whose headers are already scanned. (CHAR.)** +`Kont` (`std::vector`, `Interpreter` member → header on the stack), `Frame::Done`, `Frame::Marks`, and `ContinuationMarkSet::Frames` (which are traced transitively once they sit inside the scanned `Kont` buffer). Element *types* are unchanged (still `unique_ptr`/`shared_ptr`); only the allocator changes, so the buffers are now scanned while elements are still RAII-destructed normally by their stack-resident owners. +**The `Environment` map is deliberately NOT touched here.** Its header lives inside a `Scope` created by `make_shared` (malloc, `Interpreter.cpp:44`); `gc_allocator`-ing its nodes would move the red-black-tree nodes to GC memory whose only inbound edge (`std::map::_M_header`) lives in unscanned malloc — an unrooted collectable component that `gc_allocator`'s own `GC_MALLOC` could reclaim mid-run → UAF in `envLookup`. The env map becomes GC storage only in S17, in the same slice its owning `Scope` becomes a scanned GC cell. Characterization: full suite + a debug assert that `Kont.data()` is a GC pointer. `GC_add_roots` is rejected (vectors relocate on growth); interior-pointer scanning is rejected (§6, R6). Depends-on: S0. + +### Phase 1 — Machine speaks `Value` (100% legacy inside) + +**S2 — `Value` in registers. (CHAR.)** `Val`, `Result`, `getResult()` internals → `Value`; legacy alternative held RAII inside the stack-resident handle. Assertion: whole suite; a legacy `Value` is behaviorally identical to today's `unique_ptr`. Depends-on: S1. + +**S3 — `Value` in `Environment`; kill clone-on-lookup. (CHAR + new aliasing test.)** `add`/`lookup`/`envLookup`/`envSet` store and return `Value`; lookup **shares** instead of `clone()`ing. Sharing an immutable value equals cloning it; mutable values (`Box`/`Pair`) already share via their inner cell. Audit (must all be pinned by an assertion): `+`/`-`/`*` build fresh accumulators; `SubtractFunction` clones the first arg before `-=` (`Runtime.cpp:52`); no primitive mutates a looked-up value in place; **argument aliasing** — `(f x x)` now passes one shared object twice (correct for Racket — pin it); the WCM key/value paths. Standing guard adopted here for the *rest of the milestone*: at the moment **any** type becomes shared, add a "mutate through one reference, observe through another" test (template: the existing `set-box!`/`set-car!` tests). Depends-on: S2. + +**S4 — `Value` in frames. (CHAR.)** `Frame::{Done,Saved,Callee,WcmKeyV}` and `MarkFrame` entries → `Value`. Depends-on: S3. + +**S5 — Test-seam helpers. (CHAR/refactor.)** Add `expectInt(Run,42)`, `expectBool(Run,true)`, `expectResult(...)` wrapping today's `dyn_cast` assertions in `test/unit/test_interpreter.cpp`, localizing the eventual seam flip to one place. Depends-on: S2. + +### Phase 2 — Immediates (no allocation, no roots) + +**S6 — Booleans → `NR_TRUE`/`NR_FALSE`. (CHAR; forcing: `#f` result `== NR_FALSE`.)** `IfBranch` reads truthiness via `nr_truthy`; `getResult()`/`write()` materialize a `BooleanLiteral` view at the public seam. Depends-on: S4, S5. + +**S7 — Char / Void / Null / eof → immediates. (CHAR.)** Same pattern; retires `Char`'s `SmallString<8>` (`AST.h:435`). Guarded by char/void `.rkt` tests + the seam shims. Depends-on: S6. + +**S8 — Fixnums → `nr_fixnum`. (RED — arithmetic/overflow.)** In-range `Integer` becomes an immediate; `+`,`-`,`*`,`zero?` get a word fast path with `__builtin_*_overflow`; **overflow promotes to a still-legacy `ast::Integer` bignum** (unchanged malloc object via the legacy path). RED driver: the deep tail-loop now runs on immediate arithmetic, plus an overflow-promotion test. Depends-on: S7. + +### Phase 3 — Erect the scaffolding; collectable leaves + +**S9 — Symbol → interned `GC_MALLOC_UNCOLLECTABLE` cell. (CHAR + identity tests.)** Name via `GC_MALLOC_ATOMIC_UNCOLLECTABLE`; `eq?` = pointer identity. Deletes `Symbol::Uninterned` (`AST.h:235`) and the static `unordered_set InternTable` (`AST.cpp:96`); `gensym`/`string->uninterned-symbol` = a distinct GC symbol whose address is its identity. Interned symbols are *permanent, uncollectable roots*, so binding one anywhere needs no env scanning and nothing leaks (Boehm's uncollectable list roots them; LSan cannot see them). Guarded by `symbol eq? is identity`, `gensym`, and uninterned tests. Depends-on: S8. + +**S10 — Erect the scaffolding + String/Keyword → GC-atomic cells. (RED — `eq?`-on-string + GC-survival.)** Switch the legacy `Value` representation to the pin-index encoding and stand up the **legacy pin table** and the **GC keep-alive root** (§3). Make `String`/`Keyword` `ObjHeader`-less GC cells (`len` + `GC_MALLOC_ATOMIC` bytes), retiring their `SmallString`s. **This is the first *collectable* value:** every collectable word written into the still-malloc env map or a still-legacy container is now registered in the keep-alive root, so it survives a collection. RED drivers: a new `eq?`-on-string test documenting the intended post-migration pointer identity (NORA currently routes `String` through *structural* `valueEq`, so the flip is otherwise invisible), and a "bind a string, run a GC-forcing expression, read the string back" test under asan. Depends-on: S9, S1. + +**S11 — Bignum `Integer` → GC cell over GC-atomic `mpz_t`; install the GMP hook. (RED — bignum-through-GC + `test_parse` stays leak-clean.)** In one slice: make `Integer` a `GC_MALLOC` (scanned) cell holding a **raw `mpz_t`**, drop `~Integer`/`mpz_clear` (`AST.cpp:178`), and install `mp_set_memory_functions(gmp_alloc, gmp_realloc, gmp_free)` routing limbs through `GC_MALLOC_ATOMIC`, with the custom free being **`GC_FREE` or a no-op — never system `free`** (an `mpz` allocated before the hook and freed after would otherwise cross allocators and crash). The hook is process-global and must be installed **before the first GMP allocation in all three entry points** — `norac` `main` **and both** `CATCH_CONFIG_RUNNER` test mains, including `test_parse`, which constructs `Integer` literals at parse time and links `gmpxx`; if its main omits the hook, dropping `mpz_clear` leaks limbs and `test_parse` (which does not disable LSan) turns red. Verify no `static`/global `Integer` and no `gmpxx` temporary predates the install. Bundling is mandatory: installing the hook while `Integer` is still a malloc object would let Boehm collect limbs referenced only from unscanned memory (D). The scanned cell follows the (atomic, unscanned) limb pointer, so no false retention. Fold in the remaining leaves (`RuntimeFunction` index, `VariableReference`) as POD cells/immediates. After S11 **every leaf is a word.** Depends-on: S10. + +### Phase 4 — Containers become GC cells (each individually green via the scaffolding) + +Every container flips to a `GC_MALLOC`'d **polymorphic** C++ cell (vptr at offset 0, safe under Boehm; *not* `nrt.h`'s `NrPair`/`NrBox` — §1) whose slots are `nr_value` words. Cross-tier references are safe by construction: legacy elements ride the pin-index + pin table; migrated words stored into any still-legacy container ride the keep-alive root. No GC cell embeds a `unique_ptr`/`shared_ptr`/`mpz`/`SmallString`. + +**S12 — Box → GC cell. (CHAR + `set-box!` identity.)** Single `nr_value` slot; retires `shared_ptr` (`ASTRuntime.h:93`). Shared identity falls out of pointer identity. Depends-on: S11. + +**S13 — Pair → GC cell. (RED — value-garbage capstone, §5.)** The hot allocation site; retires `shared_ptr` (`ASTRuntime.h:127`). The capstone loop churns *transient, unbound* `cons`/`box` garbage while binding **only immediates** (`n`, `acc` are fixnums), so nothing it produces is parked in the env or a legacy container and thus nothing is held by the keep-alive root — the garbage is genuinely collectable and the GC heap plateaus. RED before this slice (malloc `shared_ptr` cells → GC churn ≈ 0), GREEN after. Guarded also by `set-car!`/`set-cdr!` identity tests. Depends-on: S12. + +**S14 — Values / List / Vector → variable-length GC cells. (CHAR.)** Length in a POD field; elements already words. Retires their `SmallVector`s. `quote8.rkt`'s `'((1 2 3) #("z" x) . the-end)` — a `List` of a `List` and a `Vector`-of-`String`+`Symbol` — crosses tiers freely and stays green because the scaffolding roots both directions. Depends-on: S13. + +**S15 — ContinuationMarkSet / MarkFrame → GC cells. (CHAR + WCM-across-GC test.)** Elements are words. Guard: a `with-continuation-mark` whose result expression forces a collection, then the mark is read back (R4; the existing `with-continuation-mark{1..6}.rkt` do not force a collection). Depends-on: S14. + +### Phase 5 — Closures, the scope cutover, demolish the scaffolding + +**S16 — Closure / CaseLambdaClosure → flat GC cells. (CHAR + GC-mid-loop correctness.)** Captures are words; the body points at the **shared, immortal AST `Lambda`** owned by `main`'s `unique_ptr` (which outlives all evaluation), retiring `unique_ptr` (`ASTRuntime.h:36`) and `unique_ptr` (`ASTRuntime.h:58`). This makes `Frame::Call::Callee` no longer load-bearing for `Control`-into-body validity, so its slot is dropped. **No intermediate may exist where the closure is a GC cell while its body is still a per-closure `new`'d clone** — Boehm never scans that clone, so a `const ASTNode* Control` interior pointer into it would dangle on collection (and the clone would leak, dtor skipped). Point at the shared AST in the *same* slice that GC-allocates the closure. Guards: peak-Kont tests (`Deep == Shallow < 16`) stay green; a mutual-tail-recursion loop that forces a GC mid-loop and asserts the result, under asan+LSan. Depends-on: S15. + +**S17 — The scope cutover (coupled by design) + demolish the scaffolding. (RED — scope-garbage capstone.)** This slice is *deliberately* one atomic step because its parts are interlocked: +- `Scope` → GC cell; `Parent` becomes a raw GC pointer. +- `Environment`'s map nodes → `gc_allocator`. This is now safe (unlike S1): the GC-cell `Scope` holding the map header is itself scanned, so the header, the tree nodes, and their `Value` words are all traced. The mapped type is a POD `nr_value` word and the key `ast::Identifier` is trivially destructible, so **a GC `Scope` whose `~map` never runs leaks nothing** — the dtor-leak trap is closed by construction, not by requiring "all values migrated first." +- Atomically delete `AllScopes` (`Interpreter.h:197`), `newScope`'s accumulation (`Interpreter.cpp:47`), and `~Interpreter`'s cycle-break (`Interpreter.cpp:31-42`). Going straight to `GC_MALLOC` (not `allocate_shared`) avoids the split-brain where a `shared_ptr` scope is reachable only through the malloc `AllScopes` buffer that Boehm frees under it. Boehm now reclaims the closure↔scope cycles that the teardown used to sever by hand. +- **Demolish the scaffolding:** every value type is now a word or GC cell, so the legacy pin table is empty and is deleted; the env map and all containers are now scanned, so the GC keep-alive root is no longer needed and is deleted. +- Add `"environment": { "ASAN_OPTIONS": "detect_leaks=0" }` to the `asan`/`ubsan` `testPresets` (the GC heap is now intentionally never torn down). + +RED driver: the scope-garbage capstone (§5) — a deep tail loop creating one scope per iteration — shows O(depth) heap while `AllScopes` roots every scope, and a bounded, depth-independent heap once scopes are GC and `AllScopes` is gone. Depends-on: S16. + +**S18 — Collapse the seam; delete `clone()`. (CHAR/refactor, suite updated same commit.)** Rewrite the S5 helpers and `getResult()`/`write()` to the `nr_value` seam; delete the `Value` legacy path and pin-index materialization, `ValueNode::clone()`, and every `ClonableNode` override. End state: `clone()`/`AllScopes`/scaffolding all gone, fully GC'd value model over the reused `nr_value` immediate encoding. Depends-on: S17. + +--- + +## 5. The forcing GC-heap seam — heap-size hook, not RSS + +Expose `getGCHeapSize()`/`getGCTotalBytes()` (`GC_get_heap_size`/`GC_get_total_bytes`) at the interpreter unit seam (mirroring the existing `getPeakKont()` at `Interpreter.h:81`) and assert a **depth-independent live-heap plateau against unbounded churn**. RSS is rejected: it is a monotonic high-water peak (cannot observe in-process reclamation), is polluted by the LLVM/binary footprint and glibc arenas, is meaningless under ASan shadow memory (so it cannot run in the sanitizer presets), and is contaminated by sibling Catch tests in one process. The GC-heap ratio is deterministic, preset-robust, and isolated to GC bytes. + +**Two distinct capstones, each scoped to the garbage it can actually observe:** + +*Value-garbage capstone (S13).* The loop churns transient, **unbound** `cons`/`box` garbage while binding only immediates, so the scaffolding's keep-alive root never retains it — it is genuinely collectable. Because the env map and scopes are still malloc through Phase 4 (and thus invisible to `getGCHeapSize`), this capstone measures **value** garbage only. + +*Scope-garbage capstone (S17).* A deep tail loop creating one scope per iteration; asserts `getGCHeapSize()` bounded and depth-independent once scopes are GC and `AllScopes` is deleted. + +This resolves the earlier heap-accounting contradiction: during Phases 3–5 the scaffolding intentionally holds env-bound and parked values (memory is O(work) by design), so the value capstone must **not** assert a globally bounded heap — only that its own transient garbage is reclaimed. Global bounded memory is asserted only at the scope capstone. + +**Robust assertion shape** (avoid magic-number flake): + +```cpp +static size_t liveHeapAfter(long depth) { + Interpreter I(Diag); + (void)GC_malloc(1 << 16); // warm-up: don't let the initial heap block dominate + AST_for(depth)->accept(I); + GC_gcollect(); // force a collection before sampling + return I.getGCHeapSize(); +} +// Assert a PLATEAU across two LARGE depths, holding machine-structure constant, +// rather than anchoring to a small baseline: +auto hA = liveHeapAfter(1'000'000); +auto hB = liveHeapAfter(8'000'000); // 8x the work +REQUIRE(hB < hA * 2); // 8x work, < 2x live heap => reclamation happened +``` + +Keep the existing `norac` end-to-end deep-loop `.rkt`/FileCheck as a coarse no-OOM backstop. + +--- + +## 6. Risks and guards + +- **R1 — Unscanned parking storage → mid-eval UAF.** In-flight registers are stack-scanned, but values parked between steps live in container storage Boehm never sees. *Guard:* S1 scans the stack-header-rooted containers (`Kont`/`Done`/`Marks`/CMS); the **env map is not scanned until S17** (scanning its nodes while its header is in malloc is itself the UAF — see S1); collectable values parked in still-malloc storage between S10 and S17 are held by the GC keep-alive root. Plus a GC-forcing many-argument `App`/`values` correctness test under asan. +- **R2 — Closure body vs. `Control` interior pointer.** *Guard:* S16 repoints closures at the shared immortal AST in the same slice it GC-allocates them — never a mixed intermediate; mutual-tail-recursion GC-mid-loop test under asan+LSan. +- **R3 — Scope-GC / `AllScopes` deletion interlock.** Removing `AllScopes` before scopes are GC reintroduces the `shared_ptr` cycle leak; GC-allocating a `Scope` whose map still held `shared_ptr` values would leak. *Guard:* S17 does all of it atomically, and the POD `Value`-word map makes a GC `Scope` dtor-free. +- **R4 — Continuation-mark value collected from an unscanned frame.** *Guard:* S15 GC-cells the marks; a set-mark → GC-forcing-result → read-mark test. +- **R5 — Silent `eq?`/identity drift when a type moves from clone to shared.** *Guard:* the S10 `eq?`-on-string test pins the intended identity before the flip; the standing "mutate through one reference, observe through another" guard is applied to **every** type at the slice it becomes shared, and the S3 audit explicitly covers argument aliasing (`(f x x)`) and the WCM key/value paths. +- **R6 — `GC_set_all_interior_pointers(1)` (spike default) worsens false retention** and never makes malloc memory scanned. Heap pointers stay at tag `000`; `Control` interior pointers are into the *non-GC* immortal AST. *Guard:* S0 drops it. +- **R7 — Heap-allocated `Interpreter` unscans the registers.** *Guard:* keep it stack-resident (it already is everywhere); note at its definition. +- **R8 — Half-migrated destructor leak (a GC cell embedding `unique_ptr`/`mpz`/`shared_ptr`), invisible to LSan.** *Guard:* the POD-of-words invariant, the pin/keep-alive scaffolding (so a GC cell never embeds a legacy owner), and the S13/S17 `GC_gcollect()` heap-ratio tests, which surface any residual leak as an unbounded live heap. Reviewers verify specifically S11 (drop `mpz_clear` only with the GMP-hook flip, in all three mains, with a GC-safe free) and S16 (drop `unique_ptr` only with the shared-AST repoint). +- **R9 — Boehm/LLVM signal-handler and threading interaction.** `GC_INIT()` runs before `llvm::InitLLVM` (which installs signal handlers). Keep Boehm **non-incremental** (the default) so there is no `SIGSEGV` dirty-bit handler for LLVM to clobber; document this constraint (it also bears on the future `call/cc` note). +- **Future (out of scope).** `call/cc`/delimited continuations may introduce real C++ exceptions or stack copying, reopening Boehm↔unwinder and stack-scanning questions. Today the eval path has no exceptions (errors go through `Diag.error` + `abortEval`'s manual `Kont.erase`), which is GC-friendly. + +--- + +## First slice: do this now — S0 + +**Failing test first** (`test/unit/test_gc.cpp`; it exercises only the reusable *immediate* ABI layer and the collector — never a polymorphic-cell object accessor): + +```cpp +#include +#include "nora_rt.h" +#include + +TEST_CASE("libgc links, inits, and the nr_value immediate ABI is usable", "[m2][gc]") { + REQUIRE(nr_fixnum_val(nr_fixnum(42)) == 42); // RED: nora_rt / libgc not linked yet + REQUIRE(nr_truthy(nr_bool(true))); + REQUIRE_FALSE(nr_truthy(NR_FALSE)); + void *p = GC_MALLOC(64); // exercise the collector + REQUIRE(p != nullptr); + REQUIRE(GC_get_heap_size() > 0); +} +``` + +New shared test main (both existing mains switch to it): + +```cpp +#define CATCH_CONFIG_RUNNER +#include +#include +int main(int argc, char **argv) { + GC_INIT(); // records the main-thread stack bottom; GMP hook is S11 + return Catch::Session().run(argc, argv); +} +``` + +**Minimal implementation:** + +1. **Promote the runtime.** `git mv spike/r1-value-model/nrt.{h,cpp}` → `src/nora_rt.{h,cpp}`, keeping the `nr_*` immediate/tag layer. **Strip or `#ifdef NORA_FLAT_ABI`-gate the `ObjHeader`-based object entry points** (`nrt_cons`/`nrt_car`/`nrt_box`/`nrt_unbox`, `NrPair`/`NrBox`/`NrClosure`) so they cannot be wired onto M2's polymorphic cells. In `nrt_init`, **remove `GC_set_all_interior_pointers(1)`**. +2. **CMake.** In the root `CMakeLists.txt` (beside the LLVM/GMP `find_package`): + ```cmake + find_package(PkgConfig REQUIRED) + pkg_check_modules(BDWGC REQUIRED IMPORTED_TARGET bdw-gc) # verified: bdw-gc 8.2.12 present + ``` + `src/CMakeLists.txt` — add `PkgConfig::BDWGC` to `LIBS` and add `nora_rt.cpp` to the `norac` sources. `test/unit/CMakeLists.txt` — add `${PROJECT_SOURCE_DIR}/src/nora_rt.cpp` to **both** exes and append `PkgConfig::BDWGC` to both `target_link_libraries` (`test_parse` compiles `AST.cpp`, which routes GMP through GC in S11) and add `test_gc.cpp` to the `test_interpreter` sources. +3. **`GC_INIT()` placement.** `src/main.cpp` — first statement of `main()`, before `llvm::InitLLVM` and `Parse::parseLinklet`. Convert `test/unit/test_interpreter.cpp:1` and `test/unit/test_parse.cpp:2` from `CATCH_CONFIG_MAIN` to `CATCH_CONFIG_RUNNER` with the shared `main` above, so `GC_INIT()` runs on the main thread before any test allocates. (The GMP hook is **not** added here — it lands in S11 in all three mains.) +4. **Heap hooks** on `Interpreter` (test-only, beside `getPeakKont()`): + ```cpp + size_t getGCHeapSize() const { return GC_get_heap_size(); } + size_t getGCTotalBytes() const { return GC_get_total_bytes(); } + ``` +5. **CI.** Add `libgc-dev` to the apt install lists in `.github/workflows/`. No new preset — GC is a hard dependency of the default build (unlike opt-in MLIR), inherited by every preset via `LIBS`. + +**Acceptance:** the whole suite (both Catch2 mains + the integration corpus) green under `debug`/`asan`/`ubsan`. If LSan reports Boehm-internal allocations under `asan`, add `ASAN_OPTIONS=detect_leaks=0` to the `asan`/`ubsan` `testPresets` immediately; otherwise defer that flag to S17. Proving Boehm ↔ sanitizer ↔ `InitLLVM` coexistence here, before any value depends on the collector, is the entire point of doing S0 first. \ No newline at end of file diff --git a/spike/r1-value-model/.gitignore b/spike/r1-value-model/.gitignore new file mode 100644 index 0000000..0a69395 --- /dev/null +++ b/spike/r1-value-model/.gitignore @@ -0,0 +1,2 @@ +spike +spike-asan diff --git a/spike/r1-value-model/README.md b/spike/r1-value-model/README.md new file mode 100644 index 0000000..a8a0758 --- /dev/null +++ b/spike/r1-value-model/README.md @@ -0,0 +1,36 @@ +# Spike R1 — value model / GC ABI + +Throwaway proof-of-concept for issue #88. **Not part of the `norac` build.** It +freezes the shared runtime value representation that milestone M2 (interpreter) +and B0/B2 (compiler + `libnora_rt`) both consume. The design it validates is +documented in [`docs/value-model-abi.md`](../../docs/value-model-abi.md). + +## What it proves + +1. **One representation, two drivers.** A tiny tree-walking interpreter + (`interp_loop`) and a statically compiled C++ function (`compiled_loop`, + standing in for codegen output) compute the identical result while calling + the *same* runtime entry points (`nrt.h`) on the *same* heap. +2. **Flat closures work in both.** A closure capturing a value is applied via + the same `nrt_apply` from the interpreter and from compiled code. +3. **Boehm GC keeps a garbage loop bounded.** ~2 allocations/iteration for tens + of millions of iterations churn multiple GiB through a sub-MiB live heap. + +## Run + +``` +make # build with pkg-config bdw-gc +./spike # defaults: 5,000,000 interp iters; 50,000,000 GC-pressure iters +./spike 2000000 20000000 +make asan # AddressSanitizer build (LSan off; the GC heap is intentionally "leaked") +``` + +Expected tail: `ALL PROOFS PASSED`. + +## Files + +- `nrt.h` — the frozen ABI (tagged `nr_value`, `ObjHeader`, flat `NrClosure`, + `nr_code` signature) + runtime API. +- `nrt.cpp` — runtime over Boehm GC (constructors, `eq?`, interned symbols, + fixnum arithmetic, flat-closure apply). +- `spike.cpp` — mini interpreter, compiled-equivalent, and the two proofs. diff --git a/spike/r1-value-model/nrt.cpp b/spike/r1-value-model/nrt.cpp new file mode 100644 index 0000000..957571a --- /dev/null +++ b/spike/r1-value-model/nrt.cpp @@ -0,0 +1,152 @@ +// R1 spike — runtime implementation over the Boehm-Demers-Weiser collector. +#include "nrt.h" + +#include +#include +#include +#include +#include + +#include + +void nrt_init(void) { + GC_INIT(); + // Deterministic behaviour for the spike; Boehm still collects on demand. + GC_set_all_interior_pointers(1); +} + +// --- allocation helpers ---------------------------------------------------- +static void *alloc(size_t n) { + void *p = GC_MALLOC(n); // zero-filled, scanned for pointers + assert(p && "GC_MALLOC returned null"); + assert(((uintptr_t)p & NR_TAG_MASK) == 0 && "heap object not 8-aligned"); + return p; +} + +// --- pairs ----------------------------------------------------------------- +nr_value nrt_cons(nr_value a, nr_value d) { + auto *p = (NrPair *)alloc(sizeof(NrPair)); + p->h = {OBJ_PAIR, 0}; + p->car = a; + p->cdr = d; + return (nr_value)p; +} +nr_value nrt_car(nr_value p) { + assert(nr_has_type(p, OBJ_PAIR) && "car: not a pair"); + return ((NrPair *)p)->car; +} +nr_value nrt_cdr(nr_value p) { + assert(nr_has_type(p, OBJ_PAIR) && "cdr: not a pair"); + return ((NrPair *)p)->cdr; +} + +// --- boxes ----------------------------------------------------------------- +nr_value nrt_box(nr_value v) { + auto *b = (NrBox *)alloc(sizeof(NrBox)); + b->h = {OBJ_BOX, 0}; + b->val = v; + return (nr_value)b; +} +nr_value nrt_unbox(nr_value b) { + assert(nr_has_type(b, OBJ_BOX) && "unbox: not a box"); + return ((NrBox *)b)->val; +} +void nrt_set_box(nr_value b, nr_value v) { + assert(nr_has_type(b, OBJ_BOX) && "set-box!: not a box"); + ((NrBox *)b)->val = v; +} + +// --- symbols (interned; permanent, so uncollectable) ----------------------- +static std::unordered_map &intern_table() { + static std::unordered_map t; + return t; +} +nr_value nrt_intern(const char *name) { + auto &t = intern_table(); + std::string key(name); + auto it = t.find(key); + if (it != t.end()) + return it->second; + // Interned symbols live forever: allocate uncollectable so the intern table + // (a plain malloc'd container the GC does not scan) cannot dangle. + auto *s = (NrSymbol *)GC_MALLOC_UNCOLLECTABLE(sizeof(NrSymbol)); + size_t n = key.size() + 1; + char *buf = (char *)GC_MALLOC_ATOMIC_UNCOLLECTABLE(n); + memcpy(buf, name, n); + s->h = {OBJ_SYMBOL, 1}; + s->name = buf; + s->hash = std::hash{}(key); + nr_value w = (nr_value)s; + t.emplace(std::move(key), w); + return w; +} + +// --- closures (flat capture) ---------------------------------------------- +nr_value nrt_make_closure(nr_code code, uint32_t nfree, const nr_value *freev) { + auto *c = + (NrClosure *)alloc(sizeof(NrClosure) + (size_t)nfree * sizeof(nr_value)); + c->h = {OBJ_CLOSURE, nfree}; + c->code = code; + c->nfree = nfree; + c->pad = 0; + for (uint32_t i = 0; i < nfree; ++i) + c->free[i] = freev[i]; + return (nr_value)c; +} +nr_value nrt_apply(nr_value clos, int64_t argc, const nr_value *argv) { + assert(nr_has_type(clos, OBJ_CLOSURE) && "apply: not a closure"); + return ((NrClosure *)clos)->code(clos, argc, argv); +} + +// --- fixnum arithmetic (spike: assert instead of promoting to bignum) ------ +nr_value nrt_fx_add(nr_value a, nr_value b) { + assert(nr_is_fixnum(a) && nr_is_fixnum(b)); + int64_t r; + bool ovf = __builtin_add_overflow(nr_fixnum_val(a), nr_fixnum_val(b), &r); + assert(!ovf && "fixnum add overflow (bignum promotion is M4, not R1)"); + (void)ovf; + return nr_fixnum(r); +} +nr_value nrt_fx_sub(nr_value a, nr_value b) { + assert(nr_is_fixnum(a) && nr_is_fixnum(b)); + int64_t r; + bool ovf = __builtin_sub_overflow(nr_fixnum_val(a), nr_fixnum_val(b), &r); + assert(!ovf && "fixnum sub overflow"); + (void)ovf; + return nr_fixnum(r); +} + +// --- debug printer --------------------------------------------------------- +void nrt_write(nr_value w) { + if (nr_is_fixnum(w)) { + printf("%lld", (long long)nr_fixnum_val(w)); + } else if (nr_is_char(w)) { + printf("#\\%u", nr_char_val(w)); + } else if (w == NR_FALSE) { + printf("#f"); + } else if (w == NR_TRUE) { + printf("#t"); + } else if (w == NR_NULL) { + printf("()"); + } else if (w == NR_VOID) { + printf("#"); + } else if (nr_has_type(w, OBJ_PAIR)) { + printf("("); + nrt_write(nrt_car(w)); + printf(" . "); + nrt_write(nrt_cdr(w)); + printf(")"); + } else if (nr_has_type(w, OBJ_BOX)) { + printf("#&"); + nrt_write(nrt_unbox(w)); + } else if (nr_has_type(w, OBJ_SYMBOL)) { + printf("%s", ((NrSymbol *)w)->name); + } else if (nr_has_type(w, OBJ_CLOSURE)) { + printf("#"); + } else { + printf("#<0x%llx>", (unsigned long long)w); + } +} + +size_t nrt_gc_heap_size(void) { return GC_get_heap_size(); } +size_t nrt_gc_total_bytes(void) { return GC_get_total_bytes(); } diff --git a/spike/r1-value-model/nrt.h b/spike/r1-value-model/nrt.h new file mode 100644 index 0000000..6e7f8e6 --- /dev/null +++ b/spike/r1-value-model/nrt.h @@ -0,0 +1,173 @@ +// R1 spike — NORA runtime value ABI (nr_value) + Boehm-GC heap. +// +// This is the *frozen* representation that milestone M2 (interpreter value +// model) and B0/B2 (NIR concrete types + libnora_rt) are both meant to consume, +// so that the tree-walking interpreter and statically compiled code share one +// heap and one object layout. See docs/value-model-abi.md for the rationale. +// +// Throwaway spike: standalone, not wired into the norac build. +#ifndef NRT_H +#define NRT_H + +#include +#include + +// --------------------------------------------------------------------------- +// nr_value: a tagged 64-bit word. +// +// bit 0 == 1 -> fixnum (value = (int64)w >> 1; 63-bit) +// low 3 bits == 0b000 (w != 0) -> heap pointer (8-byte-aligned ObjHeader*) +// low 3 bits == 0b010 -> singleton immediate (subtype in w >> 3) +// low 3 bits == 0b110 -> character (codepoint in w >> 3) +// low 3 bits == 0b100 -> reserved +// +// Fixnums are odd, so they never collide with the even-tagged immediates or the +// 8-aligned heap pointers. Heap pointers keep tag 000 so an ObjHeader* is its +// own nr_value with no masking on dereference. +// --------------------------------------------------------------------------- +typedef uint64_t nr_value; + +static constexpr uint64_t NR_TAG_MASK = 0x7; +static constexpr uint64_t NR_TAG_PTR = 0x0; // heap pointer +static constexpr uint64_t NR_TAG_FIX = 0x1; // fixnum (any odd word) +static constexpr uint64_t NR_TAG_IMM = 0x2; // singleton immediate +static constexpr uint64_t NR_TAG_CHR = 0x6; // character + +// Singleton immediates: subtype << 3 | NR_TAG_IMM. +enum NrImm : uint64_t { + NR_IMM_FALSE = 0, + NR_IMM_TRUE = 1, + NR_IMM_NULL = 2, // '() + NR_IMM_VOID = 3, // (void) + NR_IMM_EOF = 4, + NR_IMM_UNDEF = 5, // unsafe-undefined sentinel (729x in the expander) + NR_IMM_UNINIT = 6, // letrec pre-initialisation hole +}; + +#define NR_MK_IMM(sub) (((uint64_t)(sub) << 3) | NR_TAG_IMM) +static constexpr nr_value NR_FALSE = NR_MK_IMM(NR_IMM_FALSE); +static constexpr nr_value NR_TRUE = NR_MK_IMM(NR_IMM_TRUE); +static constexpr nr_value NR_NULL = NR_MK_IMM(NR_IMM_NULL); +static constexpr nr_value NR_VOID = NR_MK_IMM(NR_IMM_VOID); +static constexpr nr_value NR_EOF = NR_MK_IMM(NR_IMM_EOF); +static constexpr nr_value NR_UNDEF = NR_MK_IMM(NR_IMM_UNDEF); +static constexpr nr_value NR_UNINIT = NR_MK_IMM(NR_IMM_UNINIT); + +// --- immediate predicates / (un)boxing ------------------------------------- +static inline bool nr_is_fixnum(nr_value w) { return (w & NR_TAG_FIX) != 0; } +static inline bool nr_is_ptr(nr_value w) { + return w != 0 && (w & NR_TAG_MASK) == NR_TAG_PTR; +} +static inline bool nr_is_imm(nr_value w) { + return (w & NR_TAG_MASK) == NR_TAG_IMM; +} +static inline bool nr_is_char(nr_value w) { + return (w & NR_TAG_MASK) == NR_TAG_CHR; +} + +static inline nr_value nr_fixnum(int64_t v) { + return (nr_value)((uint64_t)v << 1) | NR_TAG_FIX; +} +static inline int64_t nr_fixnum_val(nr_value w) { + return (int64_t)w >> 1; // arithmetic shift keeps the sign +} +static inline nr_value nr_char(uint32_t cp) { + return ((nr_value)cp << 3) | NR_TAG_CHR; +} +static inline uint32_t nr_char_val(nr_value w) { return (uint32_t)(w >> 3); } +static inline nr_value nr_bool(bool b) { return b ? NR_TRUE : NR_FALSE; } + +// Racket truthiness: only #f is false. +static inline bool nr_truthy(nr_value w) { return w != NR_FALSE; } + +// --------------------------------------------------------------------------- +// Heap objects. Every heap object begins with an 8-byte ObjHeader so payloads +// stay 8-aligned. `meta` carries a length / arity / flags per type. (A precise +// GC would also live here; the spike uses Boehm conservative GC and needs only +// the type tag — see the ABI doc.) +// --------------------------------------------------------------------------- +enum NrObjType : uint32_t { + OBJ_PAIR = 1, + OBJ_BOX, + OBJ_CLOSURE, + OBJ_SYMBOL, +}; + +struct ObjHeader { + uint32_t type; // NrObjType + uint32_t meta; // length / arity / flags +}; + +struct NrPair { + ObjHeader h; + nr_value car; + nr_value cdr; +}; + +struct NrBox { + ObjHeader h; + nr_value val; +}; + +// A flat closure: header + code pointer + inline captured cells. This layout is +// byte-for-byte what B2's compiled closures use, and `nr_code` is exactly B2's +// planned code signature (self is the closure; it reads free[] for captures). +typedef nr_value (*nr_code)(nr_value self, int64_t argc, const nr_value *argv); + +struct NrClosure { + ObjHeader h; + nr_code code; + uint32_t nfree; + uint32_t pad; + nr_value free[]; // flexible array member, 8-aligned +}; + +struct NrSymbol { + ObjHeader h; + const char *name; + uint64_t hash; +}; + +static inline ObjHeader *nr_obj(nr_value w) { return (ObjHeader *)w; } +static inline bool nr_has_type(nr_value w, NrObjType t) { + return nr_is_ptr(w) && nr_obj(w)->type == t; +} + +// --------------------------------------------------------------------------- +// Runtime API (implemented in nrt.cpp). Deliberately identical entry points for +// the interpreter and for compiled code. +// --------------------------------------------------------------------------- +void nrt_init(void); // GC_INIT + intern table + +// pairs +nr_value nrt_cons(nr_value a, nr_value d); +nr_value nrt_car(nr_value p); +nr_value nrt_cdr(nr_value p); + +// boxes +nr_value nrt_box(nr_value v); +nr_value nrt_unbox(nr_value b); +void nrt_set_box(nr_value b, nr_value v); + +// symbols (interned; eq? works by pointer identity) +nr_value nrt_intern(const char *name); + +// closures +nr_value nrt_make_closure(nr_code code, uint32_t nfree, const nr_value *freev); +nr_value nrt_apply(nr_value clos, int64_t argc, const nr_value *argv); + +// identity / equality +static inline bool nrt_eq(nr_value a, nr_value b) { return a == b; } // eq? + +// fixnum arithmetic (spike: no bignum promotion; asserts on overflow) +nr_value nrt_fx_add(nr_value a, nr_value b); +nr_value nrt_fx_sub(nr_value a, nr_value b); + +// debug +void nrt_write(nr_value w); + +// GC statistics passthrough (Boehm) +size_t nrt_gc_heap_size(void); +size_t nrt_gc_total_bytes(void); + +#endif // NRT_H diff --git a/spike/r1-value-model/spike.cpp b/spike/r1-value-model/spike.cpp new file mode 100644 index 0000000..aaa4543 --- /dev/null +++ b/spike/r1-value-model/spike.cpp @@ -0,0 +1,293 @@ +// R1 spike driver. Proves two things about the shared nr_value ABI: +// +// (1) EQUIVALENCE: a tree-walking interpreter and a statically compiled C++ +// function (standing in for codegen output) compute identical results +// while calling the *same* runtime entry points on the *same* heap. +// (2) GC: a garbage-generating tail loop runs in bounded heap — Boehm collects +// the per-iteration boxes/pairs, so cumulative bytes allocated dwarf the +// live heap. +// +// Build: make (see Makefile). Run: ./spike [N_interp] [N_gc] +#include "nrt.h" + +#include +#include +#include +#include +#include + +#include +#include + +// --------------------------------------------------------------------------- +// A tiny AST + trampolining tree-walker. Enough to express a tail-recursive +// loop that allocates (and drops) a box and a pair each iteration. +// --------------------------------------------------------------------------- +enum EK { + LIT, + CLOSLIT, + VAR, + IFE, + ADD, + SUB, + BOXV, + UNBOXV, + CONSV, + CARV, + EQ0, + RECUR, + CALLV, +}; + +struct Expr { + EK k; + nr_value lit = 0; // LIT / CLOSLIT + int var = 0; // VAR: env index + const Expr *a = nullptr; // first child + const Expr *b = nullptr; // second child + const Expr *c = nullptr; // IFE else-branch + std::vector args; // RECUR / CALLV + explicit Expr(EK kk) : k(kk) {} // ctor -> no aggregate-init warnings +}; + +struct RecurSig { + bool active = false; + std::vector next; +}; + +static nr_value eval(const Expr *e, const std::vector &env, + RecurSig &rec); + +// Non-tail evaluation: recur must not escape a non-tail position. +static nr_value evalNT(const Expr *e, const std::vector &env) { + RecurSig local; + nr_value v = eval(e, env, local); + assert(!local.active && "recur in non-tail position"); + return v; +} + +static nr_value eval(const Expr *e, const std::vector &env, + RecurSig &rec) { + switch (e->k) { + case LIT: + case CLOSLIT: + return e->lit; + case VAR: + return env[(size_t)e->var]; + case ADD: + return nrt_fx_add(evalNT(e->a, env), evalNT(e->b, env)); + case SUB: + return nrt_fx_sub(evalNT(e->a, env), evalNT(e->b, env)); + case BOXV: + return nrt_box(evalNT(e->a, env)); + case UNBOXV: + return nrt_unbox(evalNT(e->a, env)); + case CONSV: + return nrt_cons(evalNT(e->a, env), evalNT(e->b, env)); + case CARV: + return nrt_car(evalNT(e->a, env)); + case EQ0: + return nr_bool(nrt_eq(evalNT(e->a, env), nr_fixnum(0))); + case IFE: + // condition is non-tail; the chosen branch is tail (recur may propagate). + return nr_truthy(evalNT(e->a, env)) ? eval(e->b, env, rec) + : eval(e->c, env, rec); + case CALLV: { + nr_value f = evalNT(e->a, env); + std::vector as; + as.reserve(e->args.size()); + for (auto *ae : e->args) + as.push_back(evalNT(ae, env)); + return nrt_apply(f, (int64_t)as.size(), as.data()); + } + case RECUR: { + rec.next.clear(); + rec.next.reserve(e->args.size()); + for (auto *ae : e->args) + rec.next.push_back(evalNT(ae, env)); + rec.active = true; + return NR_VOID; + } + } + abort(); +} + +static nr_value run_loop(const Expr *body, std::vector env) { + for (;;) { + RecurSig rec; + nr_value v = eval(body, env, rec); + if (!rec.active) + return v; + env = std::move(rec.next); + } +} + +// --------------------------------------------------------------------------- +// The interpreted loop: (rec loop ([n N] [sum 0]) +// (if (= n 0) sum (loop (- n 1) (+ (unbox (box n)) (car (cons sum sum)))))) +// sum' = sum + n, with a box and a pair allocated (and dropped) per iteration. +// --------------------------------------------------------------------------- +static nr_value interp_loop(int64_t N) { + Expr n{VAR}; + n.var = 0; + Expr sum{VAR}; + sum.var = 1; + Expr zero{EQ0}; + zero.a = &n; + + Expr one{LIT}; + one.lit = nr_fixnum(1); + Expr nm1{SUB}; + nm1.a = &n; + nm1.b = &one; + + Expr boxn{BOXV}; + boxn.a = &n; // (box n) -> garbage + Expr ubox{UNBOXV}; + ubox.a = &boxn; // (unbox (box n)) = n + Expr consss{CONSV}; + consss.a = ∑ + consss.b = ∑ // (cons sum sum) -> garbage + Expr carp{CARV}; + carp.a = &consss; // (car (cons sum sum)) = sum + Expr sump{ADD}; + sump.a = &ubox; + sump.b = &carp; // n + sum + + Expr recur{RECUR}; + recur.args = {&nm1, &sump}; + Expr body{IFE}; + body.a = &zero; + body.b = ∑ + body.c = &recur; + + return run_loop(&body, {nr_fixnum(N), nr_fixnum(0)}); +} + +// The compiled-equivalent loop: identical runtime calls, no interpreter. +static nr_value compiled_loop(int64_t N) { + nr_value n = nr_fixnum(N), sum = nr_fixnum(0); + while (!nrt_eq(n, nr_fixnum(0))) { + nr_value tmp = nrt_unbox(nrt_box(n)); // box garbage + nr_value s = nrt_car(nrt_cons(sum, sum)); // pair garbage + sum = nrt_fx_add(tmp, s); + n = nrt_fx_sub(n, nr_fixnum(1)); + } + return sum; +} + +// --------------------------------------------------------------------------- +// Flat-closure proof: a closure capturing one value, applied in both paths. +// --------------------------------------------------------------------------- +static nr_value add_captured(nr_value self, int64_t argc, + const nr_value *argv) { + assert(argc == 1); + auto *c = (NrClosure *)self; + return nrt_fx_add(c->free[0], argv[0]); +} + +static long rss_kb() { + struct rusage ru; + getrusage(RUSAGE_SELF, &ru); + return ru.ru_maxrss; // KiB on Linux +} + +static void hr(const char *label, size_t bytes) { + printf(" %-22s %10.2f MiB\n", label, (double)bytes / (1024 * 1024)); +} + +int main(int argc, char **argv) { + nrt_init(); + int64_t N = argc > 1 ? atoll(argv[1]) : 5000000; // interpreter loop + int64_t Ngc = argc > 2 ? atoll(argv[2]) : 50000000; // GC-pressure loop + int fails = 0; + + printf("== R1 value-model / GC spike ==\n"); + printf( + "nr_value = %zu bytes; ObjHeader = %zu; NrPair = %zu; NrBox = %zu; " + "NrClosure = %zu\n\n", + sizeof(nr_value), sizeof(ObjHeader), sizeof(NrPair), sizeof(NrBox), + sizeof(NrClosure)); + + // --- immediate/tag sanity ------------------------------------------------- + assert(nr_is_fixnum(nr_fixnum(-42)) && nr_fixnum_val(nr_fixnum(-42)) == -42); + assert(nr_is_fixnum(nr_fixnum(0)) && nr_fixnum_val(nr_fixnum(0)) == 0); + assert(nr_is_char(nr_char('Z')) && nr_char_val(nr_char('Z')) == 'Z'); + assert(!nr_truthy(NR_FALSE) && nr_truthy(NR_TRUE) && nr_truthy(nr_fixnum(0))); + assert(nr_is_ptr(nrt_cons(nr_fixnum(1), NR_NULL))); + // identity: interned symbols eq?, fresh pairs not eq?. + assert(nrt_eq(nrt_intern("lambda"), nrt_intern("lambda"))); + assert(!nrt_eq(nrt_intern("lambda"), nrt_intern("if"))); + nr_value p = nrt_cons(nr_fixnum(1), nr_fixnum(2)); + assert(nrt_eq(p, p) && !nrt_eq(p, nrt_cons(nr_fixnum(1), nr_fixnum(2)))); + // mutation through a shared reference (no clone!). + nr_value b = nrt_box(nr_fixnum(7)); + nr_value alias = b; + nrt_set_box(alias, nr_fixnum(99)); + assert(nr_fixnum_val(nrt_unbox(b)) == 99); + printf("[ok] tagging, identity, interning, in-place mutation\n"); + + // --- PROOF 1: interpreter == compiled ------------------------------------ + int64_t expected = N % 2 == 0 ? (N / 2) * (N + 1) : N * ((N + 1) / 2); + nr_value ri = interp_loop(N); + nr_value rc = compiled_loop(N); + printf("[%.4s] interp(%lld) = ", nrt_eq(ri, rc) ? "ok" : "FAIL", + (long long)N); + nrt_write(ri); + printf(" ; compiled = "); + nrt_write(rc); + printf(" ; expected N(N+1)/2 = %lld\n", (long long)expected); + if (!nrt_eq(ri, rc) || nr_fixnum_val(ri) != expected) + fails++; + + // flat closure applied through both paths + nr_value cap = nr_fixnum(10); + nr_value clos = nrt_make_closure(add_captured, 1, &cap); + nr_value five = nr_fixnum(5); + nr_value r_compiled = nrt_apply(clos, 1, &five); + Expr cl{CLOSLIT}; + cl.lit = clos; + Expr a5{LIT}; + a5.lit = five; + Expr call{CALLV}; + call.a = &cl; + call.args = {&a5}; + nr_value r_interp = evalNT(&call, {}); + printf( + "[%.4s] closure (capture 10) applied to 5: interp=%lld compiled=%lld\n", + (nrt_eq(r_interp, r_compiled) && nr_fixnum_val(r_interp) == 15) ? "ok" + : "FAIL", + (long long)nr_fixnum_val(r_interp), (long long)nr_fixnum_val(r_compiled)); + if (!nrt_eq(r_interp, r_compiled) || nr_fixnum_val(r_interp) != 15) + fails++; + + // --- PROOF 2: GC keeps a garbage loop bounded ---------------------------- + size_t heap_before = nrt_gc_heap_size(); + size_t total_before = nrt_gc_total_bytes(); + nr_value g = compiled_loop(Ngc); // ~2 allocations per iteration + (void)g; + size_t heap_after = nrt_gc_heap_size(); + size_t total_after = nrt_gc_total_bytes(); + size_t churn = total_after - total_before; + printf("\nGC pressure loop: %lld iterations (~2 allocs each)\n", + (long long)Ngc); + hr("cumulative allocated", churn); + hr("live heap after loop", heap_after); + hr("heap before loop", heap_before); + printf(" %-22s %10ld MiB (getrusage peak)\n", "process RSS", + rss_kb() / 1024); + bool collected = churn > (size_t)1 << 30 && // > 1 GiB churned + heap_after < churn / 10; // live heap < 10% of churn + printf("[%.4s] GC collected: churned %.2f GiB into a %.1f MiB live heap " + "(%.0fx)\n", + collected ? "ok" : "FAIL", (double)churn / (1 << 30), + (double)heap_after / (1024 * 1024), + (double)churn / (double)heap_after); + if (!collected) + fails++; + (void)total_before; + (void)heap_before; + + printf("\n%s\n", fails == 0 ? "ALL PROOFS PASSED" : "SPIKE FAILED"); + return fails == 0 ? 0 : 1; +} diff --git a/src/AST.cpp b/src/AST.cpp index 2dbdfea..6225a0b 100644 --- a/src/AST.cpp +++ b/src/AST.cpp @@ -1,5 +1,8 @@ #include "AST.h" +#include +#include + #include #include #include @@ -84,6 +87,22 @@ void Symbol::dump() const { llvm::dbgs() << "#"; } // the other value writers (List, Integer via gmp_printf). void Symbol::write() const { std::cout << getName().str(); } +const void *Symbol::identity() const { + if (Uninterned) { + return Uninterned.get(); + } + // Interned symbols are canonical by name: a global table hands out one stable + // pointer per name (unordered_set never invalidates element pointers). + static std::unordered_set InternTable; + return &*InternTable.insert(getName().str()).first; +} + +std::unique_ptr Symbol::makeUninterned(llvm::StringRef Name) { + auto S = std::make_unique(Name); + S->Uninterned = std::make_shared(); // fresh unique identity token + return S; +} + // // Implementation of Keyword node. // diff --git a/src/ASTRuntime.cpp b/src/ASTRuntime.cpp index 0bfddba..590db69 100644 --- a/src/ASTRuntime.cpp +++ b/src/ASTRuntime.cpp @@ -44,6 +44,56 @@ void CaseLambdaClosure::dump() const { } void CaseLambdaClosure::write() const {} +Box::Box(std::unique_ptr V) + : ClonableNode(ASTNodeKind::AST_Box), C(std::make_shared()) { + C->Value = std::move(V); +} + +Box::Box(const Box &Other) : ClonableNode(ASTNodeKind::AST_Box), C(Other.C) {} + +std::unique_ptr Box::get() const { + return std::unique_ptr(C->Value->clone()); +} + +void Box::set(std::unique_ptr V) const { C->Value = std::move(V); } + +void Box::dump() const { llvm::dbgs() << "#&\n"; } + +void Box::write() const { + std::cout << "#&"; + C->Value->write(); +} + +Pair::Pair(std::unique_ptr Car, std::unique_ptr Cdr) + : ClonableNode(ASTNodeKind::AST_Pair), C(std::make_shared()) { + C->Car = std::move(Car); + C->Cdr = std::move(Cdr); +} + +Pair::Pair(const Pair &Other) + : ClonableNode(ASTNodeKind::AST_Pair), C(Other.C) {} + +std::unique_ptr Pair::car() const { + return std::unique_ptr(C->Car->clone()); +} + +std::unique_ptr Pair::cdr() const { + return std::unique_ptr(C->Cdr->clone()); +} + +void Pair::setCar(std::unique_ptr V) const { C->Car = std::move(V); } +void Pair::setCdr(std::unique_ptr V) const { C->Cdr = std::move(V); } + +void Pair::dump() const { llvm::dbgs() << "#\n"; } + +void Pair::write() const { + std::cout << "("; + C->Car->write(); + std::cout << " . "; + C->Cdr->write(); + std::cout << ")"; +} + // // Continuation marks // diff --git a/src/AnalysisFreeVars.cpp b/src/AnalysisFreeVars.cpp index a8a51f0..dfa6803 100644 --- a/src/AnalysisFreeVars.cpp +++ b/src/AnalysisFreeVars.cpp @@ -142,6 +142,16 @@ void AnalysisFreeVars::visit(ast::BooleanLiteral const &Bool) { // Nothing to do. } +void AnalysisFreeVars::visit(ast::Box const &B) { + // A box is a runtime value with no free variables. + // Nothing to do. +} + +void AnalysisFreeVars::visit(ast::Pair const &P) { + // A pair is a runtime value with no free variables. + // Nothing to do. +} + void AnalysisFreeVars::visit(ast::Char const &C) { // Characters have no free variables. // Nothing to do. diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 529cd4e..abf7867 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -3,7 +3,7 @@ include_directories(${CMAKE_CURRENT_BINARY_DIR}/include/) add_subdirectory(include) -set(LIBS gmp) +set(LIBS gmp PkgConfig::BDWGC) if(NORA_ENABLE_MLIR) add_subdirectory(mlir) @@ -25,6 +25,7 @@ set(LLVM_LINK_COMPONENTS Support) add_llvm_executable(norac AnalysisFreeVars.cpp AST.cpp + nora_rt.cpp ASTRuntime.cpp Diagnostics.cpp Environment.cpp diff --git a/src/Interpreter.cpp b/src/Interpreter.cpp index 015795d..c842be7 100644 --- a/src/Interpreter.cpp +++ b/src/Interpreter.cpp @@ -112,7 +112,7 @@ void Interpreter::visit(ast::Linklet const &Linklet) { Val = nullptr; M = Mode::Eval; run(); - Last = std::move(Val); + Last = Val.takeLegacy(); if (Diag.hadError()) { break; } @@ -122,6 +122,9 @@ void Interpreter::visit(ast::Linklet const &Linklet) { void Interpreter::run() { while (true) { + if (Kont.size() > PeakKont) { + PeakKont = Kont.size(); + } if (M == Mode::Eval) { Control->accept(*this); } else { @@ -146,16 +149,32 @@ void Interpreter::continueStep() { case Frame::Seq: { if (Top.Begin0 && Top.Idx == 1) { - Top.Saved = std::move(Val); + Top.Saved = Val.takeLegacy(); } if (Top.Idx < Top.Exprs.size()) { - Control = Top.Exprs[Top.Idx]; - Env = Top.Env; - Top.Idx++; - M = Mode::Eval; + const bool IsLast = Top.Idx + 1 == Top.Exprs.size(); + if (IsLast && !Top.Begin0) { + // Tail position: drop the sequence frame before its final expression + // (mirrors IfBranch) so a tail call there reuses the enclosing + // activation frame rather than stacking a new one. + const ast::ExprNode *E = Top.Exprs[Top.Idx]; + EnvPtr SeqEnv = Top.Env; + Kont.pop_back(); + Control = E; + Env = SeqEnv; + M = Mode::Eval; + } else { + Control = Top.Exprs[Top.Idx]; + Env = Top.Env; + Top.Idx++; + M = Mode::Eval; + } } else { + // Only begin0 reaches here: its frame persists to the end to return the + // saved first value; a plain sequence's final expression is handled + // above. std::unique_ptr R = - Top.Begin0 ? std::move(Top.Saved) : std::move(Val); + Top.Begin0 ? std::move(Top.Saved) : Val.takeLegacy(); Kont.pop_back(); deliver(std::move(R)); } @@ -166,7 +185,7 @@ void Interpreter::continueStep() { const ast::ExprNode *ThenE = Top.ThenE; const ast::ExprNode *ElseE = Top.ElseE; EnvPtr E = Top.Env; - std::unique_ptr Cond = std::move(Val); + std::unique_ptr Cond = Val.takeLegacy(); Kont.pop_back(); auto *B = llvm::dyn_cast_or_null(Cond.get()); Control = (B && !B->value()) ? ElseE : ThenE; @@ -176,7 +195,7 @@ void Interpreter::continueStep() { } case Frame::App: { - Top.Done.push_back(std::move(Val)); + Top.Done.push_back(Val.takeLegacy()); if (Top.Done.size() < Top.Exprs.size()) { Control = Top.Exprs[Top.Done.size()]; Env = Top.Env; @@ -192,7 +211,7 @@ void Interpreter::continueStep() { } case Frame::MkValues: { - Top.Done.push_back(std::move(Val)); + Top.Done.push_back(Val.takeLegacy()); if (Top.Done.size() < Top.Exprs.size()) { Control = Top.Exprs[Top.Done.size()]; Env = Top.Env; @@ -215,7 +234,7 @@ void Interpreter::continueStep() { } case Frame::LetBind: { - Top.Done.push_back(std::move(Val)); + Top.Done.push_back(Val.takeLegacy()); const ast::LetValues *Let = Top.Let; if (Top.Done.size() < Let->exprsCount()) { Control = &Let->getBindingExpr(Top.Done.size()); @@ -254,7 +273,7 @@ void Interpreter::continueStep() { const ast::LetValues *Let = Top.Let; EnvPtr RecScope = Top.RecScope; if (!bindValues(Diag, Let->getLoc(), RecScope->Vars, - Let->getBindingIds(Top.Idx), std::move(Val))) { + Let->getBindingIds(Top.Idx), Val.takeLegacy())) { abortEval(); return; } @@ -278,7 +297,7 @@ void Interpreter::continueStep() { case Frame::Define: { const ast::DefineValues *DV = Top.Def; EnvPtr DefEnv = Top.DefEnv; - std::unique_ptr V = std::move(Val); + std::unique_ptr V = Val.takeLegacy(); Kont.pop_back(); if (DV->countIds() == 1) { @@ -315,7 +334,7 @@ void Interpreter::continueStep() { case Frame::Set: { const ast::Identifier *Id = Top.SetId; EnvPtr E = Top.Env; - std::unique_ptr V = std::move(Val); + std::unique_ptr V = Val.takeLegacy(); Kont.pop_back(); if (!envSet(E, *Id, std::move(V))) { Diag.error(Id->getLoc(), llvm::Twine("cannot set unbound identifier: ") + @@ -331,7 +350,7 @@ void Interpreter::continueStep() { const ast::ExprNode *ValE = Top.WcmValE; const ast::ExprNode *ResultE = Top.WcmResultE; EnvPtr E = Top.Env; - std::unique_ptr KeyV = std::move(Val); + std::unique_ptr KeyV = Val.takeLegacy(); Kont.pop_back(); Kont.emplace_back(Frame::WcmVal); Frame &WV = Kont.back(); @@ -348,15 +367,30 @@ void Interpreter::continueStep() { const ast::ExprNode *ResultE = Top.WcmResultE; EnvPtr E = Top.Env; std::unique_ptr KeyV = std::move(Top.WcmKeyV); - std::unique_ptr ValV = std::move(Val); + std::unique_ptr ValV = Val.takeLegacy(); Kont.pop_back(); - // Push a dedicated mark-bearing frame holding this key/value, and evaluate - // the result expression under it. The frame is popped when the result - // produces a value (see the WcmMark case), so the mark's dynamic extent is - // exactly the result expression - a with-continuation-mark in non-tail - // position no longer leaks its mark into later expressions. - Kont.emplace_back(Frame::WcmMark); - ast::setMark(Kont.back().Marks, std::move(KeyV), std::move(ValV)); + // The result expression is in tail position with respect to whatever + // frame is now on top. Call/WcmMark/Halt frames are exactly the frames + // that a tail call is later allowed to reuse (see applyProcedure), so + // installing the mark directly onto that same frame - instead of pushing + // a new one - shares one continuation frame across a chain of + // tail-nested with-continuation-marks, the same way tail calls already + // share one frame across a self-recursive loop: a later mark for the + // same key in that frame correctly replaces this one (setMark) rather + // than stacking a second entry, and a tail call out of this expression + // reuses the frame with the mark already on it (O(1) space). A + // genuinely non-tail with-continuation-mark (Kont.back() is anything + // else - Seq, App, LetBind, ...) still gets its own frame, popped when + // the result produces a value, so the mark's dynamic extent is exactly + // the result expression and doesn't leak into later expressions. + if (!Kont.empty() && + (Kont.back().K == Frame::Call || Kont.back().K == Frame::WcmMark || + Kont.back().K == Frame::Halt)) { + ast::setMark(Kont.back().Marks, std::move(KeyV), std::move(ValV)); + } else { + Kont.emplace_back(Frame::WcmMark); + ast::setMark(Kont.back().Marks, std::move(KeyV), std::move(ValV)); + } Control = ResultE; Env = E; M = Mode::Eval; @@ -366,7 +400,7 @@ void Interpreter::continueStep() { case Frame::WcmMark: { // The result expression has produced a value; discard the mark frame and // pass the value through to the enclosing continuation. - std::unique_ptr V = std::move(Val); + std::unique_ptr V = Val.takeLegacy(); Kont.pop_back(); deliver(std::move(V)); break; @@ -375,7 +409,7 @@ void Interpreter::continueStep() { case Frame::Call: { // The activation's body has produced a value; its frame (and marks) is // discarded and the value flows to the caller's continuation. - std::unique_ptr V = std::move(Val); + std::unique_ptr V = Val.takeLegacy(); Kont.pop_back(); deliver(std::move(V)); break; @@ -507,6 +541,31 @@ void Interpreter::applyProcedure( } } + // Tail call: if the enclosing continuation frame is one this call can + // reuse instead of stacking a new one, do so. Frame::Call is the caller's + // own activation; Frame::WcmMark and Frame::Halt are also reusable because + // WcmVal now installs a tail-position mark directly onto whichever of + // these three kinds is on top (see the WcmVal case) rather than always + // pushing a fresh frame, so any of them may already be carrying marks + // whose dynamic extent covers this call. Together with popping + // Seq/if/let-body frames before their tail sub-expression, this makes + // self- and mutual tail recursion - including through + // with-continuation-mark - run in O(1) continuation space. Marks are + // intentionally not cleared on reuse: they persist on the shared frame + // until a later with-continuation-mark for the same key overwrites them + // (setMark), exactly as if every tail-recursive step of the loop were + // still the same continuation frame - which, after reuse, it is. + if (!Kont.empty() && + (Kont.back().K == Frame::Call || Kont.back().K == Frame::WcmMark || + Kont.back().K == Frame::Halt)) { + Frame &Enc = Kont.back(); + Enc.Callee = std::move(Op); // frees the previous activation's closure + Control = &Clause->getBody(); + Env = CalleeScope; + M = Mode::Eval; + return; + } + Kont.emplace_back(Frame::Call); // The Call frame takes ownership of the closure so its (cloned) lambda body, // into which Control now points, outlives this function. @@ -722,6 +781,14 @@ void Interpreter::visit(ast::BooleanLiteral const &Bool) { deliver(std::unique_ptr(Bool.clone())); } +void Interpreter::visit(ast::Box const &B) { + deliver(std::unique_ptr(B.clone())); +} + +void Interpreter::visit(ast::Pair const &P) { + deliver(std::unique_ptr(P.clone())); +} + void Interpreter::visit(ast::Char const &C) { deliver(std::unique_ptr(C.clone())); } diff --git a/src/Parse.cpp b/src/Parse.cpp index e128d95..833797c 100644 --- a/src/Parse.cpp +++ b/src/Parse.cpp @@ -225,6 +225,13 @@ std::unique_ptr Parse::parseExpr(SourceStream &S) { return Bool; } + // A string literal is a self-evaluating expression (its leading '"' matches + // no other expression parser). + std::unique_ptr Str = parseString(S); + if (Str) { + return Str; + } + // If the expression is quoted, then identifier is a symbol. std::unique_ptr Id = parseIdentifier(S); if (Id) { diff --git a/src/Runtime.cpp b/src/Runtime.cpp index f582281..7b5a7c4 100644 --- a/src/Runtime.cpp +++ b/src/Runtime.cpp @@ -184,6 +184,307 @@ class ContinuationMarkSetToListFunction : public ast::RuntimeFunction { void accept(ASTVisitor &V) const override { V.visit(*this); } }; +// (zero? n) is a minimal integer predicate. It exists so a terminating +// tail-recursive loop can be written to exercise proper tail calls (M1); the +// full numeric tower and its predicates arrive in M4. +class ZeroPredicateFunction : public ast::RuntimeFunction { +public: + ZeroPredicateFunction(const std::string &Name) : RuntimeFunction(Name) {} + + std::unique_ptr operator()( + const llvm::SmallVector &Args) const override { + if (Args.size() != 1) { + return nullptr; + } + if (auto const *I = llvm::dyn_cast(Args[0])) { + return std::make_unique(*I == 0); + } + return nullptr; + } + + ast::RuntimeFunction *clone() const override { + return new ZeroPredicateFunction(*this); + } + + void accept(ASTVisitor &V) const override { V.visit(*this); } +}; + +// (box v) allocates a fresh mutable cell holding v. (unbox b) reads it. The +// box's cell is shared across copies of the Box value, so mutation and identity +// survive the interpreter's clone-on-lookup - the start of M2's shared value +// model. +class BoxFunction : public ast::RuntimeFunction { +public: + BoxFunction(const std::string &Name) : RuntimeFunction(Name) {} + + std::unique_ptr operator()( + const llvm::SmallVector &Args) const override { + if (Args.size() != 1) { + return nullptr; + } + return std::make_unique( + std::unique_ptr(Args[0]->clone())); + } + + ast::RuntimeFunction *clone() const override { + return new BoxFunction(*this); + } + void accept(ASTVisitor &V) const override { V.visit(*this); } +}; + +class UnboxFunction : public ast::RuntimeFunction { +public: + UnboxFunction(const std::string &Name) : RuntimeFunction(Name) {} + + std::unique_ptr operator()( + const llvm::SmallVector &Args) const override { + if (Args.size() != 1) { + return nullptr; + } + if (auto const *B = llvm::dyn_cast(Args[0])) { + return B->get(); + } + return nullptr; + } + + ast::RuntimeFunction *clone() const override { + return new UnboxFunction(*this); + } + void accept(ASTVisitor &V) const override { V.visit(*this); } +}; + +class SetBoxFunction : public ast::RuntimeFunction { +public: + SetBoxFunction(const std::string &Name) : RuntimeFunction(Name) {} + + std::unique_ptr operator()( + const llvm::SmallVector &Args) const override { + if (Args.size() != 2) { + return nullptr; + } + if (auto const *B = llvm::dyn_cast(Args[0])) { + B->set(std::unique_ptr(Args[1]->clone())); + return std::make_unique(); + } + return nullptr; + } + + ast::RuntimeFunction *clone() const override { + return new SetBoxFunction(*this); + } + void accept(ASTVisitor &V) const override { V.visit(*this); } +}; + +// (eq? a b): object identity. Heap objects with a cell (boxes) compare by cell +// pointer; other values fall back to the structural valueEq approximation +// (interned symbols, fixnums, chars, booleans). This is the identity operation +// the clone-everything model could not provide. +class EqFunction : public ast::RuntimeFunction { +public: + EqFunction(const std::string &Name) : RuntimeFunction(Name) {} + + std::unique_ptr operator()( + const llvm::SmallVector &Args) const override { + if (Args.size() != 2) { + return nullptr; + } + // A quoted datum such as 'k evaluates to a QuotedExpr wrapping the + // symbol/box/pair rather than the bare value, which would otherwise skip + // the identity branches below and fall through to valueEq's structural + // (name-only) symbol comparison - losing the distinction between an + // interned and an uninterned symbol of the same name. + const ast::ValueNode *A = Args[0]; + const ast::ValueNode *B = Args[1]; + while (auto const *QA = llvm::dyn_cast(A)) { + A = &QA->getQuotedExpr(); + } + while (auto const *QB = llvm::dyn_cast(B)) { + B = &QB->getQuotedExpr(); + } + bool Eq; + if (auto const *BA = llvm::dyn_cast(A)) { + auto const *BB = llvm::dyn_cast(B); + Eq = (BB != nullptr) && BA->identity() == BB->identity(); + } else if (auto const *PA = llvm::dyn_cast(A)) { + auto const *PB = llvm::dyn_cast(B); + Eq = (PB != nullptr) && PA->identity() == PB->identity(); + } else if (auto const *SA = llvm::dyn_cast(A)) { + auto const *SB = llvm::dyn_cast(B); + Eq = (SB != nullptr) && SA->identity() == SB->identity(); + } else { + Eq = ast::valueEq(*A, *B); + } + return std::make_unique(Eq); + } + + ast::RuntimeFunction *clone() const override { return new EqFunction(*this); } + void accept(ASTVisitor &V) const override { V.visit(*this); } +}; + +// (cons a d) allocates a fresh mutable pair; (car p)/(cdr p) read its fields. +// The pair's cell is shared across copies of the Pair value, so mutation and +// identity survive the interpreter's clone-on-lookup. +class ConsFunction : public ast::RuntimeFunction { +public: + ConsFunction(const std::string &Name) : RuntimeFunction(Name) {} + + std::unique_ptr operator()( + const llvm::SmallVector &Args) const override { + if (Args.size() != 2) { + return nullptr; + } + return std::make_unique( + std::unique_ptr(Args[0]->clone()), + std::unique_ptr(Args[1]->clone())); + } + + ast::RuntimeFunction *clone() const override { + return new ConsFunction(*this); + } + void accept(ASTVisitor &V) const override { V.visit(*this); } +}; + +class CarFunction : public ast::RuntimeFunction { +public: + CarFunction(const std::string &Name) : RuntimeFunction(Name) {} + + std::unique_ptr operator()( + const llvm::SmallVector &Args) const override { + if (Args.size() != 1) { + return nullptr; + } + if (auto const *P = llvm::dyn_cast(Args[0])) { + return P->car(); + } + return nullptr; + } + + ast::RuntimeFunction *clone() const override { + return new CarFunction(*this); + } + void accept(ASTVisitor &V) const override { V.visit(*this); } +}; + +class CdrFunction : public ast::RuntimeFunction { +public: + CdrFunction(const std::string &Name) : RuntimeFunction(Name) {} + + std::unique_ptr operator()( + const llvm::SmallVector &Args) const override { + if (Args.size() != 1) { + return nullptr; + } + if (auto const *P = llvm::dyn_cast(Args[0])) { + return P->cdr(); + } + return nullptr; + } + + ast::RuntimeFunction *clone() const override { + return new CdrFunction(*this); + } + void accept(ASTVisitor &V) const override { V.visit(*this); } +}; + +class SetCarFunction : public ast::RuntimeFunction { +public: + SetCarFunction(const std::string &Name) : RuntimeFunction(Name) {} + + std::unique_ptr operator()( + const llvm::SmallVector &Args) const override { + if (Args.size() != 2) { + return nullptr; + } + if (auto const *P = llvm::dyn_cast(Args[0])) { + P->setCar(std::unique_ptr(Args[1]->clone())); + return std::make_unique(); + } + return nullptr; + } + + ast::RuntimeFunction *clone() const override { + return new SetCarFunction(*this); + } + void accept(ASTVisitor &V) const override { V.visit(*this); } +}; + +class SetCdrFunction : public ast::RuntimeFunction { +public: + SetCdrFunction(const std::string &Name) : RuntimeFunction(Name) {} + + std::unique_ptr operator()( + const llvm::SmallVector &Args) const override { + if (Args.size() != 2) { + return nullptr; + } + if (auto const *P = llvm::dyn_cast(Args[0])) { + P->setCdr(std::unique_ptr(Args[1]->clone())); + return std::make_unique(); + } + return nullptr; + } + + ast::RuntimeFunction *clone() const override { + return new SetCdrFunction(*this); + } + void accept(ASTVisitor &V) const override { V.visit(*this); } +}; + +// (string->uninterned-symbol s) makes a fresh uninterned symbol: distinct from +// every other symbol (interned or not), even one with the same name. Interned +// symbols, by contrast, are canonical by name, so eq? on symbols is identity. +class StringToUninternedSymbolFunction : public ast::RuntimeFunction { +public: + StringToUninternedSymbolFunction(const std::string &Name) + : RuntimeFunction(Name) {} + + std::unique_ptr operator()( + const llvm::SmallVector &Args) const override { + if (Args.size() != 1) { + return nullptr; + } + if (auto const *S = llvm::dyn_cast(Args[0])) { + return ast::Symbol::makeUninterned(S->getValue()); + } + return nullptr; + } + + ast::RuntimeFunction *clone() const override { + return new StringToUninternedSymbolFunction(*this); + } + void accept(ASTVisitor &V) const override { V.visit(*this); } +}; + +// (gensym [base]) returns a fresh uninterned symbol, never eq? to any other. +// A monotonic counter gives it a readable, unique name; distinctness comes from +// its uninterned identity, not the name. +class GensymFunction : public ast::RuntimeFunction { +public: + GensymFunction(const std::string &Name) : RuntimeFunction(Name) {} + + std::unique_ptr operator()( + const llvm::SmallVector &Args) const override { + if (Args.size() > 1) { + return nullptr; + } + static unsigned Counter = 0; + std::string Base = "g"; + if (Args.size() == 1) { + if (auto const *S = llvm::dyn_cast(Args[0])) { + Base = S->getName().str(); + } else if (auto const *Str = llvm::dyn_cast(Args[0])) { + Base = Str->getValue().str(); + } + } + return ast::Symbol::makeUninterned(Base + std::to_string(++Counter)); + } + + ast::RuntimeFunction *clone() const override { + return new GensymFunction(*this); + } + void accept(ASTVisitor &V) const override { V.visit(*this); } +}; + #define RUNTIME_FUNC(Identifier, Name) \ RuntimeFunctions[Identifier] = std::make_shared(Identifier); Runtime::Runtime() { @@ -195,6 +496,18 @@ Runtime::Runtime() { RUNTIME_FUNC("continuation-mark-set-first", ContinuationMarkSetFirstFunction); RUNTIME_FUNC("continuation-mark-set->list", ContinuationMarkSetToListFunction); + RUNTIME_FUNC("zero?", ZeroPredicateFunction); + RUNTIME_FUNC("box", BoxFunction); + RUNTIME_FUNC("unbox", UnboxFunction); + RUNTIME_FUNC("set-box!", SetBoxFunction); + RUNTIME_FUNC("eq?", EqFunction); + RUNTIME_FUNC("cons", ConsFunction); + RUNTIME_FUNC("car", CarFunction); + RUNTIME_FUNC("cdr", CdrFunction); + RUNTIME_FUNC("set-car!", SetCarFunction); + RUNTIME_FUNC("set-cdr!", SetCdrFunction); + RUNTIME_FUNC("string->uninterned-symbol", StringToUninternedSymbolFunction); + RUNTIME_FUNC("gensym", GensymFunction); } std::unique_ptr diff --git a/src/include/AST.h b/src/include/AST.h index f2142f5..0bffc92 100644 --- a/src/include/AST.h +++ b/src/include/AST.h @@ -46,6 +46,7 @@ class ASTNode { AST_WithContinuationMark, First_ValueNode, // all ValueNodes must be after this AST_BooleanLiteral, + AST_Box, // result of (box v) AST_CaseLambda, AST_CaseLambdaClosure, // result of evaluating a CaseLambda expression AST_Char, @@ -55,6 +56,7 @@ class ASTNode { AST_Keyword, AST_Lambda, AST_List, + AST_Pair, // result of (cons a d) AST_String, AST_Symbol, AST_Values, @@ -203,17 +205,24 @@ class Symbol : public ClonableNode { explicit Symbol(llvm::StringRef Name) : ClonableNode(ASTNodeKind::AST_Symbol), Name(Name) {} Symbol(const Symbol &S) - : ClonableNode(ASTNodeKind::AST_Symbol), Name(S.Name) {} + : ClonableNode(ASTNodeKind::AST_Symbol), Name(S.Name), + Uninterned(S.Uninterned) {} Symbol(Symbol &&) = default; Symbol &operator=(const Symbol &S) = delete; Symbol &operator=(Symbol &&S) = delete; virtual ~Symbol() = default; - // FIXME: symbols are not interned yet, so eq?/eqv? identity is approximated - // by comparing names. Interning is future work. + // Structural (name) comparison, used by valueEq for quoted-datum equality. bool operator==(const Symbol &S) const { return getName() == S.getName(); } [[nodiscard]] llvm::StringRef getName() const { return Name; } + // Object identity for eq?: interned symbols are canonical by name; an + // uninterned symbol (gensym / string->uninterned-symbol) carries a unique + // token shared across its clones. + [[nodiscard]] const void *identity() const; + [[nodiscard]] bool isInterned() const { return Uninterned == nullptr; } + // A fresh uninterned symbol with the given (cosmetic) name. + static std::unique_ptr makeUninterned(llvm::StringRef Name); LLVM_DUMP_METHOD void dump() const override; void write() const override; @@ -223,6 +232,7 @@ class Symbol : public ClonableNode { private: llvm::SmallString<32> Name; + std::shared_ptr Uninterned; // null => interned }; // A keyword datum, e.g. #:foo. Name holds the bare keyword (without the leading diff --git a/src/include/ASTRuntime.h b/src/include/ASTRuntime.h index 20efaf8..1e09411 100644 --- a/src/include/ASTRuntime.h +++ b/src/include/ASTRuntime.h @@ -59,6 +59,74 @@ class CaseLambdaClosure : public ClonableNode { EnvPtr Env; }; +// A box is a mutable single-slot cell. Its cell is heap-allocated and shared: +// copying a Box (which the interpreter does on every environment lookup) shares +// the same cell, so a set-box! through one reference is visible through all of +// them and (eq? b b) holds. This is the first piece of M2's shared, identity- +// bearing value model; the cell moves onto the GC heap in a later M2 slice. +class Box : public ClonableNode { +public: + explicit Box(std::unique_ptr V); + Box(const Box &Other); // shares the cell (shallow copy) + ~Box() = default; + + static bool classof(const ASTNode *N) { + return N->getKind() == ASTNodeKind::AST_Box; + } + + // Current contents as a fresh value (following the interpreter's value + // model); used by unbox. + std::unique_ptr get() const; + // Replace the shared cell's contents (set-box!). Const because the Box + // wrapper is immutable; the cell it references is not. + void set(std::unique_ptr V) const; + // Cell identity for eq?: two Box values are eq? iff they share a cell. + const void *identity() const { return C.get(); } + + LLVM_DUMP_METHOD void dump() const override; + void write() const override; + +private: + struct Cell { + std::unique_ptr Value; + }; + std::shared_ptr C; +}; + +// A mutable pair (cons cell). Like Box, its car/cdr cell is heap-allocated and +// shared across copies, so set-car!/set-cdr! and eq? observe one identity +// through the interpreter's clone-on-lookup. (Interim shared_ptr cell; moves +// onto the GC heap in a later M2 slice.) +class Pair : public ClonableNode { +public: + Pair(std::unique_ptr Car, std::unique_ptr Cdr); + Pair(const Pair &Other); // shares the cell (shallow copy) + ~Pair() = default; + + static bool classof(const ASTNode *N) { + return N->getKind() == ASTNodeKind::AST_Pair; + } + + // car/cdr, each a fresh value (following the interpreter's value model). + std::unique_ptr car() const; + std::unique_ptr cdr() const; + // set-car!/set-cdr!: mutate the shared cell in place. + void setCar(std::unique_ptr V) const; + void setCdr(std::unique_ptr V) const; + // Cell identity for eq?: two Pair values are eq? iff they share a cell. + const void *identity() const { return C.get(); } + + LLVM_DUMP_METHOD void dump() const override; + void write() const override; + +private: + struct Cell { + std::unique_ptr Car; + std::unique_ptr Cdr; + }; + std::shared_ptr C; +}; + // A single continuation mark is a key/value pair. A MarkFrame collects the // marks belonging to one continuation frame; within a frame each key appears // at most once (setMark overwrites an existing entry for the same key). diff --git a/src/include/ASTVisitor.h b/src/include/ASTVisitor.h index 8691fbf..01bff1b 100644 --- a/src/include/ASTVisitor.h +++ b/src/include/ASTVisitor.h @@ -11,6 +11,7 @@ class ASTVisitor { virtual void visit(ast::Application const &A) = 0; virtual void visit(ast::Begin const &B) = 0; virtual void visit(ast::BooleanLiteral const &Bool) = 0; + virtual void visit(ast::Box const &B) = 0; virtual void visit(ast::CaseLambda const &CL) = 0; virtual void visit(ast::CaseLambdaClosure const &CL) = 0; virtual void visit(ast::Char const &C) = 0; @@ -25,6 +26,7 @@ class ASTVisitor { virtual void visit(ast::LetValues const &LV) = 0; virtual void visit(ast::Linklet const &Linklet) = 0; virtual void visit(ast::List const &L) = 0; + virtual void visit(ast::Pair const &P) = 0; virtual void visit(ast::QuotedExpr const &QE) = 0; virtual void visit(ast::RuntimeFunction const &RF) = 0; virtual void visit(ast::SetBang const &SB) = 0; diff --git a/src/include/AST_fwd.h b/src/include/AST_fwd.h index eb0a5b5..cbc30f8 100644 --- a/src/include/AST_fwd.h +++ b/src/include/AST_fwd.h @@ -5,6 +5,7 @@ namespace ast { class Application; class Begin; class BooleanLiteral; +class Box; class CaseLambda; class CaseLambdaClosure; class Char; @@ -19,6 +20,7 @@ class Lambda; class LetValues; class Linklet; class List; +class Pair; class QuotedExpr; class RuntimeFunction; class SetBang; diff --git a/src/include/AnalysisFreeVars.h b/src/include/AnalysisFreeVars.h index 9c6bbbd..f5c03a9 100644 --- a/src/include/AnalysisFreeVars.h +++ b/src/include/AnalysisFreeVars.h @@ -19,6 +19,7 @@ class AnalysisFreeVars : public ASTVisitor { virtual void visit(ast::Application const &A) override; virtual void visit(ast::Begin const &B) override; virtual void visit(ast::BooleanLiteral const &Bool) override; + virtual void visit(ast::Box const &B) override; virtual void visit(ast::CaseLambda const &CL) override; virtual void visit(ast::CaseLambdaClosure const &CL) override; virtual void visit(ast::Char const &C) override; @@ -33,6 +34,7 @@ class AnalysisFreeVars : public ASTVisitor { virtual void visit(ast::LetValues const &LV) override; virtual void visit(ast::Linklet const &Linklet) override; virtual void visit(ast::List const &L) override; + virtual void visit(ast::Pair const &P) override; virtual void visit(ast::QuotedExpr const &QE) override; virtual void visit(ast::RuntimeFunction const &LV) override; virtual void visit(ast::SetBang const &SB) override; diff --git a/src/include/Interpreter.h b/src/include/Interpreter.h index 2396044..7979d6f 100644 --- a/src/include/Interpreter.h +++ b/src/include/Interpreter.h @@ -25,6 +25,9 @@ #include "Diagnostics.h" #include "Environment.h" #include "Runtime.h" +#include "Value.h" +#include "gc_alloc.h" +#include "nora_rt.h" class Interpreter : public ASTVisitor { public: @@ -38,6 +41,7 @@ class Interpreter : public ASTVisitor { virtual void visit(ast::Application const &A) override; virtual void visit(ast::Begin const &B) override; virtual void visit(ast::BooleanLiteral const &Bool) override; + virtual void visit(ast::Box const &B) override; virtual void visit(ast::CaseLambda const &CL) override; virtual void visit(ast::CaseLambdaClosure const &CL) override; virtual void visit(ast::Char const &C) override; @@ -52,6 +56,7 @@ class Interpreter : public ASTVisitor { virtual void visit(ast::LetValues const &LV) override; virtual void visit(ast::Linklet const &Linklet) override; virtual void visit(ast::List const &L) override; + virtual void visit(ast::Pair const &P) override; virtual void visit(ast::QuotedExpr const &L) override; virtual void visit(ast::RuntimeFunction const &LV) override; virtual void visit(ast::SetBang const &SB) override; @@ -72,8 +77,15 @@ class Interpreter : public ASTVisitor { if (!Result) { return nullptr; } - return std::unique_ptr(Result->clone()); + return std::unique_ptr(Result.get()->clone()); }; + // Peak continuation depth reached across every top-level form run so far. + // Exposed for the tail-call tests: proper tail calls keep this bounded. + size_t getPeakKont() const { return PeakKont; } + // Boehm GC live-heap / cumulative-bytes, for the M2 GC forcing seam (a + // depth-independent live-heap plateau against unbounded churn). + size_t getGCHeapSize() const { return nrt_gc_heap_size(); } + size_t getGCTotalBytes() const { return nrt_gc_total_bytes(); } std::unique_ptr callFunction(const std::string &Name, const llvm::SmallVector &Args) { @@ -177,12 +189,17 @@ class Interpreter : public ASTVisitor { // Machine state. Mode M = Mode::Eval; - const ast::ASTNode *Control = nullptr; // expression under evaluation - EnvPtr Env; // current environment - std::vector Kont; // continuation (top == back()) - std::unique_ptr Val; // value register (Continue mode) - EnvPtr GlobalEnv; // top-level scope, persists per form - std::unique_ptr Result; // result of the whole linklet + const ast::ASTNode *Control = nullptr; // expression under evaluation + EnvPtr Env; // current environment + // The continuation. Its backing store is GC-allocated (and scanned) so that, + // as values migrate onto the GC heap, in-flight values held in frames stay + // reachable — the Kont header lives in this stack-resident Interpreter, so + // Boehm's stack scan roots the buffer. (M2/GC S1.) + std::vector> Kont; // continuation (top == back()) + Value Val; // value register (Continue mode) + EnvPtr GlobalEnv; // top-level scope, persists per form + Value Result; // result of the whole linklet + size_t PeakKont = 0; // peak |Kont| seen (tail-call tests) // Every scope created during evaluation, so their bindings can be cleared in // the destructor. Live-environment closures capture the scope that binds them diff --git a/src/include/Value.h b/src/include/Value.h new file mode 100644 index 0000000..861fd3d --- /dev/null +++ b/src/include/Value.h @@ -0,0 +1,36 @@ +#ifndef NORA_VALUE_H +#define NORA_VALUE_H + +#include + +#include "AST.h" + +// A machine value handle — the vehicle for the value-model + GC migration +// (docs/value-model-gc-migration.md §3). It will become a bare nr_value word +// (immediate | GC pointer | legacy pin-index) so GC cells can hold it; in the +// current phase it simply carries a legacy heap ValueNode by unique_ptr and is +// behaviourally identical to it. Move-only, like the unique_ptr it wraps. +class Value { +public: + Value() = default; + // NOLINTNEXTLINE(google-explicit-constructor): implicit, for `reg = nullptr`. + Value(std::nullptr_t) {} + // NOLINTNEXTLINE(google-explicit-constructor): implicit boundary from legacy. + Value(std::unique_ptr V) : Legacy(std::move(V)) {} + Value(Value &&) = default; + Value &operator=(Value &&) = default; + Value(const Value &) = delete; + Value &operator=(const Value &) = delete; + ~Value() = default; + + explicit operator bool() const { return static_cast(Legacy); } + ast::ValueNode *get() const { return Legacy.get(); } + // Move the legacy value out, emptying this handle. Used at boundaries with + // the still-unique_ptr frame/env slots until they migrate in later slices. + std::unique_ptr takeLegacy() { return std::move(Legacy); } + +private: + std::unique_ptr Legacy; +}; + +#endif // NORA_VALUE_H diff --git a/src/include/gc_alloc.h b/src/include/gc_alloc.h new file mode 100644 index 0000000..8094bf6 --- /dev/null +++ b/src/include/gc_alloc.h @@ -0,0 +1,40 @@ +#ifndef NORA_GC_ALLOC_H +#define NORA_GC_ALLOC_H + +#include + +#include + +// A minimal, exception-free C++ allocator over the Boehm-Demers-Weiser GC. +// +// It allocates *scanned* memory (GC_MALLOC), so any pointers stored in the +// container's backing buffer are traced by the collector — the point of moving +// a container onto the GC heap in the value-model migration. Boehm's own +// gc_allocator is unusable here because it requires -fexceptions, which this +// codebase disables (LLVM default); this is the exception-free equivalent. +// +// deallocate() is intentionally a no-op: the collector reclaims a buffer once +// nothing references it, which is the conservative-GC-safe choice (an explicit +// GC_FREE could free a buffer a stale conservative pointer still appears to +// reference). Element construction/destruction still happen normally via +// std::allocator_traits, so RAII members of legacy elements are not leaked. +template struct GcAllocator { + using value_type = T; + + GcAllocator() = default; + template GcAllocator(const GcAllocator &) noexcept {} + + T *allocate(std::size_t N) { + return static_cast(GC_MALLOC(N * sizeof(T))); + } + void deallocate(T *, std::size_t) noexcept {} + + template bool operator==(const GcAllocator &) const noexcept { + return true; + } + template bool operator!=(const GcAllocator &) const noexcept { + return false; + } +}; + +#endif // NORA_GC_ALLOC_H diff --git a/src/include/nir/CMakeLists.txt b/src/include/nir/CMakeLists.txt index 2dfa4fc..ac4c683 100644 --- a/src/include/nir/CMakeLists.txt +++ b/src/include/nir/CMakeLists.txt @@ -1,6 +1,8 @@ set(LLVM_TARGET_DEFINITIONS NirOps.td) mlir_tablegen(NirOps.h.inc -gen-op-decls) mlir_tablegen(NirOps.cpp.inc -gen-op-defs) -mlir_tablegen(Dialect.h.inc -gen-dialect-decls) -mlir_tablegen(Dialect.cpp.inc -gen-dialect-defs) +# -dialect=nir: NirOps.td transitively includes the builtin dialect (via +# BuiltinTypes.td), so the dialect generators must be told which to emit. +mlir_tablegen(Dialect.h.inc -gen-dialect-decls -dialect=nir) +mlir_tablegen(Dialect.cpp.inc -gen-dialect-defs -dialect=nir) add_public_tablegen_target(NirIncGen) diff --git a/src/include/nir/Dialect.h b/src/include/nir/Dialect.h index f77d43a..ff8525a 100644 --- a/src/include/nir/Dialect.h +++ b/src/include/nir/Dialect.h @@ -2,6 +2,7 @@ #include "mlir/IR/BuiltinOps.h" #include "mlir/IR/Dialect.h" +#include "mlir/Interfaces/ControlFlowInterfaces.h" #include "mlir/Interfaces/SideEffectInterfaces.h" #include "nora.h" diff --git a/src/include/nir/NirOps.td b/src/include/nir/NirOps.td index 17a0ba5..47afc08 100644 --- a/src/include/nir/NirOps.td +++ b/src/include/nir/NirOps.td @@ -1,6 +1,13 @@ -//===- Ops.td - NIR dialect operation definitions ----------*- tablegen -*-===// +//===- NirOps.td - NIR dialect operation definitions -------*- tablegen -*-===// // -// Defines the operations of the Nora IR dialect. +// NIR (NORA IR): an MLIR dialect kept close to Racket linklets so that +// Racket-level optimizations happen here, before lowering NIR -> LLVM IR. +// +// R5 / B0 scope: a *minimal, representation-agnostic* skeleton — just enough to +// round-trip through nir-opt and prove the MLIR toolchain integrates. Concrete +// nr_value-tagged types (!nir.fixnum, !nir.box, !nir.closure, ...) are added +// after the M2 ABI freeze (docs/value-model-abi.md); they are absent here on +// purpose. // //===----------------------------------------------------------------------===// @@ -8,92 +15,61 @@ #define NIR_OPS include "mlir/IR/OpBase.td" +include "mlir/IR/BuiltinTypes.td" include "mlir/Interfaces/SideEffectInterfaces.td" +include "mlir/Interfaces/ControlFlowInterfaces.td" + +//===----------------------------------------------------------------------===// +// Dialect +//===----------------------------------------------------------------------===// -// Provide a definition of the 'nir' dialect in the ODS framework so that we -// can define our operations. def NIR_Dialect : Dialect { - let summary = "NORA IR Dialect"; + let name = "nir"; + let summary = "NORA IR — a Racket-linklet-close MLIR dialect"; let description = [{ - NIR (NORA IR) is specified here as an MLIR dialect. + NIR (NORA IR) models Racket's fully-expanded / linklet layer as an MLIR + dialect so that Racket-level optimizations (primitive inlining, fixnum + unboxing, known-call devirtualization, dead-export elimination) can run + before lowering to LLVM IR. }]; - - // The namespace of our dialect. - let name = "nir"; - - // The C++ namespace that the dialect class definition resides in. let cppNamespace = "::mlir::nir"; } -// def NIR_NoraType : DialectType()">, "NIR Nora type">; - -// def NIR_NoraValueAttr : Attr()">, -// "NIR Nora Value attribute"> { -// let storageType = [{ NoraValueAttr }]; -// let returnType = [{ nr_value_t * }]; -// } - -// Base class for nir dialect operations. This operation inherits from the base -// `Op` class in OpBase.td, and provides: -// * The parent dialect of the operation. -// * The mnemonic for the operation, or the name without the dialect prefix. -// * A list of traits for the operation. +// Base class for NIR ops. NOTE: `list` is the current ODS spelling; the +// old scaffold's `list` was removed from MLIR years ago. class NIR_Op traits = []> : Op; - //===----------------------------------------------------------------------===// -// NIR Operations +// Operations //===----------------------------------------------------------------------===// -def UnimplementedOp : NIR_Op<"unimplemented"> { - let summary = "unimplemented operation"; +def NIR_ConstantOp : NIR_Op<"constant", [Pure]> { + let summary = "materialise a constant"; let description = [{ - unimplemented + Representation-agnostic skeleton constant. The payload is a builtin integer + attribute for now; the nr_value-tagged constant lands after the M2 ABI + freeze (docs/value-model-abi.md). + + ```mlir + %0 = nir.constant 42 + ``` + (The result type is a fixed i64, so the custom form omits `: i64`.) }]; - - let results = (outs); - // let builders = [ - // OpBuilder<(ins "nr_datatype_t *":$type)> - // ]; + let arguments = (ins I64Attr:$value); + let results = (outs I64:$result); + let assemblyFormat = "$value attr-dict"; } -// def ConstantOp : NIR_Op<"constant", [ConstantLike, NoSideEffect]> { -// let summary = "constant"; -// let description = [{ -// Constant operation turns a literal into an SSA value. The data is attached -// to the operation as an attribute. For example: - -// ```mlir -// %0 = nir.constant dense<2> -// ``` -// }]; - -// // The constant operation takes an attribute as the only input. -// let arguments = (ins); - -// // The constant operation returns a single value of NumberType -// let results = (outs); - -// // Add custom build methods for the constant operation. This method -// // populates the `state` that MLIR uses to create operations i.e. these -// // are used when using `builder.create(...)`. -// // let builders = [ -// // // Build a constant with a given constant numeric value. -// // OpBuilder<(ins "nr_value_t *":$value, "nr_datatype_t *":$type)> -// // ]; - -// // Set the folder bit so that we can implement constant folders. -// //let hasFolder = 1; -// } - -// def CallWithValues : NIR_Op<"call-with-values", [ConstantLike, NoSideEffect]> { -// } - -// def Lambda : NIR_Op<"lambda", [ConstantLike, NoSideEffect]> { -// } - -// def PrintValues : NIR_Op<"print-values", [ConstantLike, NoSideEffect]> { -// } +def NIR_ReturnOp : NIR_Op<"return", [Pure, Terminator, ReturnLike]> { + let summary = "return terminator"; + let description = [{ + Returns zero or more values from a NIR region. Defined here for B0's + lowering; it is exercised once a NIR region op (nir.func / nir.linklet) + exists to host it. + }]; + let arguments = (ins Variadic:$operands); + let assemblyFormat = "attr-dict ($operands^ `:` type($operands))?"; +} #endif // NIR_OPS diff --git a/src/include/nir/Ops.td b/src/include/nir/Ops.td deleted file mode 100644 index 4b4c2de..0000000 --- a/src/include/nir/Ops.td +++ /dev/null @@ -1,28 +0,0 @@ -//===- Ops.td - NIR dialect operation definitions ----------*- tablegen -*-===// -// -// Defines the operations of the Nora IR dialect. -// -//===----------------------------------------------------------------------===// - -#ifndef NIR_OPS -#define NIR_OPS - -include "mlir/IR/OpBase.td" -include "mlir/Interfaces/SideEffectInterfaces.td" - -// Provide a definition of the 'nir' dialect in the ODS framework so that we -// can define our operations. -def NIR_Dialect : Dialect { - let name = "nir"; - let cppNamespace = "::mlir::nir"; -} - -// Base class for nir dialect operations. This operation inherits from the base -// `Op` class in OpBase.td, and provides: -// * The parent dialect of the operation. -// * The mnemonic for the operation, or the name without the dialect prefix. -// * A list of traits for the operation. -class NIR_Op traits = []> : - Op; - -#endif // NIR_OPS \ No newline at end of file diff --git a/src/include/nora_rt.h b/src/include/nora_rt.h new file mode 100644 index 0000000..7c77a8c --- /dev/null +++ b/src/include/nora_rt.h @@ -0,0 +1,80 @@ +// NORA runtime — the immediate (tagged nr_value) ABI layer. +// +// Promoted from the R1 spike (spike/r1-value-model) and frozen in +// docs/value-model-abi.md. M2 reuses ONLY this immediate/tag layer over the +// existing polymorphic ValueNode hierarchy: heap values are GC-allocated C++ +// cells whose vptr sits at offset 0 — NOT an ObjHeader. R1's ObjHeader-based +// object accessors (NrPair/NrBox/nrt_cons/…) are deliberately absent here so +// nothing reads a vtable pointer as a type tag; flattening the object layout to +// ObjHeader/nr_value is a post-M2 step. See docs/value-model-gc-migration.md. +#ifndef NORA_RT_H +#define NORA_RT_H + +#include +#include + +// A tagged 64-bit word: +// bit 0 == 1 fixnum (value = (int64)w >> 1; 63-bit) +// low 3 bits == 0b000 (w != 0) heap pointer (an 8-byte-aligned cell) +// low 3 bits == 0b010 singleton immediate (subtype in w >> 3) +// low 3 bits == 0b110 character (codepoint in w >> 3) +typedef uint64_t nr_value; + +static constexpr uint64_t NR_TAG_MASK = 0x7; +static constexpr uint64_t NR_TAG_PTR = 0x0; +static constexpr uint64_t NR_TAG_FIX = 0x1; +static constexpr uint64_t NR_TAG_IMM = 0x2; +static constexpr uint64_t NR_TAG_CHR = 0x6; + +enum NrImm : uint64_t { + NR_IMM_FALSE = 0, + NR_IMM_TRUE = 1, + NR_IMM_NULL = 2, + NR_IMM_VOID = 3, + NR_IMM_EOF = 4, + NR_IMM_UNDEF = 5, + NR_IMM_UNINIT = 6, +}; + +#define NR_MK_IMM(sub) (((uint64_t)(sub) << 3) | NR_TAG_IMM) +static constexpr nr_value NR_FALSE = NR_MK_IMM(NR_IMM_FALSE); +static constexpr nr_value NR_TRUE = NR_MK_IMM(NR_IMM_TRUE); +static constexpr nr_value NR_NULL = NR_MK_IMM(NR_IMM_NULL); +static constexpr nr_value NR_VOID = NR_MK_IMM(NR_IMM_VOID); +static constexpr nr_value NR_EOF = NR_MK_IMM(NR_IMM_EOF); +static constexpr nr_value NR_UNDEF = NR_MK_IMM(NR_IMM_UNDEF); +static constexpr nr_value NR_UNINIT = NR_MK_IMM(NR_IMM_UNINIT); + +static inline bool nr_is_fixnum(nr_value w) { return (w & NR_TAG_FIX) != 0; } +static inline bool nr_is_ptr(nr_value w) { + return w != 0 && (w & NR_TAG_MASK) == NR_TAG_PTR; +} +static inline bool nr_is_imm(nr_value w) { + return (w & NR_TAG_MASK) == NR_TAG_IMM; +} +static inline bool nr_is_char(nr_value w) { + return (w & NR_TAG_MASK) == NR_TAG_CHR; +} + +static inline nr_value nr_fixnum(int64_t v) { + return (nr_value)((uint64_t)v << 1) | NR_TAG_FIX; +} +static inline int64_t nr_fixnum_val(nr_value w) { + return (int64_t)w >> 1; // arithmetic shift keeps the sign +} +static inline nr_value nr_char(uint32_t cp) { + return ((nr_value)cp << 3) | NR_TAG_CHR; +} +static inline uint32_t nr_char_val(nr_value w) { return (uint32_t)(w >> 3); } +static inline nr_value nr_bool(bool b) { return b ? NR_TRUE : NR_FALSE; } + +// Racket truthiness: only #f is false. +static inline bool nr_truthy(nr_value w) { return w != NR_FALSE; } + +// Boehm GC heap statistics — the M2 forcing seam (a depth-independent live-heap +// plateau against unbounded churn). Declared without pulling into every +// includer; implemented in nora_rt.cpp. +size_t nrt_gc_heap_size(void); +size_t nrt_gc_total_bytes(void); + +#endif // NORA_RT_H diff --git a/src/main.cpp b/src/main.cpp index a1ef7a1..2b4335a 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -5,6 +5,8 @@ #include #include +#include + #include #include #include @@ -37,6 +39,10 @@ static cl::opt } // namespace int main(int argc, char *argv[]) { + // Bring up the Boehm collector before anything allocates so it records the + // main-thread stack bottom (M2 value model). Non-incremental (the default), + // and before llvm::InitLLVM installs its signal handlers. + GC_INIT(); llvm::InitLLVM X(argc, argv); cl::ParseCommandLineOptions(argc, argv, "norac\n"); diff --git a/src/mlir/CMakeLists.txt b/src/mlir/CMakeLists.txt index 3d3bb10..33426d8 100644 --- a/src/mlir/CMakeLists.txt +++ b/src/mlir/CMakeLists.txt @@ -2,8 +2,36 @@ add_mlir_dialect_library(nirLib nora.cpp Dialect.cpp - ADDITIONAL_HEADER_DIRS + PARTIAL_SOURCES_INTENDED + + ADDITIONAL_HEADER_DIRS ${PROJECT_SOURCE_DIR}/include/nir - DEPENDS - NirIncGen) \ No newline at end of file + DEPENDS + NirIncGen + + LINK_LIBS PUBLIC + MLIRIR + MLIRSupport +) + +# R5 / B0: a minimal nir-opt driver that registers the NIR dialect so .mlir +# files can be parsed, verified and printed (round-trip). Built only when +# NORA_ENABLE_MLIR is ON (this whole directory is added under that guard). +get_property(dialect_libs GLOBAL PROPERTY MLIR_DIALECT_LIBS) +get_property(conversion_libs GLOBAL PROPERTY MLIR_CONVERSION_LIBS) +add_llvm_executable(nir-opt nir-opt.cpp PARTIAL_SOURCES_INTENDED) +llvm_update_compile_flags(nir-opt) +target_link_libraries(nir-opt + PRIVATE + nirLib + ${dialect_libs} + ${conversion_libs} + MLIROptLib + MLIRRegisterAllDialects + MLIRRegisterAllPasses + MLIRIR + MLIRParser + MLIRSupport +) +mlir_check_all_link_libraries(nir-opt) diff --git a/src/mlir/Dialect.cpp b/src/mlir/Dialect.cpp index 5d0b30c..ba60b71 100644 --- a/src/mlir/Dialect.cpp +++ b/src/mlir/Dialect.cpp @@ -2,6 +2,7 @@ #include "mlir/IR/BuiltinTypes.h" #include "mlir/IR/OpImplementation.h" +#include "mlir/Interfaces/ControlFlowInterfaces.h" using namespace mlir; using namespace mlir::nir; diff --git a/src/mlir/README.md b/src/mlir/README.md new file mode 100644 index 0000000..2f7047d --- /dev/null +++ b/src/mlir/README.md @@ -0,0 +1,47 @@ +# NIR MLIR dialect (spike R5 / seed of B0) + +The NIR (NORA IR) dialect is the Racket-linklet-close MLIR layer where +Racket-level optimizations run before lowering NIR → LLVM IR. This directory is +**opt-in** (`-DNORA_ENABLE_MLIR=ON`, the `mlir` CMake preset) and is **not** part +of the default `norac` build. + +## Status: built and round-trip-verified (MLIR 22) + +R5 is complete: the dialect builds under the `mlir` preset and +`test/mlir/nir-roundtrip.mlir` round-trips through `nir-opt | nir-opt` (FileCheck +passes). Requires MLIR 22 (`find_package(MLIR REQUIRED CONFIG)`); the default +`norac` build is unaffected and stays green. + +What the rewrite changed vs. the old scaffold: + +- Deleted the stale duplicate `Ops.td` (used the long-removed `list` API). +- `NirOps.td`: a real, minimal, **representation-agnostic** dialect — `nir.constant` + (`i64` skeleton payload) and a `nir.return` terminator. No `nr_value`-tagged + types yet: those land after the M2 ABI freeze (see `docs/value-model-abi.md`). +- Added `nir-opt.cpp`, a minimal `mlir-opt` clone registering the NIR dialect, and + wired it in `CMakeLists.txt`. +- Added the round-trip acceptance test `test/mlir/nir-roundtrip.mlir`. + +## Verify (once MLIR is installed) + +```sh +# Arch: MLIR is the AUR `mlir` package (needs sudo/AUR; the maintainer runs it): +yay -S mlir +# ...or point CMake at an existing build: -DMLIR_DIR=/path/to/lib/cmake/mlir + +cmake --preset mlir +cmake --build --preset mlir --target nir-opt +build/mlir/bin/nir-opt test/mlir/nir-roundtrip.mlir | build/mlir/bin/nir-opt +# expect: the module prints back with `nir.constant 42 : i64` +``` + +## Known items to confirm at first build + +Because this was authored without a live `mlir-tblgen`, the first `cmake --preset +mlir` build may surface small ODS/version nits to fix in place — most likely the +generated dialect **C++ class name** (`def NIR_Dialect` is assumed to generate +`mlir::nir::NIRDialect`, matching `Dialect.cpp`) and the exact link-library list +in `CMakeLists.txt`. The op set, assembly formats, and traits (`Pure`, +`Terminator`, `ReturnLike`) follow current MLIR conventions. + +Tracking issue: #90. diff --git a/src/mlir/nir-opt.cpp b/src/mlir/nir-opt.cpp new file mode 100644 index 0000000..21b92d2 --- /dev/null +++ b/src/mlir/nir-opt.cpp @@ -0,0 +1,23 @@ +//===- nir-opt.cpp - NIR optimizer / round-trip driver --------------------===// +// +// A minimal `mlir-opt` clone that registers the NIR dialect alongside the +// upstream dialects, so .mlir files using `nir.*` can be parsed, verified, and +// printed. This is the R5 spike's verification tool (parse->print round-trip) +// and the seed of B0's opt driver. +// +//===----------------------------------------------------------------------===// +#include "nir/Dialect.h" + +#include "mlir/IR/DialectRegistry.h" +#include "mlir/InitAllDialects.h" +#include "mlir/InitAllPasses.h" +#include "mlir/Tools/mlir-opt/MlirOptMain.h" + +int main(int argc, char **argv) { + mlir::DialectRegistry registry; + registry.insert(); + mlir::registerAllDialects(registry); // func, builtin, arith, ... + mlir::registerAllPasses(); + return mlir::asMainReturnCode(mlir::MlirOptMain( + argc, argv, "NIR optimizer / round-trip driver\n", registry)); +} diff --git a/src/nora_rt.cpp b/src/nora_rt.cpp new file mode 100644 index 0000000..f7172af --- /dev/null +++ b/src/nora_rt.cpp @@ -0,0 +1,6 @@ +#include "nora_rt.h" + +#include + +size_t nrt_gc_heap_size(void) { return GC_get_heap_size(); } +size_t nrt_gc_total_bytes(void) { return GC_get_total_bytes(); } diff --git a/test/integration/box.rkt b/test/integration/box.rkt new file mode 100644 index 0000000..237f95f --- /dev/null +++ b/test/integration/box.rkt @@ -0,0 +1,10 @@ +;; RUN: norac %s | FileCheck %s +;; A box is a shared mutable cell: set-box! through one reference to `b` is +;; visible when `b` is read again, even though the interpreter clones values on +;; every lookup (M2's shared value model). +;; CHECK: 10 +(linklet + () + () + (let-values (((b) (box 1))) + (begin (set-box! b 10) (unbox b)))) diff --git a/test/integration/lit.cfg.py b/test/integration/lit.cfg.py index bf81216..d2dfefb 100644 --- a/test/integration/lit.cfg.py +++ b/test/integration/lit.cfg.py @@ -2,7 +2,10 @@ import lit.formats config.name = "NORA lit tests" -config.test_format = lit.formats.ShTest(True) +# execute_external=True is deprecated as of lit 23 (removed in 24); every RUN +# line here is a single `tool %s | FileCheck %s` pipe, which lit's internal +# shell (the default, execute_external=False) already handles. +config.test_format = lit.formats.ShTest(execute_external=False) config.suffixes = ['.rkt'] diff --git a/test/integration/pair.rkt b/test/integration/pair.rkt new file mode 100644 index 0000000..b30c670 --- /dev/null +++ b/test/integration/pair.rkt @@ -0,0 +1,9 @@ +;; RUN: norac %s | FileCheck %s +;; A pair is a shared mutable cons cell: set-car!/set-cdr! through one reference +;; to `p` are visible when its fields are read again (M2's shared value model). +;; CHECK: 30 +(linklet + () + () + (let-values (((p) (cons 1 2))) + (begin (set-car! p 10) (set-cdr! p 20) (+ (car p) (cdr p))))) diff --git a/test/integration/symbol.rkt b/test/integration/symbol.rkt new file mode 100644 index 0000000..83e25ce --- /dev/null +++ b/test/integration/symbol.rkt @@ -0,0 +1,6 @@ +;; RUN: norac %s | FileCheck %s +;; gensym returns a fresh uninterned symbol, never eq? to any other, so eq? on +;; symbols is object identity (interned symbols with the same name are eq?; a +;; gensym is not) — M2's shared value model. +;; CHECK: #f +(linklet () () (eq? (gensym) (gensym))) diff --git a/test/integration/tailcall.rkt b/test/integration/tailcall.rkt new file mode 100644 index 0000000..0985cad --- /dev/null +++ b/test/integration/tailcall.rkt @@ -0,0 +1,10 @@ +;; RUN: norac %s | FileCheck %s +;; A deep self-tail-recursive loop runs end to end and returns its result; +;; proper tail calls keep it in bounded continuation space (see the unit tests +;; for the peak-depth assertion). +;; CHECK: 42 +(linklet + () + () + (letrec-values (((loop) (lambda (n) (if (zero? n) 42 (loop (- n 1)))))) + (loop 100000))) diff --git a/test/integration/with-continuation-mark4.rkt b/test/integration/with-continuation-mark4.rkt index bb84e1b..0e75bf7 100644 --- a/test/integration/with-continuation-mark4.rkt +++ b/test/integration/with-continuation-mark4.rkt @@ -1,7 +1,9 @@ ;; RUN: norac %s | FileCheck %s -;; Marks for the same key set in different continuation frames (a caller and a -;; callee) accumulate; continuation-mark-set->list returns them innermost first. -;; CHECK: (2 1) +;; (f 0) is a tail call of the enclosing with-continuation-mark, which is +;; itself in tail position of the linklet body, so f's activation reuses the +;; same continuation frame as the outer mark: installing 'k again replaces +;; the outer value (1) rather than stacking alongside it. +;; CHECK: (2) (linklet () () (define-values (f) (lambda (x) diff --git a/test/integration/with-continuation-mark7.rkt b/test/integration/with-continuation-mark7.rkt new file mode 100644 index 0000000..bf9403e --- /dev/null +++ b/test/integration/with-continuation-mark7.rkt @@ -0,0 +1,14 @@ +;; RUN: norac %s | FileCheck %s +;; Marks for the same key set in genuinely different continuation frames (a +;; caller and a callee) accumulate; continuation-mark-set->list returns them +;; innermost first. Unlike with-continuation-mark4.rkt, (f 0) here is bound by +;; let-values rather than called in tail position, so it does NOT reuse the +;; outer with-continuation-mark's frame. +;; CHECK: (2 1) +(linklet () () + (define-values (f) + (lambda (x) + (with-continuation-mark 'k 2 + (continuation-mark-set->list (current-continuation-marks) 'k)))) + (with-continuation-mark 'k 1 + (let-values ([(r) (f 0)]) r))) diff --git a/test/mlir/nir-roundtrip.mlir b/test/mlir/nir-roundtrip.mlir new file mode 100644 index 0000000..a789310 --- /dev/null +++ b/test/mlir/nir-roundtrip.mlir @@ -0,0 +1,21 @@ +// RUN: nir-opt %s | nir-opt | FileCheck %s +// +// Round-trips the minimal NIR skeleton (parse -> verify -> print, twice) to +// prove the dialect is registered and its assembly format is stable. This is +// the R5 acceptance test. It is NOT wired into ctest yet (nir-opt only exists +// under the `mlir` preset); run it manually: +// +// cmake --preset mlir && cmake --build --preset mlir --target nir-opt +// build/mlir/bin/nir-opt test/mlir/nir-roundtrip.mlir | build/mlir/bin/nir-opt \ +// | FileCheck test/mlir/nir-roundtrip.mlir +// +// nir.constant's result type is a fixed i64, so the custom form prints the +// value without a trailing `: i64`. + +// CHECK-LABEL: func.func @const_return +func.func @const_return() -> i64 { + // CHECK: %[[C:.*]] = nir.constant 42 + %0 = nir.constant 42 + // CHECK: return %[[C]] : i64 + return %0 : i64 +} diff --git a/test/unit/CMakeLists.txt b/test/unit/CMakeLists.txt index d3ad0f1..87a99be 100644 --- a/test/unit/CMakeLists.txt +++ b/test/unit/CMakeLists.txt @@ -19,6 +19,7 @@ llvm_map_components_to_libnames(LLVM_LIBS support) add_executable(test_parse test_parse.cpp + test_main.cpp ${PROJECT_SOURCE_DIR}/src/AnalysisFreeVars.cpp ${PROJECT_SOURCE_DIR}/src/AST.cpp ${PROJECT_SOURCE_DIR}/src/ASTRuntime.cpp @@ -30,7 +31,30 @@ add_executable(test_parse ${PROJECT_SOURCE_DIR}/src/Runtime.cpp ${PROJECT_SOURCE_DIR}/src/SourceStream.cpp ${PROJECT_SOURCE_DIR}/src/UTF8.cpp + ${PROJECT_SOURCE_DIR}/src/nora_rt.cpp ) -target_link_libraries(test_parse PRIVATE Catch2::Catch2 gmp gmpxx ${LLVM_LIBS}) +target_link_libraries(test_parse PRIVATE Catch2::Catch2 gmp gmpxx PkgConfig::BDWGC ${LLVM_LIBS}) -catch_discover_tests(test_parse) \ No newline at end of file +catch_discover_tests(test_parse) + +add_executable(test_interpreter + test_interpreter.cpp + test_main.cpp + test_gc.cpp + ${PROJECT_SOURCE_DIR}/src/AnalysisFreeVars.cpp + ${PROJECT_SOURCE_DIR}/src/AST.cpp + ${PROJECT_SOURCE_DIR}/src/ASTRuntime.cpp + ${PROJECT_SOURCE_DIR}/src/Diagnostics.cpp + ${PROJECT_SOURCE_DIR}/src/Environment.cpp + ${PROJECT_SOURCE_DIR}/src/IdPool.cpp + ${PROJECT_SOURCE_DIR}/src/Interpreter.cpp + ${PROJECT_SOURCE_DIR}/src/Lex.cpp + ${PROJECT_SOURCE_DIR}/src/Parse.cpp + ${PROJECT_SOURCE_DIR}/src/Runtime.cpp + ${PROJECT_SOURCE_DIR}/src/SourceStream.cpp + ${PROJECT_SOURCE_DIR}/src/UTF8.cpp + ${PROJECT_SOURCE_DIR}/src/nora_rt.cpp +) +target_link_libraries(test_interpreter PRIVATE Catch2::Catch2 gmp gmpxx PkgConfig::BDWGC ${LLVM_LIBS}) + +catch_discover_tests(test_interpreter) \ No newline at end of file diff --git a/test/unit/test_gc.cpp b/test/unit/test_gc.cpp new file mode 100644 index 0000000..c1e2150 --- /dev/null +++ b/test/unit/test_gc.cpp @@ -0,0 +1,23 @@ +#include + +#include "nora_rt.h" + +#include + +// S0 of the value-model + GC migration: the Boehm collector is linked and +// initialised, and the reusable nr_value immediate ABI is usable. This test +// touches only immediates and the collector — never a polymorphic-cell object +// accessor (M2 cells have a vptr at offset 0, not an ObjHeader). +TEST_CASE("libgc links, inits, and the nr_value immediate ABI is usable", + "[m2][gc]") { + REQUIRE(nr_fixnum_val(nr_fixnum(42)) == 42); + REQUIRE(nr_fixnum_val(nr_fixnum(-7)) == -7); + REQUIRE(nr_truthy(nr_bool(true))); + REQUIRE_FALSE(nr_truthy(NR_FALSE)); + REQUIRE(nr_char_val(nr_char('Z')) == 'Z'); + + void *P = GC_MALLOC(64); + REQUIRE(P != nullptr); + REQUIRE(GC_get_heap_size() > 0); + REQUIRE(nrt_gc_heap_size() > 0); +} diff --git a/test/unit/test_interpreter.cpp b/test/unit/test_interpreter.cpp new file mode 100644 index 0000000..ee747cc --- /dev/null +++ b/test/unit/test_interpreter.cpp @@ -0,0 +1,335 @@ +#include + +#include "AST.h" +#include "ASTRuntime.h" +#include "Diagnostics.h" +#include "Interpreter.h" +#include "Parse.h" +#include "SourceStream.h" + +#include + +#include + +#include +#include + +namespace { + +// Parse + interpret a linklet source string at the interpreter's public seam. +struct Run { + bool ok = false; // no diagnostics were reported + std::unique_ptr result; // Interpreter::getResult() +}; + +Run runLinklet(const std::string &Src) { + nora::DiagnosticEngine Diag; + SourceStream S(Src.c_str(), &Diag); + std::unique_ptr AST = Parse::parseLinklet(S); + REQUIRE(AST); + Interpreter I(Diag); + AST->accept(I); + Run R; + R.ok = !Diag.hadError(); + R.result = I.getResult(); + return R; +} + +} // namespace + +TEST_CASE("tail-recursive loop computes the correct value", "[interp][tco]") { + Run R = runLinklet("(linklet () () " + "(letrec-values ([(loop) " + " (lambda (n) (if (zero? n) 42 (loop (- n 1))))]) " + "(loop 1000)))"); + REQUIRE(R.ok); + REQUIRE(R.result); + auto *Int = llvm::dyn_cast(R.result.get()); + REQUIRE(Int); + REQUIRE(*Int == 42); +} + +namespace { +// Peak continuation depth of a self-tail-recursive countdown of `depth` steps. +size_t tailLoopPeak(int Depth) { + nora::DiagnosticEngine Diag; + std::string Src = "(linklet () () (letrec-values ([(loop) " + "(lambda (n) (if (zero? n) 0 (loop (- n 1))))]) (loop " + + std::to_string(Depth) + ")))"; + SourceStream S(Src.c_str(), &Diag); + std::unique_ptr AST = Parse::parseLinklet(S); + REQUIRE(AST); + Interpreter I(Diag); + AST->accept(I); + REQUIRE_FALSE(Diag.hadError()); + return I.getPeakKont(); +} +} // namespace + +TEST_CASE("tail recursion runs in bounded continuation space", + "[interp][tco]") { + // Proper tail calls: the same loop at wildly different iteration counts must + // reach the *same* peak continuation depth (O(1)), and that depth is small. + const size_t Shallow = tailLoopPeak(100); + const size_t Deep = tailLoopPeak(100000); + REQUIRE(Deep == Shallow); + REQUIRE(Deep < 16); +} + +namespace { +// Non-tail countdown: the recursive call sits under a pending (+ 1 ...), so it +// is not a tail call and must retain a continuation frame per level. +size_t nonTailLoopPeak(int Depth) { + nora::DiagnosticEngine Diag; + std::string Src = "(linklet () () (letrec-values ([(loop) " + "(lambda (n) (if (zero? n) 0 (+ 1 (loop (- n 1)))))]) " + "(loop " + + std::to_string(Depth) + ")))"; + SourceStream S(Src.c_str(), &Diag); + std::unique_ptr AST = Parse::parseLinklet(S); + REQUIRE(AST); + Interpreter I(Diag); + AST->accept(I); + REQUIRE_FALSE(Diag.hadError()); + return I.getPeakKont(); +} +} // namespace + +TEST_CASE("non-tail recursion still grows the continuation", "[interp][tco]") { + // Contrast: proper tail calls must not collapse genuinely non-tail calls. + // Peak depth grows with the recursion count and dwarfs the tail loop's. + REQUIRE(nonTailLoopPeak(2000) > nonTailLoopPeak(1000)); + REQUIRE(nonTailLoopPeak(1000) > 10 * tailLoopPeak(1000)); +} + +TEST_CASE("the continuation lives in the GC heap", "[m2][gc]") { + // A deep non-tail recursion grows the Kont vector to thousands of frames. + // Nothing else is GC-allocated during evaluation yet, so the cumulative GC + // bytes churned by the run measure Kont's backing store: ~0 while Kont is + // malloc'd, large once it is GC-allocated. (GC-heap seam, per the plan.) + const size_t Before = GC_get_total_bytes(); + (void)nonTailLoopPeak(4000); + const size_t Churned = GC_get_total_bytes() - Before; + REQUIRE(Churned > 100000); +} + +TEST_CASE("a box round-trips its contents", "[interp][m2]") { + Run R = runLinklet("(linklet () () (unbox (box 5)))"); + REQUIRE(R.ok); + REQUIRE(R.result); + auto *Int = llvm::dyn_cast(R.result.get()); + REQUIRE(Int); + REQUIRE(*Int == 5); +} + +TEST_CASE("set-box! mutates through a shared reference", "[interp][m2]") { + // `b` is looked up three times (each lookup clones the value), yet the + // mutation is visible: the box's cell is shared across the clones. This is + // the behaviour the old clone-everything value model could not express. + Run R = runLinklet("(linklet () () " + "(let-values ([(b) (box 1)]) " + "(begin (set-box! b 10) (unbox b))))"); + REQUIRE(R.ok); + REQUIRE(R.result); + auto *Int = llvm::dyn_cast(R.result.get()); + REQUIRE(Int); + REQUIRE(*Int == 10); +} + +TEST_CASE("eq? distinguishes box identity", "[interp][m2]") { + // A box is eq? to itself; two freshly allocated boxes are not. + Run Same = + runLinklet("(linklet () () (let-values ([(b) (box 0)]) (eq? b b)))"); + REQUIRE(Same.ok); + REQUIRE(Same.result); + auto *S = llvm::dyn_cast(Same.result.get()); + REQUIRE(S); + REQUIRE(S->value()); + + Run Diff = runLinklet("(linklet () () (eq? (box 0) (box 0)))"); + REQUIRE(Diff.ok); + REQUIRE(Diff.result); + auto *D = llvm::dyn_cast(Diff.result.get()); + REQUIRE(D); + REQUIRE_FALSE(D->value()); +} + +TEST_CASE("cons/car/cdr round-trip", "[interp][m2]") { + Run Ca = runLinklet("(linklet () () (car (cons 1 2)))"); + REQUIRE(Ca.ok); + REQUIRE(Ca.result); + auto *A = llvm::dyn_cast(Ca.result.get()); + REQUIRE(A); + REQUIRE(*A == 1); + + Run Cd = runLinklet("(linklet () () (cdr (cons 1 2)))"); + REQUIRE(Cd.ok); + REQUIRE(Cd.result); + auto *D = llvm::dyn_cast(Cd.result.get()); + REQUIRE(D); + REQUIRE(*D == 2); +} + +TEST_CASE("set-car!/set-cdr! mutate through a shared reference", + "[interp][m2]") { + Run R = runLinklet( + "(linklet () () (let-values ([(p) (cons 1 2)]) " + "(begin (set-car! p 10) (set-cdr! p 20) (+ (car p) (cdr p)))))"); + REQUIRE(R.ok); + REQUIRE(R.result); + auto *Int = llvm::dyn_cast(R.result.get()); + REQUIRE(Int); + REQUIRE(*Int == 30); +} + +TEST_CASE("eq? distinguishes pair identity", "[interp][m2]") { + Run Same = + runLinklet("(linklet () () (let-values ([(p) (cons 1 2)]) (eq? p p)))"); + REQUIRE(Same.ok); + REQUIRE(Same.result); + auto *S = llvm::dyn_cast(Same.result.get()); + REQUIRE(S); + REQUIRE(S->value()); + + Run Diff = runLinklet("(linklet () () (eq? (cons 1 2) (cons 1 2)))"); + REQUIRE(Diff.ok); + REQUIRE(Diff.result); + auto *D = llvm::dyn_cast(Diff.result.get()); + REQUIRE(D); + REQUIRE_FALSE(D->value()); +} + +TEST_CASE("symbol eq? is identity, not name", "[interp][m2]") { + // Two uninterned symbols with the same name are distinct objects... + Run Un = runLinklet("(linklet () () (eq? (string->uninterned-symbol \"s\") " + "(string->uninterned-symbol \"s\")))"); + REQUIRE(Un.ok); + REQUIRE(Un.result); + auto *U = llvm::dyn_cast(Un.result.get()); + REQUIRE(U); + REQUIRE_FALSE(U->value()); + + // ...while interned symbols with the same name are eq?. + Run In = runLinklet("(linklet () () (eq? 'a 'a))"); + REQUIRE(In.ok); + REQUIRE(In.result); + auto *I = llvm::dyn_cast(In.result.get()); + REQUIRE(I); + REQUIRE(I->value()); +} + +TEST_CASE("eq? unwraps a quoted symbol before comparing identity", + "[interp][m2]") { + // A quoted symbol literal like 'probe evaluates to a QuotedExpr wrapping + // the interned symbol, not a bare Symbol; eq? must unwrap it before its + // identity dispatch rather than falling through to valueEq's structural + // (name-only) comparison, which would wrongly equate it with an + // uninterned symbol of the same name. Built directly against the Runtime + // seam (rather than parsed source) so the comparison has same-named + // operands deterministically, independent of gensym's shared counter. + auto Quoted = std::make_unique(); + Quoted->setQuotedExpr(std::make_unique("probe")); + std::unique_ptr Uninterned = + ast::Symbol::makeUninterned("probe"); + + llvm::SmallVector Args = {Quoted.get(), + Uninterned.get()}; + std::unique_ptr Result = + Runtime::getInstance().callFunction("eq?", Args); + REQUIRE(Result); + auto *B = llvm::dyn_cast(Result.get()); + REQUIRE(B); + REQUIRE_FALSE(B->value()); + + // Same check with operands swapped. + llvm::SmallVector ArgsRev = {Uninterned.get(), + Quoted.get()}; + std::unique_ptr ResultRev = + Runtime::getInstance().callFunction("eq?", ArgsRev); + REQUIRE(ResultRev); + auto *BR = llvm::dyn_cast(ResultRev.get()); + REQUIRE(BR); + REQUIRE_FALSE(BR->value()); +} + +TEST_CASE("gensym produces fresh distinct symbols", "[interp][m2]") { + Run R = runLinklet("(linklet () () (eq? (gensym) (gensym)))"); + REQUIRE(R.ok); + REQUIRE(R.result); + auto *B = llvm::dyn_cast(R.result.get()); + REQUIRE(B); + REQUIRE_FALSE(B->value()); +} + +TEST_CASE("gensym rejects more than one argument", "[interp][m2]") { + Run R = runLinklet("(linklet () () (gensym 'a 'b))"); + REQUIRE_FALSE(R.ok); +} + +TEST_CASE("mutual tail recursion is bounded and correct", "[interp][tco]") { + // ev/od tail-call each other: the reused activation frame belongs to a + // *different* closure than the caller, so this exercises tail-call handling + // in its general (non-self) form. + Run R = runLinklet("(linklet () () (letrec-values (" + " ((ev) (lambda (n) (if (zero? n) 1 (od (- n 1)))))" + " ((od) (lambda (n) (if (zero? n) 0 (ev (- n 1))))))" + " (ev 100000)))"); + REQUIRE(R.ok); + REQUIRE(R.result); + auto *Int = llvm::dyn_cast(R.result.get()); + REQUIRE(Int); + REQUIRE(*Int == 1); // ev(100000): 100000 is even +} + +namespace { +// Peak continuation depth of a self-tail-recursive countdown of `depth` steps +// whose body is wrapped in a with-continuation-mark around the tail call. +size_t wcmTailLoopPeak(int Depth) { + nora::DiagnosticEngine Diag; + std::string Src = "(linklet () () (letrec-values ([(loop) " + "(lambda (n) (with-continuation-mark 'k n " + " (if (zero? n) 0 (loop (- n 1)))))]) (loop " + + std::to_string(Depth) + ")))"; + SourceStream S(Src.c_str(), &Diag); + std::unique_ptr AST = Parse::parseLinklet(S); + REQUIRE(AST); + Interpreter I(Diag); + AST->accept(I); + REQUIRE_FALSE(Diag.hadError()); + return I.getPeakKont(); +} +} // namespace + +TEST_CASE("a tail call through with-continuation-mark runs in bounded space", + "[interp][tco][m2]") { + // A with-continuation-mark wrapping a self-tail-recursive loop's body must + // not defeat frame reuse: the WcmMark frame it installs is still on top of + // Kont when the tail call happens, so this must reach the same *small*, + // depth-independent peak as an unwrapped tail loop. + const size_t Shallow = wcmTailLoopPeak(100); + const size_t Deep = wcmTailLoopPeak(100000); + REQUIRE(Deep == Shallow); + REQUIRE(Deep < 16); +} + +TEST_CASE("a tail-position with-continuation-mark replaces, not " + "accumulates, a same-key mark across loop iterations", + "[interp][m2]") { + // Each iteration's with-continuation-mark is in tail position, so + // successive iterations share one continuation frame; installing the same + // key there again must replace the previous value (as real Scheme/Racket + // does), not stack a second entry alongside it. + Run R = runLinklet("(linklet () () (letrec-values ([(loop) " + "(lambda (n) (with-continuation-mark 'k n " + " (if (zero? n) (continuation-mark-set->list " + " (current-continuation-marks) 'k) " + " (loop (- n 1)))))]) (loop 5)))"); + REQUIRE(R.ok); + REQUIRE(R.result); + auto *L = llvm::dyn_cast(R.result.get()); + REQUIRE(L); + REQUIRE(L->length() == 1); + auto *Elem = llvm::dyn_cast(&(*L)[0]); + REQUIRE(Elem); + REQUIRE(*Elem == 0); +} diff --git a/test/unit/test_main.cpp b/test/unit/test_main.cpp new file mode 100644 index 0000000..22bd411 --- /dev/null +++ b/test/unit/test_main.cpp @@ -0,0 +1,12 @@ +// Shared Catch2 entry point for the unit test executables. GC_INIT() must run +// on the main thread before any test allocates, so the Boehm collector records +// the correct stack bottom. (The GMP allocator hook lands in a later GC slice.) +#define CATCH_CONFIG_RUNNER +#include + +#include + +int main(int argc, char **argv) { + GC_INIT(); + return Catch::Session().run(argc, argv); +} diff --git a/test/unit/test_parse.cpp b/test/unit/test_parse.cpp index 3631476..8446687 100644 --- a/test/unit/test_parse.cpp +++ b/test/unit/test_parse.cpp @@ -1,5 +1,4 @@ -#define CATCH_CONFIG_MAIN #include #include "Diagnostics.h" diff --git a/tools/freevars.py b/tools/freevars.py new file mode 100644 index 0000000..e82b0b6 --- /dev/null +++ b/tools/freevars.py @@ -0,0 +1,354 @@ +#!/usr/bin/env python3 +"""R3-static spike: enumerate the free (primitive) identifiers referenced by a +flattened linklet such as expander/expander.rktl. + +The production C++ parser cannot yet ingest expander.rktl (rational/flonum and +`1/foo` identifier lexing are M3 work), so this standalone, tolerant, +scope-aware s-expression walker produces the primitive worklist that seeds +M4-M7. It performs a real free-variable analysis: it tracks lexical scope for +the FEP binding forms (lambda, case-lambda, let-values, letrec-values, +define-values), skips quoted data, and reports every identifier referenced but +neither locally bound nor defined by the linklet, ranked by frequency. + +Usage: python3 tools/freevars.py expander/expander.rktl [out.tsv] +""" +import re +import sys +from collections import Counter + +# --------------------------------------------------------------------------- +# Tolerant reader: produces a nested tree. Nodes are: +# list -> a Python list of child nodes +# ("S", name) -> a symbol (identifier) +# ("A",) -> any non-symbol atom (number/string/char/bool/kw/bytes) +# Reader-shorthand and self-quoting literals are wrapped as ["quote", ...] or +# tagged data so the walker skips them. +# --------------------------------------------------------------------------- + +OPEN = set("([{") +CLOSE = set(")]}") +DELIM = set(" \t\n\r\f\v()[]{}\"';`,") + +NUM_RE = re.compile( + r"""^[+-]?( + \d+ | # integer + \d+/\d+ | # rational + (\d+\.\d*|\.\d+|\d+)([eE][+-]?\d+)? | # decimal / float + (inf|nan)\.[0f] # +inf.0 -nan.0 etc (sign required, handled above) + )$""", + re.VERBOSE, +) + + +def is_number(tok: str) -> bool: + if NUM_RE.match(tok): + return True + if tok in ("+inf.0", "-inf.0", "+nan.0", "-nan.0", "+inf.f", "-inf.f", + "+nan.f", "-nan.f"): + return True + return False + + +def read_tree(s: str): + """Parse the whole string into a root list of top-level datums.""" + n = len(s) + i = 0 + root = [] + stack = [root] # stack[-1] is the current list being filled + wrap = [0] # pending quote-wraps for the next datum at this level + childwrap = [] # wrap to apply to a child list when it closes + + def add(datum): + w = wrap[-1] + wrap[-1] = 0 + for _ in range(w): + datum = [("S", "quote"), datum] + stack[-1].append(datum) + + while i < n: + c = s[i] + if c in " \t\n\r\f\v": + i += 1 + continue + if c == ";": # line comment + j = s.find("\n", i) + i = n if j < 0 else j + 1 + continue + if c in OPEN: + childwrap.append(wrap[-1]); wrap[-1] = 0 + stack.append([]); wrap.append(0) + i += 1 + continue + if c in CLOSE: + lst = stack.pop(); wrap.pop() + w = childwrap.pop() if childwrap else 0 + for _ in range(w): + lst = [("S", "quote"), lst] + stack[-1].append(lst) + i += 1 + continue + if c == '"': # string + i = skip_string(s, i + 1, n); add(("A",)); continue + if c == "'": # quote + wrap[-1] += 1; i += 1; continue + if c == "`": # quasiquote -> treat as data, skip + wrap[-1] += 1; i += 1; continue + if c == ",": # unquote / unquote-splicing -> data + i += 2 if s[i:i + 2] == ",@" else 1 + wrap[-1] += 1; continue + if c == "#": + i = read_hash(s, i, n, stack, wrap, childwrap, add) + continue + # default: a constituent token (symbol or number) + j = i + while j < n and s[j] not in DELIM: + j += 1 + tok = s[i:j]; i = j + add(("A",) if is_number(tok) else ("S", tok)) + return root + + +def skip_string(s, i, n): + while i < n: + c = s[i] + if c == "\\": + i += 2; continue + if c == '"': + return i + 1 + i += 1 + return n + + +def read_hash(s, i, n, stack, wrap, childwrap, add): + """Handle a token starting with '#'. Returns the new index.""" + c1 = s[i + 1] if i + 1 < n else "" + if c1 == "|": # block comment (nestable) + depth = 1; j = i + 2 + while j < n and depth: + two = s[j:j + 2] + if two == "#|": + depth += 1; j += 2 + elif two == "|#": + depth -= 1; j += 2 + else: + j += 1 + return j + if c1 == ";": # datum comment: drop next datum + # emit a throwaway wrap that we cancel by adding to a scratch list + # (none present in expander.rktl; handled defensively) + wrap[-1] += 0 + return i + 2 # best-effort: skip the '#;' marker only + if c1 == "\\": # char literal + j = i + 2 + if j < n and (s[j].isalpha()): + k = j + while k < n and (s[k].isalnum() or s[k] == "-"): + k += 1 + j = k if k > j + 1 else j + 1 # named char, else single char + else: + j = j + 1 if j < n else j + add(("A",)); return j + if c1 in OPEN: # #( vector literal -> data + childwrap.append(wrap[-1] + 1); wrap[-1] = 0 # +1 so it is skipped as data + stack.append([]); wrap.append(0) + return i + 2 + if c1 == '"': # #"..." byte string + j = skip_string(s, i + 2, n); add(("A",)); return j + if c1 == "%": # #%... symbol (identifier) + j = i + while j < n and s[j] not in DELIM: + j += 1 + add(("S", s[i:j])); return j + if c1 == ":": # #:keyword atom + j = i + while j < n and s[j] not in DELIM: + j += 1 + add(("A",)); return j + # #hash( / #hasheq( / #s( / #N( ... -> data-open; else #t/#f/#x.. atom + j = i + 1 + while j < n and (s[j].isalnum()): + j += 1 + if j < n and s[j] in OPEN: # #word( -> data literal + childwrap.append(wrap[-1] + 1); wrap[-1] = 0 + stack.append([]); wrap.append(0) + return j + 1 + # plain #-atom (boolean / radix number / etc.) + k = i + while k < n and s[k] not in DELIM: + k += 1 + add(("A",)); return k + + +# --------------------------------------------------------------------------- +# Free-variable analysis +# --------------------------------------------------------------------------- + +SKIP_HEADS = {"quote", "quote-syntax", "quasiquote", "unquote", + "unquote-splicing"} +SEQ_HEADS = {"if", "begin", "begin0", "with-continuation-mark", "#%expression", + "#%app", "#%plain-app", "begin-for-syntax"} +IGNORE_REFS = {"."} + + +def sym(node): + return node[1] if isinstance(node, tuple) and node[0] == "S" else None + + +def formals_set(node): + """Bound identifiers of a lambda/case-lambda formal spec.""" + out = set() + s = sym(node) + if s is not None: # rest-arg: (lambda x ...) + out.add(s) + return out + if isinstance(node, list): # (a b . rest) / (a b) / () + for e in node: + es = sym(e) + if es is not None and es != ".": + out.add(es) + return out + + +def collect_defined(body): + """First pass: every identifier bound by a define-values, ignoring quoted + data. These are the linklet's own module-level bindings.""" + G = set() + stack = list(body) + while stack: + node = stack.pop() + if not isinstance(node, list) or not node: + continue + h = sym(node[0]) + if h in SKIP_HEADS: + continue + if h in ("define-values", "define-syntaxes") and len(node) >= 2: + ids = node[1] + if isinstance(ids, list): + for e in ids: + es = sym(e) + if es: + G.add(es) + else: + es = sym(ids) + if es: + G.add(es) + stack.extend(node) + return G + + +def analyze(body, G): + """Iterative scope-aware walk. Returns Counter of free identifiers.""" + free = Counter() + # work items: (node, scopes) where scopes is a tuple of frozenset + work = [(f, ()) for f in reversed(body)] + while work: + node, scopes = work.pop() + # atom + if isinstance(node, tuple): + if node[0] == "S": + name = node[1] + if name in IGNORE_REFS: + continue + if name in G: + continue + if any(name in fr for fr in scopes): + continue + free[name] += 1 + continue + if not isinstance(node, list) or not node: + continue + h = sym(node[0]) + if h in SKIP_HEADS: + continue + if h in ("lambda", "#%plain-lambda") and len(node) >= 2: + b = frozenset(formals_set(node[1])) + ns = scopes + (b,) + for e in node[2:]: + work.append((e, ns)) + continue + if h == "case-lambda": + for clause in node[1:]: + if isinstance(clause, list) and clause: + b = frozenset(formals_set(clause[0])) + ns = scopes + (b,) + for e in clause[1:]: + work.append((e, ns)) + continue + if h in ("let-values", "letrec-values", "let*-values") and len(node) >= 2: + binds = node[1] if isinstance(node[1], list) else [] + ids = set() + for bnd in binds: + if isinstance(bnd, list) and bnd: + for e in (bnd[0] if isinstance(bnd[0], list) else [bnd[0]]): + es = sym(e) + if es and es != ".": + ids.add(es) + bset = frozenset(ids) + rec = h != "let-values" + rhs_scope = scopes + (bset,) if rec else scopes + body_scope = scopes + (bset,) + for bnd in binds: + if isinstance(bnd, list) and len(bnd) >= 2: + work.append((bnd[1], rhs_scope)) + for e in node[2:]: + work.append((e, body_scope)) + continue + if h in ("define-values", "define-syntaxes") and len(node) >= 3: + work.append((node[2], scopes)) # ids already in G + continue + if h == "set!" and len(node) >= 3: + work.append((node[1], scopes)) # target is a reference too + work.append((node[2], scopes)) + continue + if h == "#%variable-reference": + for e in node[1:]: + work.append((e, scopes)) + continue + if h in SEQ_HEADS: + for e in node[1:]: + work.append((e, scopes)) + continue + # application (operator + operands are all expressions) + for e in node: + work.append((e, scopes)) + return free + + +def main(): + path = sys.argv[1] if len(sys.argv) > 1 else "expander/expander.rktl" + out = sys.argv[2] if len(sys.argv) > 2 else None + s = open(path, encoding="utf-8", errors="replace").read() + root = read_tree(s) + linklet = None + for d in root: + if isinstance(d, list) and d and sym(d[0]) == "linklet": + linklet = d + break + if linklet is None: + print("error: no (linklet ...) form found", file=sys.stderr) + sys.exit(1) + imports, exports = linklet[1], linklet[2] + body = linklet[3:] + G = collect_defined(body) + free = analyze(body, G) + + n_exports = len(exports) if isinstance(exports, list) else 0 + print(f"file: {path}") + print(f"top-level body forms: {len(body)}") + print(f"exports: {n_exports}") + print(f"linklet-defined ids: {len(G)}") + print(f"distinct FREE ids: {len(free)} (total refs: {sum(free.values())})") + print() + print("Top 60 free identifiers by reference count:") + for name, c in free.most_common(60): + print(f" {c:6d} {name}") + + if out: + with open(out, "w") as fh: + for name, c in free.most_common(): + fh.write(f"{c}\t{name}\n") + print(f"\nfull ranked list -> {out}") + + +if __name__ == "__main__": + main()