feat(studio): correlate logs with traces and expand span detail

api-snapshots/do.api.md

@@ -367,13 +367,14 @@ abstract class ShardDO {
     protected resolveShape(_name: string, _args: Record<string, unknown>, _identity?: SubscriptionIdentity): ResolvedShape | undefined;
     protected isShapeRelayUniform(name: string, args: Record<string, unknown>): boolean;
     protected readGlobalShapeRows(_resolved: ResolvedShape, _identity?: SubscriptionIdentity): Promise<ShapeRow[]>;
-    protected pollExternalSources(): Promise<number | undefined>;
+    protected pollExternalSources(_trace?: TraceRefLike): Promise<number | undefined>;
     protected scheduleSourcePoll(): Promise<void>;
     protected ttlSweeps(): ReadonlyArray<TtlSweepSpec>;
-    protected pollTtlSweeps(): Promise<number | undefined>;
+    protected pollTtlSweeps(trace?: TraceRefLike): Promise<number | undefined>;
     protected scheduleTtlSweep(): Promise<void>;
     protected currentShardKey(): string;
-    protected recordExternalSourceError(table: string, error: unknown): void;
+    protected recordExternalSourceError(table: string, error: unknown, trace?: TraceRefLike): void;
+    protected recordExternalSourceWarning(table: string, message: string, trace?: TraceRefLike): void;
     protected executeStream(_functionPath: string, _args: Record<string, unknown>): null | {
         iterator: (signal: AbortSignal) => AsyncIterable<unknown>;
     };
@@ -509,6 +510,14 @@ interface TelemetrySink {
 }
 ```
 
+### `TraceRefLike` (interface)
+
+```ts
+interface TraceRefLike {
+    traceId: string;
+}
+```
+
 ### `TransactionHeadroomTracker` (class)
 
 Re-exported from `@lunora/shard-engine` — signature tracked at its source.

api-snapshots/lunora.api.md

@@ -2237,6 +2237,10 @@ Re-exported from `@lunora/shard-engine` — signature tracked at its source.
 
 Re-exported from `@lunora/do` — signature tracked at its source.
 
+### `TraceRefLike` (interface)
+
+Re-exported from `@lunora/do` — signature tracked at its source.
+
 ### `TransactionHeadroomTracker` (class)
 
 Re-exported from `@lunora/shard-engine` — signature tracked at its source.

api-snapshots/observability.api.md

@@ -56,6 +56,7 @@ interface AppendRequestLogEntry {
     subscriptionsReRun?: number;
     tablesRead?: string[];
     tablesWritten?: string[];
+    traceId?: string;
     ts: number;
     userId?: string;
 }
@@ -425,6 +426,7 @@ interface LogEntry {
     level: LogLevel;
     message: string;
     timestamp: number;
+    traceId?: string;
 }
 ```
 
@@ -680,6 +682,7 @@ interface RequestLogEntry {
     subscriptionsReRun: number;
     tablesRead: string[];
     tablesWritten: string[];
+    traceId?: string;
     ts: number;
     userId?: string;
 }
@@ -870,6 +873,8 @@ interface TraceSpan {
         message: string;
         type: string;
     };
+    events?: SpanEventPoint[];
+    kind?: OtlpSpanKind;
     name: string;
     offsetMs: number;
     ok: boolean;

api-snapshots/studio.api.md

@@ -650,6 +650,7 @@ interface LogEntry {
     level: LogLevel;
     message: string;
     timestamp: number;
+    traceId?: string;
 }
 ```
 

apps/docs/src/content/docs/concepts/observability.mdx

@@ -395,6 +395,25 @@ hibernation. It is a development readout, not a trace store: for retention and
 cross-instance search, point [`otlpSink`](#otlpsink) at a real collector, where
 each span is exported as an OTLP `INTERNAL` span carrying its `parentSpanId`.
 
+Expanding a trace gives an elapsed-time ruler over the bars, a per-span detail
+block (span id, kind, the full attribute bag, the error, and any
+`span.addEvent` / `span.recordException` events), and the `ctx.log` lines the
+same dispatch emitted, joined by trace id. **Errors only** narrows the list to
+traces where something threw; the search box matches a span name or span id as
+well as the trace's own identifiers, so an id pasted from an error report finds
+the trace containing it. Both **Logs** views carry a **Trace** link on rows
+emitted inside a dispatch, opening the waterfall on the shard it came from.
+
+:::caution[The Requests view's Trace link is best-effort]
+The durable request log records each dispatch's trace id, but it **outlives** the
+span ring it points at: the log survives hibernation and the ring does not. A
+Trace link on a row older than the current DO instance therefore lands on an
+empty Traces panel — the trace has aged out locally. The id itself stays valid
+wherever [`otlpSink`](#otlpsink) ships spans, which is where a deployed app's
+traces are retained; it is also emitted on the `Logpush` console event, so a SIEM
+can join a request to its spans.
+:::
+
 ### Generation spans (AI)
 
 An AI model call is a span with the OpenTelemetry

apps/studio/src/mock/dev-client.ts

@@ -127,6 +127,11 @@ const TRACES = [
                 attributes: { method: "POST", url: "api.stripe.com/v1/charges" },
                 depth: 2,
                 durationMs: 200,
+                // A retried-then-succeeded outbound call: `kind` and the recorded
+                // events are what the span-detail block exists to show, and a
+                // handled retry is invisible anywhere else in the Studio.
+                events: [{ attributes: { "retry.attempt": 1 }, name: "http.retry", ts: now - 3900 }],
+                kind: "client",
                 name: "http.post",
                 offsetMs: 28,
                 ok: true,
@@ -170,6 +175,8 @@ const TRACES = [
                 depth: 1,
                 durationMs: 165,
                 error: { message: "card_declined: Your card was declined.", type: "StripeError" },
+                events: [{ attributes: { "exception.type": "StripeError" }, name: "exception", ts: now - 14_900 }],
+                kind: "client",
                 name: "stripe.charge",
                 offsetMs: 10,
                 ok: false,
@@ -379,10 +386,20 @@ const dataFor = (reference: string, args: unknown): unknown => {
         }
         case ADMIN_FUNCTIONS.getLogs: {
             return {
+                // `traceId` matches the `TRACES` fixtures above, so the Traces
+                // panel's correlated-log section and the Logs panel's Trace link
+                // both have something to resolve. The last line deliberately has
+                // none — a container-lifecycle entry, emitted outside any dispatch.
                 entries: [
-                    { functionPath: "posts:publish", level: "error", message: "Rate limited (429) from upstream", timestamp: now - 4000 },
-                    { functionPath: "messages:list", level: "info", message: "Served 128 rows in 12ms", timestamp: now - 9000 },
-                    { functionPath: "messages:send", level: "warn", message: "Slow write: 28ms", timestamp: now - 15_000 },
+                    {
+                        functionPath: "posts:publish",
+                        level: "error",
+                        message: "Rate limited (429) from upstream",
+                        timestamp: now - 4000,
+                        traceId: "trace-publish",
+                    },
+                    { functionPath: "messages:list", level: "info", message: "Served 128 rows in 12ms", timestamp: now - 9000, traceId: "trace-list" },
+                    { functionPath: "messages:send", level: "warn", message: "Slow write: 28ms", timestamp: now - 15_000, traceId: "trace-send" },
                     { level: "debug", message: "Reactive cache warm (312 entries)", timestamp: now - 22_000 },
                 ],
             };
@@ -420,6 +437,7 @@ const dataFor = (reference: string, args: unknown): unknown => {
                         subscriptionsReRun: 0,
                         tablesRead: ["messages"],
                         tablesWritten: [],
+                        traceId: "trace-list",
                         ts: now - 4000,
                     },
                     {
@@ -431,8 +449,22 @@ const dataFor = (reference: string, args: unknown): unknown => {
                         subscriptionsReRun: 0,
                         tablesRead: ["posts"],
                         tablesWritten: [],
+                        traceId: "trace-publish",
                         ts: now - 9000,
                     },
+                    // No `traceId`: a row written before the column existed. The
+                    // Requests view must render it with no Trace link rather than
+                    // an empty button.
+                    {
+                        durationMs: 8,
+                        functionPath: "messages:send",
+                        outcome: "ok",
+                        seq: 8999,
+                        subscriptionsReRun: 2,
+                        tablesRead: [],
+                        tablesWritten: ["messages"],
+                        ts: now - 14_000,
+                    },
                 ],
             };
         }

packages/codegen/__tests__/emit-external-source.test.ts

@@ -45,7 +45,7 @@ describe("emitShard — external-source ingest", () => {
 
         const shard = emitShard({ schema: discover(SOURCED) });
 
-        expect(shard).toContain("protected override async pollExternalSources()");
+        expect(shard).toContain("protected override async pollExternalSources(trace?: TraceRefLike)");
         // The per-table work delegates to the tested @lunora/do helper, not an inline projection.
         expect(shard).toContain("pullExternalSourceTick(this.sql as SqlExec, writer, client");
         // Incremental mode (plan 136) branches to the durable-watermark helper.
@@ -62,7 +62,7 @@ describe("emitShard — external-source ingest", () => {
         // Plan 148: the override reports the earliest NEXT-DUE timestamp (not a
         // bare active count), so the shared alarm can sleep until a source is
         // actually due instead of spinning at the 2 s global-shape floor.
-        expect(shard).toContain("protected override async pollExternalSources(): Promise<number | undefined>");
+        expect(shard).toContain("protected override async pollExternalSources(trace?: TraceRefLike): Promise<number | undefined>");
         expect(shard).toContain("nextDueAt = nextDueAt === undefined ? sourceNextDueAt : Math.min(nextDueAt, sourceNextDueAt);");
         // Plan 207 step 2: each table's writer gets its OWN fresh per-work-item
         // meter — an alarm tick has no `/rpc` dispatch to fall back to.
@@ -71,7 +71,49 @@ describe("emitShard — external-source ingest", () => {
         // warns (not `recordExternalSourceError`) and leaves the table due so the
         // shared alarm re-arms promptly instead of throttling to `refresh.everyMs`.
         expect(shard).toContain('error instanceof LunoraError && error.code === "TRANSACTION_LIMIT_EXCEEDED"');
-        expect(shard).toContain('level: "warn"');
+        // The `warn` level itself is the base class's (`recordExternalSourceWarning`);
+        // what the emitter owns is routing the back-off there rather than to
+        // `recordExternalSourceError`, which would group it as a real Issue.
+        expect(shard).toContain("this.recordExternalSourceWarning(");
+    });
+
+    it("forwards the alarm's trace to every contained ingest failure it records", () => {
+        expect.assertions(4);
+
+        const shard = emitShard({ schema: discover(SOURCED) });
+
+        // The parameter is only worth having if all three log sites in the
+        // generated poll loop actually pass it on — a signature that accepts a
+        // trace and drops it would leave these lines uncorrelated exactly as
+        // before, and the type-checker would not notice.
+        expect(shard).toContain("this.recordExternalSourceError(table, error, trace);");
+        // A regex so the emitted `${source.binding}` interpolation stays out of a
+        // string literal (`no-template-curly-in-string`).
+        expect(shard).toMatch(/no sourceClient resolved for binding[^\n]*, trace\);/u);
+        expect(shard).toContain("this.recordExternalSourceWarning(");
+        // Structural projection from the base package, so a generated app never
+        // takes on an `@lunora/observability` dependency for this.
+        expect(shard).toContain("import type { ExternalSourceLike, SourceClientLike, TraceRefLike }");
+    });
+
+    it("only touches base-class members the compile-time contract covers", () => {
+        expect.assertions(1);
+
+        // The emitted shard is a string, so nothing type-checks it — which is how
+        // `this.logs.push(...)` (private on `ShardDO`) shipped in the poll loop for
+        // the life of this feature. `emitted-shard-contract.ts` compiles a subclass
+        // using each member below, so `lint:types` fails if one changes visibility
+        // or signature; this asserts the emitter still restricts itself to that set.
+        //
+        // Adding a member here means adding it to the contract file too — that
+        // pairing is the whole guard. Members read off `this` inside the emitted
+        // poll loop, ignoring locals and the `schema`/`config` module closures.
+        const membersOf = (shard: string): Set<string> => new Set([...shard.matchAll(/\bthis\.([A-Za-z_]\w*)/gu)].map((match) => match[1] as string));
+
+        const plain = membersOf(emitShard({ schema: discover(PLAIN) }));
+        const added = [...membersOf(emitShard({ schema: discover(SOURCED) }))].filter((member) => !plain.has(member)).toSorted((a, b) => a.localeCompare(b));
+
+        expect(added).toStrictEqual(["alarmHeadroom", "currentShardKey", "recordExternalSourceError", "recordExternalSourceWarning", "scheduleSourcePoll"]);
     });
 
     it("stays byte-identical (none of the ingest surface) for a non-sourced schema", () => {

packages/codegen/__tests__/emitted-shard-contract.ts

@@ -0,0 +1,59 @@
+/**
+ * A compile-time contract for the shard subclass `emitShard` generates.
+ *
+ * **Why this file exists.** The emitter produces TypeScript as a *string*, and
+ * every test around it asserts on substrings of that string. Nothing compiles
+ * it. So the generated code can reference a base-class member that a subclass
+ * cannot legally touch, and the whole suite stays green — which is exactly what
+ * happened: the external-source poll loop emitted `this.logs.push(...)` against
+ * a `private logs`, so every project declaring a `.source()` table emitted a
+ * shard that failed `tsc`. No fixture or example declares one, so nobody ran
+ * into it.
+ *
+ * Golden fixtures cannot close that gap: `tsconfig.json` deliberately excludes
+ * `__tests__/fixtures/**`, because generated output only type-checks inside a
+ * whole app (its schema, its `_generated` siblings, its bindings). This file is
+ * the cheap half of the job — it mirrors the base-class surface the emitted
+ * shard depends on, in a real subclass, so `lint:types` fails HERE, in the
+ * package that owns the emitter, the moment one of those members changes
+ * visibility or signature.
+ *
+ * It is not a test and is never executed. Keeping it honest is a two-part
+ * contract with `emit-external-source.test.ts`, which asserts the emitted text
+ * really does call these members (and really does not touch `this.logs`):
+ * this file proves the calls compile, that one proves they are the calls made.
+ *
+ * When the emitter starts using a new base member, add it here.
+ */
+import type { TraceRefLike } from "@lunora/do";
+import { ShardDO } from "@lunora/do";
+
+class EmittedShardContract extends ShardDO {
+    // eslint-disable-next-line class-methods-use-this -- required abstract override; this file exercises the base surface, it never dispatches
+    public override handleRpc(): Promise<unknown> {
+        return Promise.resolve(undefined);
+    }
+
+    /**
+     * Mirrors the `hasSourcedTables` branch of the generated shard: the override
+     * signature, and every base member its poll loop touches.
+     */
+    protected override async pollExternalSources(trace?: TraceRefLike): Promise<number | undefined> {
+        await Promise.resolve();
+
+        const shardKey = this.currentShardKey();
+
+        this.alarmHeadroom();
+        this.recordChangedTable("documents");
+
+        // The two contained-failure seams. `recordExternalSourceWarning` exists
+        // precisely so this loop never has to reach for the private log ring.
+        this.recordExternalSourceError("documents", new Error(shardKey), trace);
+        this.recordExternalSourceWarning("documents", "hit the transaction limit mid-batch", trace);
+
+        await this.scheduleSourcePoll();
+
+        return undefined;
+    }
+}
+export default EmittedShardContract;