From a81f1775f64b242ca219420d51ba947dc6cdf78b Mon Sep 17 00:00:00 2001 From: Nathan Disidore Date: Wed, 12 Aug 2026 11:17:35 -0500 Subject: [PATCH 1/2] Prove reconnects before publishing them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit handleBroken published a fresh stub every backoff cycle before it had round-tripped: capnweb queues sends while the socket is CONNECTING, so the unproven stub looked fine until every effect pipelined onto it failed at once, once per cycle. And because useAuth pipelines authenticate() without awaiting, markConnectionRestored() fired instantly and the Reconnecting chip flickered off/on all outage long. Rework handleBroken into a recovery loop that probes each candidate with getServerConfig() (20s timeout) and only publishes it once the probe round-trips. During the outage the published stub stays the dead one, so stub-keyed effects don't re-fire per attempt; subscribers hear exactly twice per outage — lost, then restored. The proof replaces markConnectionRestored(), so delete it and its __root.tsx effect. Keeps today's fast recovery from one-off blips: the first backoff is skipped when the dying connection had been up longer than the initial backoff. The workspace editor's Reconnecting chip previously appeared only as a side effect of the churn: each republish re-fired the workspace-open effect, whose failure set the workspace-level connectionLost. With the dead stub staying published, drive the chip from the socket-level flag too. --- .../workshop-frontend/src/GadgetEditor.tsx | 6 +- packages/workshop-frontend/src/main.tsx | 98 ++++++++++++------- .../workshop-frontend/src/routes/__root.tsx | 6 -- 3 files changed, 65 insertions(+), 45 deletions(-) diff --git a/packages/workshop-frontend/src/GadgetEditor.tsx b/packages/workshop-frontend/src/GadgetEditor.tsx index 4eec39a2..03e56be6 100644 --- a/packages/workshop-frontend/src/GadgetEditor.tsx +++ b/packages/workshop-frontend/src/GadgetEditor.tsx @@ -15,6 +15,7 @@ import { } from '@phosphor-icons/react' import { RpcStub, RpcTarget } from 'capnweb' import { useAuthenticatedApi } from './AuthContext' +import { useConnectionLost } from './RpcContext' import UserMenu from './components/UserMenu' import SiteLogo from './components/SiteLogo' @@ -447,6 +448,9 @@ export default function GadgetEditor() { isEditingTitleRef.current = isEditingTitle const [titleInput, setTitleInput] = useState('') + // The workspace-level flag covers reopen failures; the socket-level flag covers the outage + // window itself, during which the dead stub stays published and no reopen is attempted yet. + const rpcConnectionLost = useConnectionLost() const { overseer, metadata, @@ -1418,7 +1422,7 @@ export default function GadgetEditor() { onViewActivity={openActivity} /> - {connectionLost && } + {(connectionLost || rpcConnectionLost) && } setShareModalOpen(true)} diff --git a/packages/workshop-frontend/src/main.tsx b/packages/workshop-frontend/src/main.tsx index 625eff89..66ce8474 100644 --- a/packages/workshop-frontend/src/main.tsx +++ b/packages/workshop-frontend/src/main.tsx @@ -56,7 +56,23 @@ async function devAutoLogin(stub: RpcStub): Promise { // // Anyway, I pulled the connection management out into these globals instead. let lastConnectTime: number = 0; -let backoff: number = 1000; + +const INITIAL_BACKOFF_MS = 1000; +const MAX_BACKOFF_MS = 10000; +// The probe is getServerConfig — the same call the app needs to boot anyway. A generous +// deadline lets a slow-but-alive backend settle instead of connect/dispose looping. +const RECONNECT_PROBE_TIMEOUT_MS = 20000; + +// Callbacks to call whenever `currentStub` or connection state is updated. +const subscribers = new Set<() => void>(); +const notifySubscribers = () => subscribers.forEach(cb => cb()); +let isConnectionLost = false; +let reconnecting = false; + +const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); + +const withTimeout = (promise: Promise, ms: number): Promise => + Promise.race([promise, sleep(ms).then((): never => { throw new Error(`timed out after ${ms}ms`); })]); function getBackendHost(): string { // Only the Vite dev server is hosted separately from the backend. Built assets are served from @@ -74,45 +90,51 @@ function startConnection(): RpcStub { return newWebSocketRpcSession(wsUrl); } -async function handleBroken(error: any) { - console.warn('RPC connection lost:', error); +const disposeQuietly = (stub: RpcStub) => { + try { stub[Symbol.dispose](); } catch { /* already broken */ } +}; - isConnectionLost = true; - for (let cb of notifyCurrentStubUpdated) { cb(); } - - let timeSinceConnect = Date.now() - lastConnectTime; - if (timeSinceConnect < backoff) { - let waitTime = backoff - timeSinceConnect; - console.warn(`Will try again in ${Math.round(waitTime / 1000)} seconds...`) - await new Promise(resolve => setTimeout(resolve, waitTime)); - console.warn(`Retrying connection...`); - backoff = Math.min(backoff * 2, 10000); - } else { - backoff = 1000; - } +// A reconnect attempt only becomes `currentStub` after a probe RPC round-trips: capnweb queues +// sends while the socket is still CONNECTING, so an unproven stub looks fine until everything +// pipelined onto it fails at once. Proving first means subscribers hear exactly twice per +// outage — lost, then restored — instead of once per failed attempt. +async function handleBroken(error: unknown) { + if (reconnecting) return; // stale/disposed stub, or recovery already underway + reconnecting = true; - currentStub = startConnection(); - currentStub.onRpcBroken(handleBroken); - - // Don't clear isConnectionLost here — the new connection hasn't proven - // it works yet. It gets cleared by markConnectionRestored() once the - // app successfully communicates with the backend. - for (let cb of notifyCurrentStubUpdated) { - cb(); + console.warn('RPC connection lost:', error); + isConnectionLost = true; + notifySubscribers(); // `currentStub` stays the dead one, so stub-keyed effects don't re-fire + + // Fast recovery from one-off blips: skip the first backoff if the dying connection was up a while. + let skipSleep = Date.now() - lastConnectTime >= INITIAL_BACKOFF_MS; + let backoff = INITIAL_BACKOFF_MS; + for (;;) { + if (!skipSleep) { + await sleep(backoff * (0.85 + 0.3 * Math.random())); // jittered against stampedes + backoff = Math.min(backoff * 2, MAX_BACKOFF_MS); + } + skipSleep = false; + + const candidate = startConnection(); + try { + await withTimeout(candidate.getServerConfig(), RECONNECT_PROBE_TIMEOUT_MS); + } catch (probeError) { + console.debug('Reconnect attempt failed:', probeError); + disposeQuietly(candidate); + continue; + } + + candidate.onRpcBroken(handleBroken); + currentStub = candidate; + reconnecting = false; + isConnectionLost = false; + console.warn('RPC connection restored.'); + notifySubscribers(); + return; } } -// Callbacks to call whenever `currentStub` or connection state is updated. -let notifyCurrentStubUpdated: Set<() => void> = new Set(); -let isConnectionLost = false; - -// Called externally (e.g., by auth) to indicate the connection is alive. -export function markConnectionRestored() { - if (!isConnectionLost) return; - isConnectionLost = false; - for (let cb of notifyCurrentStubUpdated) { cb(); } -} - // Current stub. handleBroken() will replace this on disconnect. installWorkshopErrorReporting() let currentStub = startConnection(); @@ -130,9 +152,9 @@ function AppWithConnection() { const [serverConfigError, setServerConfigError] = useState(false); useEffect(() => { - let cb = () => setRpcState({ stub: currentStub, connectionLost: isConnectionLost }); - notifyCurrentStubUpdated.add(cb); - return () => { notifyCurrentStubUpdated.delete(cb); }; + const cb = () => setRpcState({ stub: currentStub, connectionLost: isConnectionLost }); + subscribers.add(cb); + return () => { subscribers.delete(cb); }; }, []); // Fetch deployment config once the (re)connected stub is available. Re-fetch on reconnect so a diff --git a/packages/workshop-frontend/src/routes/__root.tsx b/packages/workshop-frontend/src/routes/__root.tsx index 0d6019d1..e1686169 100644 --- a/packages/workshop-frontend/src/routes/__root.tsx +++ b/packages/workshop-frontend/src/routes/__root.tsx @@ -5,7 +5,6 @@ import { TooltipProvider, Toasty } from '@cloudflare/kumo' import { RpcStub } from 'capnweb' import { AuthenticatedApi } from '@gadgets/workshop-shared/api' import { useRpcStub, useConnectionLost } from '../RpcContext' -import { markConnectionRestored } from '../main' import { useAuth, CF_ACCESS_MODE } from '../useAuth' import { AuthProvider } from '../AuthContext' import { FeatureFlagsProvider } from '../FeatureFlagsContext' @@ -25,11 +24,6 @@ function RootComponent() { const { isAuthenticated, authenticatedApi, isLoading, error, logout, login } = useAuth(rpcStub) const pathname = useRouterState({ select: (s) => s.location.pathname }) - // When authenticatedApi becomes available, the connection is proven alive. - useEffect(() => { - if (authenticatedApi) markConnectionRestored() - }, [authenticatedApi]) - // Routes that don't require auth (public routes) const isSignup = pathname === '/signup' const isBlueprint = pathname.startsWith('/blueprint/') From a7b93ef8f55ae9454b91652d359690152b5c9ca9 Mon Sep 17 00:00:00 2001 From: Nathan Disidore Date: Wed, 12 Aug 2026 11:19:12 -0500 Subject: [PATCH 2/2] Probe the connection on tab wake MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nothing detected a socket killed during laptop sleep or background tab throttling: the user woke the tab and their first action hung on a zombie socket before recovery even started. On tab-visible and network-online signals, probe currentStub with getServerConfig() under a 10s timeout — skipped while reconnecting, while a probe is already in flight, or when the connection was proven alive within the last 15s (so rapid tab switches stay silent). On timeout, dispose the stub; that fires onRpcBroken and the recovery loop takes over, whose skip-first-backoff path retries immediately — right for "the network just came back". --- packages/workshop-frontend/src/main.tsx | 36 +++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/packages/workshop-frontend/src/main.tsx b/packages/workshop-frontend/src/main.tsx index 66ce8474..ad740570 100644 --- a/packages/workshop-frontend/src/main.tsx +++ b/packages/workshop-frontend/src/main.tsx @@ -59,15 +59,20 @@ let lastConnectTime: number = 0; const INITIAL_BACKOFF_MS = 1000; const MAX_BACKOFF_MS = 10000; -// The probe is getServerConfig — the same call the app needs to boot anyway. A generous -// deadline lets a slow-but-alive backend settle instead of connect/dispose looping. +// The probe is getServerConfig — the same call the app needs to boot anyway. Generous deadlines +// let a slow-but-alive backend settle instead of connect/dispose looping (or, on wake, tearing +// down a healthy socket under load). const RECONNECT_PROBE_TIMEOUT_MS = 20000; +const WAKE_PROBE_TIMEOUT_MS = 10000; +const WAKE_PROBE_MIN_IDLE_MS = 15000; // Callbacks to call whenever `currentStub` or connection state is updated. const subscribers = new Set<() => void>(); const notifySubscribers = () => subscribers.forEach(cb => cb()); let isConnectionLost = false; let reconnecting = false; +let probing = false; +let lastProvenAt = Date.now(); const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); @@ -127,6 +132,7 @@ async function handleBroken(error: unknown) { candidate.onRpcBroken(handleBroken); currentStub = candidate; + lastProvenAt = Date.now(); reconnecting = false; isConnectionLost = false; console.warn('RPC connection restored.'); @@ -135,6 +141,32 @@ async function handleBroken(error: unknown) { } } +// Passive close detection misses sockets killed during laptop sleep or tab throttling, so on +// tab-visible / network-online signals probe the connection instead of letting the user's next +// action hang on a zombie socket. +async function probeOnWake() { + if (reconnecting || probing || Date.now() - lastProvenAt < WAKE_PROBE_MIN_IDLE_MS) return; + probing = true; + const suspect = currentStub; + try { + await withTimeout(suspect.getServerConfig(), WAKE_PROBE_TIMEOUT_MS); + lastProvenAt = Date.now(); + } catch (error) { + if (currentStub !== suspect || reconnecting) return; // a real broken event won the race + console.warn('Connection unresponsive after wake:', error); + // Disposal fires onRpcBroken → handleBroken recovers. Its skip-first-backoff path retries + // immediately — right for "the network just came back". + disposeQuietly(suspect); + } finally { + probing = false; + } +} + +document.addEventListener('visibilitychange', () => { + if (document.visibilityState === 'visible') void probeOnWake(); +}); +window.addEventListener('online', () => void probeOnWake()); + // Current stub. handleBroken() will replace this on disconnect. installWorkshopErrorReporting() let currentStub = startConnection();