From 85f2da2875541ca3bf28a985527330a4091a8ccc Mon Sep 17 00:00:00 2001 From: Hydra Date: Sat, 14 Nov 2026 21:45:14 +0200 Subject: [PATCH] fix(instance): fence late event streams during controller handoff --- CHANGELOG.md | 6 ++ apps/api/src/app.ts | 2 + apps/api/src/lib/sse.ts | 15 ++++ apps/api/test/e2e/fixtures/instance-api.ts | 2 +- .../api/test/e2e/instance-handoff.e2e.test.ts | 43 ++++++++++- .../api/test/lib/controller-sse-drain.test.ts | 71 +++++++++++++++++++ .../components/instance/InstanceLocation.tsx | 4 +- .../instance/instance-location.test.tsx | 10 +++ docs/dashboard-design.md | 2 + 9 files changed, 151 insertions(+), 4 deletions(-) create mode 100644 apps/api/test/lib/controller-sse-drain.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index dcd6b4bab..5785ef043 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,12 @@ Runtime security notices use the signed feed published from `security-advisories - Run the container-to-host connectivity probe with Node.js, matching the API image, so `openship up` and `openship doctor` can diagnose the host channel without Bun. +### Instance relocation + +- Fence event-stream admission while moving the controller so a late subscriber cannot + keep the source draining indefinitely. Restore stream access when the controller resumes, + and link server preparation to its build view while deployment reads remain available. + ## 0.9.12 **Critical update** for build-secret protection, database upgrade repairs and diff --git a/apps/api/src/app.ts b/apps/api/src/app.ts index 264549493..0363ab976 100644 --- a/apps/api/src/app.ts +++ b/apps/api/src/app.ts @@ -23,6 +23,7 @@ import { registerControllerLifecycle, } from "./modules/system/instance/controller-state"; import { resumeControllerSockets } from "./lib/ws"; +import { resumeControllerStreams } from "./lib/sse"; import { trackBackgroundWork } from "@repo/platform/engine/lib/background-work"; import { instanceAuthorityGuard } from "./modules/system/instance/authority-guard"; import { quiesceController } from "./modules/system/instance/quiescence"; @@ -408,6 +409,7 @@ async function startControllerBackground(): Promise { backgroundStarted = true; try { resumeControllerSockets(); + resumeControllerStreams(); // ─── Backup job runner + boot reconcile ───────────────────────────── // // One JobRunner powers all backup work — BullMQ when Redis is diff --git a/apps/api/src/lib/sse.ts b/apps/api/src/lib/sse.ts index 210f16549..469f5dde6 100644 --- a/apps/api/src/lib/sse.ts +++ b/apps/api/src/lib/sse.ts @@ -41,7 +41,12 @@ import { SYSTEM } from "@repo/core"; import { trackBackgroundWork } from "@repo/platform/engine/lib/background-work"; const controllerStreams = new Set(); +let accepting = true; +export function resumeControllerStreams(): void { + accepting = true; +} export function closeControllerStreams(): void { + accepting = false; for (const stream of controllerStreams) stream.abort(); } @@ -182,6 +187,16 @@ export function streamSSE( c: Context, cb: (stream: SSEStreamingApi) => Promise, ) { + // An admitted request can finish its permission/metadata read after the HTTP + // fence closed existing streams. Do not give the drain a new live subscriber. + if (!accepting) { + c.header("Retry-After", "5"); + c.header("Cache-Control", "no-store"); + return c.json({ + code: "INSTANCE_CONTROLLER_INACTIVE", + error: "Your instance is moving. Reconnect after the handoff.", + }, 503); + } const context = currentErrorContext(); // Disable reverse-proxy response buffering. nginx/OpenResty buffer proxied // responses by default, which holds SSE events back until the buffer fills — diff --git a/apps/api/test/e2e/fixtures/instance-api.ts b/apps/api/test/e2e/fixtures/instance-api.ts index 97f198dea..7bf882126 100644 --- a/apps/api/test/e2e/fixtures/instance-api.ts +++ b/apps/api/test/e2e/fixtures/instance-api.ts @@ -135,7 +135,7 @@ export async function startApi(input: { } export async function stopApi(api: RunningApi): Promise { - if (api.child.exitCode === null) { + if (api.child.exitCode === null && api.child.signalCode === null) { api.child.kill("SIGTERM"); const exited = new Promise((resolve) => api.child.once("exit", () => resolve(true))); if (!(await Promise.race([exited, delay(15_000).then(() => false)]))) { diff --git a/apps/api/test/e2e/instance-handoff.e2e.test.ts b/apps/api/test/e2e/instance-handoff.e2e.test.ts index 0bb185c46..e39c61903 100644 --- a/apps/api/test/e2e/instance-handoff.e2e.test.ts +++ b/apps/api/test/e2e/instance-handoff.e2e.test.ts @@ -5,7 +5,7 @@ import { createRequire } from "node:module"; import { execFile } from "node:child_process"; import { promisify } from "node:util"; import { afterAll, expect, it } from "vitest"; -import { createServer, type Server } from "node:http"; +import { createServer, request as httpRequest, type Server } from "node:http"; import { createHmac } from "node:crypto"; import { freePort, startApi, stopApi, jsonRequest, type RunningApi } from "./fixtures/instance-api"; import type { DataTransferFile } from "../../src/modules/system/data-transfer/types"; @@ -433,6 +433,47 @@ it("resumes after a lost activation acknowledgement without reimporting or start ).toBe(newer.id); }, 120_000); +it.each(["resume", "cancel"] as const)("recovers a persisted quiescing source after restart with %s", async action => { + const { source, target } = await pair(); + const original = await createProject(source, "Before interrupted drain"); + const offer = await post<{ code: string }>(target, "/api/system/instance/offer", { direction: "target" }); + // A real admitted mutation waits for its remaining request body. The move must + // keep waiting for it, rather than treating an incomplete drain as a snapshot. + const held = httpRequest(`${source.baseUrl}/api/projects`, { + method: "POST", headers: { "content-type": "application/json" }, + }); + held.on("error", () => { /* The fixture intentionally interrupts this request. */ }); + held.write('{"name":'); + let restarted: RunningApi | undefined; + try { + await post(source, "/api/system/instance/move", { code: offer.code, confirmReplace: true }); + await expect.poll(async () => (await state(source)).role, { timeout: 15_000 }).toBe("quiescing"); + const exited = new Promise(resolve => source.child.once("exit", () => resolve())); + source.child.kill("SIGKILL"); // simulate a crash after the durable fence + await exited; + held.destroy(); + restarted = await startApi({ dbDir: source.dbDir, port: source.port, secret: source.secret }); + apis.push(restarted); + expect(await state(restarted)).toMatchObject({ role: "quiescing", handoff: { running: false } }); + expect((await fetch(`${restarted.baseUrl}/api/health`)).status).toBe(200); + expect((await fetch(`${restarted.baseUrl}/api/projects`)).status).toBe(503); + await post(restarted, `/api/system/instance/${action}`); + if (action === "resume") { + await waitMove(restarted); + expect((await state(target)).role).toBe("active"); + expect((await jsonRequest<{ data: { id: string } }>(target.baseUrl, `/api/projects/${original.id}`)).data.id).toBe(original.id); + } else { + expect(await state(restarted)).toMatchObject({ role: "active", handoff: { status: "aborted" } }); + expect((await jsonRequest<{ data: { id: string } }>(restarted.baseUrl, `/api/projects/${original.id}`)).data.id).toBe(original.id); + } + } finally { + held.destroy(); + await stopApi(source); + await stopApi(target); + if (restarted) await stopApi(restarted); + } +}, 120_000); + it("cancels a prepared but unactivated receiver and restores its recovery archive", async () => { const { source, target } = await pair(); const old = await createProject(target, "Receiver original"); diff --git a/apps/api/test/lib/controller-sse-drain.test.ts b/apps/api/test/lib/controller-sse-drain.test.ts new file mode 100644 index 000000000..fd2e2ffcc --- /dev/null +++ b/apps/api/test/lib/controller-sse-drain.test.ts @@ -0,0 +1,71 @@ +import { afterEach, beforeEach, expect, it, vi } from "vitest"; +import { Hono } from "hono"; +import type { SSEStreamingApi } from "hono/streaming"; +import { closeControllerStreams, resumeControllerStreams, streamSSE } from "../../src/lib/sse"; +import { drainBackgroundWork } from "@repo/platform/engine/lib/background-work"; + +beforeEach(() => resumeControllerStreams()); +afterEach(async () => { + closeControllerStreams(); + await drainBackgroundWork(); + resumeControllerStreams(); +}); + +it("refuses a stream admitted before the move but initialized after streams were closed", async () => { + let enter!: () => void; + let proceed!: () => void; + const entered = new Promise(resolve => { enter = resolve; }); + const admitted = new Promise(resolve => { proceed = resolve; }); + const callback = vi.fn(async (stream: SSEStreamingApi) => { + await new Promise(resolve => stream.onAbort(resolve)); + }); + const app = new Hono(); + app.get("/events", async c => { + enter(); + await admitted; // permission/metadata read already admitted by the HTTP guard + return streamSSE(c, callback); + }); + const pending = app.request("/events"); + await entered; + closeControllerStreams(); + proceed(); + const response = await pending; + try { + expect(response.status).toBe(503); + expect(await response.json()).toMatchObject({ code: "INSTANCE_CONTROLLER_INACTIVE" }); + expect(callback).not.toHaveBeenCalled(); + } finally { + closeControllerStreams(); + if (!response.bodyUsed) await response.body?.cancel(); + await drainBackgroundWork(); + } +}); + +it("aborts an existing subscriber so the controller can finish draining", async () => { + const aborted = vi.fn(); + const app = new Hono(); + app.get("/events", c => streamSSE(c, async stream => { + await new Promise(resolve => stream.onAbort(() => { aborted(); resolve(); })); + })); + const response = await app.request("/events"); + expect(response.status).toBe(200); + closeControllerStreams(); + await drainBackgroundWork(); + expect(aborted).toHaveBeenCalledOnce(); + await response.body?.cancel(); +}); + +it("accepts streams again when the active controller is resumed after cancellation", async () => { + const callback = vi.fn(async () => {}); + const app = new Hono(); + app.get("/events", c => streamSSE(c, callback)); + closeControllerStreams(); + const refused = await app.request("/events"); + expect(refused.status).toBe(503); + expect(callback).not.toHaveBeenCalled(); + resumeControllerStreams(); + const response = await app.request("/events"); + expect(response.status).toBe(200); + expect(callback).toHaveBeenCalledOnce(); + await response.body?.cancel(); +}); diff --git a/apps/dashboard/src/components/instance/InstanceLocation.tsx b/apps/dashboard/src/components/instance/InstanceLocation.tsx index 80efb4396..5b893c744 100644 --- a/apps/dashboard/src/components/instance/InstanceLocation.tsx +++ b/apps/dashboard/src/components/instance/InstanceLocation.tsx @@ -129,9 +129,9 @@ export function InstanceProgress({ Cancel move )} - {h?.provisioning?.deploymentId && ( + {state.role === "active" && h?.provisioning?.deploymentId && ( )} diff --git a/apps/dashboard/src/components/instance/instance-location.test.tsx b/apps/dashboard/src/components/instance/instance-location.test.tsx index 171377bd4..37ab3fa0e 100644 --- a/apps/dashboard/src/components/instance/instance-location.test.tsx +++ b/apps/dashboard/src/components/instance/instance-location.test.tsx @@ -41,4 +41,14 @@ describe("instance handoff recovery UI", () => { expect(html).not.toContain("Cancel move"); expect(instanceProgress(retired)).toBe("Connecting to the new control plane"); }); + it("links server preparation to its build view while the source controller is active", () => { + const preparing: InstanceStatus = { ...state, role: "active", handoff: { + ...state.handoff!, provisioning: { projectId: "project", deploymentId: "deploy", access: "desktop" }, + } }; + const html = renderToStaticMarkup( {}} />); + expect(html).toContain('href="/build/deploy"'); + expect(html).not.toContain('href="/deployments/deploy"'); + const frozen = renderToStaticMarkup( {}} />); + expect(frozen).not.toContain("View deployment"); + }); }); diff --git a/docs/dashboard-design.md b/docs/dashboard-design.md index c71a2d330..8b07fd51a 100644 --- a/docs/dashboard-design.md +++ b/docs/dashboard-design.md @@ -509,6 +509,8 @@ Instance relocation has one entry in **Settings → Instance → Instance locati the control plane, connecting a Desktop, and backup/archive recovery distinct. Reuse the shared server selector and option cards for API-only versus API + dashboard. Show progress and recovery in the same card; do not add another direct-transfer or Team migration wizard. +During server preparation, View deployment opens the saved build view. Hide that action +while the source is fenced for transfer, when ordinary deployment reads are unavailable. Keep **Connect to an existing instance** and **Move to a server** visible on Desktop. A connected Desktop shows **Reconnect** and **Move back to Desktop**. An active server shows **Connect a Desktop**, **Move to another server**, and **Move to Desktop**.