From 9a54e57971229167524cdcf9ec8df139cd42c5e1 Mon Sep 17 00:00:00 2001 From: Arthur Fiorette Date: Sat, 14 Nov 2026 01:50:40 -0300 Subject: [PATCH 01/10] fix(core): give a restarted foreach's resume data only to the resumed item Before, a restart handed the stored payload to every item, so suspended siblings ran with another item's approval. The running foreach record now stores which item its resumePayload belongs to (an index, 'all' for a resume without an index, 'none' otherwise). A restart copies it through, hands the payload only to that item, and keeps the other suspended items suspended. Records without a target hand the payload to no item. A resumed foreach now stores its running record even when the same instance suspended the run. A resumed nested workflow step, and a foreach whose items are nested workflows, store what the resume targeted inside the nested run, so a restart that has to resume the nested run again resumes only that. When that target is missing (records from earlier versions), the nested run stays suspended. A resume of a foreach item inside a nested workflow by index now reaches only that item. --- .changeset/goofy-icons-pull.md | 9 + .../durable-agent-approval-restart.test.ts | 319 +++++++++ .../durable/__tests__/restart-harness.ts | 22 +- packages/core/src/workflows/constants.ts | 5 + .../workflows/foreach-resume-restart.test.ts | 636 ++++++++++++++++++ .../src/workflows/handlers/control-flow.ts | 58 +- .../workflows/handlers/foreach-progress.ts | 69 +- packages/core/src/workflows/handlers/step.ts | 44 +- .../src/workflows/restart-mid-block.test.ts | 51 +- .../src/workflows/test-utils/run-tables.ts | 52 ++ packages/core/src/workflows/workflow.ts | 36 +- 11 files changed, 1234 insertions(+), 67 deletions(-) create mode 100644 .changeset/goofy-icons-pull.md create mode 100644 packages/core/src/agent/durable/__tests__/durable-agent-approval-restart.test.ts create mode 100644 packages/core/src/workflows/foreach-resume-restart.test.ts create mode 100644 packages/core/src/workflows/test-utils/run-tables.ts diff --git a/.changeset/goofy-icons-pull.md b/.changeset/goofy-icons-pull.md new file mode 100644 index 00000000000..6717145cf04 --- /dev/null +++ b/.changeset/goofy-icons-pull.md @@ -0,0 +1,9 @@ +--- +'@mastra/core': patch +--- + +Fixed `foreach` items receiving another item's resume data when a run is restarted while one item is being resumed. Items that were still suspended stay suspended until they are resumed, and items the `foreach` had not reached start without resume data. This also holds for a `foreach` inside a nested workflow, for a `foreach` whose items are nested workflows, and when the resume was handled by the same `Mastra` instance that suspended the run. For a `DurableAgent` that delegates to several sub-agents whose tools need approval, approving one delegation and then recovering the run after a crash no longer runs the other sub-agents' tools without their approval. + +Fixed resuming one item of a `foreach` inside a nested workflow by index, for example `run.resume({ step: ['child-workflow', 'approve'], forEachIndex: 1, resumeData })`, resuming every suspended item of that `foreach` with the same resume data. + +Runs interrupted while resuming a `foreach` item or a nested workflow on an earlier version don't record what was resumed. On restart, the suspended items and steps stay suspended, so resume the interrupted one again. diff --git a/packages/core/src/agent/durable/__tests__/durable-agent-approval-restart.test.ts b/packages/core/src/agent/durable/__tests__/durable-agent-approval-restart.test.ts new file mode 100644 index 00000000000..949fc933fde --- /dev/null +++ b/packages/core/src/agent/durable/__tests__/durable-agent-approval-restart.test.ts @@ -0,0 +1,319 @@ +/** + * Recovering a durable agent run that crashed while an approved tool call ran. + * + * The model asks for three approval-gated tool calls in one step. The first is approved through + * the same agent instance that suspended the run, and the process crashes while it runs. The + * recovered run must run only that call again: the other two still wait for their own approval. + */ + +import { createOpenAI } from '@ai-sdk/openai-v5'; +import { LLMock } from '@copilotkit/aimock'; +import { MockLanguageModelV2, convertArrayToReadableStream } from '@internal/ai-sdk-v5/test'; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest'; +import { z } from 'zod'; +import { DurableStepIds } from '../constants'; +import type { Gate } from './restart-harness'; +import { createGate, createRestartScenario } from './restart-harness'; + +let llm: LLMock; +const gates: Gate[] = []; +const scenarios: { stop(): Promise }[] = []; + +beforeAll(async () => { + llm = new LLMock({ port: 0 }); + await llm.start(); + llm.on( + { endpoint: 'chat', hasToolResult: false }, + { toolCalls: [0, 1, 2].map(index => ({ id: `call_${index}`, name: 'charge', arguments: { index } })) }, + ); + llm.on({ endpoint: 'chat', hasToolResult: true }, { content: 'all charged' }); +}); + +afterAll(async () => { + await llm.stop(); +}); + +afterEach(async () => { + for (const gate of gates.splice(0)) gate.release(); + for (const scenario of scenarios.splice(0)) await scenario.stop(); +}); + +describe('durable agent: restart while an approved tool call runs', () => { + it('runs only the approved tool call, not the calls still waiting for approval', async () => { + const runId = 'approval-restart'; + const hold = createGate(); + gates.push(hold); + const charges: { generation: number; index: number }[] = []; + const scenario = createRestartScenario({ + kind: 'durable', + runId, + build: ({ core, generation }) => + new core.Agent({ + id: 'approval-agent', + name: 'approval-agent', + instructions: 'Charge every card.', + model: createOpenAI({ apiKey: 'test-key', baseURL: `${llm.url}/v1` }).chat('gpt-4o-mini'), + tools: { + charge: core.createTool({ + id: 'charge', + description: 'Charges a card', + inputSchema: z.object({ index: z.number() }), + requireApproval: true, + execute: async ({ index }) => { + charges.push({ generation, index }); + if (generation === 1) await hold.wait(); + return { charged: index }; + }, + }), + }, + }), + }); + scenarios.push(scenario); + + let approvedId: string | undefined; + const original = await scenario.start(async ({ agent }) => { + const first = await agent.stream('Charge the cards', { runId }); + for await (const chunk of first.fullStream) { + if (chunk.type === 'tool-call-approval') { + approvedId = chunk.payload.toolCallId; + break; + } + } + // Approved through the instance that suspended the run. + return agent.approveToolCall({ runId, toolCallId: approvedId }); + }); + await hold.reached; + expect(approvedId).toBe('call_0'); + expect(charges).toEqual([{ generation: 1, index: 0 }]); + + const recovered = await scenario.restart(await original.checkpoint(), { closeOnSuspend: true }); + + expect(recovered.streamErrors).toEqual([]); + expect(charges.filter(c => c.generation === 2)).toEqual([{ generation: 2, index: 0 }]); + // The next call asks for its own approval. + expect(recovered.chunkTypes).toContain('tool-call-approval'); + }, 30_000); +}); + +/** A sub-agent that calls its approval-gated tool once, then answers. */ +function subAgentModel(label: string) { + return new MockLanguageModelV2({ + doStream: async ({ prompt }) => { + const done = JSON.stringify(prompt).includes(`${label} done`); + return { + rawCall: { rawPrompt: null, rawSettings: {} }, + warnings: [], + stream: convertArrayToReadableStream([ + { type: 'stream-start', warnings: [] }, + { type: 'response-metadata', id: `${label}-${done ? 1 : 0}`, modelId: 'mock-model', timestamp: new Date(0) }, + ...(done + ? [ + { type: 'text-start', id: `${label}-text` }, + { type: 'text-delta', id: `${label}-text`, delta: `${label} finished.` }, + { type: 'text-end', id: `${label}-text` }, + ] + : [ + { + type: 'tool-call', + toolCallType: 'function', + toolCallId: `inner-${label}`, + toolName: 'gatedTool', + input: JSON.stringify({ query: label }), + providerExecuted: false, + }, + ]), + { + type: 'finish', + finishReason: done ? 'stop' : 'tool-calls', + usage: { inputTokens: 5, outputTokens: 10, totalTokens: 15 }, + }, + ]), + }; + }, + }); +} + +/** A supervisor that delegates to both sub-agents in one step, then answers. */ +function supervisorModel() { + return new MockLanguageModelV2({ + doStream: async ({ prompt }) => { + const text = JSON.stringify(prompt); + const done = text.includes('alpha finished') && text.includes('beta finished'); + return { + rawCall: { rawPrompt: null, rawSettings: {} }, + warnings: [], + stream: convertArrayToReadableStream([ + { type: 'stream-start', warnings: [] }, + { type: 'response-metadata', id: `sup-${done ? 1 : 0}`, modelId: 'mock-model', timestamp: new Date(0) }, + ...(done + ? [ + { type: 'text-start', id: 'sup-text' }, + { type: 'text-delta', id: 'sup-text', delta: 'Both delegations complete.' }, + { type: 'text-end', id: 'sup-text' }, + ] + : ['Alpha', 'Beta'].map(name => ({ + type: 'tool-call', + toolCallType: 'function', + toolCallId: `sup-tc-${name[0]}`, + toolName: `agent-sub${name}`, + input: JSON.stringify({ prompt: `do ${name.toLowerCase()}` }), + providerExecuted: false, + }))), + { + type: 'finish', + finishReason: done ? 'stop' : 'tool-calls', + usage: { inputTokens: 10, outputTokens: 20, totalTokens: 30 }, + }, + ]), + }; + }, + }); +} + +describe('durable agent: restart while an approved delegation is being resumed', () => { + // Delegations to sub-agents run concurrently, so both suspend at once when each sub-agent's tool + // needs approval. The approval of one goes through the instance that suspended the run, and the + // process crashes before the loop's iteration workflow stores its resumed tool calls. + const crashPoints = [ + { + point: 'before the iteration workflow claims its resume', + hold: { + when: (snapshot: any) => + snapshot.context?.[DurableStepIds.AGENTIC_EXECUTION]?.status === 'running' && + snapshot.context[DurableStepIds.AGENTIC_EXECUTION].resumePayload !== undefined, + }, + }, + { + point: 'after the iteration workflow claims its resume, before its tool calls are stored', + hold: { + workflowName: DurableStepIds.AGENTIC_EXECUTION, + when: (snapshot: any) => + snapshot.status === 'running' && snapshot.context?.[DurableStepIds.TOOL_CALL]?.status === 'suspended', + }, + }, + ]; + + /** Both delegations suspended, `sup-tc-A` approved, and a copy of the rows at the crash. */ + async function crashWhileApproving(hold: (typeof crashPoints)[number]['hold']) { + const runId = 'delegation-approval-restart'; + const memory = { thread: 'delegation-thread', resource: 'delegation-resource' }; + const executions: { generation: number; query: string }[] = []; + // Memory keeps the pending approvals; both graphs use one store for it. + let memoryStore: any; + const scenario = createRestartScenario({ + kind: 'durable', + runId, + hold, + // A sub-agent's snapshot holds a function, which the harness can't copy and which a + // persistent store drops when it serializes the row. + storageMode: 'serializing', + build: ({ core, generation }) => { + const gatedTool = () => + core.createTool({ + id: 'gatedTool', + description: 'Approval-gated tool', + inputSchema: z.object({ query: z.string() }), + requireApproval: true, + execute: async ({ query }: { query: string }) => { + executions.push({ generation, query }); + return { result: `${query} done` }; + }, + }); + const subAgent = (name: string) => + new core.Agent({ + id: `sub${name}`, + name: `sub${name}`, + description: `Does ${name.toLowerCase()}`, + instructions: 'Use your tool.', + model: subAgentModel(name.toLowerCase()) as any, + tools: { gatedTool: gatedTool() }, + }); + return new core.Agent({ + id: 'supervisor', + name: 'Supervisor', + instructions: 'Delegate to both sub-agents.', + model: supervisorModel() as any, + agents: { subAlpha: subAgent('Alpha'), subBeta: subAgent('Beta') }, + memory: new core.MockMemory({ storage: (memoryStore ??= new core.InMemoryStore()) }), + }); + }, + }); + scenarios.push(scenario); + + const original = await scenario.start(async ({ agent, workflows }) => { + const first = await agent.stream('Do alpha and beta', { runId, memory, maxSteps: 5 }); + const asked = new Set(); + for await (const chunk of first.fullStream) { + if (chunk.type === 'tool-call-approval') asked.add(chunk.payload.toolCallId); + if (asked.size === 2) break; + } + expect([...asked].sort()).toEqual(['sup-tc-A', 'sup-tc-B']); + await vi.waitFor(async () => { + const row = await workflows.getWorkflowRunById({ runId, workflowName: DurableStepIds.AGENTIC_LOOP }); + expect(row?.snapshot?.status).toBe('suspended'); + }); + // Approved through the instance that suspended the run. + return agent.approveToolCall({ runId, toolCallId: 'sup-tc-A', memory }); + }); + await original.held; + const checkpoint = await original.checkpoint(); + expect(executions).toEqual([]); + return { scenario, checkpoint, executions, runId, memory }; + } + + const approve = async (agent: any, runId: string, toolCallId: string, memory: object) => { + const resumed = await agent.approveToolCall({ runId, toolCallId, memory, closeOnSuspend: true }); + let text = ''; + for await (const chunk of resumed.fullStream) { + if (chunk.type === 'text-delta') text += chunk.payload.text; + } + return text; + }; + + it.each(crashPoints)( + 'runs only the approved delegation after a crash $point', + async ({ hold }) => { + const { scenario, checkpoint, executions, runId, memory } = await crashWhileApproving(hold); + + const recovered = await scenario.restart(checkpoint, { closeOnSuspend: true }); + + expect(recovered.streamErrors).toEqual([]); + expect(executions).toEqual([{ generation: 2, query: 'alpha' }]); + + // Beta's own approval is still pending, and still works. + expect(await approve(recovered.graph.agent, runId, 'sup-tc-B', memory)).toBe('Both delegations complete.'); + expect(executions).toEqual([ + { generation: 2, query: 'alpha' }, + { generation: 2, query: 'beta' }, + ]); + }, + 60_000, + ); + + it.each(crashPoints)( + 'runs no delegation when the stored resume recorded no target, after a crash $point', + async ({ hold }) => { + // Runs stored before the target was recorded carry the approval but not which call it is for. + const { scenario, checkpoint, executions, runId, memory } = await crashWhileApproving(hold); + const loop = checkpoint.rows.find(row => row.workflowName === DurableStepIds.AGENTIC_LOOP)!; + const record = loop.snapshot.context[DurableStepIds.AGENTIC_EXECUTION] as any; + expect(record.metadata.nestedResumeTarget).toEqual({ steps: [], label: 'sup-tc-A' }); + delete record.metadata.nestedResumeTarget; + + const recovered = await scenario.restart(checkpoint, { closeOnSuspend: true }); + + expect(recovered.streamErrors).toEqual([]); + expect(executions).toEqual([]); + + // Both approvals are still pending: approving each again runs it. + await approve(recovered.graph.agent, runId, 'sup-tc-A', memory); + expect(executions).toEqual([{ generation: 2, query: 'alpha' }]); + expect(await approve(recovered.graph.agent, runId, 'sup-tc-B', memory)).toBe('Both delegations complete.'); + expect(executions).toEqual([ + { generation: 2, query: 'alpha' }, + { generation: 2, query: 'beta' }, + ]); + }, + 60_000, + ); +}); diff --git a/packages/core/src/agent/durable/__tests__/restart-harness.ts b/packages/core/src/agent/durable/__tests__/restart-harness.ts index e6735c598b5..5780384eb6e 100644 --- a/packages/core/src/agent/durable/__tests__/restart-harness.ts +++ b/packages/core/src/agent/durable/__tests__/restart-harness.ts @@ -102,6 +102,7 @@ export async function loadGraph() { { createEventedAgent }, { globalRunRegistry }, { __resetExecutionFencesForTests }, + { MockMemory }, workflows, eventedWorkflows, ] = await Promise.all([ @@ -113,6 +114,7 @@ export async function loadGraph() { import('../create-evented-agent'), import('../run-registry'), import('../execution-fence'), + import('../../../memory/mock'), import('../../../workflows'), import('../../../workflows/evented'), ]); @@ -126,6 +128,7 @@ export async function loadGraph() { globalRunRegistry, /** Stops this graph's claim heartbeats and forgets its local executions. */ resetExecutionFences: __resetExecutionFencesForTests, + MockMemory, createWorkflow: workflows.createWorkflow, createStep: workflows.createStep, createEventedWorkflow: eventedWorkflows.createWorkflow, @@ -182,7 +185,8 @@ export function createGate() { export type Gate = ReturnType; -type BuildContext = { core: CoreGraph; generation: number }; +/** `storage` is the graph's store: a memory built on it keeps its data across the restart. */ +type BuildContext = { core: CoreGraph; generation: number; storage: any }; type AgentBuild = (ctx: BuildContext) => Agent; type WorkflowBuild = (ctx: BuildContext) => any; @@ -235,7 +239,7 @@ async function buildGraph(options: ScenarioOptions, ge options.storageMode === 'serializing' ? (await import('../../../storage/domains/workflows/serializing-store.test-utils')).createSerializingStore().store : new core.InMemoryStore(); - const built = options.build({ core, generation } as BuildContext); + const built = options.build({ core, generation, storage } as BuildContext); let mastra: any; try { if (options.kind === 'workflow') { @@ -453,7 +457,14 @@ export function createRestartScenario(options: ScenarioOp * recovers the run. Fails if the checkpoint has no running row to recover * from, so a test cannot pass by recovering nothing. */ - async restart(checkpoint: Checkpoint): Promise { + async restart( + checkpoint: Checkpoint, + /** + * Agent kinds: close the recovered stream when the run suspends again; it stays open by + * default, so a test that expects the run to suspend sets this. + */ + { closeOnSuspend }: { closeOnSuspend?: boolean } = {}, + ): Promise { const graph = await buildGraph(options, ++generation); graphs.push(graph); // Graph 1 is the one that ran; every later graph is seeded from it, not from a previous restart. @@ -485,7 +496,10 @@ export function createRestartScenario(options: ScenarioOp ); let onFinishCalls = 0; const attempt = (async () => { - const recovered = await agent.recover(options.runId, { onFinish: () => void onFinishCalls++ }); + const recovered = await agent.recover(options.runId, { + onFinish: () => void onFinishCalls++, + ...(closeOnSuspend ? { closeOnSuspend } : {}), + }); // Read while the run is live: the registry entry is cleared on cleanup. const execution = graph.core.globalRunRegistry.get(options.runId)?.workflowExecution; const streamErrors: string[] = []; diff --git a/packages/core/src/workflows/constants.ts b/packages/core/src/workflows/constants.ts index ae780c940e4..0a8dde2c1d1 100644 --- a/packages/core/src/workflows/constants.ts +++ b/packages/core/src/workflows/constants.ts @@ -13,6 +13,11 @@ export const PARENT_STATE_SYMBOL = Symbol('parent_state'); // the state its changes are relative to, so they're applied to the parent's state then. export const HAND_BACK_STATE_SYMBOL = Symbol('hand_back_state'); +// Symbol used to give a nested workflow step, restarted while it was being resumed, what that +// resume targeted inside the nested run (step path, label, foreach item), for a restart that +// resumes the nested run. +export const RESTART_RESUME_SYMBOL = Symbol('restart_resume'); + // Abort reason used when workflow code explicitly cancels a run. export const WORKFLOW_CANCELLED_SYMBOL = Symbol('workflow_cancelled'); diff --git a/packages/core/src/workflows/foreach-resume-restart.test.ts b/packages/core/src/workflows/foreach-resume-restart.test.ts new file mode 100644 index 00000000000..c79650ac522 --- /dev/null +++ b/packages/core/src/workflows/foreach-resume-restart.test.ts @@ -0,0 +1,636 @@ +/** + * A restart of a run that crashed while one `foreach` item was being resumed must hand that + * item's resume data to that item only. Items that were still suspended, or never reached, must + * not receive it: in an agent loop each item is a tool call waiting for its own approval. + * + * Each case suspends the items, resumes some of them either through the `Mastra` instance that + * suspended the run or through a second instance on the same storage, copies what storage holds + * while a resumed item runs (or at a chosen write), and restarts from that copy in a fresh + * module graph. + */ + +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { z } from 'zod/v4'; +import { createGate, createRestartScenario, findRow } from '../agent/durable/__tests__/restart-harness'; +import type { Checkpoint, CoreGraph, Gate, HoldWrite } from '../agent/durable/__tests__/restart-harness'; +import { Mastra } from '../mastra'; +import { MockStore } from '../storage/mock'; +import { createWorkflow } from './create'; +import { copyRunTables, interceptWrites, restoreRunTables, runTablesOf } from './test-utils/run-tables'; +import type { RunTables } from './test-utils/run-tables'; +import { createStep } from './workflow'; + +type Seen = { generation: number; k: number; resumeData: unknown; suspendData: unknown }; + +const ITEMS = [{ k: 0 }, { k: 1 }, { k: 2 }]; +const ITEM = z.object({ k: z.number() }); +const APPROVED = { approved: true }; + +const gates: Gate[] = []; +const scenarios: { stop(): Promise }[] = []; +afterEach(async () => { + for (const gate of gates.splice(0)) gate.release(); + await Promise.all(scenarios.splice(0).map(s => s.stop())); +}); + +function newGate() { + const gate = createGate(); + gates.push(gate); + return gate; +} + +/** An approval step: suspends until resumed; a resumed call in generation `blockIn` waits on `gate`. */ +function approvalStep( + createStepFn: any, + { + id, + generation, + seen, + gate, + blockIn = 1, + }: { id: string; generation: number; seen: Seen[]; gate: Gate; blockIn?: number }, +) { + return createStepFn({ + id, + inputSchema: ITEM, + outputSchema: z.object({ k: z.number(), approved: z.boolean() }), + resumeSchema: z.object({ approved: z.boolean() }), + suspendSchema: z.object({ ask: z.number() }), + execute: async ({ inputData, resumeData, suspendData, suspend }: any) => { + seen.push({ generation, k: inputData.k, resumeData, suspendData }); + if (resumeData === undefined) return suspend({ ask: inputData.k }, { resumeLabel: `item-${inputData.k}` }); + if (generation === blockIn) await gate.wait(); + return { k: inputData.k, approved: resumeData.approved }; + }, + }); +} + +type Build = (core: CoreGraph, generation: number) => any; + +const leafForeach = + (id: string, concurrency: number, seen: Seen[], gate: Gate): Build => + (core, generation) => + core + .createWorkflow({ id, inputSchema: z.array(ITEM), outputSchema: z.any() }) + .foreach(approvalStep(core.createStep, { id: 'approve', generation, seen, gate }), { concurrency }) + .commit(); + +/** A foreach of approval steps inside a nested workflow: the shape of a durable agent's tool calls. */ +const childForeach = + (id: string, seen: Seen[], gate: Gate): Build => + (core, generation) => + core + .createWorkflow({ id, inputSchema: z.array(ITEM), outputSchema: z.any() }) + .then( + core + .createWorkflow({ id: `${id}-child`, inputSchema: z.array(ITEM), outputSchema: z.any() }) + .foreach(approvalStep(core.createStep, { id: 'approve', generation, seen, gate }), { concurrency: 3 }) + .commit(), + ) + .commit(); + +/** + * A restart scenario whose graph 1 suspends the run and resumes it, through the same `Mastra` + * instance, or through a second instance on the same storage when `elsewhere` is set. + */ +function resumeScenario( + id: string, + build: Build, + resume: Record, + { elsewhere = false, hold }: { elsewhere?: boolean; hold?: HoldWrite } = {}, +) { + const runId = `${id}-run`; + const scenario = createRestartScenario({ + kind: 'workflow', + runId, + build: ({ core, generation }) => build(core, generation), + hold, + }); + scenarios.push(scenario); + const start = () => + scenario.start(async ({ core, storage, workflow }) => { + let suspender = workflow; + if (elsewhere) { + suspender = build(core, 0); + new core.Mastra({ logger: false, storage, workflows: { [suspender.id]: suspender } }); + } + expect((await (await suspender.createRun({ runId })).start({ inputData: ITEMS })).status).toBe('suspended'); + return (await workflow.createRun({ runId })).resume(resume); + }); + return { scenario, start, runId }; +} + +const resumedIn = (seen: Seen[], generation: number) => + seen + .filter(s => s.generation === generation && s.resumeData !== undefined) + .map(s => s.k) + .sort(); + +const foreachRecord = (checkpoint: Checkpoint, workflowName: string, runId: string, stepId: string) => + findRow(checkpoint, workflowName, runId)?.snapshot.context?.[stepId] as any; + +/** Copies graph 1's rows once the resumed foreach's running record, with the payload, is stored. */ +async function checkpointWhileResuming( + original: { checkpoint(): Promise }, + { workflowName, runId, stepId }: { workflowName: string; runId: string; stepId: string }, +) { + let checkpoint!: Checkpoint; + await vi.waitFor(async () => { + checkpoint = await original.checkpoint(); + expect(foreachRecord(checkpoint, workflowName, runId, stepId)).toMatchObject({ + status: 'running', + resumePayload: APPROVED, + }); + }); + return checkpoint; +} + +/** + * A leaf foreach whose items all suspend, resumed as `resume` says, cut while `waiters` resumed + * items run. + */ +async function crashWhileResuming( + id: string, + { + concurrency, + resume, + waiters = 1, + elsewhere = true, + }: { concurrency: number; resume: Record; waiters?: number; elsewhere?: boolean }, +) { + const seen: Seen[] = []; + const gate = newGate(); + const { scenario, start, runId } = resumeScenario( + id, + leafForeach(id, concurrency, seen, gate), + { ...resume, resumeData: APPROVED }, + { elsewhere }, + ); + const original = await start(); + await vi.waitFor(() => expect(gate.waiters).toBe(waiters)); + const checkpoint = await checkpointWhileResuming(original, { workflowName: id, runId, stepId: 'approve' }); + return { seen, scenario, checkpoint, runId }; +} + +function itemOutputs(steps: any, stepId: string) { + return (steps[stepId].suspendPayload.__workflow_meta.foreachOutput as any[]).map(r => ({ + status: r?.status, + ...(r?.status === 'suspended' ? { suspendPayload: r.suspendPayload } : {}), + })); +} + +async function expectChildItems(graph: any, child: string, runId: string, statuses: string[]) { + const childRun = await graph.workflows.loadWorkflowSnapshot({ workflowName: child, runId }); + expect(itemOutputs(childRun.context, 'approve').map(o => o.status)).toEqual(statuses); +} + +describe('restart while a resumed foreach item runs', () => { + it('gives the approved item its decision and keeps the other suspended items waiting for theirs', async () => { + const id = 'foreach-resume-leaf'; + const { seen, scenario, checkpoint, runId } = await crashWhileResuming(id, { + concurrency: 3, + resume: { step: 'approve', forEachIndex: 1 }, + }); + + const { result, graph } = await scenario.restart(checkpoint); + + expect(resumedIn(seen, 2)).toEqual([1]); + expect(seen.find(s => s.generation === 2 && s.k === 1)).toMatchObject({ + resumeData: APPROVED, + suspendData: { ask: 1 }, + }); + expect(result.status).toBe('suspended'); + expect(itemOutputs(result.steps, 'approve')).toEqual([ + { status: 'suspended', suspendPayload: { ask: 0 } }, + { status: 'success' }, + { status: 'suspended', suspendPayload: { ask: 2 } }, + ]); + const stored = await graph.workflows.loadWorkflowSnapshot({ workflowName: id, runId }); + expect(JSON.stringify(result)).not.toContain('foreachResumeTarget'); + expect(JSON.stringify(stored)).not.toContain('foreachResumeTarget'); + + // Item 0 still answers to its own resume. + const next = await ( + await graph.workflow.createRun({ runId }) + ).resume({ + step: 'approve', + forEachIndex: 0, + resumeData: { approved: false }, + }); + expect(seen.filter(s => s.generation === 2 && s.k === 0)).toEqual([ + { generation: 2, k: 0, resumeData: { approved: false }, suspendData: { ask: 0 } }, + ]); + expect(next.status).toBe('suspended'); + expect(itemOutputs(next.steps, 'approve').map(o => o.status)).toEqual(['success', 'success', 'suspended']); + }); + + it('does not hand the resumed item decision to items the foreach had not reached', async () => { + const { seen, scenario, checkpoint } = await crashWhileResuming('foreach-resume-unreached', { + concurrency: 1, + resume: { step: 'approve', forEachIndex: 0 }, + }); + + const { result } = await scenario.restart(checkpoint); + + expect(resumedIn(seen, 2)).toEqual([0]); + expect(seen.find(s => s.generation === 2 && s.k === 1)).toEqual({ + generation: 2, + k: 1, + resumeData: undefined, + suspendData: undefined, + }); + expect(result.status).toBe('suspended'); + expect(itemOutputs(result.steps, 'approve')).toEqual([ + { status: 'success' }, + { status: 'suspended', suspendPayload: { ask: 1 } }, + ]); + }); + + it('resumes only the approved nested workflow when the foreach body is a workflow', async () => { + const id = 'foreach-resume-nested'; + const child = `${id}-child`; + const seen: Seen[] = []; + const gate = newGate(); + const build: Build = (core, generation) => + core + .createWorkflow({ id, inputSchema: z.array(ITEM), outputSchema: z.any() }) + .foreach( + core + .createWorkflow({ id: child, inputSchema: ITEM, outputSchema: z.any() }) + .then(approvalStep(core.createStep, { id: 'ask', generation, seen, gate })) + .commit(), + { concurrency: 3 }, + ) + .commit(); + const { scenario, start, runId } = resumeScenario( + id, + build, + { step: [child, 'ask'], forEachIndex: 1, resumeData: APPROVED }, + { elsewhere: true }, + ); + const original = await start(); + await gate.reached; + const checkpoint = await checkpointWhileResuming(original, { workflowName: id, runId, stepId: child }); + + const { result } = await scenario.restart(checkpoint); + + expect(resumedIn(seen, 2)).toEqual([1]); + // The suspended siblings' nested runs are neither resumed nor started over. + expect(seen.filter(s => s.generation === 2).map(s => s.k)).toEqual([1]); + expect(result.status).toBe('suspended'); + expect(itemOutputs(result.steps, child).map(o => o.status)).toEqual(['suspended', 'success', 'suspended']); + }); + + it('gives the decision to every item a resume without an index reached, and to no other', async () => { + // Concurrency 2: items 0 and 1 suspend and item 2 is never reached. A resume with no index + // resumes 0 and 1; item 2 is still queued behind them at the crash. + const { seen, scenario, checkpoint } = await crashWhileResuming('foreach-resume-all', { + concurrency: 2, + resume: { step: 'approve' }, + waiters: 2, + }); + + const { result } = await scenario.restart(checkpoint); + + expect(resumedIn(seen, 2)).toEqual([0, 1]); + expect(seen.find(s => s.generation === 2 && s.k === 2)).toMatchObject({ resumeData: undefined }); + expect(result.status).toBe('suspended'); + expect(itemOutputs(result.steps, 'approve')).toEqual([ + { status: 'success' }, + { status: 'success' }, + { status: 'suspended', suspendPayload: { ask: 2 } }, + ]); + }); + + it('keeps every suspended item waiting when the stored run does not say which item was resumed', async () => { + // Runs stored before the resumed item was recorded carry the payload but no target. + const id = 'foreach-resume-untargeted'; + const { seen, scenario, checkpoint, runId } = await crashWhileResuming(id, { + concurrency: 3, + resume: { step: 'approve', forEachIndex: 1 }, + }); + const record = foreachRecord(checkpoint, id, runId, 'approve'); + expect(record.metadata.foreachResumeTarget).toBe(1); + delete record.metadata.foreachResumeTarget; + + const { result } = await scenario.restart(checkpoint); + + expect(resumedIn(seen, 2)).toEqual([]); + expect(result.status).toBe('suspended'); + expect(itemOutputs(result.steps, 'approve')).toEqual([ + { status: 'suspended', suspendPayload: { ask: 0 } }, + { status: 'suspended', suspendPayload: { ask: 1 } }, + { status: 'suspended', suspendPayload: { ask: 2 } }, + ]); + }); + + it('gives the decision only to the approved item when the instance that suspended the run resumes it', async () => { + const { seen, scenario, checkpoint } = await crashWhileResuming('foreach-resume-in-place', { + concurrency: 3, + resume: { label: 'item-1' }, + elsewhere: false, + }); + + const { result } = await scenario.restart(checkpoint); + + expect(resumedIn(seen, 2)).toEqual([1]); + expect(result.status).toBe('suspended'); + expect(itemOutputs(result.steps, 'approve').map(o => o.status)).toEqual(['suspended', 'success', 'suspended']); + }); + + it('gives the decision only to the approved item of a foreach inside a nested workflow', async () => { + // The instance that suspended the run resumes one item by its label. + const id = 'foreach-resume-in-child'; + const seen: Seen[] = []; + const gate = newGate(); + const { scenario, start, runId } = resumeScenario(id, childForeach(id, seen, gate), { + label: 'item-1', + resumeData: APPROVED, + }); + const original = await start(); + await gate.reached; + const checkpoint = await checkpointWhileResuming(original, { + workflowName: `${id}-child`, + runId, + stepId: 'approve', + }); + + const { result, graph } = await scenario.restart(checkpoint); + + expect(resumedIn(seen, 2)).toEqual([1]); + expect(result.status).toBe('suspended'); + await expectChildItems(graph, `${id}-child`, runId, ['suspended', 'success', 'suspended']); + }); + + it('resumes only the chosen item of a foreach inside a nested workflow', async () => { + const id = 'foreach-resume-child-index'; + const seen: Seen[] = []; + const gate = newGate(); + const { start } = resumeScenario(id, childForeach(id, seen, gate), { + step: [`${id}-child`, 'approve'], + forEachIndex: 1, + resumeData: APPROVED, + }); + const original = await start(); + gate.release(); + + const result = (await original.driven) as any; + + expect(resumedIn(seen, 1)).toEqual([1]); + expect(result.status).toBe('suspended'); + await expectChildItems(original.graph, `${id}-child`, `${id}-run`, ['suspended', 'success', 'suspended']); + }); + + // The two crash points before the nested run stores its resumed foreach: the parent has stored + // its resumed step, and the nested run is still suspended, or has only claimed its resume. + const crashPoints = [ + { + point: 'before the nested run claims its resume', + hold: (id: string) => ({ + when: (snapshot: any) => + snapshot.context?.[`${id}-child`]?.status === 'running' && + snapshot.context[`${id}-child`].resumePayload !== undefined, + }), + childStatus: 'suspended', + }, + { + point: 'after the nested run claims its resume, before its foreach is stored', + hold: (id: string) => ({ + workflowName: `${id}-child`, + when: (snapshot: any) => snapshot.status === 'running' && snapshot.context?.approve?.status === 'suspended', + }), + childStatus: 'running', + }, + ]; + const selectors = [ + { by: 'label', resume: (_child: string) => ({ label: 'item-1' }) }, + { by: 'index', resume: (child: string) => ({ step: [child, 'approve'], forEachIndex: 1 }) }, + ]; + it.each(crashPoints.flatMap(point => selectors.map(selector => ({ ...point, ...selector }))))( + 'gives the decision only to the item resumed by $by after a crash $point', + async ({ point, hold, childStatus, by, resume }) => { + const id = `foreach-resume-child-${childStatus}-${by}`; + const seen: Seen[] = []; + const gate = newGate(); + const { scenario, start, runId } = resumeScenario( + id, + childForeach(id, seen, gate), + { ...resume(`${id}-child`), resumeData: APPROVED }, + { hold: hold(id) }, + ); + const original = await start(); + await original.held; + const checkpoint = await original.checkpoint(); + expect(findRow(checkpoint, id, runId)?.snapshot.context?.[`${id}-child`], point).toMatchObject({ + status: 'running', + resumePayload: APPROVED, + }); + expect(findRow(checkpoint, `${id}-child`, runId)?.snapshot).toMatchObject({ + status: childStatus, + context: { approve: { status: 'suspended' } }, + }); + + const { result, graph } = await scenario.restart(checkpoint); + + expect(resumedIn(seen, 1)).toEqual([]); + expect(resumedIn(seen, 2)).toEqual([1]); + expect(result.status).toBe('suspended'); + await expectChildItems(graph, `${id}-child`, runId, ['suspended', 'success', 'suspended']); + const stored = await graph.workflows.loadWorkflowSnapshot({ workflowName: id, runId }); + expect(JSON.stringify([result, stored])).not.toContain('nestedResumeTarget'); + }, + ); + + it.each(crashPoints)( + 'keeps every item of a nested foreach suspended when the stored resume recorded no target, after a crash $point', + async ({ hold, childStatus }) => { + // Runs stored before the nested target was recorded carry the payload but no target. + const id = `foreach-resume-child-untargeted-${childStatus}`; + const child = `${id}-child`; + const seen: Seen[] = []; + const gate = newGate(); + const { scenario, start, runId } = resumeScenario( + id, + childForeach(id, seen, gate), + { label: 'item-1', resumeData: APPROVED }, + { hold: hold(id) }, + ); + const original = await start(); + await original.held; + const checkpoint = await original.checkpoint(); + const record = findRow(checkpoint, id, runId)!.snapshot.context[child] as any; + expect(record.metadata.nestedResumeTarget).toEqual({ steps: [], label: 'item-1' }); + delete record.metadata.nestedResumeTarget; + // The state the nested run had handed back when it parked, which a crash during a later + // resume rebases from. + const startState = record.suspendPayload.__workflow_meta.startState; + expect(startState).toBeDefined(); + + const { result, graph } = await scenario.restart(checkpoint); + + expect(seen.filter(s => s.generation === 2)).toEqual([]); + expect(result.status).toBe('suspended'); + await expectChildItems(graph, child, runId, ['suspended', 'suspended', 'suspended']); + const parked = await graph.workflows.loadWorkflowSnapshot({ workflowName: id, runId }); + expect(parked.context[child].suspendPayload.__workflow_meta.startState).toEqual(startState); + + // The interrupted item is resumed again, and only it runs. + const next = await (await graph.workflow.createRun({ runId })).resume({ label: 'item-1', resumeData: APPROVED }); + expect(resumedIn(seen, 2)).toEqual([1]); + expect(next.status).toBe('suspended'); + await expectChildItems(graph, child, runId, ['suspended', 'success', 'suspended']); + }, + ); + + // A foreach whose items are nested workflows that each run their own concurrent foreach. The + // label names one inner item of outer item 0; outer item 1 waits for its own approvals. + const foreachOfChildForeach = + (id: string, seen: Seen[], gate: Gate): Build => + (core, generation) => + core + .createWorkflow({ id, inputSchema: z.array(z.array(ITEM)), outputSchema: z.any() }) + .foreach( + core + .createWorkflow({ id: `${id}-child`, inputSchema: z.array(ITEM), outputSchema: z.any() }) + .foreach(approvalStep(core.createStep, { id: 'approve', generation, seen, gate }), { concurrency: 2 }) + .commit(), + { concurrency: 2 }, + ) + .commit(); + const OUTER_ITEMS = [ + [{ k: 0 }, { k: 1 }], + [{ k: 2 }, { k: 3 }], + ]; + // The outer foreach's running record doesn't hold its items back, so the cut parks the nested + // run's resume claim. A crash just before that claim leaves the same rows with the nested run + // still `suspended`. + it.each([ + { point: 'before the nested run claims its resume', claimed: false, recorded: true }, + { point: 'after the nested run claims its resume', claimed: true, recorded: true }, + { point: 'after the nested run claims its resume, with no recorded target', claimed: true, recorded: false }, + ])( + 'gives the decision only to the labelled item of a foreach in a foreach item, after a crash $point', + async ({ claimed, recorded }) => { + const id = `foreach-of-child-foreach-${claimed}-${recorded}`; + const child = `${id}-child`; + const runId = `${id}-run`; + const seen: Seen[] = []; + const gate = newGate(); + const build = foreachOfChildForeach(id, seen, gate); + const scenario = createRestartScenario({ + kind: 'workflow', + runId, + build: ({ core, generation }) => build(core, generation), + hold: { + workflowName: child, + when: snapshot => snapshot.status === 'running' && snapshot.context?.approve?.status === 'suspended', + }, + }); + scenarios.push(scenario); + const original = await scenario.start(async ({ workflow }) => { + const run = await workflow.createRun({ runId }); + expect((await run.start({ inputData: OUTER_ITEMS })).status).toBe('suspended'); + return run.resume({ label: 'item-1', resumeData: APPROVED }); + }); + await original.held; + let checkpoint!: Checkpoint; + await vi.waitFor(async () => { + checkpoint = await original.checkpoint(); + expect(foreachRecord(checkpoint, id, runId, child)).toMatchObject({ + status: 'running', + resumePayload: APPROVED, + }); + }); + const childRows = checkpoint.rows.filter(row => row.workflowName === child); + const claimedRow = childRows.find(row => row.snapshot.status === 'running')!; + expect(claimedRow.snapshot.context.approve).toMatchObject({ status: 'suspended' }); + if (!claimed) claimedRow.snapshot.status = 'suspended'; + const record = foreachRecord(checkpoint, id, runId, child); + expect(record.metadata.nestedResumeTarget).toEqual({ steps: [], label: 'item-1' }); + if (!recorded) delete record.metadata.nestedResumeTarget; + + const { result, graph } = await scenario.restart(checkpoint); + + expect(resumedIn(seen, 1)).toEqual([]); + expect(resumedIn(seen, 2)).toEqual(recorded ? [1] : []); + expect(result.status).toBe('suspended'); + if (recorded) return; + + // The interrupted item is resumed again, and only it runs. + const next = await (await graph.workflow.createRun({ runId })).resume({ label: 'item-1', resumeData: APPROVED }); + expect(resumedIn(seen, 2)).toEqual([1]); + expect(next.status).toBe('suspended'); + }, + ); + + // The second copy is taken inside the first restart, which the restart harness can't cut, so + // this copies the store's tables (which also covers storage v2). + describe.each([1, 2] as const)('storage v%s', storageVersion => { + it('still gives only the approved item its decision after a second crash during the restart', async () => { + const id = 'foreach-resume-twice'; + const runId = 'twice'; + const seen: Seen[] = []; + const crashes = [newGate(), newGate()]; + const build = (generation: number) => + createWorkflow({ id, inputSchema: z.array(ITEM), outputSchema: z.any(), options: { storageVersion } }) + .foreach( + approvalStep(createStep, { + id: 'approve', + generation, + seen, + // Generations 1 and 2 crash while item 1 runs; generation 3 finishes. + gate: crashes[generation - 1] ?? newGate(), + blockIn: generation === 1 || generation === 2 ? generation : -1, + }), + { concurrency: 3 }, + ) + .commit(); + /** One process with its own store, seeded with `tables`, counting writes of the resumed foreach. */ + const boot = async (generation: number, tables?: RunTables) => { + const storage = new MockStore(); + const workflow = build(generation); + new Mastra({ logger: false, storage, workflows: { [id]: workflow } }); + if (tables) restoreRunTables(await runTablesOf(storage), tables); + const store = (await storage.getStore('workflows')) as any; + let writes = 0; + interceptWrites(store, async (args, write) => { + const written = await write(args); + if (JSON.stringify(args ?? {}).includes('"resumePayload"')) writes++; + return written; + }); + /** Copies the tables once this process stored the running foreach record itself. */ + const crashCopy = async () => { + await vi.waitFor(async () => { + expect(writes).toBeGreaterThan(0); + const snapshot = await store.loadWorkflowSnapshot({ workflowName: id, runId }); + expect(snapshot?.context.approve).toMatchObject({ status: 'running', resumePayload: APPROVED }); + }); + return copyRunTables(await runTablesOf(storage)); + }; + return { workflow, crashCopy }; + }; + + const first = await boot(1); + const run1 = await first.workflow.createRun({ runId }); + expect((await run1.start({ inputData: ITEMS })).status).toBe('suspended'); + void run1.resume({ step: 'approve', forEachIndex: 1, resumeData: APPROVED }); + await crashes[0]!.reached; + const second = await boot(2, await first.crashCopy()); + void (await second.workflow.createRun({ runId })).restart(); + await crashes[1]!.reached; + const third = await boot(3, await second.crashCopy()); + const run = await third.workflow.createRun({ runId }); + const events: unknown[] = []; + run.watch(event => void events.push(event)); + const result = await run.restart(); + + expect(resumedIn(seen, 2)).toEqual([1]); + expect(resumedIn(seen, 3)).toEqual([1]); + expect(result.status).toBe('suspended'); + const steps = typeof result.steps === 'function' ? await (result.steps as any)() : result.steps; + expect(itemOutputs(steps, 'approve').map(o => o.status)).toEqual(['suspended', 'success', 'suspended']); + expect(events.length).toBeGreaterThan(0); + expect(JSON.stringify(events)).not.toContain('foreachResumeTarget'); + }); + }); +}); diff --git a/packages/core/src/workflows/handlers/control-flow.ts b/packages/core/src/workflows/handlers/control-flow.ts index 48a9e477a80..59cb60657a9 100644 --- a/packages/core/src/workflows/handlers/control-flow.ts +++ b/packages/core/src/workflows/handlers/control-flow.ts @@ -40,8 +40,13 @@ import { resolveForeachConcurrency, } from '../utils'; import { isWorkflowRunCanceled } from './entry'; -import { getForeachProgress, omitForeachProgress } from './foreach-progress'; -import type { ForeachProgress } from './foreach-progress'; +import { + getForeachProgress, + getForeachResumeTarget, + getNestedResumeTarget, + omitForeachProgress, +} from './foreach-progress'; +import type { ForeachProgress, ForeachResumeTarget, NestedResumeTarget } from './foreach-progress'; import { emitStepResultEvents } from './step'; import type { ExecuteStepParams } from './step'; @@ -1050,6 +1055,7 @@ export interface ExecuteForeachParams extends ObservabilityContext { resumePayload: any; resumePath: number[]; forEachIndex?: number; + label?: string; }; executionContext: ExecutionContext; pubsub: PubSub; @@ -1130,9 +1136,25 @@ export async function executeForeach( const startTime = resume?.steps[0] === stepId ? undefined : Date.now(); const resumeTime = resume?.steps[0] === stepId ? Date.now() : undefined; + const restartsRunning = !!restart && stepResults[stepId]?.status === 'running'; // Items that finished before a crash, checkpointed on the running foreach result (#26214). - const restartProgress = - restart && stepResults[stepId]?.status === 'running' ? getForeachProgress(stepResults[stepId]) : undefined; + const restartProgress = restartsRunning ? getForeachProgress(stepResults[stepId]) : undefined; + // Which items the stored `resumePayload` belongs to. A restart keeps the stored target so it + // survives another crash; results stored before targets were recorded have none (undefined). + const resumeTarget: ForeachResumeTarget | undefined = + resume?.steps[0] === stepId + ? (resume.forEachIndex ?? 'all') + : restartsRunning + ? getForeachResumeTarget(stepResults[stepId]) + : 'none'; + // What the resume targeted inside an item that is a nested workflow, for a restart that has to + // resume that item's nested run again. + const nestedResumeTarget: NestedResumeTarget | undefined = + resume?.steps[0] === stepId + ? { steps: resume.steps.slice(1), ...(resume.label ? { label: resume.label } : {}) } + : restartsRunning + ? getNestedResumeTarget(stepResults[stepId]) + : undefined; const stepInfo = { // Same as executeStep: strip prior completion/suspend fields on re-entry. @@ -1189,8 +1211,11 @@ export async function executeForeach( type ForeachStepResult = StepResult | StepBailed; type PersistedForeachStepResult = ForeachStepResult & { suspendPayload?: any }; - const prevForeachOutput = (prevPayload?.suspendPayload?.__workflow_meta?.foreachOutput || - []) as PersistedForeachStepResult[]; + // A copy: the running record keeps the stored suspend state, which progress checkpoints don't + // rewrite, while items record their results here. + const prevForeachOutput = [ + ...(prevPayload?.suspendPayload?.__workflow_meta?.foreachOutput || []), + ] as PersistedForeachStepResult[]; const foreachProgress: ForeachProgress = { ...restartProgress }; for (const [index, { output, nestedRunId }] of Object.entries(foreachProgress)) { prevForeachOutput[Number(index)] = { @@ -1222,7 +1247,17 @@ export async function executeForeach( status: 'running', }; const runningWithProgress = (): StepResult => - ({ ...runningFields, metadata: { ...stepInfo.metadata, foreachProgress } }) as StepResult; + ({ + ...runningFields, + metadata: { + ...stepInfo.metadata, + // A copy: items that finish while a write is in flight belong to the next write, whose + // patch on storage v2 is built from the items unsent when it starts. + foreachProgress: { ...foreachProgress }, + ...(resumeTarget !== undefined ? { foreachResumeTarget: resumeTarget } : {}), + ...(nestedResumeTarget ? { nestedResumeTarget } : {}), + }, + }) as StepResult; /** Items finished since the latest progress write started. */ let unsentItems: number[] = []; @@ -1256,6 +1291,9 @@ export async function executeForeach( workflowStatus: 'running', requestContext, phase: `foreach-progress.${checkpointCount++}`, + // Also when this instance last stored the run as suspended: a restart must find the + // resumed foreach running with its target, or it can't tell which items were resumed. + recordResumedStepStart: resume?.steps[0] === stepId, }); } catch (err) { checkpointError = err instanceof Error ? err : new Error(String(err)); @@ -1484,7 +1522,11 @@ export async function executeForeach( const prevItemResult = prevForeachOutput[k]; if ( prevItemResult?.status === 'success' || - (prevItemResult?.status === 'suspended' && resume?.forEachIndex !== k && resume?.forEachIndex !== undefined) + (prevItemResult?.status === 'suspended' && resume?.forEachIndex !== k && resume?.forEachIndex !== undefined) || + // On a restart, a suspended item stays suspended when the stored resume targeted another item or recorded no target. + (prevItemResult?.status === 'suspended' && + restartsRunning && + (resumeTarget === undefined || (typeof resumeTarget === 'number' && resumeTarget !== k))) ) { if (prevItemResult?.status === 'success') { // Already succeeded in a previous run – clean up resume label diff --git a/packages/core/src/workflows/handlers/foreach-progress.ts b/packages/core/src/workflows/handlers/foreach-progress.ts index ef5656e2774..b7d31f68789 100644 --- a/packages/core/src/workflows/handlers/foreach-progress.ts +++ b/packages/core/src/workflows/handlers/foreach-progress.ts @@ -27,12 +27,77 @@ export function getForeachProgress(stepResult: { metadata?: unknown } | undefine return isForeachProgress(progress) ? progress : undefined; } -/** Copy of `stepInfo` without `metadata.foreachProgress`; drops `metadata` if nothing else is left. */ +/** + * Which items the `resumePayload` on a foreach step's running result belongs to, stored next to + * the progress under `metadata.foreachResumeTarget`: the index of the resumed item, `'all'` for a + * resume without an index (every item suspended at the time), `'none'` when the foreach was + * entered without a resume. A restart hands the stored payload only to those items. + */ +export type ForeachResumeTarget = number | 'all' | 'none'; + +/** The stored resume target on a foreach step's running result; undefined if none was recorded. */ +export function getForeachResumeTarget( + stepResult: { metadata?: unknown } | undefined, +): ForeachResumeTarget | undefined { + const metadata = stepResult?.metadata; + if (!isPlainRecord(metadata)) return undefined; + const target = metadata.foreachResumeTarget; + return target === 'all' || target === 'none' || (typeof target === 'number' && Number.isInteger(target)) + ? target + : undefined; +} + +/** Whether a restart hands the stored `resumePayload` of a running foreach result to item `index`. */ +export function takesStoredForeachResume( + stepResult: { metadata?: unknown; suspendPayload?: any } | undefined, + index: number, +): boolean { + const target = getForeachResumeTarget(stepResult); + if (target === 'all') { + return stepResult?.suspendPayload?.__workflow_meta?.foreachOutput?.[index]?.status === 'suspended'; + } + return target === index; +} + +/** + * What the resume of a nested workflow step targeted inside the nested run (the step path below + * the nested workflow, the resume label, the foreach item), stored under + * `metadata.nestedResumeTarget` on the step's running result. A restart that has to resume the + * nested run resumes only that. + */ +export type NestedResumeTarget = { steps: string[]; label?: string; forEachIndex?: number }; + +/** The stored nested resume target on a nested workflow step's running result, if any. */ +export function getNestedResumeTarget(stepResult: { metadata?: unknown } | undefined): NestedResumeTarget | undefined { + const metadata = stepResult?.metadata; + if (!isPlainRecord(metadata)) return undefined; + const target = metadata.nestedResumeTarget; + if (!isPlainRecord(target) || !Array.isArray(target.steps)) return undefined; + if (!target.steps.every(step => typeof step === 'string')) return undefined; + if (target.label !== undefined && typeof target.label !== 'string') return undefined; + if (target.forEachIndex !== undefined && !Number.isInteger(target.forEachIndex)) return undefined; + return { + steps: target.steps, + ...(target.label !== undefined ? { label: target.label } : {}), + ...(target.forEachIndex !== undefined ? { forEachIndex: target.forEachIndex as number } : {}), + }; +} + +/** + * Copy of `stepInfo` without the keys only a running result carries (`metadata.foreachProgress`, + * `metadata.foreachResumeTarget`, `metadata.nestedResumeTarget`); drops `metadata` if nothing + * else is left. + */ export function omitForeachProgress( stepInfo: T, ): Omit & { metadata?: Record } { const { metadata, ...rest } = stepInfo; if (!isPlainRecord(metadata)) return rest; - const { foreachProgress: _foreachProgress, ...others } = metadata; + const { + foreachProgress: _foreachProgress, + foreachResumeTarget: _foreachResumeTarget, + nestedResumeTarget: _nestedResumeTarget, + ...others + } = metadata; return Object.keys(others).length > 0 ? { ...rest, metadata: others } : rest; } diff --git a/packages/core/src/workflows/handlers/step.ts b/packages/core/src/workflows/handlers/step.ts index 664445d089a..c570082eb8c 100644 --- a/packages/core/src/workflows/handlers/step.ts +++ b/packages/core/src/workflows/handlers/step.ts @@ -21,6 +21,7 @@ import { HAND_BACK_STATE_SYMBOL, PARENT_STATE_SYMBOL, PUBSUB_SYMBOL, + RESTART_RESUME_SYMBOL, STREAM_FORMAT_SYMBOL, WORKFLOW_CANCELLED_SYMBOL, } from '../constants'; @@ -49,7 +50,8 @@ import { validateStepStateData, validateStepRequestContext, } from '../utils'; -import { omitForeachProgress } from './foreach-progress'; +import { getNestedResumeTarget, omitForeachProgress, takesStoredForeachResume } from './foreach-progress'; +import type { NestedResumeTarget } from './foreach-progress'; export interface ExecuteStepParams extends ObservabilityContext { workflowId: string; @@ -150,6 +152,7 @@ export async function executeStep( }); let resumeDataToUse: unknown; + let restartsResume = false; if (timeTravelResumeData !== undefined && !timeTravelResumeValidationError) { resumeDataToUse = timeTravelResumeData; } else if (timeTravelResumeData !== undefined && timeTravelResumeValidationError) { @@ -159,9 +162,33 @@ export async function executeStep( }); } else if (resume?.steps[0] === step.id) { resumeDataToUse = resume?.resumePayload; - } else if (restart?.activeStepsPath?.[step.id] && stepResults[step.id]?.status === 'running') { + } else if ( + restart?.activeStepsPath?.[step.id] && + stepResults[step.id]?.status === 'running' && + // A foreach item's record is the foreach's own: its payload belongs only to the resumed items. + (executionContext.foreachIndex === undefined || + takesStoredForeachResume(stepResults[step.id], executionContext.foreachIndex)) + ) { // A resumed step that was in flight when the process died re-runs with its original resume data. resumeDataToUse = (stepResults[step.id] as { resumePayload?: unknown }).resumePayload; + restartsResume = true; + } + + // What a resumed nested workflow's resume targeted inside the nested run, kept on its running + // record: a restart that finds the nested run still suspended, or only claimed, resumes it + // again, and without the target it would resume every suspended step there. A foreach item's + // record is the foreach's own, which stores the target for its items. + let nestedResumeTarget: NestedResumeTarget | undefined; + if (step.component === 'WORKFLOW') { + if (resume?.steps[0] === step.id && executionContext.foreachIndex === undefined) { + nestedResumeTarget = { + steps: resume.steps.slice(1), + ...(resume.label ? { label: resume.label } : {}), + ...(resume.forEachIndex !== undefined ? { forEachIndex: resume.forEachIndex } : {}), + }; + } else if (restartsResume) { + nestedResumeTarget = getNestedResumeTarget(stepResults[step.id]); + } } // Extract suspend data if this step was previously suspended @@ -261,7 +288,10 @@ export async function executeStep( serializedStepGraph, stepResults: { ...stepResults, - [step.id]: stepInfo, + [step.id]: + nestedResumeTarget && executionContext.foreachIndex === undefined + ? { ...stepInfo, metadata: { ...stepInfo.metadata, nestedResumeTarget } } + : stepInfo, } as Record>, executionContext, workflowStatus: 'running', @@ -591,7 +621,9 @@ export async function executeStep( resumePayload: resume?.resumePayload, runId: (nestedIterationSuspendPayload ?? stepResults[step.id]?.suspendPayload)?.__workflow_meta?.runId, label: resume?.label, - forEachIndex: resume?.forEachIndex, + // Inside a foreach the index picked this item; otherwise it picks an item of a + // foreach inside the nested run. + forEachIndex: foreachIndex === undefined ? resume?.forEachIndex : undefined, } : undefined, // Only pass restart data if this step is part of activeStepsPath @@ -615,6 +647,10 @@ export async function executeStep( ? (((state: unknown) => journal.baseFor(state)) satisfies ParentStateLink) : undefined, [HAND_BACK_STATE_SYMBOL]: isNestedWorkflow ? setState : undefined, + // `null` tells the nested workflow the restarted resume's target is unknown (a record stored + // before targets were): it then leaves the nested run suspended. + [RESTART_RESUME_SYMBOL]: + restartsResume && step.component === 'WORKFLOW' ? (nestedResumeTarget ?? null) : undefined, engine: engine.getEngineContext(), abortSignal: abortController?.signal, outputWriter, diff --git a/packages/core/src/workflows/restart-mid-block.test.ts b/packages/core/src/workflows/restart-mid-block.test.ts index 8dd2a162ede..27f2e9dd14b 100644 --- a/packages/core/src/workflows/restart-mid-block.test.ts +++ b/packages/core/src/workflows/restart-mid-block.test.ts @@ -21,6 +21,8 @@ import { Mastra } from '../mastra'; import { cloneRunData } from '../storage/domains/workflows/inmemory'; import { MockStore } from '../storage/mock'; import { createWorkflow } from './create'; +import { copyRunTables, interceptWrites, restoreRunTables, runTablesOf } from './test-utils/run-tables'; +import type { RunTables } from './test-utils/run-tables'; import type { WorkflowRunState } from './types'; import { createStep } from './workflow'; @@ -47,20 +49,6 @@ function progressCount(snapshot: WorkflowRunState | undefined, stepId: string) { return Object.keys((snapshot?.context?.[stepId] as any)?.metadata?.foreachProgress ?? {}).length; } -/** - * Replaces every workflows-store write with `intercept`, which makes the write by calling `write`. - * Covers the writes of both storage versions. - */ -function interceptWrites( - store: any, - intercept: (args: any, write: (args: any) => Promise) => Promise, -) { - for (const method of WRITE_METHODS) { - const write = store[method].bind(store); - vi.spyOn(store, method).mockImplementation((args: any) => intercept(args, write)); - } -} - /** The run's snapshot as storage holds it, rebuilt from the journal on storage v2. */ async function storedSnapshot(store: any, workflowName: string, runId: string) { return (await store.loadWorkflowSnapshot({ workflowName, runId })) as WorkflowRunState | null; @@ -255,9 +243,12 @@ describe.each([1, 2] as const)( expect(restarted.status).toBe('success'); expect(restarted.result).toEqual(original.result); expect(restarted.steps.item.output).toEqual(original.steps.item.output); - // Per-item progress is a crash-recovery record only; it must not survive into results. + // Per-item progress and the resume target are crash-recovery records only; they must not + // survive into results. expect(JSON.stringify(restarted)).not.toContain('foreachProgress'); expect(JSON.stringify(stored)).not.toContain('foreachProgress'); + expect(JSON.stringify(restarted)).not.toContain('foreachResumeTarget'); + expect(JSON.stringify(stored)).not.toContain('foreachResumeTarget'); }, ); @@ -535,36 +526,6 @@ describe.each([1, 2] as const)( }, ); -const RUN_TABLES = ['workflows', 'workflowRunHeaders', 'workflowRunEntries'] as const; -const WRITE_METHODS = [ - 'persistWorkflowSnapshot', - 'updateWorkflowResults', - 'updateWorkflowState', - 'updateWorkflowRunHeader', - 'writeWorkflowRunCheckpoint', - 'appendWorkflowRunEntries', - 'appendWorkflowRunStateDraft', - 'deleteWorkflowRunById', -] as const; - -type RunTables = Record<(typeof RUN_TABLES)[number], Map>; - -async function runTablesOf(storage: InstanceType): Promise { - return ((await storage.getStore('workflows')) as any).db; -} - -function copyRunTables(tables: RunTables): RunTables { - return Object.fromEntries(RUN_TABLES.map(table => [table, cloneRunData(tables[table])])) as RunTables; -} - -/** Makes `tables` hold exactly what `copy` holds. */ -function restoreRunTables(tables: RunTables, copy: RunTables) { - for (const table of RUN_TABLES) { - tables[table].clear(); - for (const [key, value] of copy[table]) tables[table].set(key, value); - } -} - function seeded(seed: number) { return () => (seed = (seed * 1103515245 + 12345) % 2 ** 31) / 2 ** 31; } diff --git a/packages/core/src/workflows/test-utils/run-tables.ts b/packages/core/src/workflows/test-utils/run-tables.ts new file mode 100644 index 00000000000..2b998b42db8 --- /dev/null +++ b/packages/core/src/workflows/test-utils/run-tables.ts @@ -0,0 +1,52 @@ +/** + * Copies of a `MockStore`'s workflow run tables, for tests that crash a run by copying what + * storage holds at one moment and restarting from that copy in fresh storage. The copy is of the + * tables, so it works for both storage versions. + */ + +import { vi } from 'vitest'; +import { cloneRunData } from '../../storage/domains/workflows/inmemory'; +import type { MockStore } from '../../storage/mock'; + +const RUN_TABLES = ['workflows', 'workflowRunHeaders', 'workflowRunEntries'] as const; + +/** Every workflows-store method that writes, across both storage versions. */ +export const WRITE_METHODS = [ + 'persistWorkflowSnapshot', + 'updateWorkflowResults', + 'updateWorkflowState', + 'updateWorkflowRunHeader', + 'writeWorkflowRunCheckpoint', + 'appendWorkflowRunEntries', + 'appendWorkflowRunStateDraft', + 'deleteWorkflowRunById', +] as const; + +export type RunTables = Record<(typeof RUN_TABLES)[number], Map>; + +export async function runTablesOf(storage: InstanceType): Promise { + return ((await storage.getStore('workflows')) as any).db; +} + +export function copyRunTables(tables: RunTables): RunTables { + return Object.fromEntries(RUN_TABLES.map(table => [table, cloneRunData(tables[table])])) as RunTables; +} + +/** Makes `tables` hold exactly what `copy` holds. */ +export function restoreRunTables(tables: RunTables, copy: RunTables) { + for (const table of RUN_TABLES) { + tables[table].clear(); + for (const [key, value] of copy[table]) tables[table].set(key, value); + } +} + +/** Replaces every workflows-store write with `intercept`, which makes the write by calling `write`. */ +export function interceptWrites( + store: any, + intercept: (args: any, write: (args: any) => Promise) => Promise, +) { + for (const method of WRITE_METHODS) { + const write = store[method].bind(store); + vi.spyOn(store, method).mockImplementation((args: any) => intercept(args, write)); + } +} diff --git a/packages/core/src/workflows/workflow.ts b/packages/core/src/workflows/workflow.ts index c3ce0d16d12..5e2d357dd72 100644 --- a/packages/core/src/workflows/workflow.ts +++ b/packages/core/src/workflows/workflow.ts @@ -82,11 +82,18 @@ import { isMastraTool } from '../tools/toolchecks'; import type { ToolExecutionContext } from '../tools/types'; import type { DynamicArgument } from '../types'; import { resolveDynamicArgument } from '../utils/dynamic-argument'; -import { HAND_BACK_STATE_SYMBOL, PARENT_STATE_SYMBOL, PUBSUB_SYMBOL, WORKFLOW_CANCELLED_SYMBOL } from './constants'; +import { + HAND_BACK_STATE_SYMBOL, + PARENT_STATE_SYMBOL, + PUBSUB_SYMBOL, + RESTART_RESUME_SYMBOL, + WORKFLOW_CANCELLED_SYMBOL, +} from './constants'; import { DefaultExecutionEngine } from './default'; import type { ClassifierStepOutput } from './entry-executors'; import { withoutAuthToken } from './evented/helpers'; import type { ExecutionEngine, ExecutionGraph } from './execution-engine'; +import type { NestedResumeTarget } from './handlers/foreach-progress'; import { validateTemplate } from './mapping-template'; import { cancelNestedRun, @@ -3402,6 +3409,7 @@ export class Workflow< [PUBSUB_SYMBOL]: pubsub, [PARENT_STATE_SYMBOL]: parentState, [HAND_BACK_STATE_SYMBOL]: handBackState, + [RESTART_RESUME_SYMBOL]: restartResume, mastra, requestContext, abort, @@ -3443,6 +3451,8 @@ export class Workflow< [PUBSUB_SYMBOL]: PubSub; [PARENT_STATE_SYMBOL]?: ParentStateLink; [HAND_BACK_STATE_SYMBOL]?: (state: TState, base: Record) => Promise; + /** `null`: the restarted resume didn't record its target inside the nested run. */ + [RESTART_RESUME_SYMBOL]?: NestedResumeTarget | null; mastra: Mastra; requestContext?: RequestContext; engine: DefaultEngineType; @@ -3566,6 +3576,8 @@ export class Workflow< // What the nested run's hand-back is relative to. A nested run a restarting parent resumes may // have changes its parent never received, so it hands back its whole state. let handBackBase: Record | undefined; + // The state a nested run left suspended by a restart had handed back when it parked. + let parkedState: Record | undefined; try { let restartNested = !!restart; let resumeNested = isResume; @@ -3625,16 +3637,32 @@ export class Workflow< outputWriter, outputOptions: { includeState: true, includeResumeLabels: true }, }); + } else if (resumeNested && !resume && restartResume === null) { + // The interrupted resume didn't record what it targeted inside the nested run, so its data + // can't be handed to the right step there: leave the nested run suspended as stored, and + // this step suspends with it, to be resumed again. + const stored = await run.runStorage.loadFull(); + res = { + status: 'suspended', + steps: (stored?.context ?? {}) as any, + resumeLabels: stored?.resumeLabels, + } as WorkflowResult; + // The nested run hasn't run since it parked, so the state it had handed back then is its + // stored state; it hands nothing back now. + if (!parentState) parkedState = toJsonState(stored?.value); } else if (resumeNested) { + // A restart that resumes the nested run resumes what the interrupted resume targeted. + const target = resume ?? restartResume ?? undefined; res = await run.resumeNestedByParent({ resumeData, - step: resume?.steps?.length ? (resume.steps as any) : undefined, + step: target?.steps?.length ? (target.steps as any) : undefined, requestContext, actor, ...observabilityContext, outputWriter, outputOptions: { includeState: true, includeResumeLabels: true }, - label: resume?.label, + label: target?.label, + forEachIndex: target?.forEachIndex, perStep, }); if (isResume) handBackBase = run.resumedFromState; @@ -3676,7 +3704,7 @@ export class Workflow< // parent's current state from here, so the changes it already handed back aren't applied twice. // Storage v2 keeps the state each drive started from, so a v2 parent reads it from there // instead of storing a copy. - const handedBack = handBackBase && !parentState ? toJsonState(res.state) : undefined; + const handedBack = parkedState ?? (handBackBase && !parentState ? toJsonState(res.state) : undefined); if (suspendedSteps?.length) { for (const [stepName, stepResult] of suspendedSteps) { From e63d59f36dd60fc6ea9d633d7e844b09adac2963 Mon Sep 17 00:00:00 2001 From: Arthur Fiorette Date: Sat, 14 Nov 2026 03:21:33 -0300 Subject: [PATCH 02/10] fix(core): read a foreach item's stored record before its input validation Foreach items share the foreach's record in memory, and a sibling that ends replaces it with its own result. An item that read the record after its validation awaits could lose its stored resume data on a restart and read the sibling's suspend payload instead. executeStep now reads the record once, on entry. The durable approval restart tests also assert that the recovered execution did not fail. --- .../durable-agent-approval-restart.test.ts | 3 + .../workflows/foreach-resume-restart.test.ts | 71 +++++++++++++++++++ packages/core/src/workflows/handlers/step.ts | 15 ++-- 3 files changed, 83 insertions(+), 6 deletions(-) diff --git a/packages/core/src/agent/durable/__tests__/durable-agent-approval-restart.test.ts b/packages/core/src/agent/durable/__tests__/durable-agent-approval-restart.test.ts index 949fc933fde..53d15bf1ee5 100644 --- a/packages/core/src/agent/durable/__tests__/durable-agent-approval-restart.test.ts +++ b/packages/core/src/agent/durable/__tests__/durable-agent-approval-restart.test.ts @@ -89,6 +89,7 @@ describe('durable agent: restart while an approved tool call runs', () => { const recovered = await scenario.restart(await original.checkpoint(), { closeOnSuspend: true }); expect(recovered.streamErrors).toEqual([]); + expect(recovered.executionError).toBeUndefined(); expect(charges.filter(c => c.generation === 2)).toEqual([{ generation: 2, index: 0 }]); // The next call asks for its own approval. expect(recovered.chunkTypes).toContain('tool-call-approval'); @@ -278,6 +279,7 @@ describe('durable agent: restart while an approved delegation is being resumed', const recovered = await scenario.restart(checkpoint, { closeOnSuspend: true }); expect(recovered.streamErrors).toEqual([]); + expect(recovered.executionError).toBeUndefined(); expect(executions).toEqual([{ generation: 2, query: 'alpha' }]); // Beta's own approval is still pending, and still works. @@ -303,6 +305,7 @@ describe('durable agent: restart while an approved delegation is being resumed', const recovered = await scenario.restart(checkpoint, { closeOnSuspend: true }); expect(recovered.streamErrors).toEqual([]); + expect(recovered.executionError).toBeUndefined(); expect(executions).toEqual([]); // Both approvals are still pending: approving each again runs it. diff --git a/packages/core/src/workflows/foreach-resume-restart.test.ts b/packages/core/src/workflows/foreach-resume-restart.test.ts index c79650ac522..d4d336314ae 100644 --- a/packages/core/src/workflows/foreach-resume-restart.test.ts +++ b/packages/core/src/workflows/foreach-resume-restart.test.ts @@ -633,4 +633,75 @@ describe('restart while a resumed foreach item runs', () => { expect(JSON.stringify(events)).not.toContain('foreachResumeTarget'); }); }); + + it('gives the resumed item its decision when a sibling suspends while it is still validating its input', async () => { + // Concurrency 2: items 0 and 1 suspend and item 2 is never reached; item 1 is resumed. On the + // restart items 1 and 2 run together, and item 2 suspends before item 1's input validation + // finishes. Items share the foreach's record, which item 2's suspension replaces in memory. + const id = 'foreach-resume-sibling-suspends'; + const runId = 'sibling-suspends'; + const seen: Seen[] = []; + const crash = newGate(); + const validation = newGate(); + const build = (generation: number) => { + const approve = createStep({ + id: 'approve', + // On the restart, item 1's validation waits until item 2 has suspended. + inputSchema: ITEM.refine(async ({ k }) => { + if (generation === 2 && k === 1) await validation.wait(); + return true; + }), + outputSchema: z.object({ k: z.number(), approved: z.boolean() }), + resumeSchema: z.object({ approved: z.boolean() }), + suspendSchema: z.object({ ask: z.number() }), + execute: async ({ inputData, resumeData, suspendData, suspend }) => { + seen.push({ generation, k: inputData.k, resumeData, suspendData }); + if (resumeData === undefined) return suspend({ ask: inputData.k }); + if (generation === 1) await crash.wait(); + return { k: inputData.k, approved: resumeData.approved }; + }, + }); + return createWorkflow({ id, inputSchema: z.array(ITEM), outputSchema: z.any() }) + .foreach(approve, { concurrency: 2 }) + .commit(); + }; + + const storage1 = new MockStore(); + const first = build(1); + new Mastra({ logger: false, storage: storage1, workflows: { [id]: first } }); + const run1 = await first.createRun({ runId }); + expect((await run1.start({ inputData: ITEMS })).status).toBe('suspended'); + void run1.resume({ step: 'approve', forEachIndex: 1, resumeData: APPROVED }); + await crash.reached; + const store1 = (await storage1.getStore('workflows'))!; + await vi.waitFor(async () => { + const snapshot = await store1.loadWorkflowSnapshot({ workflowName: id, runId }); + expect(snapshot?.context.approve).toMatchObject({ status: 'running', resumePayload: APPROVED }); + }); + const atCrash = copyRunTables(await runTablesOf(storage1)); + + const storage2 = new MockStore(); + const second = build(2); + new Mastra({ logger: false, storage: storage2, workflows: { [id]: second } }); + restoreRunTables(await runTablesOf(storage2), atCrash); + const run2 = await second.createRun({ runId }); + run2.watch(event => { + const payload = (event as any).payload; + if (event.type === 'workflow-step-progress' && payload?.currentIndex === 2) { + expect(payload.iterationStatus).toBe('suspended'); + validation.release(); + } + }); + const result = await run2.restart(); + + expect(seen.filter(s => s.generation === 2).map(s => s.k)).toEqual([2, 1]); + expect(resumedIn(seen, 2)).toEqual([1]); + expect(seen.find(s => s.generation === 2 && s.k === 1)).toMatchObject({ + resumeData: APPROVED, + suspendData: { ask: 1 }, + }); + expect(result.status).toBe('suspended'); + if (result.status !== 'suspended') return; + expect(itemOutputs(result.steps, 'approve').map(o => o.status)).toEqual(['suspended', 'success', 'suspended']); + }); }); diff --git a/packages/core/src/workflows/handlers/step.ts b/packages/core/src/workflows/handlers/step.ts index c570082eb8c..8a83930c79c 100644 --- a/packages/core/src/workflows/handlers/step.ts +++ b/packages/core/src/workflows/handlers/step.ts @@ -120,6 +120,9 @@ export async function executeStep( const journal = engine.getRunJournal(runId); const stepCallId = globalThis.crypto.randomUUID(); + // Read before the awaits below: a foreach's items share this record, and a sibling that ends + // meanwhile replaces it with its own result. + const storedResult = stepResults[step.id]; const persistedNestedRunId = executionContext.foreachIndex !== undefined ? stepResults[step.id]?.suspendPayload?.__workflow_meta?.foreachOutput?.[executionContext.foreachIndex]?.metadata @@ -164,13 +167,13 @@ export async function executeStep( resumeDataToUse = resume?.resumePayload; } else if ( restart?.activeStepsPath?.[step.id] && - stepResults[step.id]?.status === 'running' && + storedResult?.status === 'running' && // A foreach item's record is the foreach's own: its payload belongs only to the resumed items. (executionContext.foreachIndex === undefined || - takesStoredForeachResume(stepResults[step.id], executionContext.foreachIndex)) + takesStoredForeachResume(storedResult, executionContext.foreachIndex)) ) { // A resumed step that was in flight when the process died re-runs with its original resume data. - resumeDataToUse = (stepResults[step.id] as { resumePayload?: unknown }).resumePayload; + resumeDataToUse = (storedResult as { resumePayload?: unknown }).resumePayload; restartsResume = true; } @@ -187,16 +190,16 @@ export async function executeStep( ...(resume.forEachIndex !== undefined ? { forEachIndex: resume.forEachIndex } : {}), }; } else if (restartsResume) { - nestedResumeTarget = getNestedResumeTarget(stepResults[step.id]); + nestedResumeTarget = getNestedResumeTarget(storedResult); } } // Extract suspend data if this step was previously suspended // A resumed step re-run after a crash is stored as 'running' but still carries its suspendPayload. - const priorStatus = stepResults[step.id]?.status; + const priorStatus = storedResult?.status; let suspendDataToUse = priorStatus === 'suspended' || (priorStatus === 'running' && resumeDataToUse !== undefined) - ? stepResults[step.id]?.suspendPayload + ? storedResult?.suspendPayload : undefined; // A suspended foreach step's step-level suspendPayload only carries the FIRST suspended From 23df37d1003fec7dc8116636f1ce99751e703da7 Mon Sep 17 00:00:00 2001 From: Arthur Fiorette Date: Sat, 14 Nov 2026 12:22:43 -0300 Subject: [PATCH 03/10] fix(core): decide a foreach item's nested resume from its stored record A foreach item that is a nested workflow decided whether to resume its nested run from the shared foreach record after its input validation. When a sibling item suspended in that window, the record read as suspended, the item resumed its nested run with no label, and every suspended item inside it received the stored resume data on a restart. The item now uses the record read on entry, as the rest of executeStep does. --- .../workflows/foreach-resume-restart.test.ts | 80 +++++++++++++++++++ packages/core/src/workflows/handlers/step.ts | 6 +- 2 files changed, 83 insertions(+), 3 deletions(-) diff --git a/packages/core/src/workflows/foreach-resume-restart.test.ts b/packages/core/src/workflows/foreach-resume-restart.test.ts index d4d336314ae..777beba39ea 100644 --- a/packages/core/src/workflows/foreach-resume-restart.test.ts +++ b/packages/core/src/workflows/foreach-resume-restart.test.ts @@ -704,4 +704,84 @@ describe('restart while a resumed foreach item runs', () => { if (result.status !== 'suspended') return; expect(itemOutputs(result.steps, 'approve').map(o => o.status)).toEqual(['suspended', 'success', 'suspended']); }); + + it('resumes only the labelled inner item when a sibling nested run suspends while the resumed one validates', async () => { + // A foreach of nested workflows, each with its own foreach. Outer items 0 and 1 suspend and + // outer item 2 is never reached; inner item 1 of outer item 0 is resumed by label, and the + // run crashes once that nested run has claimed its resume. On the restart outer items 0 and 2 + // run together, and outer item 2 suspends before outer item 0's input validation finishes. + const id = 'foreach-of-child-foreach-sibling'; + const child = `${id}-child`; + const runId = 'child-sibling-suspends'; + const seen: Seen[] = []; + const validation = newGate(); + const parked = newGate(); + const build = (generation: number) => { + const approve = approvalStep(createStep, { id: 'approve', generation, seen, gate: newGate(), blockIn: -1 }); + const nested = createWorkflow({ + id: child, + // On the restart, outer item 0's validation waits until outer item 2 has suspended. + inputSchema: z.array(ITEM).refine(async items => { + if (generation === 2 && items[0]?.k === 0) await validation.wait(); + return true; + }), + outputSchema: z.any(), + }) + .foreach(approve, { concurrency: 2 }) + .commit(); + return createWorkflow({ id, inputSchema: z.array(z.array(ITEM)), outputSchema: z.any() }) + .foreach(nested, { concurrency: 2 }) + .commit(); + }; + + // Generation 1 is parked inside the nested run's resume claim, after copying the tables. + const storage1 = new MockStore(); + const first = build(1); + new Mastra({ logger: false, storage: storage1, workflows: { [id]: first } }); + const store1 = (await storage1.getStore('workflows')) as any; + let atCrash: RunTables | undefined; + let resuming = false; + interceptWrites(store1, async (args, write) => { + const written = await write(args); + if (resuming && !atCrash && args.workflowName === child) { + const snapshot = await store1.loadWorkflowSnapshot({ workflowName: child, runId: args.runId }); + if (snapshot?.status === 'running' && snapshot.context?.approve?.status === 'suspended') { + atCrash = copyRunTables(await runTablesOf(storage1)); + parked.wait(); + await new Promise(() => {}); + } + } + return written; + }); + const run1 = await first.createRun({ runId }); + const inputData = [ + [{ k: 0 }, { k: 1 }], + [{ k: 2 }, { k: 3 }], + [{ k: 4 }, { k: 5 }], + ]; + expect((await run1.start({ inputData })).status).toBe('suspended'); + resuming = true; + void run1.resume({ label: 'item-1', resumeData: APPROVED }); + await parked.reached; + const outer = (await store1.loadWorkflowSnapshot({ workflowName: id, runId }))?.context?.[child]; + expect(outer).toMatchObject({ status: 'running', resumePayload: APPROVED }); + + const storage2 = new MockStore(); + const second = build(2); + new Mastra({ logger: false, storage: storage2, workflows: { [id]: second } }); + restoreRunTables(await runTablesOf(storage2), atCrash!); + const run2 = await second.createRun({ runId }); + run2.watch(event => { + const payload = (event as any).payload; + if (event.type === 'workflow-step-progress' && payload?.id === child && payload.currentIndex === 2) { + expect(payload.iterationStatus).toBe('suspended'); + validation.release(); + } + }); + const result = await run2.restart(); + + expect(resumedIn(seen, 1)).toEqual([]); + expect(resumedIn(seen, 2)).toEqual([1]); + expect(result.status).toBe('suspended'); + }); }); diff --git a/packages/core/src/workflows/handlers/step.ts b/packages/core/src/workflows/handlers/step.ts index 8a83930c79c..6106177b877 100644 --- a/packages/core/src/workflows/handlers/step.ts +++ b/packages/core/src/workflows/handlers/step.ts @@ -502,7 +502,7 @@ export async function executeStep( }); if (snapshot?.status === 'success') { // The state the nested run started from, recorded when this run parked on it. - const meta = stepResults[step.id]?.suspendPayload?.__workflow_meta; + const meta = storedResult?.suspendPayload?.__workflow_meta; completedNestedOutput = { found: true, value: snapshot.result, @@ -618,11 +618,11 @@ export async function executeStep( // In foreach, stepResults[step.id] is shared by all iterations, so also require that // this iteration itself suspended; otherwise it would claim a sibling's child run. resume: - stepResults[step.id]?.status === 'suspended' && (foreachIndex === undefined || nestedIterationSuspendPayload) + storedResult?.status === 'suspended' && (foreachIndex === undefined || nestedIterationSuspendPayload) ? { steps: resume?.steps?.slice(1) || [], resumePayload: resume?.resumePayload, - runId: (nestedIterationSuspendPayload ?? stepResults[step.id]?.suspendPayload)?.__workflow_meta?.runId, + runId: (nestedIterationSuspendPayload ?? storedResult?.suspendPayload)?.__workflow_meta?.runId, label: resume?.label, // Inside a foreach the index picked this item; otherwise it picks an item of a // foreach inside the nested run. From f8132b8e8e9c1a89e2ef91103cdda3aa8fe1c2c4 Mon Sep 17 00:00:00 2001 From: Arthur Fiorette Date: Mon, 16 Nov 2026 18:50:58 -0300 Subject: [PATCH 04/10] test(core): table the foreach resume restart cases The cases now share one scenario builder: each row names its shape, how the resume picks its target, the instance, the crash window and what every restarted item must receive, and every restart asserts that the resume targets never reach results or the stored run. The two sibling-race cases are rows of these tables instead of hand-built stores, and a gate cut restarts from the first write that stored the resumed record with its step active. The durable delegation case scripts its supervisor and sub-agents on the aimock server the direct-tool case already uses. --- .../durable-agent-approval-restart.test.ts | 231 ++-- .../durable/__tests__/restart-harness.ts | 9 +- .../workflows/foreach-resume-restart.test.ts | 1019 +++++++---------- 3 files changed, 511 insertions(+), 748 deletions(-) diff --git a/packages/core/src/agent/durable/__tests__/durable-agent-approval-restart.test.ts b/packages/core/src/agent/durable/__tests__/durable-agent-approval-restart.test.ts index 53d15bf1ee5..780063660c6 100644 --- a/packages/core/src/agent/durable/__tests__/durable-agent-approval-restart.test.ts +++ b/packages/core/src/agent/durable/__tests__/durable-agent-approval-restart.test.ts @@ -1,32 +1,72 @@ /** - * Recovering a durable agent run that crashed while an approved tool call ran. - * - * The model asks for three approval-gated tool calls in one step. The first is approved through - * the same agent instance that suspended the run, and the process crashes while it runs. The - * recovered run must run only that call again: the other two still wait for their own approval. + * Recovering a durable agent run that crashed while an approval was being applied. The recovered + * run must run only the approved tool call: every other call still waits for its own approval. */ import { createOpenAI } from '@ai-sdk/openai-v5'; import { LLMock } from '@copilotkit/aimock'; -import { MockLanguageModelV2, convertArrayToReadableStream } from '@internal/ai-sdk-v5/test'; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest'; import { z } from 'zod'; import { DurableStepIds } from '../constants'; -import type { Gate } from './restart-harness'; +import type { Checkpoint, Gate, HoldWrite } from './restart-harness'; import { createGate, createRestartScenario } from './restart-harness'; let llm: LLMock; const gates: Gate[] = []; const scenarios: { stop(): Promise }[] = []; +/** + * Picks requests by the agent's instructions (the start of the system prompt: a sub-agent's also + * carries its supervisor's), the agent's own task (the last user message), and whether the text + * already holds `done`. + */ +const request = + (instructions: string, { task, done }: { task?: string; done?: string[] } = {}) => + (body: any) => { + const messages: any[] = body?.messages ?? []; + const text = JSON.stringify(messages); + return ( + String(messages[0]?.content).startsWith(instructions) && + (task === undefined || messages.findLast(m => m.role === 'user')?.content === task) && + (done ?? []).every(part => text.includes(part)) + ); + }; +const pending = (predicate: (body: any) => boolean, done: (body: any) => boolean) => (body: any) => + predicate(body) && !done(body); + beforeAll(async () => { llm = new LLMock({ port: 0 }); await llm.start(); + // An agent with three approval-gated tool calls in one step. llm.on( - { endpoint: 'chat', hasToolResult: false }, + { endpoint: 'chat', hasToolResult: false, predicate: request('Charge every card') }, { toolCalls: [0, 1, 2].map(index => ({ id: `call_${index}`, name: 'charge', arguments: { index } })) }, ); - llm.on({ endpoint: 'chat', hasToolResult: true }, { content: 'all charged' }); + llm.on( + { endpoint: 'chat', hasToolResult: true, predicate: request('Charge every card') }, + { content: 'all charged' }, + ); + // A supervisor that delegates to two sub-agents in one step; each calls its gated tool once. + const supervisorDone = request('Delegate to both', { done: ['alpha finished', 'beta finished'] }); + llm.on({ endpoint: 'chat', predicate: supervisorDone }, { content: 'Both delegations complete.' }); + llm.on( + { endpoint: 'chat', predicate: pending(request('Delegate to both'), supervisorDone) }, + { + toolCalls: ['Alpha', 'Beta'].map(name => ({ + id: `sup-tc-${name[0]}`, + name: `agent-sub${name}`, + arguments: { prompt: `do ${name.toLowerCase()}` }, + })), + }, + ); + for (const label of ['alpha', 'beta']) { + const done = request('Use your tool', { task: `do ${label}`, done: [`${label} done`] }); + llm.on({ endpoint: 'chat', predicate: done }, { content: `${label} finished.` }); + llm.on( + { endpoint: 'chat', predicate: pending(request('Use your tool', { task: `do ${label}` }), done) }, + { toolCalls: [{ id: `inner-${label}`, name: 'gatedTool', arguments: { query: label } }] }, + ); + } }); afterAll(async () => { @@ -38,6 +78,16 @@ afterEach(async () => { for (const scenario of scenarios.splice(0)) await scenario.stop(); }); +const model = () => createOpenAI({ apiKey: 'test-key', baseURL: `${llm.url}/v1` }).chat('gpt-4o-mini'); + +/** Recovers the run from `checkpoint`; the recovered run reports no error. */ +async function recover(scenario: ReturnType, checkpoint: Checkpoint) { + const recovered = await scenario.restart(checkpoint, { closeOnSuspend: true }); + expect(recovered.streamErrors).toEqual([]); + expect(recovered.executionError).toBeUndefined(); + return recovered; +} + describe('durable agent: restart while an approved tool call runs', () => { it('runs only the approved tool call, not the calls still waiting for approval', async () => { const runId = 'approval-restart'; @@ -52,7 +102,7 @@ describe('durable agent: restart while an approved tool call runs', () => { id: 'approval-agent', name: 'approval-agent', instructions: 'Charge every card.', - model: createOpenAI({ apiKey: 'test-key', baseURL: `${llm.url}/v1` }).chat('gpt-4o-mini'), + model: model(), tools: { charge: core.createTool({ id: 'charge', @@ -86,96 +136,19 @@ describe('durable agent: restart while an approved tool call runs', () => { expect(approvedId).toBe('call_0'); expect(charges).toEqual([{ generation: 1, index: 0 }]); - const recovered = await scenario.restart(await original.checkpoint(), { closeOnSuspend: true }); + const recovered = await recover(scenario, await original.checkpoint()); - expect(recovered.streamErrors).toEqual([]); - expect(recovered.executionError).toBeUndefined(); expect(charges.filter(c => c.generation === 2)).toEqual([{ generation: 2, index: 0 }]); // The next call asks for its own approval. expect(recovered.chunkTypes).toContain('tool-call-approval'); }, 30_000); }); -/** A sub-agent that calls its approval-gated tool once, then answers. */ -function subAgentModel(label: string) { - return new MockLanguageModelV2({ - doStream: async ({ prompt }) => { - const done = JSON.stringify(prompt).includes(`${label} done`); - return { - rawCall: { rawPrompt: null, rawSettings: {} }, - warnings: [], - stream: convertArrayToReadableStream([ - { type: 'stream-start', warnings: [] }, - { type: 'response-metadata', id: `${label}-${done ? 1 : 0}`, modelId: 'mock-model', timestamp: new Date(0) }, - ...(done - ? [ - { type: 'text-start', id: `${label}-text` }, - { type: 'text-delta', id: `${label}-text`, delta: `${label} finished.` }, - { type: 'text-end', id: `${label}-text` }, - ] - : [ - { - type: 'tool-call', - toolCallType: 'function', - toolCallId: `inner-${label}`, - toolName: 'gatedTool', - input: JSON.stringify({ query: label }), - providerExecuted: false, - }, - ]), - { - type: 'finish', - finishReason: done ? 'stop' : 'tool-calls', - usage: { inputTokens: 5, outputTokens: 10, totalTokens: 15 }, - }, - ]), - }; - }, - }); -} - -/** A supervisor that delegates to both sub-agents in one step, then answers. */ -function supervisorModel() { - return new MockLanguageModelV2({ - doStream: async ({ prompt }) => { - const text = JSON.stringify(prompt); - const done = text.includes('alpha finished') && text.includes('beta finished'); - return { - rawCall: { rawPrompt: null, rawSettings: {} }, - warnings: [], - stream: convertArrayToReadableStream([ - { type: 'stream-start', warnings: [] }, - { type: 'response-metadata', id: `sup-${done ? 1 : 0}`, modelId: 'mock-model', timestamp: new Date(0) }, - ...(done - ? [ - { type: 'text-start', id: 'sup-text' }, - { type: 'text-delta', id: 'sup-text', delta: 'Both delegations complete.' }, - { type: 'text-end', id: 'sup-text' }, - ] - : ['Alpha', 'Beta'].map(name => ({ - type: 'tool-call', - toolCallType: 'function', - toolCallId: `sup-tc-${name[0]}`, - toolName: `agent-sub${name}`, - input: JSON.stringify({ prompt: `do ${name.toLowerCase()}` }), - providerExecuted: false, - }))), - { - type: 'finish', - finishReason: done ? 'stop' : 'tool-calls', - usage: { inputTokens: 10, outputTokens: 20, totalTokens: 30 }, - }, - ]), - }; - }, - }); -} - describe('durable agent: restart while an approved delegation is being resumed', () => { // Delegations to sub-agents run concurrently, so both suspend at once when each sub-agent's tool // needs approval. The approval of one goes through the instance that suspended the run, and the // process crashes before the loop's iteration workflow stores its resumed tool calls. - const crashPoints = [ + const crashPoints: { point: string; hold: HoldWrite }[] = [ { point: 'before the iteration workflow claims its resume', hold: { @@ -195,7 +168,7 @@ describe('durable agent: restart while an approved delegation is being resumed', ]; /** Both delegations suspended, `sup-tc-A` approved, and a copy of the rows at the crash. */ - async function crashWhileApproving(hold: (typeof crashPoints)[number]['hold']) { + async function crashWhileApproving(hold: HoldWrite) { const runId = 'delegation-approval-restart'; const memory = { thread: 'delegation-thread', resource: 'delegation-resource' }; const executions: { generation: number; query: string }[] = []; @@ -209,31 +182,31 @@ describe('durable agent: restart while an approved delegation is being resumed', // persistent store drops when it serializes the row. storageMode: 'serializing', build: ({ core, generation }) => { - const gatedTool = () => - core.createTool({ - id: 'gatedTool', - description: 'Approval-gated tool', - inputSchema: z.object({ query: z.string() }), - requireApproval: true, - execute: async ({ query }: { query: string }) => { - executions.push({ generation, query }); - return { result: `${query} done` }; - }, - }); const subAgent = (name: string) => new core.Agent({ id: `sub${name}`, name: `sub${name}`, description: `Does ${name.toLowerCase()}`, instructions: 'Use your tool.', - model: subAgentModel(name.toLowerCase()) as any, - tools: { gatedTool: gatedTool() }, + model: model(), + tools: { + gatedTool: core.createTool({ + id: 'gatedTool', + description: 'Approval-gated tool', + inputSchema: z.object({ query: z.string() }), + requireApproval: true, + execute: async ({ query }: { query: string }) => { + executions.push({ generation, query }); + return { result: `${query} done` }; + }, + }), + }, }); return new core.Agent({ id: 'supervisor', name: 'Supervisor', instructions: 'Delegate to both sub-agents.', - model: supervisorModel() as any, + model: model(), agents: { subAlpha: subAgent('Alpha'), subBeta: subAgent('Beta') }, memory: new core.MockMemory({ storage: (memoryStore ??= new core.InMemoryStore()) }), }); @@ -259,35 +232,29 @@ describe('durable agent: restart while an approved delegation is being resumed', await original.held; const checkpoint = await original.checkpoint(); expect(executions).toEqual([]); - return { scenario, checkpoint, executions, runId, memory }; + /** Approves `toolCallId` on the recovered graph and returns the reply's text. */ + const approve = async (recovered: { graph: any }, toolCallId: string) => { + const resumed = await recovered.graph.agent.approveToolCall({ runId, toolCallId, memory, closeOnSuspend: true }); + let text = ''; + for await (const chunk of resumed.fullStream) { + if (chunk.type === 'text-delta') text += chunk.payload.text; + } + return text; + }; + return { scenario, checkpoint, executions, approve }; } - const approve = async (agent: any, runId: string, toolCallId: string, memory: object) => { - const resumed = await agent.approveToolCall({ runId, toolCallId, memory, closeOnSuspend: true }); - let text = ''; - for await (const chunk of resumed.fullStream) { - if (chunk.type === 'text-delta') text += chunk.payload.text; - } - return text; - }; - it.each(crashPoints)( 'runs only the approved delegation after a crash $point', async ({ hold }) => { - const { scenario, checkpoint, executions, runId, memory } = await crashWhileApproving(hold); + const { scenario, checkpoint, executions, approve } = await crashWhileApproving(hold); - const recovered = await scenario.restart(checkpoint, { closeOnSuspend: true }); + const recovered = await recover(scenario, checkpoint); - expect(recovered.streamErrors).toEqual([]); - expect(recovered.executionError).toBeUndefined(); expect(executions).toEqual([{ generation: 2, query: 'alpha' }]); - // Beta's own approval is still pending, and still works. - expect(await approve(recovered.graph.agent, runId, 'sup-tc-B', memory)).toBe('Both delegations complete.'); - expect(executions).toEqual([ - { generation: 2, query: 'alpha' }, - { generation: 2, query: 'beta' }, - ]); + expect(await approve(recovered, 'sup-tc-B')).toBe('Both delegations complete.'); + expect(executions.map(e => e.query)).toEqual(['alpha', 'beta']); }, 60_000, ); @@ -296,26 +263,20 @@ describe('durable agent: restart while an approved delegation is being resumed', 'runs no delegation when the stored resume recorded no target, after a crash $point', async ({ hold }) => { // Runs stored before the target was recorded carry the approval but not which call it is for. - const { scenario, checkpoint, executions, runId, memory } = await crashWhileApproving(hold); + const { scenario, checkpoint, executions, approve } = await crashWhileApproving(hold); const loop = checkpoint.rows.find(row => row.workflowName === DurableStepIds.AGENTIC_LOOP)!; const record = loop.snapshot.context[DurableStepIds.AGENTIC_EXECUTION] as any; expect(record.metadata.nestedResumeTarget).toEqual({ steps: [], label: 'sup-tc-A' }); delete record.metadata.nestedResumeTarget; - const recovered = await scenario.restart(checkpoint, { closeOnSuspend: true }); + const recovered = await recover(scenario, checkpoint); - expect(recovered.streamErrors).toEqual([]); - expect(recovered.executionError).toBeUndefined(); expect(executions).toEqual([]); - // Both approvals are still pending: approving each again runs it. - await approve(recovered.graph.agent, runId, 'sup-tc-A', memory); + await approve(recovered, 'sup-tc-A'); expect(executions).toEqual([{ generation: 2, query: 'alpha' }]); - expect(await approve(recovered.graph.agent, runId, 'sup-tc-B', memory)).toBe('Both delegations complete.'); - expect(executions).toEqual([ - { generation: 2, query: 'alpha' }, - { generation: 2, query: 'beta' }, - ]); + expect(await approve(recovered, 'sup-tc-B')).toBe('Both delegations complete.'); + expect(executions.map(e => e.query)).toEqual(['alpha', 'beta']); }, 60_000, ); diff --git a/packages/core/src/agent/durable/__tests__/restart-harness.ts b/packages/core/src/agent/durable/__tests__/restart-harness.ts index 25efa560fa9..01ce113f9ad 100644 --- a/packages/core/src/agent/durable/__tests__/restart-harness.ts +++ b/packages/core/src/agent/durable/__tests__/restart-harness.ts @@ -184,8 +184,7 @@ export function createGate() { export type Gate = ReturnType; -/** `storage` is the graph's store: a memory built on it keeps its data across the restart. */ -type BuildContext = { core: CoreGraph; generation: number; storage: any }; +type BuildContext = { core: CoreGraph; generation: number }; type AgentBuild = (ctx: BuildContext) => Agent; type WorkflowBuild = (ctx: BuildContext) => any; @@ -244,7 +243,7 @@ async function buildGraph(options: ScenarioOptions, ge options.storageMode === 'serializing' ? (await import('../../../storage/domains/workflows/serializing-store.test-utils')).createSerializingStore().store : new core.InMemoryStore(); - const built = options.build({ core, generation, storage } as BuildContext); + const built = options.build({ core, generation } as BuildContext); let mastra: any; try { if (options.kind === 'workflow') { @@ -467,7 +466,8 @@ export function createRestartScenario(options: ScenarioOp * Agent kinds: close the recovered stream when the run suspends again; it stays open by * default, so a test that expects the run to suspend sets this. */ - { closeOnSuspend }: { closeOnSuspend?: boolean } = {}, + /** Workflow kind: `watch` sees the restarted run's events. */ + { closeOnSuspend, watch }: { closeOnSuspend?: boolean; watch?: (event: any) => void } = {}, ): Promise { const graph = await buildGraph(options, ++generation); graphs.push(graph); @@ -490,6 +490,7 @@ export function createRestartScenario(options: ScenarioOp if (options.kind === 'workflow') { const run = await graph.workflow.createRun({ runId: options.runId }); + if (watch) run.watch(watch); const result = await withTimeout(run.restart(), timeoutMs, `restart(${options.runId})`); return { result, graph }; } diff --git a/packages/core/src/workflows/foreach-resume-restart.test.ts b/packages/core/src/workflows/foreach-resume-restart.test.ts index 777beba39ea..c7364e85a1f 100644 --- a/packages/core/src/workflows/foreach-resume-restart.test.ts +++ b/packages/core/src/workflows/foreach-resume-restart.test.ts @@ -3,10 +3,9 @@ * item's resume data to that item only. Items that were still suspended, or never reached, must * not receive it: in an agent loop each item is a tool call waiting for its own approval. * - * Each case suspends the items, resumes some of them either through the `Mastra` instance that - * suspended the run or through a second instance on the same storage, copies what storage holds - * while a resumed item runs (or at a chosen write), and restarts from that copy in a fresh - * module graph. + * Each case suspends the items, resumes some of them through the `Mastra` instance that suspended + * the run or through a second instance on the same storage, copies what storage holds while a + * resumed item runs (or at a chosen write), and restarts from that copy in a fresh module graph. */ import { afterEach, describe, expect, it, vi } from 'vitest'; @@ -20,11 +19,16 @@ import { copyRunTables, interceptWrites, restoreRunTables, runTablesOf } from '. import type { RunTables } from './test-utils/run-tables'; import { createStep } from './workflow'; -type Seen = { generation: number; k: number; resumeData: unknown; suspendData: unknown }; +type Call = { k: number; resumeData: unknown; suspendData: unknown }; +type Seen = Call & { generation: number }; const ITEMS = [{ k: 0 }, { k: 1 }, { k: 2 }]; const ITEM = z.object({ k: z.number() }); const APPROVED = { approved: true }; +/** A call that received its own approval and its own suspend payload. */ +const resumed = (k: number): Call => ({ k, resumeData: APPROVED, suspendData: { ask: k } }); +/** A call that started the item fresh. */ +const fresh = (k: number): Call => ({ k, resumeData: undefined, suspendData: undefined }); const gates: Gate[] = []; const scenarios: { stop(): Promise }[] = []; @@ -39,6 +43,24 @@ function newGate() { return gate; } +/** The calls generation `generation` made after `from`, by item. */ +const calls = (seen: Seen[], generation: number, from = 0): Call[] => + seen + .slice(from) + .filter(s => s.generation === generation) + .map(({ generation: _, ...call }) => call) + .sort((a, b) => a.k - b.k); + +type Ctx = { + id: string; + generation: number; + seen: Seen[]; + gate: Gate; + concurrency: number; + /** Runs inside input validation (of the approval step, or of the nested workflow per item). */ + validate: (input: any) => Promise; +}; + /** An approval step: suspends until resumed; a resumed call in generation `blockIn` waits on `gate`. */ function approvalStep( createStepFn: any, @@ -47,12 +69,16 @@ function approvalStep( generation, seen, gate, + validate, blockIn = 1, - }: { id: string; generation: number; seen: Seen[]; gate: Gate; blockIn?: number }, + }: Omit & { id: string; blockIn?: number }, ) { return createStepFn({ id, - inputSchema: ITEM, + inputSchema: ITEM.refine(async input => { + await validate(input); + return true; + }), outputSchema: z.object({ k: z.number(), approved: z.boolean() }), resumeSchema: z.object({ approved: z.boolean() }), suspendSchema: z.object({ ask: z.number() }), @@ -65,504 +91,429 @@ function approvalStep( }); } -type Build = (core: CoreGraph, generation: number) => any; +const statusesOf = (steps: any, stepId: string): string[] => + steps[stepId].suspendPayload.__workflow_meta.foreachOutput.map((r: any) => r?.status); -const leafForeach = - (id: string, concurrency: number, seen: Seen[], gate: Gate): Build => - (core, generation) => +type Shape = { + build(core: CoreGraph, ctx: Ctx): any; + inputData: unknown; + /** Where the resumed foreach's running record is stored: workflow name, step id. */ + record(id: string): [string, string]; + /** The approval foreach's item statuses after `result`; absent where they aren't one list. */ + statuses?(args: { id: string; runId: string; result: any; graph: any }): Promise; +}; + +const leaf: Shape = { + build: (core, ctx) => core - .createWorkflow({ id, inputSchema: z.array(ITEM), outputSchema: z.any() }) - .foreach(approvalStep(core.createStep, { id: 'approve', generation, seen, gate }), { concurrency }) - .commit(); + .createWorkflow({ id: ctx.id, inputSchema: z.array(ITEM), outputSchema: z.any() }) + .foreach(approvalStep(core.createStep, { ...ctx, id: 'approve' }), { concurrency: ctx.concurrency }) + .commit(), + inputData: ITEMS, + record: id => [id, 'approve'], + statuses: async ({ result }) => statusesOf(result.steps, 'approve'), +}; + +/** A foreach whose items are nested workflows with one approval step each. */ +const nestedBody: Shape = { + build: (core, ctx) => + core + .createWorkflow({ id: ctx.id, inputSchema: z.array(ITEM), outputSchema: z.any() }) + .foreach( + core + .createWorkflow({ id: `${ctx.id}-child`, inputSchema: ITEM, outputSchema: z.any() }) + .then(approvalStep(core.createStep, { ...ctx, id: 'ask' })) + .commit(), + { concurrency: 3 }, + ) + .commit(), + inputData: ITEMS, + record: id => [id, `${id}-child`], + statuses: async ({ id, result }) => statusesOf(result.steps, `${id}-child`), +}; /** A foreach of approval steps inside a nested workflow: the shape of a durable agent's tool calls. */ -const childForeach = - (id: string, seen: Seen[], gate: Gate): Build => - (core, generation) => +const inChild: Shape = { + build: (core, ctx) => core - .createWorkflow({ id, inputSchema: z.array(ITEM), outputSchema: z.any() }) + .createWorkflow({ id: ctx.id, inputSchema: z.array(ITEM), outputSchema: z.any() }) .then( core - .createWorkflow({ id: `${id}-child`, inputSchema: z.array(ITEM), outputSchema: z.any() }) - .foreach(approvalStep(core.createStep, { id: 'approve', generation, seen, gate }), { concurrency: 3 }) + .createWorkflow({ id: `${ctx.id}-child`, inputSchema: z.array(ITEM), outputSchema: z.any() }) + .foreach(approvalStep(core.createStep, { ...ctx, id: 'approve' }), { concurrency: 3 }) .commit(), ) - .commit(); + .commit(), + inputData: ITEMS, + record: id => [`${id}-child`, 'approve'], + statuses: async ({ id, runId, graph }) => { + const child = await graph.workflows.loadWorkflowSnapshot({ workflowName: `${id}-child`, runId }); + return statusesOf(child.context, 'approve'); + }, +}; /** - * A restart scenario whose graph 1 suspends the run and resumes it, through the same `Mastra` - * instance, or through a second instance on the same storage when `elsewhere` is set. + * A foreach whose items are nested workflows that each run their own concurrent foreach. Outer + * items 0 and 1 suspend and outer item 2 is never reached; the label names inner item 1 of outer + * item 0. */ -function resumeScenario( - id: string, - build: Build, - resume: Record, - { elsewhere = false, hold }: { elsewhere?: boolean; hold?: HoldWrite } = {}, -) { +const foreachOfInChild: Shape = { + build: (core, ctx) => + core + .createWorkflow({ id: ctx.id, inputSchema: z.array(z.array(ITEM)), outputSchema: z.any() }) + .foreach( + core + .createWorkflow({ + id: `${ctx.id}-child`, + inputSchema: z.array(ITEM).refine(async items => { + await ctx.validate(items); + return true; + }), + outputSchema: z.any(), + }) + .foreach(approvalStep(core.createStep, { ...ctx, id: 'approve', validate: async () => {} }), { + concurrency: 2, + }) + .commit(), + { concurrency: 2 }, + ) + .commit(), + inputData: [ + [{ k: 0 }, { k: 1 }], + [{ k: 2 }, { k: 3 }], + [{ k: 4 }, { k: 5 }], + ], + record: id => [id, `${id}-child`], +}; + +/** + * On the restart, the input validation of the item `holds` picks waits until item `index` of the + * foreach `step` has suspended. Items share the foreach's record in memory, and that sibling's + * suspension replaces it while the resumed item is still validating. + */ +type Race = { holds: (input: any) => boolean; step: (id: string) => string; index: number }; +const leafRace: Race = { holds: input => input.k === 1, step: () => 'approve', index: 2 }; +const outerRace: Race = { holds: input => input[0]?.k === 0, step: id => `${id}-child`, index: 2 }; + +/** A crash in the window before the nested run stores its resumed foreach. */ +type Window = { hold: (id: string) => HoldWrite; claimed: boolean; unclaim?: boolean }; +/** The parent has stored its resumed nested step; the nested run is still suspended. */ +const beforeClaim: Window = { + claimed: false, + hold: id => ({ + when: snapshot => + snapshot.context?.[`${id}-child`]?.status === 'running' && + (snapshot.context[`${id}-child`] as any).resumePayload !== undefined, + }), +}; +/** The nested run has claimed its resume. */ +const afterClaim: Window = { + claimed: true, + hold: id => ({ + workflowName: `${id}-child`, + when: snapshot => snapshot.status === 'running' && snapshot.context?.approve?.status === 'suspended', + }), +}; +/** + * The outer foreach's running record doesn't hold its items back, so the cut parks the nested + * run's claim; a crash just before it leaves the same rows with the nested run still suspended. + */ +const beforeClaimOfItem: Window = { ...afterClaim, claimed: false, unclaim: true }; + +type Row = { + name: string; + shape: Shape; + resume: (child: string) => Record; + /** Suspend through a second `Mastra` instance on the same storage. */ + elsewhere?: boolean; + concurrency?: number; + /** Without a window: cut once this many resumed items are running. */ + waiters?: number; + window?: Window; + race?: Race; +}; + +const byIndex = (k: number) => () => ({ step: 'approve', forEachIndex: k }); +const byLabel = (k: number) => () => ({ label: `item-${k}` }); +const nestedByIndex = (k: number) => (child: string) => ({ step: [child, 'approve'], forEachIndex: k }); + +/** Runs a row's graph 1 up to its cut and copies storage there. */ +async function crash(id: string, row: Row) { const runId = `${id}-run`; + const child = `${id}-child`; + const seen: Seen[] = []; + const gate = newGate(); + const validation = newGate(); + const build = (core: CoreGraph, generation: number) => + row.shape.build(core, { + id, + generation, + seen, + gate, + concurrency: row.concurrency ?? 3, + validate: async input => { + if (generation === 2 && row.race?.holds(input)) await validation.wait(); + }, + }); const scenario = createRestartScenario({ kind: 'workflow', runId, build: ({ core, generation }) => build(core, generation), - hold, + hold: row.window?.hold(id), }); scenarios.push(scenario); - const start = () => - scenario.start(async ({ core, storage, workflow }) => { - let suspender = workflow; - if (elsewhere) { - suspender = build(core, 0); - new core.Mastra({ logger: false, storage, workflows: { [suspender.id]: suspender } }); - } - expect((await (await suspender.createRun({ runId })).start({ inputData: ITEMS })).status).toBe('suspended'); - return (await workflow.createRun({ runId })).resume(resume); - }); - return { scenario, start, runId }; -} - -const resumedIn = (seen: Seen[], generation: number) => - seen - .filter(s => s.generation === generation && s.resumeData !== undefined) - .map(s => s.k) - .sort(); - -const foreachRecord = (checkpoint: Checkpoint, workflowName: string, runId: string, stepId: string) => - findRow(checkpoint, workflowName, runId)?.snapshot.context?.[stepId] as any; - -/** Copies graph 1's rows once the resumed foreach's running record, with the payload, is stored. */ -async function checkpointWhileResuming( - original: { checkpoint(): Promise }, - { workflowName, runId, stepId }: { workflowName: string; runId: string; stepId: string }, -) { - let checkpoint!: Checkpoint; - await vi.waitFor(async () => { - checkpoint = await original.checkpoint(); - expect(foreachRecord(checkpoint, workflowName, runId, stepId)).toMatchObject({ - status: 'running', - resumePayload: APPROVED, - }); + const original = await scenario.start(async ({ core, storage, workflow }) => { + let suspender = workflow; + if (row.elsewhere) { + suspender = build(core, 0); + new core.Mastra({ logger: false, storage, workflows: { [suspender.id]: suspender } }); + } + const started = await (await suspender.createRun({ runId })).start({ inputData: row.shape.inputData }); + expect(started.status).toBe('suspended'); + return (await workflow.createRun({ runId })).resume({ ...row.resume(child), resumeData: APPROVED }); }); - return checkpoint; + if (row.window) await original.held; + else await vi.waitFor(() => expect(gate.waiters).toBe(row.waiters ?? 1)); + + // The resumed record a restart reads: the foreach's, or the parent's nested step in a window. + const [workflowName, stepId] = row.window ? [id, child] : row.shape.record(id); + const resuming = (c: Checkpoint) => { + const snapshot = findRow(c, workflowName, runId)?.snapshot; + const record = snapshot?.context?.[stepId] as any; + // Outside a window, also once the run records the step as active, which is what a restart reruns. + return ( + record?.status === 'running' && + record.resumePayload !== undefined && + (!!row.window || !!snapshot?.activeStepsPath?.[stepId]) + ); + }; + // In a window, graph 1 is parked in the chosen write; otherwise the first write that stored the + // resumed record (later writes while items run can store the run without its active step). + const checkpoint = (row.window ? await original.checkpoint() : original.checkpoints.find(resuming))!; + expect(!!checkpoint && resuming(checkpoint), 'a stored running record with the resume payload').toBe(true); + if (row.window) { + // Nested runs whose foreach is still stored suspended; the claimed one is `running`. + const parked = checkpoint.rows.filter( + r => r.workflowName === child && (r.snapshot.context as any).approve?.status === 'suspended', + ); + if (row.window.unclaim) for (const r of parked) r.snapshot.status = 'suspended'; + expect(parked.filter(r => r.snapshot.status === 'running')).toHaveLength(row.window.claimed ? 1 : 0); + expect(calls(seen, 1).filter(c => c.resumeData !== undefined)).toEqual([]); + } + + const restart = async () => { + const { result, graph } = await scenario.restart(checkpoint, { + watch: event => { + const payload = event.payload; + const { race } = row; + if (!race || event.type !== 'workflow-step-progress') return; + if (payload?.id === race.step(id) && payload.currentIndex === race.index) { + expect(payload.iterationStatus).toBe('suspended'); + validation.release(); + } + }, + }); + const stored = await graph.workflows.loadWorkflowSnapshot({ workflowName: id, runId }); + expect(JSON.stringify([result, stored])).not.toMatch(/foreachResumeTarget|nestedResumeTarget/); + const statuses = (r: any) => row.shape.statuses?.({ id, runId, result: r, graph }); + const resume = (args: Record) => + graph.workflow.createRun({ runId }).then((r: any) => r.resume(args)); + return { result, graph, stored, statuses, resume }; + }; + const record = () => findRow(checkpoint, workflowName, runId)!.snapshot.context[stepId] as any; + return { seen, restart, record, child }; } -/** - * A leaf foreach whose items all suspend, resumed as `resume` says, cut while `waiters` resumed - * items run. - */ -async function crashWhileResuming( - id: string, - { - concurrency, - resume, - waiters = 1, - elsewhere = true, - }: { concurrency: number; resume: Record; waiters?: number; elsewhere?: boolean }, -) { - const seen: Seen[] = []; - const gate = newGate(); - const { scenario, start, runId } = resumeScenario( - id, - leafForeach(id, concurrency, seen, gate), - { ...resume, resumeData: APPROVED }, - { elsewhere }, - ); - const original = await start(); - await vi.waitFor(() => expect(gate.waiters).toBe(waiters)); - const checkpoint = await checkpointWhileResuming(original, { workflowName: id, runId, stepId: 'approve' }); - return { seen, scenario, checkpoint, runId }; -} - -function itemOutputs(steps: any, stepId: string) { - return (steps[stepId].suspendPayload.__workflow_meta.foreachOutput as any[]).map(r => ({ - status: r?.status, - ...(r?.status === 'suspended' ? { suspendPayload: r.suspendPayload } : {}), - })); -} - -async function expectChildItems(graph: any, child: string, runId: string, statuses: string[]) { - const childRun = await graph.workflows.loadWorkflowSnapshot({ workflowName: child, runId }); - expect(itemOutputs(childRun.context, 'approve').map(o => o.status)).toEqual(statuses); -} +const S = 'suspended'; +const OK = 'success'; describe('restart while a resumed foreach item runs', () => { - it('gives the approved item its decision and keeps the other suspended items waiting for theirs', async () => { - const id = 'foreach-resume-leaf'; - const { seen, scenario, checkpoint, runId } = await crashWhileResuming(id, { - concurrency: 3, - resume: { step: 'approve', forEachIndex: 1 }, - }); - - const { result, graph } = await scenario.restart(checkpoint); - - expect(resumedIn(seen, 2)).toEqual([1]); - expect(seen.find(s => s.generation === 2 && s.k === 1)).toMatchObject({ - resumeData: APPROVED, - suspendData: { ask: 1 }, - }); - expect(result.status).toBe('suspended'); - expect(itemOutputs(result.steps, 'approve')).toEqual([ - { status: 'suspended', suspendPayload: { ask: 0 } }, - { status: 'success' }, - { status: 'suspended', suspendPayload: { ask: 2 } }, - ]); - const stored = await graph.workflows.loadWorkflowSnapshot({ workflowName: id, runId }); - expect(JSON.stringify(result)).not.toContain('foreachResumeTarget'); - expect(JSON.stringify(stored)).not.toContain('foreachResumeTarget'); - - // Item 0 still answers to its own resume. - const next = await ( - await graph.workflow.createRun({ runId }) - ).resume({ - step: 'approve', - forEachIndex: 0, - resumeData: { approved: false }, - }); - expect(seen.filter(s => s.generation === 2 && s.k === 0)).toEqual([ - { generation: 2, k: 0, resumeData: { approved: false }, suspendData: { ask: 0 } }, - ]); - expect(next.status).toBe('suspended'); - expect(itemOutputs(next.steps, 'approve').map(o => o.status)).toEqual(['success', 'success', 'suspended']); - }); - - it('does not hand the resumed item decision to items the foreach had not reached', async () => { - const { seen, scenario, checkpoint } = await crashWhileResuming('foreach-resume-unreached', { + it.each([ + { + name: 'leaf foreach, item 1 resumed by index through a second instance', + shape: leaf, + resume: byIndex(1), + elsewhere: true, + gen2: [resumed(1)], + statuses: [S, OK, S], + }, + { + name: 'leaf foreach, item 1 resumed by label through the instance that suspended it', + shape: leaf, + resume: byLabel(1), + gen2: [resumed(1)], + statuses: [S, OK, S], + }, + { + name: 'leaf foreach at concurrency 1, item 0 resumed, items 1 and 2 never reached', + shape: leaf, concurrency: 1, - resume: { step: 'approve', forEachIndex: 0 }, - }); - - const { result } = await scenario.restart(checkpoint); - - expect(resumedIn(seen, 2)).toEqual([0]); - expect(seen.find(s => s.generation === 2 && s.k === 1)).toEqual({ - generation: 2, - k: 1, - resumeData: undefined, - suspendData: undefined, - }); - expect(result.status).toBe('suspended'); - expect(itemOutputs(result.steps, 'approve')).toEqual([ - { status: 'success' }, - { status: 'suspended', suspendPayload: { ask: 1 } }, - ]); - }); - - it('resumes only the approved nested workflow when the foreach body is a workflow', async () => { - const id = 'foreach-resume-nested'; - const child = `${id}-child`; - const seen: Seen[] = []; - const gate = newGate(); - const build: Build = (core, generation) => - core - .createWorkflow({ id, inputSchema: z.array(ITEM), outputSchema: z.any() }) - .foreach( - core - .createWorkflow({ id: child, inputSchema: ITEM, outputSchema: z.any() }) - .then(approvalStep(core.createStep, { id: 'ask', generation, seen, gate })) - .commit(), - { concurrency: 3 }, - ) - .commit(); - const { scenario, start, runId } = resumeScenario( - id, - build, - { step: [child, 'ask'], forEachIndex: 1, resumeData: APPROVED }, - { elsewhere: true }, - ); - const original = await start(); - await gate.reached; - const checkpoint = await checkpointWhileResuming(original, { workflowName: id, runId, stepId: child }); - - const { result } = await scenario.restart(checkpoint); - - expect(resumedIn(seen, 2)).toEqual([1]); - // The suspended siblings' nested runs are neither resumed nor started over. - expect(seen.filter(s => s.generation === 2).map(s => s.k)).toEqual([1]); - expect(result.status).toBe('suspended'); - expect(itemOutputs(result.steps, child).map(o => o.status)).toEqual(['suspended', 'success', 'suspended']); - }); - - it('gives the decision to every item a resume without an index reached, and to no other', async () => { - // Concurrency 2: items 0 and 1 suspend and item 2 is never reached. A resume with no index - // resumes 0 and 1; item 2 is still queued behind them at the crash. - const { seen, scenario, checkpoint } = await crashWhileResuming('foreach-resume-all', { + resume: byIndex(0), + elsewhere: true, + gen2: [resumed(0), fresh(1)], + statuses: [OK, S], + }, + { + name: 'leaf foreach at concurrency 2, both suspended items resumed without an index', + shape: leaf, concurrency: 2, - resume: { step: 'approve' }, + resume: () => ({ step: 'approve' }), + elsewhere: true, waiters: 2, - }); + gen2: [resumed(0), resumed(1), fresh(2)], + statuses: [OK, OK, S], + }, + { + name: 'leaf foreach at concurrency 2, a sibling suspends while the resumed item validates its input', + shape: leaf, + concurrency: 2, + resume: byIndex(1), + race: leafRace, + gen2: [resumed(1), fresh(2)], + statuses: [S, OK, S], + }, + { + name: 'foreach of nested workflows, item 1 resumed by index through a second instance', + shape: nestedBody, + resume: child => ({ step: [child, 'ask'], forEachIndex: 1 }), + elsewhere: true, + gen2: [resumed(1)], + statuses: [S, OK, S], + }, + { + name: 'foreach inside a nested workflow, item 1 resumed by label through the instance that suspended it', + shape: inChild, + resume: byLabel(1), + gen2: [resumed(1)], + statuses: [S, OK, S], + }, + ])('gives the decision only to the resumed items: $name', async row => { + const { seen, restart } = await crash(`ff-${row.name.replace(/\W+/g, '-')}`, row); - const { result } = await scenario.restart(checkpoint); + const { result, statuses, resume } = await restart(); - expect(resumedIn(seen, 2)).toEqual([0, 1]); - expect(seen.find(s => s.generation === 2 && s.k === 2)).toMatchObject({ resumeData: undefined }); + expect(calls(seen, 2)).toEqual(row.gen2); expect(result.status).toBe('suspended'); - expect(itemOutputs(result.steps, 'approve')).toEqual([ - { status: 'success' }, - { status: 'success' }, - { status: 'suspended', suspendPayload: { ask: 2 } }, - ]); + expect(await statuses(result)).toEqual(row.statuses); + + // An item still waiting answers to its own resume. + const k = row.statuses.indexOf(S); + const from = seen.length; + await resume({ label: `item-${k}`, resumeData: APPROVED }); + expect(calls(seen, 2, from).filter(c => c.resumeData !== undefined)).toEqual([resumed(k)]); }); - it('keeps every suspended item waiting when the stored run does not say which item was resumed', async () => { - // Runs stored before the resumed item was recorded carry the payload but no target. - const id = 'foreach-resume-untargeted'; - const { seen, scenario, checkpoint, runId } = await crashWhileResuming(id, { - concurrency: 3, - resume: { step: 'approve', forEachIndex: 1 }, - }); - const record = foreachRecord(checkpoint, id, runId, 'approve'); - expect(record.metadata.foreachResumeTarget).toBe(1); - delete record.metadata.foreachResumeTarget; + const nestedWindows: (Row & { gen2: Call[] })[] = [ + ...[beforeClaim, afterClaim].flatMap(window => + [ + { by: 'label', resume: byLabel(1) }, + { by: 'index', resume: nestedByIndex(1) }, + ].map(({ by, resume }) => ({ + name: `foreach inside a nested workflow, by ${by}, ${window.claimed ? 'after' : 'before'} the nested run claims its resume`, + shape: inChild, + resume, + window, + gen2: [resumed(1)], + })), + ), + ...[beforeClaimOfItem, afterClaim].map(window => ({ + name: `foreach in a foreach item, by label, ${window.claimed ? 'after' : 'before'} the nested run claims its resume, a sibling suspending meanwhile`, + shape: foreachOfInChild, + resume: byLabel(1), + window, + race: outerRace, + gen2: [resumed(1), fresh(4), fresh(5)], + })), + ]; - const { result } = await scenario.restart(checkpoint); + it.each(nestedWindows)('gives the decision only to the resumed inner item: $name', async row => { + const { seen, restart } = await crash(`fw-${row.name.replace(/\W+/g, '-')}`, row); - expect(resumedIn(seen, 2)).toEqual([]); + const { result, statuses } = await restart(); + + expect(calls(seen, 2)).toEqual(row.gen2); expect(result.status).toBe('suspended'); - expect(itemOutputs(result.steps, 'approve')).toEqual([ - { status: 'suspended', suspendPayload: { ask: 0 } }, - { status: 'suspended', suspendPayload: { ask: 1 } }, - { status: 'suspended', suspendPayload: { ask: 2 } }, - ]); + if (row.shape.statuses) expect(await statuses(result)).toEqual([S, OK, S]); }); - it('gives the decision only to the approved item when the instance that suspended the run resumes it', async () => { - const { seen, scenario, checkpoint } = await crashWhileResuming('foreach-resume-in-place', { - concurrency: 3, - resume: { label: 'item-1' }, - elsewhere: false, - }); + // Runs stored before targets were recorded carry the payload but no target. + it.each([ + { + name: 'leaf foreach', + shape: leaf, + resume: byIndex(1), + elsewhere: true, + gen2: [], + target: ['foreachResumeTarget', 1], + }, + ...[beforeClaim, afterClaim].map(window => ({ + name: `foreach inside a nested workflow, ${window.claimed ? 'after' : 'before'} the nested run claims its resume`, + shape: inChild, + resume: byLabel(1), + window, + gen2: [], + target: ['nestedResumeTarget', { steps: [], label: 'item-1' }] as [string, unknown], + })), + { + name: 'foreach in a foreach item, after the nested run claims its resume', + shape: foreachOfInChild, + resume: byLabel(1), + window: afterClaim, + race: outerRace, + gen2: [fresh(4), fresh(5)], + target: ['nestedResumeTarget', { steps: [], label: 'item-1' }], + }, + ])('keeps every suspended item waiting when the stored resume recorded no target: $name', async row => { + const { seen, restart, record, child } = await crash(`fo-${row.name.replace(/\W+/g, '-')}`, row); + const [key, value] = row.target; + expect(record().metadata[key]).toEqual(value); + delete record().metadata[key]; + // The state a nested run had handed back when it parked, which a later resume rebases from. + const startState = row.shape === inChild ? record().suspendPayload.__workflow_meta.startState : undefined; + if (row.shape === inChild) expect(startState).toBeDefined(); - const { result } = await scenario.restart(checkpoint); + const { result, stored, statuses, resume } = await restart(); - expect(resumedIn(seen, 2)).toEqual([1]); + expect(calls(seen, 2)).toEqual(row.gen2); expect(result.status).toBe('suspended'); - expect(itemOutputs(result.steps, 'approve').map(o => o.status)).toEqual(['suspended', 'success', 'suspended']); + if (row.shape.statuses) expect(await statuses(result)).toEqual([S, S, S]); + if (startState) expect(stored.context[child].suspendPayload.__workflow_meta.startState).toEqual(startState); + + // The interrupted item is resumed again, and only it runs. + const from = seen.length; + const next = await resume({ label: 'item-1', resumeData: APPROVED }); + expect(calls(seen, 2, from)).toEqual([resumed(1)]); + if (row.shape.statuses) expect(await statuses(next)).toEqual([S, OK, S]); }); - it('gives the decision only to the approved item of a foreach inside a nested workflow', async () => { - // The instance that suspended the run resumes one item by its label. - const id = 'foreach-resume-in-child'; - const seen: Seen[] = []; - const gate = newGate(); - const { scenario, start, runId } = resumeScenario(id, childForeach(id, seen, gate), { - label: 'item-1', - resumeData: APPROVED, - }); - const original = await start(); - await gate.reached; - const checkpoint = await checkpointWhileResuming(original, { - workflowName: `${id}-child`, - runId, - stepId: 'approve', - }); - - const { result, graph } = await scenario.restart(checkpoint); - - expect(resumedIn(seen, 2)).toEqual([1]); - expect(result.status).toBe('suspended'); - await expectChildItems(graph, `${id}-child`, runId, ['suspended', 'success', 'suspended']); - }); - - it('resumes only the chosen item of a foreach inside a nested workflow', async () => { + it('resumes only the chosen item of a foreach inside a nested workflow, by index', async () => { const id = 'foreach-resume-child-index'; const seen: Seen[] = []; - const gate = newGate(); - const { start } = resumeScenario(id, childForeach(id, seen, gate), { - step: [`${id}-child`, 'approve'], - forEachIndex: 1, - resumeData: APPROVED, + const workflow = inChild.build({ createWorkflow, createStep } as unknown as CoreGraph, { + id, + generation: 2, + seen, + gate: newGate(), + concurrency: 3, + validate: async () => {}, }); - const original = await start(); - gate.release(); + const storage = new MockStore(); + new Mastra({ logger: false, storage, workflows: { [id]: workflow } }); + const run = await workflow.createRun({ runId: id }); + expect((await run.start({ inputData: ITEMS })).status).toBe('suspended'); + const from = seen.length; - const result = (await original.driven) as any; + const result = await run.resume({ step: [`${id}-child`, 'approve'], forEachIndex: 1, resumeData: APPROVED }); - expect(resumedIn(seen, 1)).toEqual([1]); + expect(calls(seen, 2, from)).toEqual([resumed(1)]); expect(result.status).toBe('suspended'); - await expectChildItems(original.graph, `${id}-child`, `${id}-run`, ['suspended', 'success', 'suspended']); + const child = await (await storage.getStore('workflows'))!.loadWorkflowSnapshot({ + workflowName: `${id}-child`, + runId: id, + }); + expect(statusesOf(child!.context, 'approve')).toEqual([S, OK, S]); }); - // The two crash points before the nested run stores its resumed foreach: the parent has stored - // its resumed step, and the nested run is still suspended, or has only claimed its resume. - const crashPoints = [ - { - point: 'before the nested run claims its resume', - hold: (id: string) => ({ - when: (snapshot: any) => - snapshot.context?.[`${id}-child`]?.status === 'running' && - snapshot.context[`${id}-child`].resumePayload !== undefined, - }), - childStatus: 'suspended', - }, - { - point: 'after the nested run claims its resume, before its foreach is stored', - hold: (id: string) => ({ - workflowName: `${id}-child`, - when: (snapshot: any) => snapshot.status === 'running' && snapshot.context?.approve?.status === 'suspended', - }), - childStatus: 'running', - }, - ]; - const selectors = [ - { by: 'label', resume: (_child: string) => ({ label: 'item-1' }) }, - { by: 'index', resume: (child: string) => ({ step: [child, 'approve'], forEachIndex: 1 }) }, - ]; - it.each(crashPoints.flatMap(point => selectors.map(selector => ({ ...point, ...selector }))))( - 'gives the decision only to the item resumed by $by after a crash $point', - async ({ point, hold, childStatus, by, resume }) => { - const id = `foreach-resume-child-${childStatus}-${by}`; - const seen: Seen[] = []; - const gate = newGate(); - const { scenario, start, runId } = resumeScenario( - id, - childForeach(id, seen, gate), - { ...resume(`${id}-child`), resumeData: APPROVED }, - { hold: hold(id) }, - ); - const original = await start(); - await original.held; - const checkpoint = await original.checkpoint(); - expect(findRow(checkpoint, id, runId)?.snapshot.context?.[`${id}-child`], point).toMatchObject({ - status: 'running', - resumePayload: APPROVED, - }); - expect(findRow(checkpoint, `${id}-child`, runId)?.snapshot).toMatchObject({ - status: childStatus, - context: { approve: { status: 'suspended' } }, - }); - - const { result, graph } = await scenario.restart(checkpoint); - - expect(resumedIn(seen, 1)).toEqual([]); - expect(resumedIn(seen, 2)).toEqual([1]); - expect(result.status).toBe('suspended'); - await expectChildItems(graph, `${id}-child`, runId, ['suspended', 'success', 'suspended']); - const stored = await graph.workflows.loadWorkflowSnapshot({ workflowName: id, runId }); - expect(JSON.stringify([result, stored])).not.toContain('nestedResumeTarget'); - }, - ); - - it.each(crashPoints)( - 'keeps every item of a nested foreach suspended when the stored resume recorded no target, after a crash $point', - async ({ hold, childStatus }) => { - // Runs stored before the nested target was recorded carry the payload but no target. - const id = `foreach-resume-child-untargeted-${childStatus}`; - const child = `${id}-child`; - const seen: Seen[] = []; - const gate = newGate(); - const { scenario, start, runId } = resumeScenario( - id, - childForeach(id, seen, gate), - { label: 'item-1', resumeData: APPROVED }, - { hold: hold(id) }, - ); - const original = await start(); - await original.held; - const checkpoint = await original.checkpoint(); - const record = findRow(checkpoint, id, runId)!.snapshot.context[child] as any; - expect(record.metadata.nestedResumeTarget).toEqual({ steps: [], label: 'item-1' }); - delete record.metadata.nestedResumeTarget; - // The state the nested run had handed back when it parked, which a crash during a later - // resume rebases from. - const startState = record.suspendPayload.__workflow_meta.startState; - expect(startState).toBeDefined(); - - const { result, graph } = await scenario.restart(checkpoint); - - expect(seen.filter(s => s.generation === 2)).toEqual([]); - expect(result.status).toBe('suspended'); - await expectChildItems(graph, child, runId, ['suspended', 'suspended', 'suspended']); - const parked = await graph.workflows.loadWorkflowSnapshot({ workflowName: id, runId }); - expect(parked.context[child].suspendPayload.__workflow_meta.startState).toEqual(startState); - - // The interrupted item is resumed again, and only it runs. - const next = await (await graph.workflow.createRun({ runId })).resume({ label: 'item-1', resumeData: APPROVED }); - expect(resumedIn(seen, 2)).toEqual([1]); - expect(next.status).toBe('suspended'); - await expectChildItems(graph, child, runId, ['suspended', 'success', 'suspended']); - }, - ); - - // A foreach whose items are nested workflows that each run their own concurrent foreach. The - // label names one inner item of outer item 0; outer item 1 waits for its own approvals. - const foreachOfChildForeach = - (id: string, seen: Seen[], gate: Gate): Build => - (core, generation) => - core - .createWorkflow({ id, inputSchema: z.array(z.array(ITEM)), outputSchema: z.any() }) - .foreach( - core - .createWorkflow({ id: `${id}-child`, inputSchema: z.array(ITEM), outputSchema: z.any() }) - .foreach(approvalStep(core.createStep, { id: 'approve', generation, seen, gate }), { concurrency: 2 }) - .commit(), - { concurrency: 2 }, - ) - .commit(); - const OUTER_ITEMS = [ - [{ k: 0 }, { k: 1 }], - [{ k: 2 }, { k: 3 }], - ]; - // The outer foreach's running record doesn't hold its items back, so the cut parks the nested - // run's resume claim. A crash just before that claim leaves the same rows with the nested run - // still `suspended`. - it.each([ - { point: 'before the nested run claims its resume', claimed: false, recorded: true }, - { point: 'after the nested run claims its resume', claimed: true, recorded: true }, - { point: 'after the nested run claims its resume, with no recorded target', claimed: true, recorded: false }, - ])( - 'gives the decision only to the labelled item of a foreach in a foreach item, after a crash $point', - async ({ claimed, recorded }) => { - const id = `foreach-of-child-foreach-${claimed}-${recorded}`; - const child = `${id}-child`; - const runId = `${id}-run`; - const seen: Seen[] = []; - const gate = newGate(); - const build = foreachOfChildForeach(id, seen, gate); - const scenario = createRestartScenario({ - kind: 'workflow', - runId, - build: ({ core, generation }) => build(core, generation), - hold: { - workflowName: child, - when: snapshot => snapshot.status === 'running' && snapshot.context?.approve?.status === 'suspended', - }, - }); - scenarios.push(scenario); - const original = await scenario.start(async ({ workflow }) => { - const run = await workflow.createRun({ runId }); - expect((await run.start({ inputData: OUTER_ITEMS })).status).toBe('suspended'); - return run.resume({ label: 'item-1', resumeData: APPROVED }); - }); - await original.held; - let checkpoint!: Checkpoint; - await vi.waitFor(async () => { - checkpoint = await original.checkpoint(); - expect(foreachRecord(checkpoint, id, runId, child)).toMatchObject({ - status: 'running', - resumePayload: APPROVED, - }); - }); - const childRows = checkpoint.rows.filter(row => row.workflowName === child); - const claimedRow = childRows.find(row => row.snapshot.status === 'running')!; - expect(claimedRow.snapshot.context.approve).toMatchObject({ status: 'suspended' }); - if (!claimed) claimedRow.snapshot.status = 'suspended'; - const record = foreachRecord(checkpoint, id, runId, child); - expect(record.metadata.nestedResumeTarget).toEqual({ steps: [], label: 'item-1' }); - if (!recorded) delete record.metadata.nestedResumeTarget; - - const { result, graph } = await scenario.restart(checkpoint); - - expect(resumedIn(seen, 1)).toEqual([]); - expect(resumedIn(seen, 2)).toEqual(recorded ? [1] : []); - expect(result.status).toBe('suspended'); - if (recorded) return; - - // The interrupted item is resumed again, and only it runs. - const next = await (await graph.workflow.createRun({ runId })).resume({ label: 'item-1', resumeData: APPROVED }); - expect(resumedIn(seen, 2)).toEqual([1]); - expect(next.status).toBe('suspended'); - }, - ); - // The second copy is taken inside the first restart, which the restart harness can't cut, so // this copies the store's tables (which also covers storage v2). describe.each([1, 2] as const)('storage v%s', storageVersion => { @@ -581,6 +532,7 @@ describe('restart while a resumed foreach item runs', () => { // Generations 1 and 2 crash while item 1 runs; generation 3 finishes. gate: crashes[generation - 1] ?? newGate(), blockIn: generation === 1 || generation === 2 ? generation : -1, + validate: async () => {}, }), { concurrency: 3 }, ) @@ -624,164 +576,13 @@ describe('restart while a resumed foreach item runs', () => { run.watch(event => void events.push(event)); const result = await run.restart(); - expect(resumedIn(seen, 2)).toEqual([1]); - expect(resumedIn(seen, 3)).toEqual([1]); + expect(calls(seen, 2)).toEqual([resumed(1)]); + expect(calls(seen, 3)).toEqual([resumed(1)]); expect(result.status).toBe('suspended'); const steps = typeof result.steps === 'function' ? await (result.steps as any)() : result.steps; - expect(itemOutputs(steps, 'approve').map(o => o.status)).toEqual(['suspended', 'success', 'suspended']); + expect(statusesOf(steps, 'approve')).toEqual([S, OK, S]); expect(events.length).toBeGreaterThan(0); expect(JSON.stringify(events)).not.toContain('foreachResumeTarget'); }); }); - - it('gives the resumed item its decision when a sibling suspends while it is still validating its input', async () => { - // Concurrency 2: items 0 and 1 suspend and item 2 is never reached; item 1 is resumed. On the - // restart items 1 and 2 run together, and item 2 suspends before item 1's input validation - // finishes. Items share the foreach's record, which item 2's suspension replaces in memory. - const id = 'foreach-resume-sibling-suspends'; - const runId = 'sibling-suspends'; - const seen: Seen[] = []; - const crash = newGate(); - const validation = newGate(); - const build = (generation: number) => { - const approve = createStep({ - id: 'approve', - // On the restart, item 1's validation waits until item 2 has suspended. - inputSchema: ITEM.refine(async ({ k }) => { - if (generation === 2 && k === 1) await validation.wait(); - return true; - }), - outputSchema: z.object({ k: z.number(), approved: z.boolean() }), - resumeSchema: z.object({ approved: z.boolean() }), - suspendSchema: z.object({ ask: z.number() }), - execute: async ({ inputData, resumeData, suspendData, suspend }) => { - seen.push({ generation, k: inputData.k, resumeData, suspendData }); - if (resumeData === undefined) return suspend({ ask: inputData.k }); - if (generation === 1) await crash.wait(); - return { k: inputData.k, approved: resumeData.approved }; - }, - }); - return createWorkflow({ id, inputSchema: z.array(ITEM), outputSchema: z.any() }) - .foreach(approve, { concurrency: 2 }) - .commit(); - }; - - const storage1 = new MockStore(); - const first = build(1); - new Mastra({ logger: false, storage: storage1, workflows: { [id]: first } }); - const run1 = await first.createRun({ runId }); - expect((await run1.start({ inputData: ITEMS })).status).toBe('suspended'); - void run1.resume({ step: 'approve', forEachIndex: 1, resumeData: APPROVED }); - await crash.reached; - const store1 = (await storage1.getStore('workflows'))!; - await vi.waitFor(async () => { - const snapshot = await store1.loadWorkflowSnapshot({ workflowName: id, runId }); - expect(snapshot?.context.approve).toMatchObject({ status: 'running', resumePayload: APPROVED }); - }); - const atCrash = copyRunTables(await runTablesOf(storage1)); - - const storage2 = new MockStore(); - const second = build(2); - new Mastra({ logger: false, storage: storage2, workflows: { [id]: second } }); - restoreRunTables(await runTablesOf(storage2), atCrash); - const run2 = await second.createRun({ runId }); - run2.watch(event => { - const payload = (event as any).payload; - if (event.type === 'workflow-step-progress' && payload?.currentIndex === 2) { - expect(payload.iterationStatus).toBe('suspended'); - validation.release(); - } - }); - const result = await run2.restart(); - - expect(seen.filter(s => s.generation === 2).map(s => s.k)).toEqual([2, 1]); - expect(resumedIn(seen, 2)).toEqual([1]); - expect(seen.find(s => s.generation === 2 && s.k === 1)).toMatchObject({ - resumeData: APPROVED, - suspendData: { ask: 1 }, - }); - expect(result.status).toBe('suspended'); - if (result.status !== 'suspended') return; - expect(itemOutputs(result.steps, 'approve').map(o => o.status)).toEqual(['suspended', 'success', 'suspended']); - }); - - it('resumes only the labelled inner item when a sibling nested run suspends while the resumed one validates', async () => { - // A foreach of nested workflows, each with its own foreach. Outer items 0 and 1 suspend and - // outer item 2 is never reached; inner item 1 of outer item 0 is resumed by label, and the - // run crashes once that nested run has claimed its resume. On the restart outer items 0 and 2 - // run together, and outer item 2 suspends before outer item 0's input validation finishes. - const id = 'foreach-of-child-foreach-sibling'; - const child = `${id}-child`; - const runId = 'child-sibling-suspends'; - const seen: Seen[] = []; - const validation = newGate(); - const parked = newGate(); - const build = (generation: number) => { - const approve = approvalStep(createStep, { id: 'approve', generation, seen, gate: newGate(), blockIn: -1 }); - const nested = createWorkflow({ - id: child, - // On the restart, outer item 0's validation waits until outer item 2 has suspended. - inputSchema: z.array(ITEM).refine(async items => { - if (generation === 2 && items[0]?.k === 0) await validation.wait(); - return true; - }), - outputSchema: z.any(), - }) - .foreach(approve, { concurrency: 2 }) - .commit(); - return createWorkflow({ id, inputSchema: z.array(z.array(ITEM)), outputSchema: z.any() }) - .foreach(nested, { concurrency: 2 }) - .commit(); - }; - - // Generation 1 is parked inside the nested run's resume claim, after copying the tables. - const storage1 = new MockStore(); - const first = build(1); - new Mastra({ logger: false, storage: storage1, workflows: { [id]: first } }); - const store1 = (await storage1.getStore('workflows')) as any; - let atCrash: RunTables | undefined; - let resuming = false; - interceptWrites(store1, async (args, write) => { - const written = await write(args); - if (resuming && !atCrash && args.workflowName === child) { - const snapshot = await store1.loadWorkflowSnapshot({ workflowName: child, runId: args.runId }); - if (snapshot?.status === 'running' && snapshot.context?.approve?.status === 'suspended') { - atCrash = copyRunTables(await runTablesOf(storage1)); - parked.wait(); - await new Promise(() => {}); - } - } - return written; - }); - const run1 = await first.createRun({ runId }); - const inputData = [ - [{ k: 0 }, { k: 1 }], - [{ k: 2 }, { k: 3 }], - [{ k: 4 }, { k: 5 }], - ]; - expect((await run1.start({ inputData })).status).toBe('suspended'); - resuming = true; - void run1.resume({ label: 'item-1', resumeData: APPROVED }); - await parked.reached; - const outer = (await store1.loadWorkflowSnapshot({ workflowName: id, runId }))?.context?.[child]; - expect(outer).toMatchObject({ status: 'running', resumePayload: APPROVED }); - - const storage2 = new MockStore(); - const second = build(2); - new Mastra({ logger: false, storage: storage2, workflows: { [id]: second } }); - restoreRunTables(await runTablesOf(storage2), atCrash!); - const run2 = await second.createRun({ runId }); - run2.watch(event => { - const payload = (event as any).payload; - if (event.type === 'workflow-step-progress' && payload?.id === child && payload.currentIndex === 2) { - expect(payload.iterationStatus).toBe('suspended'); - validation.release(); - } - }); - const result = await run2.restart(); - - expect(resumedIn(seen, 1)).toEqual([]); - expect(resumedIn(seen, 2)).toEqual([1]); - expect(result.status).toBe('suspended'); - }); }); From 92953125363a0a359f80582c7477dbe7d9aa2804 Mon Sep 17 00:00:00 2001 From: Arthur Fiorette Date: Mon, 16 Nov 2026 18:51:01 -0300 Subject: [PATCH 05/10] refactor(core): trim comments on the foreach restart resume targets Drops comments that restated the code or repeated the JSDoc of the target types. No code change. --- packages/core/src/workflows/constants.ts | 4 +--- packages/core/src/workflows/handlers/control-flow.ts | 2 -- packages/core/src/workflows/handlers/foreach-progress.ts | 8 ++------ packages/core/src/workflows/handlers/step.ts | 6 ++---- packages/core/src/workflows/workflow.ts | 6 ++---- 5 files changed, 7 insertions(+), 19 deletions(-) diff --git a/packages/core/src/workflows/constants.ts b/packages/core/src/workflows/constants.ts index 0a8dde2c1d1..2c865f35329 100644 --- a/packages/core/src/workflows/constants.ts +++ b/packages/core/src/workflows/constants.ts @@ -13,9 +13,7 @@ export const PARENT_STATE_SYMBOL = Symbol('parent_state'); // the state its changes are relative to, so they're applied to the parent's state then. export const HAND_BACK_STATE_SYMBOL = Symbol('hand_back_state'); -// Symbol used to give a nested workflow step, restarted while it was being resumed, what that -// resume targeted inside the nested run (step path, label, foreach item), for a restart that -// resumes the nested run. +// Symbol used to give a nested workflow step restarted mid-resume what that resume targeted inside the nested run. export const RESTART_RESUME_SYMBOL = Symbol('restart_resume'); // Abort reason used when workflow code explicitly cancels a run. diff --git a/packages/core/src/workflows/handlers/control-flow.ts b/packages/core/src/workflows/handlers/control-flow.ts index ad358faf210..548bd0a0813 100644 --- a/packages/core/src/workflows/handlers/control-flow.ts +++ b/packages/core/src/workflows/handlers/control-flow.ts @@ -1156,8 +1156,6 @@ export async function executeForeach( : restartsRunning ? getForeachResumeTarget(stepResults[stepId]) : 'none'; - // What the resume targeted inside an item that is a nested workflow, for a restart that has to - // resume that item's nested run again. const nestedResumeTarget: NestedResumeTarget | undefined = resume?.steps[0] === stepId ? { steps: resume.steps.slice(1), ...(resume.label ? { label: resume.label } : {}) } diff --git a/packages/core/src/workflows/handlers/foreach-progress.ts b/packages/core/src/workflows/handlers/foreach-progress.ts index b7d31f68789..9b62952ac41 100644 --- a/packages/core/src/workflows/handlers/foreach-progress.ts +++ b/packages/core/src/workflows/handlers/foreach-progress.ts @@ -35,7 +35,6 @@ export function getForeachProgress(stepResult: { metadata?: unknown } | undefine */ export type ForeachResumeTarget = number | 'all' | 'none'; -/** The stored resume target on a foreach step's running result; undefined if none was recorded. */ export function getForeachResumeTarget( stepResult: { metadata?: unknown } | undefined, ): ForeachResumeTarget | undefined { @@ -60,14 +59,11 @@ export function takesStoredForeachResume( } /** - * What the resume of a nested workflow step targeted inside the nested run (the step path below - * the nested workflow, the resume label, the foreach item), stored under - * `metadata.nestedResumeTarget` on the step's running result. A restart that has to resume the - * nested run resumes only that. + * What a nested workflow step's resume targeted inside the nested run, stored under + * `metadata.nestedResumeTarget` on its running result. A restart that resumes the nested run resumes only that. */ export type NestedResumeTarget = { steps: string[]; label?: string; forEachIndex?: number }; -/** The stored nested resume target on a nested workflow step's running result, if any. */ export function getNestedResumeTarget(stepResult: { metadata?: unknown } | undefined): NestedResumeTarget | undefined { const metadata = stepResult?.metadata; if (!isPlainRecord(metadata)) return undefined; diff --git a/packages/core/src/workflows/handlers/step.ts b/packages/core/src/workflows/handlers/step.ts index 70c12f16f1b..52a760d3985 100644 --- a/packages/core/src/workflows/handlers/step.ts +++ b/packages/core/src/workflows/handlers/step.ts @@ -179,10 +179,8 @@ export async function executeStep( restartsResume = true; } - // What a resumed nested workflow's resume targeted inside the nested run, kept on its running - // record: a restart that finds the nested run still suspended, or only claimed, resumes it - // again, and without the target it would resume every suspended step there. A foreach item's - // record is the foreach's own, which stores the target for its items. + // Kept on the nested step's running record: without it, a restart that resumes the nested run + // again would resume every suspended step there. A foreach item's is on the foreach's record. let nestedResumeTarget: NestedResumeTarget | undefined; if (step.component === 'WORKFLOW') { if (resume?.steps[0] === step.id && executionContext.foreachIndex === undefined) { diff --git a/packages/core/src/workflows/workflow.ts b/packages/core/src/workflows/workflow.ts index 14bd11ff064..2af87a99d27 100644 --- a/packages/core/src/workflows/workflow.ts +++ b/packages/core/src/workflows/workflow.ts @@ -3699,9 +3699,8 @@ export class Workflow< outputOptions: { includeState: true, includeResumeLabels: true }, }); } else if (resumeNested && !resume && restartResume === null) { - // The interrupted resume didn't record what it targeted inside the nested run, so its data - // can't be handed to the right step there: leave the nested run suspended as stored, and - // this step suspends with it, to be resumed again. + // The interrupted resume didn't record its target inside the nested run: leave the nested + // run suspended as stored, and this step suspends with it, to be resumed again. const stored = await run.runStorage.loadFull(); res = { status: 'suspended', @@ -3712,7 +3711,6 @@ export class Workflow< // stored state; it hands nothing back now. if (!parentState) parkedState = toJsonState(stored?.value); } else if (resumeNested) { - // A restart that resumes the nested run resumes what the interrupted resume targeted. const target = resume ?? restartResume ?? undefined; res = await run.resumeNestedByParent({ resumeData, From b61e55aeb55e11f1ab0718f1a9820a2f363d2736 Mon Sep 17 00:00:00 2001 From: Arthur Fiorette Date: Mon, 16 Nov 2026 19:42:37 -0300 Subject: [PATCH 06/10] fix(core): hand back a parked nested run's unreceived state on a restart without a target When a restart finds a resumed nested step with no recorded target, it leaves the nested run suspended. The nested run may have changed the state and parked again before the parent stored that suspension. Those changes were never handed back, and the later resume rebases from the nested run's own stored state, so they were lost from the parent. On storage v1 the parent now takes them in at the restart, against the state it last received from the nested run (the startState on its stored record), or whole when the record has none. --- .../workflows/foreach-resume-restart.test.ts | 66 +++++++++++++++++++ packages/core/src/workflows/handlers/step.ts | 27 ++++---- packages/core/src/workflows/workflow.ts | 22 +++++-- 3 files changed, 94 insertions(+), 21 deletions(-) diff --git a/packages/core/src/workflows/foreach-resume-restart.test.ts b/packages/core/src/workflows/foreach-resume-restart.test.ts index c7364e85a1f..3d6eed376e8 100644 --- a/packages/core/src/workflows/foreach-resume-restart.test.ts +++ b/packages/core/src/workflows/foreach-resume-restart.test.ts @@ -486,6 +486,72 @@ describe('restart while a resumed foreach item runs', () => { if (row.shape.statuses) expect(await statuses(next)).toEqual([S, OK, S]); }); + it('keeps a state change the parent never received when the stored resume recorded no target', async () => { + // The resumed nested run ran item 1, which changed the state, and suspended again; the process + // crashed before the parent stored that suspension, on a record without a target. + const id = 'fo-unreceived-state'; + const child = `${id}-child`; + const runId = `${id}-run`; + const STATE = z.object({ approvals: z.array(z.number()) }); + const scenario = createRestartScenario({ + kind: 'workflow', + runId, + build: ({ core }) => + core + .createWorkflow({ id, inputSchema: z.array(ITEM), outputSchema: z.any(), stateSchema: STATE }) + .then( + core + .createWorkflow({ id: child, inputSchema: z.array(ITEM), outputSchema: z.any(), stateSchema: STATE }) + .foreach( + core.createStep({ + id: 'approve', + inputSchema: ITEM, + outputSchema: ITEM, + resumeSchema: z.object({ approved: z.boolean() }), + stateSchema: STATE, + execute: async ({ inputData, resumeData, suspend, state, setState }: any) => { + if (resumeData === undefined) return suspend({}, { resumeLabel: `item-${inputData.k}` }); + await setState({ approvals: [...state.approvals, inputData.k] }); + return inputData; + }, + }), + { concurrency: 3 }, + ) + .commit(), + ) + .commit(), + hold: { + workflowName: child, + when: snapshot => + snapshot.status === 'suspended' && + (snapshot.context?.approve as any)?.suspendPayload?.__workflow_meta?.foreachOutput?.[1]?.status === 'success', + }, + }); + scenarios.push(scenario); + const original = await scenario.start(async ({ workflow }) => { + const run = await workflow.createRun({ runId }); + expect((await run.start({ inputData: ITEMS, initialState: { approvals: [] } })).status).toBe('suspended'); + return run.resume({ label: 'item-1', resumeData: APPROVED }); + }); + await original.held; + const checkpoint = await original.checkpoint(); + const record = findRow(checkpoint, id, runId)!.snapshot.context[child] as any; + expect(record).toMatchObject({ status: 'running', resumePayload: APPROVED }); + // The parent last took in the nested run's state when it first parked; the nested run is ahead. + expect(record.suspendPayload.__workflow_meta.startState).toEqual({ approvals: [] }); + expect(findRow(checkpoint, child, runId)?.snapshot.value).toEqual({ approvals: [1] }); + delete record.metadata.nestedResumeTarget; + + const { result, graph } = await scenario.restart(checkpoint); + expect(result.status).toBe('suspended'); + const run = await graph.workflow.createRun({ runId }); + await run.resume({ label: 'item-0', resumeData: APPROVED }); + expect((await run.resume({ label: 'item-2', resumeData: APPROVED })).status).toBe('success'); + + const stored = await graph.workflows.loadWorkflowSnapshot({ workflowName: id, runId }); + expect([...stored.value.approvals].sort()).toEqual([0, 1, 2]); + }); + it('resumes only the chosen item of a foreach inside a nested workflow, by index', async () => { const id = 'foreach-resume-child-index'; const seen: Seen[] = []; diff --git a/packages/core/src/workflows/handlers/step.ts b/packages/core/src/workflows/handlers/step.ts index 52a760d3985..adae42e9202 100644 --- a/packages/core/src/workflows/handlers/step.ts +++ b/packages/core/src/workflows/handlers/step.ts @@ -193,6 +193,13 @@ export async function executeStep( nestedResumeTarget = getNestedResumeTarget(storedResult); } } + // The nested run's state this run took in when it last parked on it (storage v1). + const parkedMeta = storedResult?.suspendPayload?.__workflow_meta; + const receivedState: Record | undefined = ( + executionContext.foreachIndex !== undefined + ? parkedMeta?.foreachOutput?.[executionContext.foreachIndex]?.suspendPayload?.__workflow_meta + : parkedMeta + )?.startState; // Extract suspend data if this step was previously suspended // A resumed step re-run after a crash is stored as 'running' but still carries its suspendPayload. @@ -502,17 +509,7 @@ export async function executeStep( storageVersion: 1, }); if (snapshot?.status === 'success') { - // The state the nested run started from, recorded when this run parked on it. - const meta = storedResult?.suspendPayload?.__workflow_meta; - completedNestedOutput = { - found: true, - value: snapshot.result, - state: snapshot.value, - base: - executionContext.foreachIndex !== undefined - ? meta?.foreachOutput?.[executionContext.foreachIndex]?.suspendPayload?.__workflow_meta?.startState - : meta?.startState, - }; + completedNestedOutput = { found: true, value: snapshot.result, state: snapshot.value, base: receivedState }; } } } @@ -704,10 +701,12 @@ export async function executeStep( ? (((state: unknown) => journal.baseFor(state)) satisfies ParentStateLink) : undefined, [HAND_BACK_STATE_SYMBOL]: isNestedWorkflow ? setState : undefined, - // `null` tells the nested workflow the restarted resume's target is unknown (a record stored - // before targets were): it then leaves the nested run suspended. + // No target means the restarted resume's target is unknown (a record stored before targets + // were): the nested workflow then leaves the nested run suspended. [RESTART_RESUME_SYMBOL]: - restartsResume && step.component === 'WORKFLOW' ? (nestedResumeTarget ?? null) : undefined, + restartsResume && step.component === 'WORKFLOW' + ? (nestedResumeTarget ?? { received: receivedState }) + : undefined, engine: engine.getEngineContext(), abortSignal: abortController?.signal, outputWriter, diff --git a/packages/core/src/workflows/workflow.ts b/packages/core/src/workflows/workflow.ts index 2af87a99d27..c61f4861b62 100644 --- a/packages/core/src/workflows/workflow.ts +++ b/packages/core/src/workflows/workflow.ts @@ -3512,8 +3512,11 @@ export class Workflow< [PUBSUB_SYMBOL]: PubSub; [PARENT_STATE_SYMBOL]?: ParentStateLink; [HAND_BACK_STATE_SYMBOL]?: (state: TState, base: Record) => Promise; - /** `null`: the restarted resume didn't record its target inside the nested run. */ - [RESTART_RESUME_SYMBOL]?: NestedResumeTarget | null; + /** + * Without `steps`: the restarted resume didn't record its target inside the nested run; + * `received` is the nested run's state the parent last took in (storage v1). + */ + [RESTART_RESUME_SYMBOL]?: NestedResumeTarget | { received?: Record }; mastra: Mastra; requestContext?: RequestContext; engine: DefaultEngineType; @@ -3698,7 +3701,7 @@ export class Workflow< outputWriter, outputOptions: { includeState: true, includeResumeLabels: true }, }); - } else if (resumeNested && !resume && restartResume === null) { + } else if (resumeNested && !resume && restartResume && !('steps' in restartResume)) { // The interrupted resume didn't record its target inside the nested run: leave the nested // run suspended as stored, and this step suspends with it, to be resumed again. const stored = await run.runStorage.loadFull(); @@ -3707,11 +3710,16 @@ export class Workflow< steps: (stored?.context ?? {}) as any, resumeLabels: stored?.resumeLabels, } as WorkflowResult; - // The nested run hasn't run since it parked, so the state it had handed back then is its - // stored state; it hands nothing back now. - if (!parentState) parkedState = toJsonState(stored?.value); + if (!parentState) { + // The resumed nested run may have changed the state and parked again before this run + // stored that: hand its changes back against the state this run last took in from it, or + // whole without one, as a restart does. The parent then holds its stored state. + res.state = stored?.value as TState; + handBackBase = restartResume.received; + parkedState = toJsonState(stored?.value); + } } else if (resumeNested) { - const target = resume ?? restartResume ?? undefined; + const target = resume ?? (restartResume && 'steps' in restartResume ? restartResume : undefined); res = await run.resumeNestedByParent({ resumeData, step: target?.steps?.length ? (target.steps as any) : undefined, From 01593d127a4f605bdb24d1ab3f1bbcc57e6b0bd7 Mon Sep 17 00:00:00 2001 From: Arthur Fiorette Date: Mon, 16 Nov 2026 20:14:20 -0300 Subject: [PATCH 07/10] fix(core): keep plain nested restarts on the normal resume path The branch that leaves a nested run suspended when a restarted resume recorded no target also caught restarts of nested steps that were started normally and never resumed. Those have no resume payload to misapply, but on storage v2 parking them dropped the state the nested run had changed before it suspended. The unknown-target marker is now sent only when the stored record carries a resume payload; a plain restart resumes the nested run as before this PR. The test for the unknown-target hand-back now also has a sibling that changes the state while the nested run works on its copy, so it fails if the hand-back ignores the parent's stored base. --- .../workflows/foreach-resume-restart.test.ts | 118 ++++++++++++++++-- packages/core/src/workflows/handlers/step.ts | 5 +- 2 files changed, 113 insertions(+), 10 deletions(-) diff --git a/packages/core/src/workflows/foreach-resume-restart.test.ts b/packages/core/src/workflows/foreach-resume-restart.test.ts index 3d6eed376e8..3c9063d1082 100644 --- a/packages/core/src/workflows/foreach-resume-restart.test.ts +++ b/packages/core/src/workflows/foreach-resume-restart.test.ts @@ -488,18 +488,30 @@ describe('restart while a resumed foreach item runs', () => { it('keeps a state change the parent never received when the stored resume recorded no target', async () => { // The resumed nested run ran item 1, which changed the state, and suspended again; the process - // crashed before the parent stored that suspension, on a record without a target. + // crashed before the parent stored that suspension, on a record without a target. A sibling + // changed `tag` after the nested run took its copy, so only a merge against the state the + // parent last received keeps both changes. const id = 'fo-unreceived-state'; const child = `${id}-child`; const runId = `${id}-run`; - const STATE = z.object({ approvals: z.array(z.number()) }); + const STATE = z.object({ approvals: z.array(z.number()), tag: z.string() }); const scenario = createRestartScenario({ kind: 'workflow', runId, build: ({ core }) => core .createWorkflow({ id, inputSchema: z.array(ITEM), outputSchema: z.any(), stateSchema: STATE }) - .then( + .parallel([ + core.createStep({ + id: 'sibling', + inputSchema: z.any(), + outputSchema: z.object({}), + stateSchema: STATE, + execute: async ({ state, setState }: any) => { + await setState({ ...state, tag: 'sibling' }); + return {}; + }, + }), core .createWorkflow({ id: child, inputSchema: z.array(ITEM), outputSchema: z.any(), stateSchema: STATE }) .foreach( @@ -511,14 +523,14 @@ describe('restart while a resumed foreach item runs', () => { stateSchema: STATE, execute: async ({ inputData, resumeData, suspend, state, setState }: any) => { if (resumeData === undefined) return suspend({}, { resumeLabel: `item-${inputData.k}` }); - await setState({ approvals: [...state.approvals, inputData.k] }); + await setState({ ...state, approvals: [...state.approvals, inputData.k] }); return inputData; }, }), { concurrency: 3 }, ) .commit(), - ) + ]) .commit(), hold: { workflowName: child, @@ -530,7 +542,8 @@ describe('restart while a resumed foreach item runs', () => { scenarios.push(scenario); const original = await scenario.start(async ({ workflow }) => { const run = await workflow.createRun({ runId }); - expect((await run.start({ inputData: ITEMS, initialState: { approvals: [] } })).status).toBe('suspended'); + const started = await run.start({ inputData: ITEMS, initialState: { approvals: [], tag: 'initial' } }); + expect(started.status).toBe('suspended'); return run.resume({ label: 'item-1', resumeData: APPROVED }); }); await original.held; @@ -538,8 +551,9 @@ describe('restart while a resumed foreach item runs', () => { const record = findRow(checkpoint, id, runId)!.snapshot.context[child] as any; expect(record).toMatchObject({ status: 'running', resumePayload: APPROVED }); // The parent last took in the nested run's state when it first parked; the nested run is ahead. - expect(record.suspendPayload.__workflow_meta.startState).toEqual({ approvals: [] }); - expect(findRow(checkpoint, child, runId)?.snapshot.value).toEqual({ approvals: [1] }); + expect(record.suspendPayload.__workflow_meta.startState).toEqual({ approvals: [], tag: 'initial' }); + expect(findRow(checkpoint, id, runId)?.snapshot.value).toEqual({ approvals: [], tag: 'sibling' }); + expect(findRow(checkpoint, child, runId)?.snapshot.value).toEqual({ approvals: [1], tag: 'initial' }); delete record.metadata.nestedResumeTarget; const { result, graph } = await scenario.restart(checkpoint); @@ -550,8 +564,96 @@ describe('restart while a resumed foreach item runs', () => { const stored = await graph.workflows.loadWorkflowSnapshot({ workflowName: id, runId }); expect([...stored.value.approvals].sort()).toEqual([0, 1, 2]); + expect(stored.value.tag).toBe('sibling'); }); + it.each([1, 2] as const)( + 'keeps a state change of a nested run started without a resume, after a crash before the parent stored its suspension, storage v%s', + async storageVersion => { + // A plain start: the nested run changes the state and suspends; the process crashes before the + // parent stores that suspension. The parent's step is `running` with no resume payload. + const id = `plain-nested-v${storageVersion}`; + const child = `${id}-child`; + const runId = `${id}-run`; + const STATE = z.object({ log: z.array(z.string()) }); + const scenario = createRestartScenario({ + kind: 'workflow', + runId, + storageVersion, + build: ({ core }) => + core + .createWorkflow({ + id, + inputSchema: z.object({}), + outputSchema: z.any(), + stateSchema: STATE, + options: { storageVersion }, + }) + .then( + core + .createWorkflow({ id: child, inputSchema: z.object({}), outputSchema: z.any(), stateSchema: STATE }) + .then( + core.createStep({ + id: 'note', + inputSchema: z.object({}), + outputSchema: z.object({}), + stateSchema: STATE, + execute: async ({ getState, setState }: any) => { + await setState({ log: [...(await getState()).log, 'nested'] }); + return {}; + }, + }), + ) + .then( + core.createStep({ + id: 'gate', + inputSchema: z.object({}), + outputSchema: z.object({ approved: z.boolean() }), + // No resumeSchema: the restart resumes the nested run with no data, and the step + // suspends again. + execute: async ({ resumeData, suspend }: any) => + resumeData === undefined ? suspend({ ask: 'gate' }) : { approved: resumeData.approved }, + }), + ) + .commit(), + ) + .commit(), + }); + scenarios.push(scenario); + // Graph 1 parks inside the write that stores the nested run suspended, on either version. + const parked = newGate(); + const original = await scenario.start(async ({ workflow, workflows }) => { + interceptWrites(workflows, async (args, write) => { + const written = await write(args); + if (!parked.waiters && args?.workflowName === child) { + const snapshot = await workflows.loadWorkflowSnapshot({ workflowName: child, runId }); + if (snapshot?.status === 'suspended') await parked.wait(); + } + return written; + }); + return (await workflow.createRun({ runId })).start({ inputData: {}, initialState: { log: [] } }); + }); + await parked.reached; + const checkpoint = await original.checkpoint(); + expect(findRow(checkpoint, id, runId)?.snapshot.context[child]).toMatchObject({ status: 'running' }); + expect((findRow(checkpoint, id, runId)?.snapshot.context[child] as any).resumePayload).toBeUndefined(); + expect(findRow(checkpoint, child, runId)?.snapshot).toMatchObject({ + status: 'suspended', + value: { log: ['nested'] }, + }); + + const { result, graph } = await scenario.restart(checkpoint); + + expect(result.status).toBe('suspended'); + const steps = typeof result.steps === 'function' ? await (result.steps as any)() : result.steps; + expect(steps[child]).toMatchObject({ status: 'suspended', suspendPayload: { ask: 'gate' } }); + const run = await graph.workflow.createRun({ runId }); + expect((await run.resume({ step: [child, 'gate'], resumeData: APPROVED })).status).toBe('success'); + const stored = await graph.workflows.loadWorkflowSnapshot({ workflowName: id, runId }); + expect(stored.value).toEqual({ log: ['nested'] }); + }, + ); + it('resumes only the chosen item of a foreach inside a nested workflow, by index', async () => { const id = 'foreach-resume-child-index'; const seen: Seen[] = []; diff --git a/packages/core/src/workflows/handlers/step.ts b/packages/core/src/workflows/handlers/step.ts index adae42e9202..9ffb300504f 100644 --- a/packages/core/src/workflows/handlers/step.ts +++ b/packages/core/src/workflows/handlers/step.ts @@ -702,9 +702,10 @@ export async function executeStep( : undefined, [HAND_BACK_STATE_SYMBOL]: isNestedWorkflow ? setState : undefined, // No target means the restarted resume's target is unknown (a record stored before targets - // were): the nested workflow then leaves the nested run suspended. + // were): the nested workflow then leaves the nested run suspended. A restart of a step that + // was never resumed has no payload and sends nothing. [RESTART_RESUME_SYMBOL]: - restartsResume && step.component === 'WORKFLOW' + restartsResume && resumeDataToUse !== undefined && step.component === 'WORKFLOW' ? (nestedResumeTarget ?? { received: receivedState }) : undefined, engine: engine.getEngineContext(), From acea401b78cd491c7f87a6115fea6a2bda3c75f8 Mon Sep 17 00:00:00 2001 From: Arthur Fiorette Date: Mon, 16 Nov 2026 20:23:08 -0300 Subject: [PATCH 08/10] fix(core): pass a recorded nested resume target on a restart without resume data The previous commit made the whole restart marker depend on reused resume data, so a resume that named its target but carried no data lost that target on a restart: the nested run was then resumed without a step and every suspended step there continued. A recorded target is now always passed; only the unknown-target marker depends on reused resume data. --- .../workflows/foreach-resume-restart.test.ts | 32 ++++++++++++++++--- packages/core/src/workflows/handlers/step.ts | 6 ++-- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/packages/core/src/workflows/foreach-resume-restart.test.ts b/packages/core/src/workflows/foreach-resume-restart.test.ts index 3c9063d1082..e436ea5ae48 100644 --- a/packages/core/src/workflows/foreach-resume-restart.test.ts +++ b/packages/core/src/workflows/foreach-resume-restart.test.ts @@ -59,6 +59,8 @@ type Ctx = { concurrency: number; /** Runs inside input validation (of the approval step, or of the nested workflow per item). */ validate: (input: any) => Promise; + /** The approval step accepts a resume without data (it then suspends again). */ + optionalResume?: boolean; }; /** An approval step: suspends until resumed; a resumed call in generation `blockIn` waits on `gate`. */ @@ -70,6 +72,7 @@ function approvalStep( seen, gate, validate, + optionalResume, blockIn = 1, }: Omit & { id: string; blockIn?: number }, ) { @@ -80,7 +83,7 @@ function approvalStep( return true; }), outputSchema: z.object({ k: z.number(), approved: z.boolean() }), - resumeSchema: z.object({ approved: z.boolean() }), + ...(optionalResume ? {} : { resumeSchema: z.object({ approved: z.boolean() }) }), suspendSchema: z.object({ ask: z.number() }), execute: async ({ inputData, resumeData, suspendData, suspend }: any) => { seen.push({ generation, k: inputData.k, resumeData, suspendData }); @@ -231,6 +234,8 @@ type Row = { waiters?: number; window?: Window; race?: Race; + /** Resume without resume data. */ + noData?: boolean; }; const byIndex = (k: number) => () => ({ step: 'approve', forEachIndex: k }); @@ -251,6 +256,7 @@ async function crash(id: string, row: Row) { seen, gate, concurrency: row.concurrency ?? 3, + optionalResume: row.noData, validate: async input => { if (generation === 2 && row.race?.holds(input)) await validation.wait(); }, @@ -270,7 +276,11 @@ async function crash(id: string, row: Row) { } const started = await (await suspender.createRun({ runId })).start({ inputData: row.shape.inputData }); expect(started.status).toBe('suspended'); - return (await workflow.createRun({ runId })).resume({ ...row.resume(child), resumeData: APPROVED }); + const resumedRun = (await workflow.createRun({ runId })).resume({ + ...row.resume(child), + ...(row.noData ? {} : { resumeData: APPROVED }), + }); + return resumedRun; }); if (row.window) await original.held; else await vi.waitFor(() => expect(gate.waiters).toBe(row.waiters ?? 1)); @@ -283,7 +293,7 @@ async function crash(id: string, row: Row) { // Outside a window, also once the run records the step as active, which is what a restart reruns. return ( record?.status === 'running' && - record.resumePayload !== undefined && + (!!row.noData || record.resumePayload !== undefined) && (!!row.window || !!snapshot?.activeStepsPath?.[stepId]) ); }; @@ -403,7 +413,7 @@ describe('restart while a resumed foreach item runs', () => { expect(calls(seen, 2, from).filter(c => c.resumeData !== undefined)).toEqual([resumed(k)]); }); - const nestedWindows: (Row & { gen2: Call[] })[] = [ + const nestedWindows: (Row & { gen2: Call[]; statuses?: string[] })[] = [ ...[beforeClaim, afterClaim].flatMap(window => [ { by: 'label', resume: byLabel(1) }, @@ -424,6 +434,18 @@ describe('restart while a resumed foreach item runs', () => { race: outerRace, gen2: [resumed(1), fresh(4), fresh(5)], })), + { + // A resume without data continues the named item, which suspends again. Through a second + // instance: the one that suspended the run doesn't store a resume without data as running. + name: 'foreach inside a nested workflow, by label with no resume data, after the nested run claims its resume', + shape: inChild, + resume: byLabel(1), + noData: true, + elsewhere: true, + window: afterClaim, + gen2: [{ k: 1, resumeData: undefined, suspendData: { ask: 1 } }], + statuses: [S, S, S], + }, ]; it.each(nestedWindows)('gives the decision only to the resumed inner item: $name', async row => { @@ -433,7 +455,7 @@ describe('restart while a resumed foreach item runs', () => { expect(calls(seen, 2)).toEqual(row.gen2); expect(result.status).toBe('suspended'); - if (row.shape.statuses) expect(await statuses(result)).toEqual([S, OK, S]); + if (row.shape.statuses) expect(await statuses(result)).toEqual(row.statuses ?? [S, OK, S]); }); // Runs stored before targets were recorded carry the payload but no target. diff --git a/packages/core/src/workflows/handlers/step.ts b/packages/core/src/workflows/handlers/step.ts index 9ffb300504f..bfa12d01c77 100644 --- a/packages/core/src/workflows/handlers/step.ts +++ b/packages/core/src/workflows/handlers/step.ts @@ -703,10 +703,10 @@ export async function executeStep( [HAND_BACK_STATE_SYMBOL]: isNestedWorkflow ? setState : undefined, // No target means the restarted resume's target is unknown (a record stored before targets // were): the nested workflow then leaves the nested run suspended. A restart of a step that - // was never resumed has no payload and sends nothing. + // was never resumed has no target and no payload, and sends nothing. [RESTART_RESUME_SYMBOL]: - restartsResume && resumeDataToUse !== undefined && step.component === 'WORKFLOW' - ? (nestedResumeTarget ?? { received: receivedState }) + restartsResume && step.component === 'WORKFLOW' + ? (nestedResumeTarget ?? (resumeDataToUse !== undefined ? { received: receivedState } : undefined)) : undefined, engine: engine.getEngineContext(), abortSignal: abortController?.signal, From 309b797346006c5b564211519fb419cd66cb8aa5 Mon Sep 17 00:00:00 2001 From: Arthur Fiorette Date: Tue, 17 Nov 2026 00:11:08 -0300 Subject: [PATCH 09/10] fix(core): keep a foreach item that used the resume at its new request on a restart A running foreach record keeps the suspend state from before the resume and checkpoints only finished items, so an item that took the stored resume data and suspended again looked, after a restart, like an item that never got it: the restart handed it the same resume data again, running the approved work a second time and answering the item's new request with the old answer. Such an item's new suspension (payload, time, metadata, resume labels) is now checkpointed as a progress entry on the running record. A restart restores it and leaves the item suspended at that request. On storage v1 the progress write also stores the run's active steps as they were when the write started: an item ending while the write was in flight removed the foreach from them, so a restart could not reuse the resume data at all, and inside a nested workflow the nested run was then resumed again from its first suspension. --- .../workflows/foreach-resume-restart.test.ts | 127 +++++++++++++++++- .../src/workflows/handlers/control-flow.ts | 42 +++++- .../workflows/handlers/foreach-progress.ts | 19 ++- 3 files changed, 182 insertions(+), 6 deletions(-) diff --git a/packages/core/src/workflows/foreach-resume-restart.test.ts b/packages/core/src/workflows/foreach-resume-restart.test.ts index e436ea5ae48..4a9fef0b471 100644 --- a/packages/core/src/workflows/foreach-resume-restart.test.ts +++ b/packages/core/src/workflows/foreach-resume-restart.test.ts @@ -26,7 +26,7 @@ const ITEMS = [{ k: 0 }, { k: 1 }, { k: 2 }]; const ITEM = z.object({ k: z.number() }); const APPROVED = { approved: true }; /** A call that received its own approval and its own suspend payload. */ -const resumed = (k: number): Call => ({ k, resumeData: APPROVED, suspendData: { ask: k } }); +const resumed = (k: number, ask: unknown = k): Call => ({ k, resumeData: APPROVED, suspendData: { ask } }); /** A call that started the item fresh. */ const fresh = (k: number): Call => ({ k, resumeData: undefined, suspendData: undefined }); @@ -458,6 +458,131 @@ describe('restart while a resumed foreach item runs', () => { if (row.shape.statuses) expect(await statuses(result)).toEqual(row.statuses ?? [S, OK, S]); }); + // A resumed item consumes the payload and suspends again with a new request while a sibling keeps + // the foreach running. The crash point is the first write that stores either item 0's new request + // or the sibling's progress: the sibling, finishing after item 0 suspends again, is not stored yet + // and runs again. + describe.each([1, 2] as const)('storage v%s', storageVersion => { + it.each( + [false, true].flatMap(nested => [ + { + name: `${nested ? 'inside a nested workflow, ' : ''}resume without an index, item 1 still running when item 0 suspends again`, + nested, + forEachIndex: undefined, + finisher: 1, + gen2: [resumed(1), fresh(2)], + statuses: [S, OK, S], + }, + { + name: `${nested ? 'inside a nested workflow, ' : ''}resume of item 0 by index, never-reached item 2 still running when item 0 suspends again`, + nested, + forEachIndex: 0, + finisher: 2, + gen2: [fresh(2)], + statuses: [S, S, S], + }, + ]), + )('keeps an item that consumed the resume at its new request: $name', async row => { + const id = `fo-consumed-${row.finisher}-${row.nested}-v${storageVersion}`; + const child = `${id}-child`; + const runId = `${id}-run`; + // The workflow whose foreach the items belong to. + const owner = row.nested ? child : id; + const seen: Seen[] = []; + const item0Suspended = newGate(); + const foreachOf = (core: CoreGraph, generation: number, workflowId: string) => + core + .createWorkflow({ + id: workflowId, + inputSchema: z.array(ITEM), + outputSchema: z.any(), + options: { storageVersion }, + }) + .foreach( + core.createStep({ + id: 'approve', + inputSchema: ITEM, + outputSchema: ITEM, + resumeSchema: z.object({ approved: z.boolean() }), + execute: async ({ inputData: { k }, resumeData, suspendData, suspend }: any) => { + seen.push({ generation, k, resumeData, suspendData }); + if (generation === 1 && k === row.finisher && (resumeData !== undefined || k > 1)) { + await item0Suspended.wait(); + return { k }; + } + if (resumeData === undefined) return suspend({ ask: k }, { resumeLabel: `item-${k}` }); + // Item 0's approval leads to a second request. + if (k === 0 && suspendData?.ask === 0) return suspend({ ask: 'second-0' }, { resumeLabel: 'second-0' }); + return { k }; + }, + }), + { concurrency: 2 }, + ) + .commit(); + const scenario = createRestartScenario({ + kind: 'workflow', + runId, + storageVersion, + build: ({ core, generation }) => + row.nested + ? core + .createWorkflow({ id, inputSchema: z.array(ITEM), outputSchema: z.any(), options: { storageVersion } }) + .then(foreachOf(core, generation, child)) + .commit() + : foreachOf(core, generation, id), + }); + scenarios.push(scenario); + const parked = newGate(); + const original = await scenario.start(async ({ workflow, workflows }) => { + const run = await workflow.createRun({ runId }); + expect((await run.start({ inputData: ITEMS })).status).toBe('suspended'); + run.watch((event: any) => { + const payload = event.payload; + if ( + event.type === 'workflow-step-progress' && + payload?.currentIndex === 0 && + payload.iterationStatus === 'suspended' + ) { + item0Suspended.release(); + } + }); + interceptWrites(workflows, async (args, write) => { + const written = await write(args); + if (!parked.waiters && args?.workflowName === owner) { + const snapshot = await workflows.loadWorkflowSnapshot({ workflowName: owner, runId }); + const progress = (snapshot?.context?.approve as any)?.metadata?.foreachProgress; + if (progress?.[0] || progress?.[row.finisher]) await parked.wait(); + } + return written; + }); + return run.resume({ + step: row.nested ? [child, 'approve'] : 'approve', + ...(row.forEachIndex !== undefined ? { forEachIndex: row.forEachIndex } : {}), + resumeData: APPROVED, + }); + }); + await parked.reached; + const checkpoint = await original.checkpoint(); + expect(calls(seen, 1).filter(c => c.k === 0 && c.resumeData !== undefined)).toEqual([resumed(0)]); + + const { result, graph } = await scenario.restart(checkpoint); + + expect(calls(seen, 2)).toEqual(row.gen2); + expect(result.status).toBe('suspended'); + const ownerRun = await graph.workflows.loadWorkflowSnapshot({ workflowName: owner, runId }); + expect(statusesOf(ownerRun.context, 'approve')).toEqual(row.statuses); + const stored = await graph.workflows.loadWorkflowSnapshot({ workflowName: id, runId }); + expect(JSON.stringify([result, stored, ownerRun])).not.toMatch( + /foreachResumeTarget|nestedResumeTarget|foreachProgress/, + ); + + // Item 0's second request is the one waiting, and its own answer reaches it. + const from = seen.length; + await (await graph.workflow.createRun({ runId })).resume({ label: 'second-0', resumeData: APPROVED }); + expect(calls(seen, 2, from)).toEqual([resumed(0, 'second-0')]); + }); + }); + // Runs stored before targets were recorded carry the payload but no target. it.each([ { diff --git a/packages/core/src/workflows/handlers/control-flow.ts b/packages/core/src/workflows/handlers/control-flow.ts index 548bd0a0813..fb9206625b1 100644 --- a/packages/core/src/workflows/handlers/control-flow.ts +++ b/packages/core/src/workflows/handlers/control-flow.ts @@ -45,7 +45,9 @@ import { getForeachProgress, getForeachResumeTarget, getNestedResumeTarget, + isResuspendedItem, omitForeachProgress, + takesStoredForeachResume, } from './foreach-progress'; import type { ForeachProgress, ForeachResumeTarget, NestedResumeTarget } from './foreach-progress'; import { emitStepResultEvents } from './step'; @@ -1224,7 +1226,15 @@ export async function executeForeach( ...(prevPayload?.suspendPayload?.__workflow_meta?.foreachOutput || []), ] as PersistedForeachStepResult[]; const foreachProgress: ForeachProgress = { ...restartProgress }; - for (const [index, { output, nestedRunId }] of Object.entries(foreachProgress)) { + for (const [index, entry] of Object.entries(foreachProgress)) { + if (isResuspendedItem(entry)) { + // It already used the stored resume data: it waits at the request it made after. + const { resumeLabels: labels, ...suspended } = entry; + prevForeachOutput[Number(index)] = { ...suspended, payload: prevOutput[Number(index)], startedAt: Date.now() }; + for (const label of labels) executionContext.resumeLabels[label] = { stepId, foreachIndex: Number(index) }; + continue; + } + const { output, nestedRunId } = entry; prevForeachOutput[Number(index)] = { status: 'success', output, @@ -1234,6 +1244,8 @@ export async function executeForeach( ...(nestedRunId ? { metadata: { nestedRunId } } : {}), }; } + /** Items that run with the resume data in this pass. */ + const takesResume = new Set(); const nestedRunIds: string[] = []; const prevResumeLabels = prevPayload?.suspendPayload?.__workflow_meta?.resumeLabels || {}; const resumeLabels = getResumeLabelsByStepId(prevResumeLabels, stepId); @@ -1294,7 +1306,9 @@ export async function executeForeach( serializedStepGraph, stepResults: { ...stepResults, [stepId]: runningWithProgress() }, foreachItems: { stepId, indices }, - executionContext, + // A copy: an item that ends while the write is in flight removes the foreach's entry + // from the shared `activeStepsPath`, and storage v1 serializes it only when it writes. + executionContext: { ...executionContext, activeStepsPath: { ...executionContext.activeStepsPath } }, workflowStatus: 'running', requestContext, phase: `foreach-progress.${checkpointCount++}`, @@ -1489,6 +1503,22 @@ export async function executeForeach( scheduleCheckpoint(); } + if (result.status === 'suspended' && checkpointProgress && takesResume.has(k)) { + // It used the resume data and asks again: checkpoint the new request, so a restart before the + // foreach ends leaves the item waiting there instead of handing it the resume data again. + foreachProgress[k] = { + status: 'suspended', + suspendPayload: result.suspendPayload, + suspendedAt: result.suspendedAt, + ...(result.metadata ? { metadata: result.metadata } : {}), + resumeLabels: Object.keys(executionContext.resumeLabels).filter( + label => executionContext.resumeLabels[label]?.foreachIndex === k, + ), + }; + unsentItems.push(k); + scheduleCheckpoint(); + } + if (result.status !== 'success') { await handleNonSuccessResult(result, k); } else { @@ -1530,10 +1560,13 @@ export async function executeForeach( if ( prevItemResult?.status === 'success' || (prevItemResult?.status === 'suspended' && resume?.forEachIndex !== k && resume?.forEachIndex !== undefined) || - // On a restart, a suspended item stays suspended when the stored resume targeted another item or recorded no target. + // On a restart, a suspended item stays suspended when the stored resume targeted another item or recorded no target, + // or when the item already used it and suspended again. (prevItemResult?.status === 'suspended' && restartsRunning && - (resumeTarget === undefined || (typeof resumeTarget === 'number' && resumeTarget !== k))) + (resumeTarget === undefined || + (typeof resumeTarget === 'number' && resumeTarget !== k) || + isResuspendedItem(foreachProgress[k]))) ) { if (prevItemResult?.status === 'success') { // Already succeeded in a previous run – clean up resume label @@ -1572,6 +1605,7 @@ export async function executeForeach( } } + if (resumeToUse || (restartsRunning && takesStoredForeachResume(prevPayload, k))) takesResume.add(k); inFlight++; queue.push({ item: prevOutput[k]!, k, resumeToUse }); } diff --git a/packages/core/src/workflows/handlers/foreach-progress.ts b/packages/core/src/workflows/handlers/foreach-progress.ts index 9b62952ac41..e95d9e4cea0 100644 --- a/packages/core/src/workflows/handlers/foreach-progress.ts +++ b/packages/core/src/workflows/handlers/foreach-progress.ts @@ -5,7 +5,24 @@ * after a crash skips items that already ran (#26214). It is removed once the * foreach ends and never appears in a final step result. */ -export type ForeachProgress = Record; +export type ForeachProgress = Record; + +/** + * A progress entry for an item that took the stored `resumePayload` and suspended again before the + * foreach ended: a restart leaves it suspended at this new request instead of handing it the + * payload again. + */ +export type ForeachResuspendedItem = { + status: 'suspended'; + suspendPayload: unknown; + suspendedAt: number; + metadata?: Record; + resumeLabels: string[]; +}; + +export function isResuspendedItem(entry: ForeachProgress[number] | undefined): entry is ForeachResuspendedItem { + return (entry as { status?: unknown } | undefined)?.status === 'suspended'; +} function isPlainRecord(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); From c5adb511e82e73891a9f5c6abbf82cc79cf1f83a Mon Sep 17 00:00:00 2001 From: Arthur Fiorette Date: Tue, 17 Nov 2026 01:04:32 -0300 Subject: [PATCH 10/10] test(core): force the order in the re-suspended foreach item restart rows The rows for an item that used the resume and suspended again now run on the shared foreach scenario builder and approval step. The item asks again only after the foreach's first running write has landed, and the write that stores its new request waits until a sibling has ended before it reaches the store, so the crash point is the same in every run: the new request is stored, the sibling's result is not, and the stored run still lists the foreach as active. New rows: a foreach whose items are nested workflows, and a second crash while the item runs with the answer to its new request. Every restart also checks its watch events, the stored run and the run that owns the foreach for running-record metadata. The progress write's comment now says that the store reads the copied active steps when it writes, on both storage versions. --- .../workflows/foreach-resume-restart.test.ts | 357 ++++++++++-------- .../src/workflows/handlers/control-flow.ts | 2 +- 2 files changed, 211 insertions(+), 148 deletions(-) diff --git a/packages/core/src/workflows/foreach-resume-restart.test.ts b/packages/core/src/workflows/foreach-resume-restart.test.ts index 4a9fef0b471..87c6ca85b92 100644 --- a/packages/core/src/workflows/foreach-resume-restart.test.ts +++ b/packages/core/src/workflows/foreach-resume-restart.test.ts @@ -61,6 +61,13 @@ type Ctx = { validate: (input: any) => Promise; /** The approval step accepts a resume without data (it then suspends again). */ optionalResume?: boolean; + /** Item `k`, resumed, asks again as `second-`; in generation 1 only after `gate` opens. */ + again?: { k: number; gate: Gate }; + /** Item `holdFresh.k`, started fresh in generation 1, waits on `holdFresh.gate` first. */ + holdFresh?: { k: number; gate: Gate }; + /** The generation whose resumed calls wait on `gate`. */ + blockIn?: number; + storageVersion?: 1 | 2; }; /** An approval step: suspends until resumed; a resumed call in generation `blockIn` waits on `gate`. */ @@ -73,8 +80,10 @@ function approvalStep( gate, validate, optionalResume, + again, + holdFresh, blockIn = 1, - }: Omit & { id: string; blockIn?: number }, + }: Omit & { id: string }, ) { return createStepFn({ id, @@ -84,18 +93,29 @@ function approvalStep( }), outputSchema: z.object({ k: z.number(), approved: z.boolean() }), ...(optionalResume ? {} : { resumeSchema: z.object({ approved: z.boolean() }) }), - suspendSchema: z.object({ ask: z.number() }), - execute: async ({ inputData, resumeData, suspendData, suspend }: any) => { - seen.push({ generation, k: inputData.k, resumeData, suspendData }); - if (resumeData === undefined) return suspend({ ask: inputData.k }, { resumeLabel: `item-${inputData.k}` }); + suspendSchema: z.object({ ask: z.union([z.number(), z.string()]) }), + execute: async ({ inputData: { k }, resumeData, suspendData, suspend }: any) => { + seen.push({ generation, k, resumeData, suspendData }); + if (resumeData === undefined) { + if (generation === 1 && k === holdFresh?.k) await holdFresh.gate.wait(); + return suspend({ ask: k }, { resumeLabel: `item-${k}` }); + } + if (k === again?.k && suspendData?.ask === k) { + if (generation === 1) await again.gate.wait(); + return suspend({ ask: `second-${k}` }, { resumeLabel: `second-${k}` }); + } if (generation === blockIn) await gate.wait(); - return { k: inputData.k, approved: resumeData.approved }; + return { k, approved: resumeData.approved }; }, }); } +const versioned = (ctx: Ctx) => (ctx.storageVersion ? { options: { storageVersion: ctx.storageVersion } } : {}); + const statusesOf = (steps: any, stepId: string): string[] => steps[stepId].suspendPayload.__workflow_meta.foreachOutput.map((r: any) => r?.status); +/** A result's steps; on storage v2 they are a loader. */ +const stepsOf = async (result: any) => (typeof result.steps === 'function' ? await result.steps() : result.steps); type Shape = { build(core: CoreGraph, ctx: Ctx): any; @@ -109,41 +129,41 @@ type Shape = { const leaf: Shape = { build: (core, ctx) => core - .createWorkflow({ id: ctx.id, inputSchema: z.array(ITEM), outputSchema: z.any() }) + .createWorkflow({ id: ctx.id, inputSchema: z.array(ITEM), outputSchema: z.any(), ...versioned(ctx) }) .foreach(approvalStep(core.createStep, { ...ctx, id: 'approve' }), { concurrency: ctx.concurrency }) .commit(), inputData: ITEMS, record: id => [id, 'approve'], - statuses: async ({ result }) => statusesOf(result.steps, 'approve'), + statuses: async ({ result }) => statusesOf(await stepsOf(result), 'approve'), }; /** A foreach whose items are nested workflows with one approval step each. */ const nestedBody: Shape = { build: (core, ctx) => core - .createWorkflow({ id: ctx.id, inputSchema: z.array(ITEM), outputSchema: z.any() }) + .createWorkflow({ id: ctx.id, inputSchema: z.array(ITEM), outputSchema: z.any(), ...versioned(ctx) }) .foreach( core .createWorkflow({ id: `${ctx.id}-child`, inputSchema: ITEM, outputSchema: z.any() }) .then(approvalStep(core.createStep, { ...ctx, id: 'ask' })) .commit(), - { concurrency: 3 }, + { concurrency: ctx.concurrency }, ) .commit(), inputData: ITEMS, record: id => [id, `${id}-child`], - statuses: async ({ id, result }) => statusesOf(result.steps, `${id}-child`), + statuses: async ({ id, result }) => statusesOf(await stepsOf(result), `${id}-child`), }; /** A foreach of approval steps inside a nested workflow: the shape of a durable agent's tool calls. */ const inChild: Shape = { build: (core, ctx) => core - .createWorkflow({ id: ctx.id, inputSchema: z.array(ITEM), outputSchema: z.any() }) + .createWorkflow({ id: ctx.id, inputSchema: z.array(ITEM), outputSchema: z.any(), ...versioned(ctx) }) .then( core .createWorkflow({ id: `${ctx.id}-child`, inputSchema: z.array(ITEM), outputSchema: z.any() }) - .foreach(approvalStep(core.createStep, { ...ctx, id: 'approve' }), { concurrency: 3 }) + .foreach(approvalStep(core.createStep, { ...ctx, id: 'approve' }), { concurrency: ctx.concurrency }) .commit(), ) .commit(), @@ -236,8 +256,21 @@ type Row = { race?: Race; /** Resume without resume data. */ noData?: boolean; + storageVersion?: 1 | 2; + /** + * Item `again` asks again after its resume, once the foreach's first running write has landed. + * The write that stores that new request waits until item `sibling` has ended, then graph 1 + * parks inside it. + */ + again?: number; + sibling?: number; + holdFresh?: number; + blockIn?: number; }; +/** What only a running record carries: it must never reach results, events or stored final records. */ +const NOT_STORED = /foreachResumeTarget|nestedResumeTarget|foreachProgress/; + const byIndex = (k: number) => () => ({ step: 'approve', forEachIndex: k }); const byLabel = (k: number) => () => ({ label: `item-${k}` }); const nestedByIndex = (k: number) => (child: string) => ({ step: [child, 'approve'], forEachIndex: k }); @@ -249,6 +282,9 @@ async function crash(id: string, row: Row) { const seen: Seen[] = []; const gate = newGate(); const validation = newGate(); + const againGate = newGate(); + const holdGate = newGate(); + const parked = newGate(); const build = (core: CoreGraph, generation: number) => row.shape.build(core, { id, @@ -257,6 +293,10 @@ async function crash(id: string, row: Row) { gate, concurrency: row.concurrency ?? 3, optionalResume: row.noData, + again: row.again !== undefined ? { k: row.again, gate: againGate } : undefined, + holdFresh: row.holdFresh !== undefined ? { k: row.holdFresh, gate: holdGate } : undefined, + blockIn: row.blockIn, + storageVersion: row.storageVersion, validate: async input => { if (generation === 2 && row.race?.holds(input)) await validation.wait(); }, @@ -264,11 +304,14 @@ async function crash(id: string, row: Row) { const scenario = createRestartScenario({ kind: 'workflow', runId, + storageVersion: row.storageVersion, build: ({ core, generation }) => build(core, generation), hold: row.window?.hold(id), }); scenarios.push(scenario); - const original = await scenario.start(async ({ core, storage, workflow }) => { + // The resumed record a restart reads: the foreach's, or the parent's nested step in a window. + const [workflowName, stepId] = row.window ? [id, child] : row.shape.record(id); + const original = await scenario.start(async ({ core, storage, workflow, workflows }) => { let suspender = workflow; if (row.elsewhere) { suspender = build(core, 0); @@ -276,17 +319,41 @@ async function crash(id: string, row: Row) { } const started = await (await suspender.createRun({ runId })).start({ inputData: row.shape.inputData }); expect(started.status).toBe('suspended'); - const resumedRun = (await workflow.createRun({ runId })).resume({ + const run = await workflow.createRun({ runId }); + if (row.again !== undefined) { + const siblingEnded = newGate(); + run.watch(event => { + if (new RegExp(`"currentIndex":${row.sibling},"iterationStatus"`).test(JSON.stringify(event))) { + siblingEnded.release(); + } + }); + // Real writes on either storage version: the harness's `hold` sees storage v1 writes only. + interceptWrites(workflows, async (args, write) => { + const own = !parked.waiters && args?.workflowName === workflowName; + if (own && JSON.stringify(args).includes(`"${AGAIN}"`)) { + // The new request's write: a sibling ends while it is in flight. + (row.holdFresh !== undefined ? holdGate : gate).release(); + await siblingEnded.wait(); + } + const written = await write(args); + if (own) { + const record = (await workflows.loadWorkflowSnapshot({ workflowName, runId }))?.context?.[stepId] as any; + if (record?.status === 'running') againGate.release(); + if (record?.metadata?.foreachProgress?.[row.again!]) await parked.wait(); + } + return written; + }); + } + const resumedRun = run.resume({ ...row.resume(child), ...(row.noData ? {} : { resumeData: APPROVED }), }); return resumedRun; }); if (row.window) await original.held; + else if (row.again !== undefined) await parked.reached; else await vi.waitFor(() => expect(gate.waiters).toBe(row.waiters ?? 1)); - // The resumed record a restart reads: the foreach's, or the parent's nested step in a window. - const [workflowName, stepId] = row.window ? [id, child] : row.shape.record(id); const resuming = (c: Checkpoint) => { const snapshot = findRow(c, workflowName, runId)?.snapshot; const record = snapshot?.context?.[stepId] as any; @@ -299,8 +366,16 @@ async function crash(id: string, row: Row) { }; // In a window, graph 1 is parked in the chosen write; otherwise the first write that stored the // resumed record (later writes while items run can store the run without its active step). - const checkpoint = (row.window ? await original.checkpoint() : original.checkpoints.find(resuming))!; + const checkpoint = ( + row.window || row.again !== undefined ? await original.checkpoint() : original.checkpoints.find(resuming) + )!; expect(!!checkpoint && resuming(checkpoint), 'a stored running record with the resume payload').toBe(true); + if (row.again !== undefined) { + // The parked write holds the item's new request alone: every sibling still runs. + const progress = (findRow(checkpoint, workflowName, runId)!.snapshot.context[stepId] as any).metadata + .foreachProgress; + expect(Object.keys(progress)).toEqual([String(row.again)]); + } if (row.window) { // Nested runs whose foreach is still stored suspended; the claimed one is `running`. const parked = checkpoint.rows.filter( @@ -312,8 +387,10 @@ async function crash(id: string, row: Row) { } const restart = async () => { + const events: unknown[] = []; const { result, graph } = await scenario.restart(checkpoint, { watch: event => { + events.push(event); const payload = event.payload; const { race } = row; if (!race || event.type !== 'workflow-step-progress') return; @@ -323,19 +400,23 @@ async function crash(id: string, row: Row) { } }, }); - const stored = await graph.workflows.loadWorkflowSnapshot({ workflowName: id, runId }); - expect(JSON.stringify([result, stored])).not.toMatch(/foreachResumeTarget|nestedResumeTarget/); + const load = (name: string) => graph.workflows.loadWorkflowSnapshot({ workflowName: name, runId }); + const stored = await load(id); + const owner = await load(row.shape.record(id)[0]); + expect(JSON.stringify([result, stored, owner, events])).not.toMatch(NOT_STORED); const statuses = (r: any) => row.shape.statuses?.({ id, runId, result: r, graph }); const resume = (args: Record) => graph.workflow.createRun({ runId }).then((r: any) => r.resume(args)); - return { result, graph, stored, statuses, resume }; + return { result, graph, stored, owner: () => load(row.shape.record(id)[0]), statuses, resume }; }; const record = () => findRow(checkpoint, workflowName, runId)!.snapshot.context[stepId] as any; - return { seen, restart, record, child }; + return { seen, restart, record, child, build, runId }; } const S = 'suspended'; const OK = 'success'; +/** The label of item 0's second request. */ +const AGAIN = 'second-0'; describe('restart while a resumed foreach item runs', () => { it.each([ @@ -458,131 +539,6 @@ describe('restart while a resumed foreach item runs', () => { if (row.shape.statuses) expect(await statuses(result)).toEqual(row.statuses ?? [S, OK, S]); }); - // A resumed item consumes the payload and suspends again with a new request while a sibling keeps - // the foreach running. The crash point is the first write that stores either item 0's new request - // or the sibling's progress: the sibling, finishing after item 0 suspends again, is not stored yet - // and runs again. - describe.each([1, 2] as const)('storage v%s', storageVersion => { - it.each( - [false, true].flatMap(nested => [ - { - name: `${nested ? 'inside a nested workflow, ' : ''}resume without an index, item 1 still running when item 0 suspends again`, - nested, - forEachIndex: undefined, - finisher: 1, - gen2: [resumed(1), fresh(2)], - statuses: [S, OK, S], - }, - { - name: `${nested ? 'inside a nested workflow, ' : ''}resume of item 0 by index, never-reached item 2 still running when item 0 suspends again`, - nested, - forEachIndex: 0, - finisher: 2, - gen2: [fresh(2)], - statuses: [S, S, S], - }, - ]), - )('keeps an item that consumed the resume at its new request: $name', async row => { - const id = `fo-consumed-${row.finisher}-${row.nested}-v${storageVersion}`; - const child = `${id}-child`; - const runId = `${id}-run`; - // The workflow whose foreach the items belong to. - const owner = row.nested ? child : id; - const seen: Seen[] = []; - const item0Suspended = newGate(); - const foreachOf = (core: CoreGraph, generation: number, workflowId: string) => - core - .createWorkflow({ - id: workflowId, - inputSchema: z.array(ITEM), - outputSchema: z.any(), - options: { storageVersion }, - }) - .foreach( - core.createStep({ - id: 'approve', - inputSchema: ITEM, - outputSchema: ITEM, - resumeSchema: z.object({ approved: z.boolean() }), - execute: async ({ inputData: { k }, resumeData, suspendData, suspend }: any) => { - seen.push({ generation, k, resumeData, suspendData }); - if (generation === 1 && k === row.finisher && (resumeData !== undefined || k > 1)) { - await item0Suspended.wait(); - return { k }; - } - if (resumeData === undefined) return suspend({ ask: k }, { resumeLabel: `item-${k}` }); - // Item 0's approval leads to a second request. - if (k === 0 && suspendData?.ask === 0) return suspend({ ask: 'second-0' }, { resumeLabel: 'second-0' }); - return { k }; - }, - }), - { concurrency: 2 }, - ) - .commit(); - const scenario = createRestartScenario({ - kind: 'workflow', - runId, - storageVersion, - build: ({ core, generation }) => - row.nested - ? core - .createWorkflow({ id, inputSchema: z.array(ITEM), outputSchema: z.any(), options: { storageVersion } }) - .then(foreachOf(core, generation, child)) - .commit() - : foreachOf(core, generation, id), - }); - scenarios.push(scenario); - const parked = newGate(); - const original = await scenario.start(async ({ workflow, workflows }) => { - const run = await workflow.createRun({ runId }); - expect((await run.start({ inputData: ITEMS })).status).toBe('suspended'); - run.watch((event: any) => { - const payload = event.payload; - if ( - event.type === 'workflow-step-progress' && - payload?.currentIndex === 0 && - payload.iterationStatus === 'suspended' - ) { - item0Suspended.release(); - } - }); - interceptWrites(workflows, async (args, write) => { - const written = await write(args); - if (!parked.waiters && args?.workflowName === owner) { - const snapshot = await workflows.loadWorkflowSnapshot({ workflowName: owner, runId }); - const progress = (snapshot?.context?.approve as any)?.metadata?.foreachProgress; - if (progress?.[0] || progress?.[row.finisher]) await parked.wait(); - } - return written; - }); - return run.resume({ - step: row.nested ? [child, 'approve'] : 'approve', - ...(row.forEachIndex !== undefined ? { forEachIndex: row.forEachIndex } : {}), - resumeData: APPROVED, - }); - }); - await parked.reached; - const checkpoint = await original.checkpoint(); - expect(calls(seen, 1).filter(c => c.k === 0 && c.resumeData !== undefined)).toEqual([resumed(0)]); - - const { result, graph } = await scenario.restart(checkpoint); - - expect(calls(seen, 2)).toEqual(row.gen2); - expect(result.status).toBe('suspended'); - const ownerRun = await graph.workflows.loadWorkflowSnapshot({ workflowName: owner, runId }); - expect(statusesOf(ownerRun.context, 'approve')).toEqual(row.statuses); - const stored = await graph.workflows.loadWorkflowSnapshot({ workflowName: id, runId }); - expect(JSON.stringify([result, stored, ownerRun])).not.toMatch( - /foreachResumeTarget|nestedResumeTarget|foreachProgress/, - ); - - // Item 0's second request is the one waiting, and its own answer reaches it. - const from = seen.length; - await (await graph.workflow.createRun({ runId })).resume({ label: 'second-0', resumeData: APPROVED }); - expect(calls(seen, 2, from)).toEqual([resumed(0, 'second-0')]); - }); - }); - // Runs stored before targets were recorded carry the payload but no target. it.each([ { @@ -832,6 +788,113 @@ describe('restart while a resumed foreach item runs', () => { // The second copy is taken inside the first restart, which the restart harness can't cut, so // this copies the store's tables (which also covers storage v2). describe.each([1, 2] as const)('storage v%s', storageVersion => { + // An item takes the resume and asks again while a sibling keeps the foreach running; graph 1 + // parks inside the write that stores the new request. + it.each( + [ + { + name: 'leaf foreach, resume without an index', + shape: leaf, + resume: () => ({ step: 'approve' }), + gen2: [resumed(1), fresh(2)], + statuses: [S, OK, S], + after: [OK, OK, S], + }, + { + name: 'leaf foreach, resume of item 0 by index', + shape: leaf, + resume: byIndex(0), + holdFresh: 2, + gen2: [fresh(2)], + statuses: [S, S, S], + after: [OK, S, S], + }, + { + name: 'foreach inside a nested workflow, resume without an index', + shape: inChild, + resume: (child: string) => ({ step: [child, 'approve'] }), + gen2: [resumed(1), fresh(2)], + statuses: [S, OK, S], + after: [OK, OK, S], + }, + { + name: 'foreach inside a nested workflow, resume of item 0 by index', + shape: inChild, + resume: nestedByIndex(0), + holdFresh: 2, + gen2: [fresh(2)], + statuses: [S, S, S], + after: [OK, S, S], + }, + { + name: 'foreach of nested workflows, resume of item 0 by index', + shape: nestedBody, + resume: (child: string) => ({ step: [child, 'ask'], forEachIndex: 0 }), + holdFresh: 2, + gen2: [fresh(2)], + statuses: [S, S, S], + after: [OK, S, S], + }, + ].map(row => ({ + ...row, + concurrency: 2, + again: 0, + // The resumed item 1, or the never-reached item that runs. + sibling: (row as { holdFresh?: number }).holdFresh ?? 1, + storageVersion, + })), + )('keeps an item that used the resume at its new request: $name', async row => { + const { seen, restart } = await crash(`fa-${storageVersion}-${row.name.replace(/\W+/g, '-')}`, row); + + const { result, statuses, owner, resume } = await restart(); + + expect(calls(seen, 2)).toEqual(row.gen2); + expect(result.status).toBe('suspended'); + expect(await statuses(result)).toEqual(row.statuses); + + // The new request is the one waiting, and its own answer reaches it. + const from = seen.length; + const next = await resume({ label: AGAIN, resumeData: APPROVED }); + expect(calls(seen, 2, from)).toEqual([resumed(0, AGAIN)]); + expect(await statuses(next)).toEqual(row.after); + expect(JSON.stringify([next, await owner()])).not.toMatch(NOT_STORED); + }); + + it('gives the answer to the new request again after a crash while the item runs with it', async () => { + const row = { + name: 'twice', + shape: leaf, + resume: byIndex(0), + holdFresh: 2, + concurrency: 2, + again: 0, + sibling: 2, + // The answer to the new request blocks in generation 2, the crash point there. + blockIn: 2, + storageVersion, + }; + const id = `fa-${storageVersion}-twice`; + const { seen, restart, build, runId } = await crash(id, row); + const { graph } = await restart(); + const SECOND = { approved: false }; + void graph.workflow.createRun({ runId }).then((r: any) => r.resume({ label: AGAIN, resumeData: SECOND })); + await vi.waitFor(async () => { + const snapshot = await graph.workflows.loadWorkflowSnapshot({ workflowName: id, runId }); + expect(snapshot?.context?.approve).toMatchObject({ status: 'running', resumePayload: SECOND }); + expect(snapshot?.activeStepsPath).toHaveProperty(['approve']); + }); + const tables = copyRunTables(await runTablesOf(graph.storage)); + + const storage = new MockStore(); + const workflow = build({ createWorkflow, createStep } as unknown as CoreGraph, 3); + new Mastra({ logger: false, storage, workflows: { [workflow.id]: workflow } }); + restoreRunTables(await runTablesOf(storage), tables); + const result = await (await workflow.createRun({ runId })).restart(); + + expect(calls(seen, 3)).toEqual([{ k: 0, resumeData: SECOND, suspendData: { ask: AGAIN } }]); + expect(statusesOf(await stepsOf(result), 'approve')).toEqual([OK, S, S]); + }); + it('still gives only the approved item its decision after a second crash during the restart', async () => { const id = 'foreach-resume-twice'; const runId = 'twice'; diff --git a/packages/core/src/workflows/handlers/control-flow.ts b/packages/core/src/workflows/handlers/control-flow.ts index fb9206625b1..049026a9ca4 100644 --- a/packages/core/src/workflows/handlers/control-flow.ts +++ b/packages/core/src/workflows/handlers/control-flow.ts @@ -1307,7 +1307,7 @@ export async function executeForeach( stepResults: { ...stepResults, [stepId]: runningWithProgress() }, foreachItems: { stepId, indices }, // A copy: an item that ends while the write is in flight removes the foreach's entry - // from the shared `activeStepsPath`, and storage v1 serializes it only when it writes. + // from the shared `activeStepsPath`, which the store reads only when it writes. executionContext: { ...executionContext, activeStepsPath: { ...executionContext.activeStepsPath } }, workflowStatus: 'running', requestContext,