diff --git a/.changeset/goofy-icons-pull.md b/.changeset/goofy-icons-pull.md new file mode 100644 index 00000000000..6717145cf04 --- /dev/null +++ b/.changeset/goofy-icons-pull.md @@ -0,0 +1,9 @@ +--- +'@mastra/core': patch +--- + +Fixed `foreach` items receiving another item's resume data when a run is restarted while one item is being resumed. Items that were still suspended stay suspended until they are resumed, and items the `foreach` had not reached start without resume data. This also holds for a `foreach` inside a nested workflow, for a `foreach` whose items are nested workflows, and when the resume was handled by the same `Mastra` instance that suspended the run. For a `DurableAgent` that delegates to several sub-agents whose tools need approval, approving one delegation and then recovering the run after a crash no longer runs the other sub-agents' tools without their approval. + +Fixed resuming one item of a `foreach` inside a nested workflow by index, for example `run.resume({ step: ['child-workflow', 'approve'], forEachIndex: 1, resumeData })`, resuming every suspended item of that `foreach` with the same resume data. + +Runs interrupted while resuming a `foreach` item or a nested workflow on an earlier version don't record what was resumed. On restart, the suspended items and steps stay suspended, so resume the interrupted one again. diff --git a/packages/core/src/agent/durable/__tests__/durable-agent-approval-restart.test.ts b/packages/core/src/agent/durable/__tests__/durable-agent-approval-restart.test.ts new file mode 100644 index 00000000000..780063660c6 --- /dev/null +++ b/packages/core/src/agent/durable/__tests__/durable-agent-approval-restart.test.ts @@ -0,0 +1,283 @@ +/** + * Recovering a durable agent run that crashed while an approval was being applied. The recovered + * run must run only the approved tool call: every other call still waits for its own approval. + */ + +import { createOpenAI } from '@ai-sdk/openai-v5'; +import { LLMock } from '@copilotkit/aimock'; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest'; +import { z } from 'zod'; +import { DurableStepIds } from '../constants'; +import type { Checkpoint, Gate, HoldWrite } from './restart-harness'; +import { createGate, createRestartScenario } from './restart-harness'; + +let llm: LLMock; +const gates: Gate[] = []; +const scenarios: { stop(): Promise }[] = []; + +/** + * Picks requests by the agent's instructions (the start of the system prompt: a sub-agent's also + * carries its supervisor's), the agent's own task (the last user message), and whether the text + * already holds `done`. + */ +const request = + (instructions: string, { task, done }: { task?: string; done?: string[] } = {}) => + (body: any) => { + const messages: any[] = body?.messages ?? []; + const text = JSON.stringify(messages); + return ( + String(messages[0]?.content).startsWith(instructions) && + (task === undefined || messages.findLast(m => m.role === 'user')?.content === task) && + (done ?? []).every(part => text.includes(part)) + ); + }; +const pending = (predicate: (body: any) => boolean, done: (body: any) => boolean) => (body: any) => + predicate(body) && !done(body); + +beforeAll(async () => { + llm = new LLMock({ port: 0 }); + await llm.start(); + // An agent with three approval-gated tool calls in one step. + llm.on( + { endpoint: 'chat', hasToolResult: false, predicate: request('Charge every card') }, + { toolCalls: [0, 1, 2].map(index => ({ id: `call_${index}`, name: 'charge', arguments: { index } })) }, + ); + llm.on( + { endpoint: 'chat', hasToolResult: true, predicate: request('Charge every card') }, + { content: 'all charged' }, + ); + // A supervisor that delegates to two sub-agents in one step; each calls its gated tool once. + const supervisorDone = request('Delegate to both', { done: ['alpha finished', 'beta finished'] }); + llm.on({ endpoint: 'chat', predicate: supervisorDone }, { content: 'Both delegations complete.' }); + llm.on( + { endpoint: 'chat', predicate: pending(request('Delegate to both'), supervisorDone) }, + { + toolCalls: ['Alpha', 'Beta'].map(name => ({ + id: `sup-tc-${name[0]}`, + name: `agent-sub${name}`, + arguments: { prompt: `do ${name.toLowerCase()}` }, + })), + }, + ); + for (const label of ['alpha', 'beta']) { + const done = request('Use your tool', { task: `do ${label}`, done: [`${label} done`] }); + llm.on({ endpoint: 'chat', predicate: done }, { content: `${label} finished.` }); + llm.on( + { endpoint: 'chat', predicate: pending(request('Use your tool', { task: `do ${label}` }), done) }, + { toolCalls: [{ id: `inner-${label}`, name: 'gatedTool', arguments: { query: label } }] }, + ); + } +}); + +afterAll(async () => { + await llm.stop(); +}); + +afterEach(async () => { + for (const gate of gates.splice(0)) gate.release(); + for (const scenario of scenarios.splice(0)) await scenario.stop(); +}); + +const model = () => createOpenAI({ apiKey: 'test-key', baseURL: `${llm.url}/v1` }).chat('gpt-4o-mini'); + +/** Recovers the run from `checkpoint`; the recovered run reports no error. */ +async function recover(scenario: ReturnType, checkpoint: Checkpoint) { + const recovered = await scenario.restart(checkpoint, { closeOnSuspend: true }); + expect(recovered.streamErrors).toEqual([]); + expect(recovered.executionError).toBeUndefined(); + return recovered; +} + +describe('durable agent: restart while an approved tool call runs', () => { + it('runs only the approved tool call, not the calls still waiting for approval', async () => { + const runId = 'approval-restart'; + const hold = createGate(); + gates.push(hold); + const charges: { generation: number; index: number }[] = []; + const scenario = createRestartScenario({ + kind: 'durable', + runId, + build: ({ core, generation }) => + new core.Agent({ + id: 'approval-agent', + name: 'approval-agent', + instructions: 'Charge every card.', + model: model(), + tools: { + charge: core.createTool({ + id: 'charge', + description: 'Charges a card', + inputSchema: z.object({ index: z.number() }), + requireApproval: true, + execute: async ({ index }) => { + charges.push({ generation, index }); + if (generation === 1) await hold.wait(); + return { charged: index }; + }, + }), + }, + }), + }); + scenarios.push(scenario); + + let approvedId: string | undefined; + const original = await scenario.start(async ({ agent }) => { + const first = await agent.stream('Charge the cards', { runId }); + for await (const chunk of first.fullStream) { + if (chunk.type === 'tool-call-approval') { + approvedId = chunk.payload.toolCallId; + break; + } + } + // Approved through the instance that suspended the run. + return agent.approveToolCall({ runId, toolCallId: approvedId }); + }); + await hold.reached; + expect(approvedId).toBe('call_0'); + expect(charges).toEqual([{ generation: 1, index: 0 }]); + + const recovered = await recover(scenario, await original.checkpoint()); + + expect(charges.filter(c => c.generation === 2)).toEqual([{ generation: 2, index: 0 }]); + // The next call asks for its own approval. + expect(recovered.chunkTypes).toContain('tool-call-approval'); + }, 30_000); +}); + +describe('durable agent: restart while an approved delegation is being resumed', () => { + // Delegations to sub-agents run concurrently, so both suspend at once when each sub-agent's tool + // needs approval. The approval of one goes through the instance that suspended the run, and the + // process crashes before the loop's iteration workflow stores its resumed tool calls. + const crashPoints: { point: string; hold: HoldWrite }[] = [ + { + point: 'before the iteration workflow claims its resume', + hold: { + when: (snapshot: any) => + snapshot.context?.[DurableStepIds.AGENTIC_EXECUTION]?.status === 'running' && + snapshot.context[DurableStepIds.AGENTIC_EXECUTION].resumePayload !== undefined, + }, + }, + { + point: 'after the iteration workflow claims its resume, before its tool calls are stored', + hold: { + workflowName: DurableStepIds.AGENTIC_EXECUTION, + when: (snapshot: any) => + snapshot.status === 'running' && snapshot.context?.[DurableStepIds.TOOL_CALL]?.status === 'suspended', + }, + }, + ]; + + /** Both delegations suspended, `sup-tc-A` approved, and a copy of the rows at the crash. */ + async function crashWhileApproving(hold: HoldWrite) { + const runId = 'delegation-approval-restart'; + const memory = { thread: 'delegation-thread', resource: 'delegation-resource' }; + const executions: { generation: number; query: string }[] = []; + // Memory keeps the pending approvals; both graphs use one store for it. + let memoryStore: any; + const scenario = createRestartScenario({ + kind: 'durable', + runId, + hold, + // A sub-agent's snapshot holds a function, which the harness can't copy and which a + // persistent store drops when it serializes the row. + storageMode: 'serializing', + build: ({ core, generation }) => { + const subAgent = (name: string) => + new core.Agent({ + id: `sub${name}`, + name: `sub${name}`, + description: `Does ${name.toLowerCase()}`, + instructions: 'Use your tool.', + model: model(), + tools: { + gatedTool: core.createTool({ + id: 'gatedTool', + description: 'Approval-gated tool', + inputSchema: z.object({ query: z.string() }), + requireApproval: true, + execute: async ({ query }: { query: string }) => { + executions.push({ generation, query }); + return { result: `${query} done` }; + }, + }), + }, + }); + return new core.Agent({ + id: 'supervisor', + name: 'Supervisor', + instructions: 'Delegate to both sub-agents.', + model: model(), + agents: { subAlpha: subAgent('Alpha'), subBeta: subAgent('Beta') }, + memory: new core.MockMemory({ storage: (memoryStore ??= new core.InMemoryStore()) }), + }); + }, + }); + scenarios.push(scenario); + + const original = await scenario.start(async ({ agent, workflows }) => { + const first = await agent.stream('Do alpha and beta', { runId, memory, maxSteps: 5 }); + const asked = new Set(); + for await (const chunk of first.fullStream) { + if (chunk.type === 'tool-call-approval') asked.add(chunk.payload.toolCallId); + if (asked.size === 2) break; + } + expect([...asked].sort()).toEqual(['sup-tc-A', 'sup-tc-B']); + await vi.waitFor(async () => { + const row = await workflows.getWorkflowRunById({ runId, workflowName: DurableStepIds.AGENTIC_LOOP }); + expect(row?.snapshot?.status).toBe('suspended'); + }); + // Approved through the instance that suspended the run. + return agent.approveToolCall({ runId, toolCallId: 'sup-tc-A', memory }); + }); + await original.held; + const checkpoint = await original.checkpoint(); + expect(executions).toEqual([]); + /** Approves `toolCallId` on the recovered graph and returns the reply's text. */ + const approve = async (recovered: { graph: any }, toolCallId: string) => { + const resumed = await recovered.graph.agent.approveToolCall({ runId, toolCallId, memory, closeOnSuspend: true }); + let text = ''; + for await (const chunk of resumed.fullStream) { + if (chunk.type === 'text-delta') text += chunk.payload.text; + } + return text; + }; + return { scenario, checkpoint, executions, approve }; + } + + it.each(crashPoints)( + 'runs only the approved delegation after a crash $point', + async ({ hold }) => { + const { scenario, checkpoint, executions, approve } = await crashWhileApproving(hold); + + const recovered = await recover(scenario, checkpoint); + + expect(executions).toEqual([{ generation: 2, query: 'alpha' }]); + // Beta's own approval is still pending, and still works. + expect(await approve(recovered, 'sup-tc-B')).toBe('Both delegations complete.'); + expect(executions.map(e => e.query)).toEqual(['alpha', 'beta']); + }, + 60_000, + ); + + it.each(crashPoints)( + 'runs no delegation when the stored resume recorded no target, after a crash $point', + async ({ hold }) => { + // Runs stored before the target was recorded carry the approval but not which call it is for. + const { scenario, checkpoint, executions, approve } = await crashWhileApproving(hold); + const loop = checkpoint.rows.find(row => row.workflowName === DurableStepIds.AGENTIC_LOOP)!; + const record = loop.snapshot.context[DurableStepIds.AGENTIC_EXECUTION] as any; + expect(record.metadata.nestedResumeTarget).toEqual({ steps: [], label: 'sup-tc-A' }); + delete record.metadata.nestedResumeTarget; + + const recovered = await recover(scenario, checkpoint); + + expect(executions).toEqual([]); + // Both approvals are still pending: approving each again runs it. + await approve(recovered, 'sup-tc-A'); + expect(executions).toEqual([{ generation: 2, query: 'alpha' }]); + expect(await approve(recovered, 'sup-tc-B')).toBe('Both delegations complete.'); + expect(executions.map(e => e.query)).toEqual(['alpha', 'beta']); + }, + 60_000, + ); +}); diff --git a/packages/core/src/agent/durable/__tests__/restart-harness.ts b/packages/core/src/agent/durable/__tests__/restart-harness.ts index f30559ba135..01ce113f9ad 100644 --- a/packages/core/src/agent/durable/__tests__/restart-harness.ts +++ b/packages/core/src/agent/durable/__tests__/restart-harness.ts @@ -101,6 +101,7 @@ export async function loadGraph() { { createEventedAgent }, { globalRunRegistry }, { __resetExecutionFencesForTests }, + { MockMemory }, workflows, eventedWorkflows, ] = await Promise.all([ @@ -112,6 +113,7 @@ export async function loadGraph() { import('../create-evented-agent'), import('../run-registry'), import('../execution-fence'), + import('../../../memory/mock'), import('../../../workflows'), import('../../../workflows/evented'), ]); @@ -125,6 +127,7 @@ export async function loadGraph() { globalRunRegistry, /** Stops this graph's claim heartbeats and forgets its local executions. */ resetExecutionFences: __resetExecutionFencesForTests, + MockMemory, createWorkflow: workflows.createWorkflow, createStep: workflows.createStep, createEventedWorkflow: eventedWorkflows.createWorkflow, @@ -457,7 +460,15 @@ export function createRestartScenario(options: ScenarioOp * recovers the run. Fails if the checkpoint has no running row to recover * from, so a test cannot pass by recovering nothing. */ - async restart(checkpoint: Checkpoint): Promise { + async restart( + checkpoint: Checkpoint, + /** + * Agent kinds: close the recovered stream when the run suspends again; it stays open by + * default, so a test that expects the run to suspend sets this. + */ + /** Workflow kind: `watch` sees the restarted run's events. */ + { closeOnSuspend, watch }: { closeOnSuspend?: boolean; watch?: (event: any) => void } = {}, + ): Promise { const graph = await buildGraph(options, ++generation); graphs.push(graph); // Graph 1 is the one that ran; every later graph is seeded from it, not from a previous restart. @@ -479,6 +490,7 @@ export function createRestartScenario(options: ScenarioOp if (options.kind === 'workflow') { const run = await graph.workflow.createRun({ runId: options.runId }); + if (watch) run.watch(watch); const result = await withTimeout(run.restart(), timeoutMs, `restart(${options.runId})`); return { result, graph }; } @@ -494,7 +506,10 @@ export function createRestartScenario(options: ScenarioOp ); let onFinishCalls = 0; const attempt = (async () => { - const recovered = await agent.recover(options.runId, { onFinish: () => void onFinishCalls++ }); + const recovered = await agent.recover(options.runId, { + onFinish: () => void onFinishCalls++, + ...(closeOnSuspend ? { closeOnSuspend } : {}), + }); // Read while the run is live: the registry entry is cleared on cleanup. const execution = graph.core.globalRunRegistry.get(options.runId)?.workflowExecution; const streamErrors: string[] = []; diff --git a/packages/core/src/workflows/constants.ts b/packages/core/src/workflows/constants.ts index ae780c940e4..2c865f35329 100644 --- a/packages/core/src/workflows/constants.ts +++ b/packages/core/src/workflows/constants.ts @@ -13,6 +13,9 @@ export const PARENT_STATE_SYMBOL = Symbol('parent_state'); // the state its changes are relative to, so they're applied to the parent's state then. export const HAND_BACK_STATE_SYMBOL = Symbol('hand_back_state'); +// Symbol used to give a nested workflow step restarted mid-resume what that resume targeted inside the nested run. +export const RESTART_RESUME_SYMBOL = Symbol('restart_resume'); + // Abort reason used when workflow code explicitly cancels a run. export const WORKFLOW_CANCELLED_SYMBOL = Symbol('workflow_cancelled'); diff --git a/packages/core/src/workflows/foreach-resume-restart.test.ts b/packages/core/src/workflows/foreach-resume-restart.test.ts new file mode 100644 index 00000000000..87c6ca85b92 --- /dev/null +++ b/packages/core/src/workflows/foreach-resume-restart.test.ts @@ -0,0 +1,966 @@ +/** + * A restart of a run that crashed while one `foreach` item was being resumed must hand that + * item's resume data to that item only. Items that were still suspended, or never reached, must + * not receive it: in an agent loop each item is a tool call waiting for its own approval. + * + * Each case suspends the items, resumes some of them through the `Mastra` instance that suspended + * the run or through a second instance on the same storage, copies what storage holds while a + * resumed item runs (or at a chosen write), and restarts from that copy in a fresh module graph. + */ + +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { z } from 'zod/v4'; +import { createGate, createRestartScenario, findRow } from '../agent/durable/__tests__/restart-harness'; +import type { Checkpoint, CoreGraph, Gate, HoldWrite } from '../agent/durable/__tests__/restart-harness'; +import { Mastra } from '../mastra'; +import { MockStore } from '../storage/mock'; +import { createWorkflow } from './create'; +import { copyRunTables, interceptWrites, restoreRunTables, runTablesOf } from './test-utils/run-tables'; +import type { RunTables } from './test-utils/run-tables'; +import { createStep } from './workflow'; + +type Call = { k: number; resumeData: unknown; suspendData: unknown }; +type Seen = Call & { generation: number }; + +const ITEMS = [{ k: 0 }, { k: 1 }, { k: 2 }]; +const ITEM = z.object({ k: z.number() }); +const APPROVED = { approved: true }; +/** A call that received its own approval and its own suspend payload. */ +const resumed = (k: number, ask: unknown = k): Call => ({ k, resumeData: APPROVED, suspendData: { ask } }); +/** A call that started the item fresh. */ +const fresh = (k: number): Call => ({ k, resumeData: undefined, suspendData: undefined }); + +const gates: Gate[] = []; +const scenarios: { stop(): Promise }[] = []; +afterEach(async () => { + for (const gate of gates.splice(0)) gate.release(); + await Promise.all(scenarios.splice(0).map(s => s.stop())); +}); + +function newGate() { + const gate = createGate(); + gates.push(gate); + return gate; +} + +/** The calls generation `generation` made after `from`, by item. */ +const calls = (seen: Seen[], generation: number, from = 0): Call[] => + seen + .slice(from) + .filter(s => s.generation === generation) + .map(({ generation: _, ...call }) => call) + .sort((a, b) => a.k - b.k); + +type Ctx = { + id: string; + generation: number; + seen: Seen[]; + gate: Gate; + concurrency: number; + /** Runs inside input validation (of the approval step, or of the nested workflow per item). */ + validate: (input: any) => Promise; + /** The approval step accepts a resume without data (it then suspends again). */ + optionalResume?: boolean; + /** Item `k`, resumed, asks again as `second-`; in generation 1 only after `gate` opens. */ + again?: { k: number; gate: Gate }; + /** Item `holdFresh.k`, started fresh in generation 1, waits on `holdFresh.gate` first. */ + holdFresh?: { k: number; gate: Gate }; + /** The generation whose resumed calls wait on `gate`. */ + blockIn?: number; + storageVersion?: 1 | 2; +}; + +/** An approval step: suspends until resumed; a resumed call in generation `blockIn` waits on `gate`. */ +function approvalStep( + createStepFn: any, + { + id, + generation, + seen, + gate, + validate, + optionalResume, + again, + holdFresh, + blockIn = 1, + }: Omit & { id: string }, +) { + return createStepFn({ + id, + inputSchema: ITEM.refine(async input => { + await validate(input); + return true; + }), + outputSchema: z.object({ k: z.number(), approved: z.boolean() }), + ...(optionalResume ? {} : { resumeSchema: z.object({ approved: z.boolean() }) }), + suspendSchema: z.object({ ask: z.union([z.number(), z.string()]) }), + execute: async ({ inputData: { k }, resumeData, suspendData, suspend }: any) => { + seen.push({ generation, k, resumeData, suspendData }); + if (resumeData === undefined) { + if (generation === 1 && k === holdFresh?.k) await holdFresh.gate.wait(); + return suspend({ ask: k }, { resumeLabel: `item-${k}` }); + } + if (k === again?.k && suspendData?.ask === k) { + if (generation === 1) await again.gate.wait(); + return suspend({ ask: `second-${k}` }, { resumeLabel: `second-${k}` }); + } + if (generation === blockIn) await gate.wait(); + return { k, approved: resumeData.approved }; + }, + }); +} + +const versioned = (ctx: Ctx) => (ctx.storageVersion ? { options: { storageVersion: ctx.storageVersion } } : {}); + +const statusesOf = (steps: any, stepId: string): string[] => + steps[stepId].suspendPayload.__workflow_meta.foreachOutput.map((r: any) => r?.status); +/** A result's steps; on storage v2 they are a loader. */ +const stepsOf = async (result: any) => (typeof result.steps === 'function' ? await result.steps() : result.steps); + +type Shape = { + build(core: CoreGraph, ctx: Ctx): any; + inputData: unknown; + /** Where the resumed foreach's running record is stored: workflow name, step id. */ + record(id: string): [string, string]; + /** The approval foreach's item statuses after `result`; absent where they aren't one list. */ + statuses?(args: { id: string; runId: string; result: any; graph: any }): Promise; +}; + +const leaf: Shape = { + build: (core, ctx) => + core + .createWorkflow({ id: ctx.id, inputSchema: z.array(ITEM), outputSchema: z.any(), ...versioned(ctx) }) + .foreach(approvalStep(core.createStep, { ...ctx, id: 'approve' }), { concurrency: ctx.concurrency }) + .commit(), + inputData: ITEMS, + record: id => [id, 'approve'], + statuses: async ({ result }) => statusesOf(await stepsOf(result), 'approve'), +}; + +/** A foreach whose items are nested workflows with one approval step each. */ +const nestedBody: Shape = { + build: (core, ctx) => + core + .createWorkflow({ id: ctx.id, inputSchema: z.array(ITEM), outputSchema: z.any(), ...versioned(ctx) }) + .foreach( + core + .createWorkflow({ id: `${ctx.id}-child`, inputSchema: ITEM, outputSchema: z.any() }) + .then(approvalStep(core.createStep, { ...ctx, id: 'ask' })) + .commit(), + { concurrency: ctx.concurrency }, + ) + .commit(), + inputData: ITEMS, + record: id => [id, `${id}-child`], + statuses: async ({ id, result }) => statusesOf(await stepsOf(result), `${id}-child`), +}; + +/** A foreach of approval steps inside a nested workflow: the shape of a durable agent's tool calls. */ +const inChild: Shape = { + build: (core, ctx) => + core + .createWorkflow({ id: ctx.id, inputSchema: z.array(ITEM), outputSchema: z.any(), ...versioned(ctx) }) + .then( + core + .createWorkflow({ id: `${ctx.id}-child`, inputSchema: z.array(ITEM), outputSchema: z.any() }) + .foreach(approvalStep(core.createStep, { ...ctx, id: 'approve' }), { concurrency: ctx.concurrency }) + .commit(), + ) + .commit(), + inputData: ITEMS, + record: id => [`${id}-child`, 'approve'], + statuses: async ({ id, runId, graph }) => { + const child = await graph.workflows.loadWorkflowSnapshot({ workflowName: `${id}-child`, runId }); + return statusesOf(child.context, 'approve'); + }, +}; + +/** + * A foreach whose items are nested workflows that each run their own concurrent foreach. Outer + * items 0 and 1 suspend and outer item 2 is never reached; the label names inner item 1 of outer + * item 0. + */ +const foreachOfInChild: Shape = { + build: (core, ctx) => + core + .createWorkflow({ id: ctx.id, inputSchema: z.array(z.array(ITEM)), outputSchema: z.any() }) + .foreach( + core + .createWorkflow({ + id: `${ctx.id}-child`, + inputSchema: z.array(ITEM).refine(async items => { + await ctx.validate(items); + return true; + }), + outputSchema: z.any(), + }) + .foreach(approvalStep(core.createStep, { ...ctx, id: 'approve', validate: async () => {} }), { + concurrency: 2, + }) + .commit(), + { concurrency: 2 }, + ) + .commit(), + inputData: [ + [{ k: 0 }, { k: 1 }], + [{ k: 2 }, { k: 3 }], + [{ k: 4 }, { k: 5 }], + ], + record: id => [id, `${id}-child`], +}; + +/** + * On the restart, the input validation of the item `holds` picks waits until item `index` of the + * foreach `step` has suspended. Items share the foreach's record in memory, and that sibling's + * suspension replaces it while the resumed item is still validating. + */ +type Race = { holds: (input: any) => boolean; step: (id: string) => string; index: number }; +const leafRace: Race = { holds: input => input.k === 1, step: () => 'approve', index: 2 }; +const outerRace: Race = { holds: input => input[0]?.k === 0, step: id => `${id}-child`, index: 2 }; + +/** A crash in the window before the nested run stores its resumed foreach. */ +type Window = { hold: (id: string) => HoldWrite; claimed: boolean; unclaim?: boolean }; +/** The parent has stored its resumed nested step; the nested run is still suspended. */ +const beforeClaim: Window = { + claimed: false, + hold: id => ({ + when: snapshot => + snapshot.context?.[`${id}-child`]?.status === 'running' && + (snapshot.context[`${id}-child`] as any).resumePayload !== undefined, + }), +}; +/** The nested run has claimed its resume. */ +const afterClaim: Window = { + claimed: true, + hold: id => ({ + workflowName: `${id}-child`, + when: snapshot => snapshot.status === 'running' && snapshot.context?.approve?.status === 'suspended', + }), +}; +/** + * The outer foreach's running record doesn't hold its items back, so the cut parks the nested + * run's claim; a crash just before it leaves the same rows with the nested run still suspended. + */ +const beforeClaimOfItem: Window = { ...afterClaim, claimed: false, unclaim: true }; + +type Row = { + name: string; + shape: Shape; + resume: (child: string) => Record; + /** Suspend through a second `Mastra` instance on the same storage. */ + elsewhere?: boolean; + concurrency?: number; + /** Without a window: cut once this many resumed items are running. */ + waiters?: number; + window?: Window; + race?: Race; + /** Resume without resume data. */ + noData?: boolean; + storageVersion?: 1 | 2; + /** + * Item `again` asks again after its resume, once the foreach's first running write has landed. + * The write that stores that new request waits until item `sibling` has ended, then graph 1 + * parks inside it. + */ + again?: number; + sibling?: number; + holdFresh?: number; + blockIn?: number; +}; + +/** What only a running record carries: it must never reach results, events or stored final records. */ +const NOT_STORED = /foreachResumeTarget|nestedResumeTarget|foreachProgress/; + +const byIndex = (k: number) => () => ({ step: 'approve', forEachIndex: k }); +const byLabel = (k: number) => () => ({ label: `item-${k}` }); +const nestedByIndex = (k: number) => (child: string) => ({ step: [child, 'approve'], forEachIndex: k }); + +/** Runs a row's graph 1 up to its cut and copies storage there. */ +async function crash(id: string, row: Row) { + const runId = `${id}-run`; + const child = `${id}-child`; + const seen: Seen[] = []; + const gate = newGate(); + const validation = newGate(); + const againGate = newGate(); + const holdGate = newGate(); + const parked = newGate(); + const build = (core: CoreGraph, generation: number) => + row.shape.build(core, { + id, + generation, + seen, + gate, + concurrency: row.concurrency ?? 3, + optionalResume: row.noData, + again: row.again !== undefined ? { k: row.again, gate: againGate } : undefined, + holdFresh: row.holdFresh !== undefined ? { k: row.holdFresh, gate: holdGate } : undefined, + blockIn: row.blockIn, + storageVersion: row.storageVersion, + validate: async input => { + if (generation === 2 && row.race?.holds(input)) await validation.wait(); + }, + }); + const scenario = createRestartScenario({ + kind: 'workflow', + runId, + storageVersion: row.storageVersion, + build: ({ core, generation }) => build(core, generation), + hold: row.window?.hold(id), + }); + scenarios.push(scenario); + // The resumed record a restart reads: the foreach's, or the parent's nested step in a window. + const [workflowName, stepId] = row.window ? [id, child] : row.shape.record(id); + const original = await scenario.start(async ({ core, storage, workflow, workflows }) => { + let suspender = workflow; + if (row.elsewhere) { + suspender = build(core, 0); + new core.Mastra({ logger: false, storage, workflows: { [suspender.id]: suspender } }); + } + const started = await (await suspender.createRun({ runId })).start({ inputData: row.shape.inputData }); + expect(started.status).toBe('suspended'); + const run = await workflow.createRun({ runId }); + if (row.again !== undefined) { + const siblingEnded = newGate(); + run.watch(event => { + if (new RegExp(`"currentIndex":${row.sibling},"iterationStatus"`).test(JSON.stringify(event))) { + siblingEnded.release(); + } + }); + // Real writes on either storage version: the harness's `hold` sees storage v1 writes only. + interceptWrites(workflows, async (args, write) => { + const own = !parked.waiters && args?.workflowName === workflowName; + if (own && JSON.stringify(args).includes(`"${AGAIN}"`)) { + // The new request's write: a sibling ends while it is in flight. + (row.holdFresh !== undefined ? holdGate : gate).release(); + await siblingEnded.wait(); + } + const written = await write(args); + if (own) { + const record = (await workflows.loadWorkflowSnapshot({ workflowName, runId }))?.context?.[stepId] as any; + if (record?.status === 'running') againGate.release(); + if (record?.metadata?.foreachProgress?.[row.again!]) await parked.wait(); + } + return written; + }); + } + const resumedRun = run.resume({ + ...row.resume(child), + ...(row.noData ? {} : { resumeData: APPROVED }), + }); + return resumedRun; + }); + if (row.window) await original.held; + else if (row.again !== undefined) await parked.reached; + else await vi.waitFor(() => expect(gate.waiters).toBe(row.waiters ?? 1)); + + const resuming = (c: Checkpoint) => { + const snapshot = findRow(c, workflowName, runId)?.snapshot; + const record = snapshot?.context?.[stepId] as any; + // Outside a window, also once the run records the step as active, which is what a restart reruns. + return ( + record?.status === 'running' && + (!!row.noData || record.resumePayload !== undefined) && + (!!row.window || !!snapshot?.activeStepsPath?.[stepId]) + ); + }; + // In a window, graph 1 is parked in the chosen write; otherwise the first write that stored the + // resumed record (later writes while items run can store the run without its active step). + const checkpoint = ( + row.window || row.again !== undefined ? await original.checkpoint() : original.checkpoints.find(resuming) + )!; + expect(!!checkpoint && resuming(checkpoint), 'a stored running record with the resume payload').toBe(true); + if (row.again !== undefined) { + // The parked write holds the item's new request alone: every sibling still runs. + const progress = (findRow(checkpoint, workflowName, runId)!.snapshot.context[stepId] as any).metadata + .foreachProgress; + expect(Object.keys(progress)).toEqual([String(row.again)]); + } + if (row.window) { + // Nested runs whose foreach is still stored suspended; the claimed one is `running`. + const parked = checkpoint.rows.filter( + r => r.workflowName === child && (r.snapshot.context as any).approve?.status === 'suspended', + ); + if (row.window.unclaim) for (const r of parked) r.snapshot.status = 'suspended'; + expect(parked.filter(r => r.snapshot.status === 'running')).toHaveLength(row.window.claimed ? 1 : 0); + expect(calls(seen, 1).filter(c => c.resumeData !== undefined)).toEqual([]); + } + + const restart = async () => { + const events: unknown[] = []; + const { result, graph } = await scenario.restart(checkpoint, { + watch: event => { + events.push(event); + const payload = event.payload; + const { race } = row; + if (!race || event.type !== 'workflow-step-progress') return; + if (payload?.id === race.step(id) && payload.currentIndex === race.index) { + expect(payload.iterationStatus).toBe('suspended'); + validation.release(); + } + }, + }); + const load = (name: string) => graph.workflows.loadWorkflowSnapshot({ workflowName: name, runId }); + const stored = await load(id); + const owner = await load(row.shape.record(id)[0]); + expect(JSON.stringify([result, stored, owner, events])).not.toMatch(NOT_STORED); + const statuses = (r: any) => row.shape.statuses?.({ id, runId, result: r, graph }); + const resume = (args: Record) => + graph.workflow.createRun({ runId }).then((r: any) => r.resume(args)); + return { result, graph, stored, owner: () => load(row.shape.record(id)[0]), statuses, resume }; + }; + const record = () => findRow(checkpoint, workflowName, runId)!.snapshot.context[stepId] as any; + return { seen, restart, record, child, build, runId }; +} + +const S = 'suspended'; +const OK = 'success'; +/** The label of item 0's second request. */ +const AGAIN = 'second-0'; + +describe('restart while a resumed foreach item runs', () => { + it.each([ + { + name: 'leaf foreach, item 1 resumed by index through a second instance', + shape: leaf, + resume: byIndex(1), + elsewhere: true, + gen2: [resumed(1)], + statuses: [S, OK, S], + }, + { + name: 'leaf foreach, item 1 resumed by label through the instance that suspended it', + shape: leaf, + resume: byLabel(1), + gen2: [resumed(1)], + statuses: [S, OK, S], + }, + { + name: 'leaf foreach at concurrency 1, item 0 resumed, items 1 and 2 never reached', + shape: leaf, + concurrency: 1, + resume: byIndex(0), + elsewhere: true, + gen2: [resumed(0), fresh(1)], + statuses: [OK, S], + }, + { + name: 'leaf foreach at concurrency 2, both suspended items resumed without an index', + shape: leaf, + concurrency: 2, + resume: () => ({ step: 'approve' }), + elsewhere: true, + waiters: 2, + gen2: [resumed(0), resumed(1), fresh(2)], + statuses: [OK, OK, S], + }, + { + name: 'leaf foreach at concurrency 2, a sibling suspends while the resumed item validates its input', + shape: leaf, + concurrency: 2, + resume: byIndex(1), + race: leafRace, + gen2: [resumed(1), fresh(2)], + statuses: [S, OK, S], + }, + { + name: 'foreach of nested workflows, item 1 resumed by index through a second instance', + shape: nestedBody, + resume: child => ({ step: [child, 'ask'], forEachIndex: 1 }), + elsewhere: true, + gen2: [resumed(1)], + statuses: [S, OK, S], + }, + { + name: 'foreach inside a nested workflow, item 1 resumed by label through the instance that suspended it', + shape: inChild, + resume: byLabel(1), + gen2: [resumed(1)], + statuses: [S, OK, S], + }, + ])('gives the decision only to the resumed items: $name', async row => { + const { seen, restart } = await crash(`ff-${row.name.replace(/\W+/g, '-')}`, row); + + const { result, statuses, resume } = await restart(); + + expect(calls(seen, 2)).toEqual(row.gen2); + expect(result.status).toBe('suspended'); + expect(await statuses(result)).toEqual(row.statuses); + + // An item still waiting answers to its own resume. + const k = row.statuses.indexOf(S); + const from = seen.length; + await resume({ label: `item-${k}`, resumeData: APPROVED }); + expect(calls(seen, 2, from).filter(c => c.resumeData !== undefined)).toEqual([resumed(k)]); + }); + + const nestedWindows: (Row & { gen2: Call[]; statuses?: string[] })[] = [ + ...[beforeClaim, afterClaim].flatMap(window => + [ + { by: 'label', resume: byLabel(1) }, + { by: 'index', resume: nestedByIndex(1) }, + ].map(({ by, resume }) => ({ + name: `foreach inside a nested workflow, by ${by}, ${window.claimed ? 'after' : 'before'} the nested run claims its resume`, + shape: inChild, + resume, + window, + gen2: [resumed(1)], + })), + ), + ...[beforeClaimOfItem, afterClaim].map(window => ({ + name: `foreach in a foreach item, by label, ${window.claimed ? 'after' : 'before'} the nested run claims its resume, a sibling suspending meanwhile`, + shape: foreachOfInChild, + resume: byLabel(1), + window, + race: outerRace, + gen2: [resumed(1), fresh(4), fresh(5)], + })), + { + // A resume without data continues the named item, which suspends again. Through a second + // instance: the one that suspended the run doesn't store a resume without data as running. + name: 'foreach inside a nested workflow, by label with no resume data, after the nested run claims its resume', + shape: inChild, + resume: byLabel(1), + noData: true, + elsewhere: true, + window: afterClaim, + gen2: [{ k: 1, resumeData: undefined, suspendData: { ask: 1 } }], + statuses: [S, S, S], + }, + ]; + + it.each(nestedWindows)('gives the decision only to the resumed inner item: $name', async row => { + const { seen, restart } = await crash(`fw-${row.name.replace(/\W+/g, '-')}`, row); + + const { result, statuses } = await restart(); + + expect(calls(seen, 2)).toEqual(row.gen2); + expect(result.status).toBe('suspended'); + if (row.shape.statuses) expect(await statuses(result)).toEqual(row.statuses ?? [S, OK, S]); + }); + + // Runs stored before targets were recorded carry the payload but no target. + it.each([ + { + name: 'leaf foreach', + shape: leaf, + resume: byIndex(1), + elsewhere: true, + gen2: [], + target: ['foreachResumeTarget', 1], + }, + ...[beforeClaim, afterClaim].map(window => ({ + name: `foreach inside a nested workflow, ${window.claimed ? 'after' : 'before'} the nested run claims its resume`, + shape: inChild, + resume: byLabel(1), + window, + gen2: [], + target: ['nestedResumeTarget', { steps: [], label: 'item-1' }] as [string, unknown], + })), + { + name: 'foreach in a foreach item, after the nested run claims its resume', + shape: foreachOfInChild, + resume: byLabel(1), + window: afterClaim, + race: outerRace, + gen2: [fresh(4), fresh(5)], + target: ['nestedResumeTarget', { steps: [], label: 'item-1' }], + }, + ])('keeps every suspended item waiting when the stored resume recorded no target: $name', async row => { + const { seen, restart, record, child } = await crash(`fo-${row.name.replace(/\W+/g, '-')}`, row); + const [key, value] = row.target; + expect(record().metadata[key]).toEqual(value); + delete record().metadata[key]; + // The state a nested run had handed back when it parked, which a later resume rebases from. + const startState = row.shape === inChild ? record().suspendPayload.__workflow_meta.startState : undefined; + if (row.shape === inChild) expect(startState).toBeDefined(); + + const { result, stored, statuses, resume } = await restart(); + + expect(calls(seen, 2)).toEqual(row.gen2); + expect(result.status).toBe('suspended'); + if (row.shape.statuses) expect(await statuses(result)).toEqual([S, S, S]); + if (startState) expect(stored.context[child].suspendPayload.__workflow_meta.startState).toEqual(startState); + + // The interrupted item is resumed again, and only it runs. + const from = seen.length; + const next = await resume({ label: 'item-1', resumeData: APPROVED }); + expect(calls(seen, 2, from)).toEqual([resumed(1)]); + if (row.shape.statuses) expect(await statuses(next)).toEqual([S, OK, S]); + }); + + it('keeps a state change the parent never received when the stored resume recorded no target', async () => { + // The resumed nested run ran item 1, which changed the state, and suspended again; the process + // crashed before the parent stored that suspension, on a record without a target. A sibling + // changed `tag` after the nested run took its copy, so only a merge against the state the + // parent last received keeps both changes. + const id = 'fo-unreceived-state'; + const child = `${id}-child`; + const runId = `${id}-run`; + const STATE = z.object({ approvals: z.array(z.number()), tag: z.string() }); + const scenario = createRestartScenario({ + kind: 'workflow', + runId, + build: ({ core }) => + core + .createWorkflow({ id, inputSchema: z.array(ITEM), outputSchema: z.any(), stateSchema: STATE }) + .parallel([ + core.createStep({ + id: 'sibling', + inputSchema: z.any(), + outputSchema: z.object({}), + stateSchema: STATE, + execute: async ({ state, setState }: any) => { + await setState({ ...state, tag: 'sibling' }); + return {}; + }, + }), + core + .createWorkflow({ id: child, inputSchema: z.array(ITEM), outputSchema: z.any(), stateSchema: STATE }) + .foreach( + core.createStep({ + id: 'approve', + inputSchema: ITEM, + outputSchema: ITEM, + resumeSchema: z.object({ approved: z.boolean() }), + stateSchema: STATE, + execute: async ({ inputData, resumeData, suspend, state, setState }: any) => { + if (resumeData === undefined) return suspend({}, { resumeLabel: `item-${inputData.k}` }); + await setState({ ...state, approvals: [...state.approvals, inputData.k] }); + return inputData; + }, + }), + { concurrency: 3 }, + ) + .commit(), + ]) + .commit(), + hold: { + workflowName: child, + when: snapshot => + snapshot.status === 'suspended' && + (snapshot.context?.approve as any)?.suspendPayload?.__workflow_meta?.foreachOutput?.[1]?.status === 'success', + }, + }); + scenarios.push(scenario); + const original = await scenario.start(async ({ workflow }) => { + const run = await workflow.createRun({ runId }); + const started = await run.start({ inputData: ITEMS, initialState: { approvals: [], tag: 'initial' } }); + expect(started.status).toBe('suspended'); + return run.resume({ label: 'item-1', resumeData: APPROVED }); + }); + await original.held; + const checkpoint = await original.checkpoint(); + const record = findRow(checkpoint, id, runId)!.snapshot.context[child] as any; + expect(record).toMatchObject({ status: 'running', resumePayload: APPROVED }); + // The parent last took in the nested run's state when it first parked; the nested run is ahead. + expect(record.suspendPayload.__workflow_meta.startState).toEqual({ approvals: [], tag: 'initial' }); + expect(findRow(checkpoint, id, runId)?.snapshot.value).toEqual({ approvals: [], tag: 'sibling' }); + expect(findRow(checkpoint, child, runId)?.snapshot.value).toEqual({ approvals: [1], tag: 'initial' }); + delete record.metadata.nestedResumeTarget; + + const { result, graph } = await scenario.restart(checkpoint); + expect(result.status).toBe('suspended'); + const run = await graph.workflow.createRun({ runId }); + await run.resume({ label: 'item-0', resumeData: APPROVED }); + expect((await run.resume({ label: 'item-2', resumeData: APPROVED })).status).toBe('success'); + + const stored = await graph.workflows.loadWorkflowSnapshot({ workflowName: id, runId }); + expect([...stored.value.approvals].sort()).toEqual([0, 1, 2]); + expect(stored.value.tag).toBe('sibling'); + }); + + it.each([1, 2] as const)( + 'keeps a state change of a nested run started without a resume, after a crash before the parent stored its suspension, storage v%s', + async storageVersion => { + // A plain start: the nested run changes the state and suspends; the process crashes before the + // parent stores that suspension. The parent's step is `running` with no resume payload. + const id = `plain-nested-v${storageVersion}`; + const child = `${id}-child`; + const runId = `${id}-run`; + const STATE = z.object({ log: z.array(z.string()) }); + const scenario = createRestartScenario({ + kind: 'workflow', + runId, + storageVersion, + build: ({ core }) => + core + .createWorkflow({ + id, + inputSchema: z.object({}), + outputSchema: z.any(), + stateSchema: STATE, + options: { storageVersion }, + }) + .then( + core + .createWorkflow({ id: child, inputSchema: z.object({}), outputSchema: z.any(), stateSchema: STATE }) + .then( + core.createStep({ + id: 'note', + inputSchema: z.object({}), + outputSchema: z.object({}), + stateSchema: STATE, + execute: async ({ getState, setState }: any) => { + await setState({ log: [...(await getState()).log, 'nested'] }); + return {}; + }, + }), + ) + .then( + core.createStep({ + id: 'gate', + inputSchema: z.object({}), + outputSchema: z.object({ approved: z.boolean() }), + // No resumeSchema: the restart resumes the nested run with no data, and the step + // suspends again. + execute: async ({ resumeData, suspend }: any) => + resumeData === undefined ? suspend({ ask: 'gate' }) : { approved: resumeData.approved }, + }), + ) + .commit(), + ) + .commit(), + }); + scenarios.push(scenario); + // Graph 1 parks inside the write that stores the nested run suspended, on either version. + const parked = newGate(); + const original = await scenario.start(async ({ workflow, workflows }) => { + interceptWrites(workflows, async (args, write) => { + const written = await write(args); + if (!parked.waiters && args?.workflowName === child) { + const snapshot = await workflows.loadWorkflowSnapshot({ workflowName: child, runId }); + if (snapshot?.status === 'suspended') await parked.wait(); + } + return written; + }); + return (await workflow.createRun({ runId })).start({ inputData: {}, initialState: { log: [] } }); + }); + await parked.reached; + const checkpoint = await original.checkpoint(); + expect(findRow(checkpoint, id, runId)?.snapshot.context[child]).toMatchObject({ status: 'running' }); + expect((findRow(checkpoint, id, runId)?.snapshot.context[child] as any).resumePayload).toBeUndefined(); + expect(findRow(checkpoint, child, runId)?.snapshot).toMatchObject({ + status: 'suspended', + value: { log: ['nested'] }, + }); + + const { result, graph } = await scenario.restart(checkpoint); + + expect(result.status).toBe('suspended'); + const steps = typeof result.steps === 'function' ? await (result.steps as any)() : result.steps; + expect(steps[child]).toMatchObject({ status: 'suspended', suspendPayload: { ask: 'gate' } }); + const run = await graph.workflow.createRun({ runId }); + expect((await run.resume({ step: [child, 'gate'], resumeData: APPROVED })).status).toBe('success'); + const stored = await graph.workflows.loadWorkflowSnapshot({ workflowName: id, runId }); + expect(stored.value).toEqual({ log: ['nested'] }); + }, + ); + + it('resumes only the chosen item of a foreach inside a nested workflow, by index', async () => { + const id = 'foreach-resume-child-index'; + const seen: Seen[] = []; + const workflow = inChild.build({ createWorkflow, createStep } as unknown as CoreGraph, { + id, + generation: 2, + seen, + gate: newGate(), + concurrency: 3, + validate: async () => {}, + }); + const storage = new MockStore(); + new Mastra({ logger: false, storage, workflows: { [id]: workflow } }); + const run = await workflow.createRun({ runId: id }); + expect((await run.start({ inputData: ITEMS })).status).toBe('suspended'); + const from = seen.length; + + const result = await run.resume({ step: [`${id}-child`, 'approve'], forEachIndex: 1, resumeData: APPROVED }); + + expect(calls(seen, 2, from)).toEqual([resumed(1)]); + expect(result.status).toBe('suspended'); + const child = await (await storage.getStore('workflows'))!.loadWorkflowSnapshot({ + workflowName: `${id}-child`, + runId: id, + }); + expect(statusesOf(child!.context, 'approve')).toEqual([S, OK, S]); + }); + + // The second copy is taken inside the first restart, which the restart harness can't cut, so + // this copies the store's tables (which also covers storage v2). + describe.each([1, 2] as const)('storage v%s', storageVersion => { + // An item takes the resume and asks again while a sibling keeps the foreach running; graph 1 + // parks inside the write that stores the new request. + it.each( + [ + { + name: 'leaf foreach, resume without an index', + shape: leaf, + resume: () => ({ step: 'approve' }), + gen2: [resumed(1), fresh(2)], + statuses: [S, OK, S], + after: [OK, OK, S], + }, + { + name: 'leaf foreach, resume of item 0 by index', + shape: leaf, + resume: byIndex(0), + holdFresh: 2, + gen2: [fresh(2)], + statuses: [S, S, S], + after: [OK, S, S], + }, + { + name: 'foreach inside a nested workflow, resume without an index', + shape: inChild, + resume: (child: string) => ({ step: [child, 'approve'] }), + gen2: [resumed(1), fresh(2)], + statuses: [S, OK, S], + after: [OK, OK, S], + }, + { + name: 'foreach inside a nested workflow, resume of item 0 by index', + shape: inChild, + resume: nestedByIndex(0), + holdFresh: 2, + gen2: [fresh(2)], + statuses: [S, S, S], + after: [OK, S, S], + }, + { + name: 'foreach of nested workflows, resume of item 0 by index', + shape: nestedBody, + resume: (child: string) => ({ step: [child, 'ask'], forEachIndex: 0 }), + holdFresh: 2, + gen2: [fresh(2)], + statuses: [S, S, S], + after: [OK, S, S], + }, + ].map(row => ({ + ...row, + concurrency: 2, + again: 0, + // The resumed item 1, or the never-reached item that runs. + sibling: (row as { holdFresh?: number }).holdFresh ?? 1, + storageVersion, + })), + )('keeps an item that used the resume at its new request: $name', async row => { + const { seen, restart } = await crash(`fa-${storageVersion}-${row.name.replace(/\W+/g, '-')}`, row); + + const { result, statuses, owner, resume } = await restart(); + + expect(calls(seen, 2)).toEqual(row.gen2); + expect(result.status).toBe('suspended'); + expect(await statuses(result)).toEqual(row.statuses); + + // The new request is the one waiting, and its own answer reaches it. + const from = seen.length; + const next = await resume({ label: AGAIN, resumeData: APPROVED }); + expect(calls(seen, 2, from)).toEqual([resumed(0, AGAIN)]); + expect(await statuses(next)).toEqual(row.after); + expect(JSON.stringify([next, await owner()])).not.toMatch(NOT_STORED); + }); + + it('gives the answer to the new request again after a crash while the item runs with it', async () => { + const row = { + name: 'twice', + shape: leaf, + resume: byIndex(0), + holdFresh: 2, + concurrency: 2, + again: 0, + sibling: 2, + // The answer to the new request blocks in generation 2, the crash point there. + blockIn: 2, + storageVersion, + }; + const id = `fa-${storageVersion}-twice`; + const { seen, restart, build, runId } = await crash(id, row); + const { graph } = await restart(); + const SECOND = { approved: false }; + void graph.workflow.createRun({ runId }).then((r: any) => r.resume({ label: AGAIN, resumeData: SECOND })); + await vi.waitFor(async () => { + const snapshot = await graph.workflows.loadWorkflowSnapshot({ workflowName: id, runId }); + expect(snapshot?.context?.approve).toMatchObject({ status: 'running', resumePayload: SECOND }); + expect(snapshot?.activeStepsPath).toHaveProperty(['approve']); + }); + const tables = copyRunTables(await runTablesOf(graph.storage)); + + const storage = new MockStore(); + const workflow = build({ createWorkflow, createStep } as unknown as CoreGraph, 3); + new Mastra({ logger: false, storage, workflows: { [workflow.id]: workflow } }); + restoreRunTables(await runTablesOf(storage), tables); + const result = await (await workflow.createRun({ runId })).restart(); + + expect(calls(seen, 3)).toEqual([{ k: 0, resumeData: SECOND, suspendData: { ask: AGAIN } }]); + expect(statusesOf(await stepsOf(result), 'approve')).toEqual([OK, S, S]); + }); + + it('still gives only the approved item its decision after a second crash during the restart', async () => { + const id = 'foreach-resume-twice'; + const runId = 'twice'; + const seen: Seen[] = []; + const crashes = [newGate(), newGate()]; + const build = (generation: number) => + createWorkflow({ id, inputSchema: z.array(ITEM), outputSchema: z.any(), options: { storageVersion } }) + .foreach( + approvalStep(createStep, { + id: 'approve', + generation, + seen, + // Generations 1 and 2 crash while item 1 runs; generation 3 finishes. + gate: crashes[generation - 1] ?? newGate(), + blockIn: generation === 1 || generation === 2 ? generation : -1, + validate: async () => {}, + }), + { concurrency: 3 }, + ) + .commit(); + /** One process with its own store, seeded with `tables`, counting writes of the resumed foreach. */ + const boot = async (generation: number, tables?: RunTables) => { + const storage = new MockStore(); + const workflow = build(generation); + new Mastra({ logger: false, storage, workflows: { [id]: workflow } }); + if (tables) restoreRunTables(await runTablesOf(storage), tables); + const store = (await storage.getStore('workflows')) as any; + let writes = 0; + interceptWrites(store, async (args, write) => { + const written = await write(args); + if (JSON.stringify(args ?? {}).includes('"resumePayload"')) writes++; + return written; + }); + /** Copies the tables once this process stored the running foreach record itself. */ + const crashCopy = async () => { + await vi.waitFor(async () => { + expect(writes).toBeGreaterThan(0); + const snapshot = await store.loadWorkflowSnapshot({ workflowName: id, runId }); + expect(snapshot?.context.approve).toMatchObject({ status: 'running', resumePayload: APPROVED }); + }); + return copyRunTables(await runTablesOf(storage)); + }; + return { workflow, crashCopy }; + }; + + const first = await boot(1); + const run1 = await first.workflow.createRun({ runId }); + expect((await run1.start({ inputData: ITEMS })).status).toBe('suspended'); + void run1.resume({ step: 'approve', forEachIndex: 1, resumeData: APPROVED }); + await crashes[0]!.reached; + const second = await boot(2, await first.crashCopy()); + void (await second.workflow.createRun({ runId })).restart(); + await crashes[1]!.reached; + const third = await boot(3, await second.crashCopy()); + const run = await third.workflow.createRun({ runId }); + const events: unknown[] = []; + run.watch(event => void events.push(event)); + const result = await run.restart(); + + expect(calls(seen, 2)).toEqual([resumed(1)]); + expect(calls(seen, 3)).toEqual([resumed(1)]); + expect(result.status).toBe('suspended'); + const steps = typeof result.steps === 'function' ? await (result.steps as any)() : result.steps; + expect(statusesOf(steps, 'approve')).toEqual([S, OK, S]); + expect(events.length).toBeGreaterThan(0); + expect(JSON.stringify(events)).not.toContain('foreachResumeTarget'); + }); + }); +}); diff --git a/packages/core/src/workflows/handlers/control-flow.ts b/packages/core/src/workflows/handlers/control-flow.ts index 4b9a4b141ab..049026a9ca4 100644 --- a/packages/core/src/workflows/handlers/control-flow.ts +++ b/packages/core/src/workflows/handlers/control-flow.ts @@ -41,8 +41,15 @@ import { resolveForeachConcurrency, } from '../utils'; import { isWorkflowRunCanceled } from './entry'; -import { getForeachProgress, omitForeachProgress } from './foreach-progress'; -import type { ForeachProgress } from './foreach-progress'; +import { + getForeachProgress, + getForeachResumeTarget, + getNestedResumeTarget, + isResuspendedItem, + omitForeachProgress, + takesStoredForeachResume, +} from './foreach-progress'; +import type { ForeachProgress, ForeachResumeTarget, NestedResumeTarget } from './foreach-progress'; import { emitStepResultEvents } from './step'; import type { ExecuteStepParams } from './step'; @@ -1059,6 +1066,7 @@ export interface ExecuteForeachParams extends ObservabilityContext { resumePayload: any; resumePath: number[]; forEachIndex?: number; + label?: string; }; executionContext: ExecutionContext; pubsub: PubSub; @@ -1139,9 +1147,23 @@ export async function executeForeach( const startTime = resume?.steps[0] === stepId ? undefined : Date.now(); const resumeTime = resume?.steps[0] === stepId ? Date.now() : undefined; + const restartsRunning = !!restart && stepResults[stepId]?.status === 'running'; // Items that finished before a crash, checkpointed on the running foreach result (#26214). - const restartProgress = - restart && stepResults[stepId]?.status === 'running' ? getForeachProgress(stepResults[stepId]) : undefined; + const restartProgress = restartsRunning ? getForeachProgress(stepResults[stepId]) : undefined; + // Which items the stored `resumePayload` belongs to. A restart keeps the stored target so it + // survives another crash; results stored before targets were recorded have none (undefined). + const resumeTarget: ForeachResumeTarget | undefined = + resume?.steps[0] === stepId + ? (resume.forEachIndex ?? 'all') + : restartsRunning + ? getForeachResumeTarget(stepResults[stepId]) + : 'none'; + const nestedResumeTarget: NestedResumeTarget | undefined = + resume?.steps[0] === stepId + ? { steps: resume.steps.slice(1), ...(resume.label ? { label: resume.label } : {}) } + : restartsRunning + ? getNestedResumeTarget(stepResults[stepId]) + : undefined; const stepInfo = { // Same as executeStep: strip prior completion/suspend fields on re-entry. @@ -1198,10 +1220,21 @@ export async function executeForeach( type ForeachStepResult = StepResult | StepBailed; type PersistedForeachStepResult = ForeachStepResult & { suspendPayload?: any }; - const prevForeachOutput = (prevPayload?.suspendPayload?.__workflow_meta?.foreachOutput || - []) as PersistedForeachStepResult[]; + // A copy: the running record keeps the stored suspend state, which progress checkpoints don't + // rewrite, while items record their results here. + const prevForeachOutput = [ + ...(prevPayload?.suspendPayload?.__workflow_meta?.foreachOutput || []), + ] as PersistedForeachStepResult[]; const foreachProgress: ForeachProgress = { ...restartProgress }; - for (const [index, { output, nestedRunId }] of Object.entries(foreachProgress)) { + for (const [index, entry] of Object.entries(foreachProgress)) { + if (isResuspendedItem(entry)) { + // It already used the stored resume data: it waits at the request it made after. + const { resumeLabels: labels, ...suspended } = entry; + prevForeachOutput[Number(index)] = { ...suspended, payload: prevOutput[Number(index)], startedAt: Date.now() }; + for (const label of labels) executionContext.resumeLabels[label] = { stepId, foreachIndex: Number(index) }; + continue; + } + const { output, nestedRunId } = entry; prevForeachOutput[Number(index)] = { status: 'success', output, @@ -1211,6 +1244,8 @@ export async function executeForeach( ...(nestedRunId ? { metadata: { nestedRunId } } : {}), }; } + /** Items that run with the resume data in this pass. */ + const takesResume = new Set(); const nestedRunIds: string[] = []; const prevResumeLabels = prevPayload?.suspendPayload?.__workflow_meta?.resumeLabels || {}; const resumeLabels = getResumeLabelsByStepId(prevResumeLabels, stepId); @@ -1231,7 +1266,17 @@ export async function executeForeach( status: 'running', }; const runningWithProgress = (): StepResult => - ({ ...runningFields, metadata: { ...stepInfo.metadata, foreachProgress } }) as StepResult; + ({ + ...runningFields, + metadata: { + ...stepInfo.metadata, + // A copy: items that finish while a write is in flight belong to the next write, whose + // patch on storage v2 is built from the items unsent when it starts. + foreachProgress: { ...foreachProgress }, + ...(resumeTarget !== undefined ? { foreachResumeTarget: resumeTarget } : {}), + ...(nestedResumeTarget ? { nestedResumeTarget } : {}), + }, + }) as StepResult; /** Items finished since the latest progress write started. */ let unsentItems: number[] = []; @@ -1261,10 +1306,15 @@ export async function executeForeach( serializedStepGraph, stepResults: { ...stepResults, [stepId]: runningWithProgress() }, foreachItems: { stepId, indices }, - executionContext, + // A copy: an item that ends while the write is in flight removes the foreach's entry + // from the shared `activeStepsPath`, which the store reads only when it writes. + executionContext: { ...executionContext, activeStepsPath: { ...executionContext.activeStepsPath } }, workflowStatus: 'running', requestContext, phase: `foreach-progress.${checkpointCount++}`, + // Also when this instance last stored the run as suspended: a restart must find the + // resumed foreach running with its target, or it can't tell which items were resumed. + recordResumedStepStart: resume?.steps[0] === stepId, }); } catch (err) { checkpointError = err instanceof Error ? err : new Error(String(err)); @@ -1453,6 +1503,22 @@ export async function executeForeach( scheduleCheckpoint(); } + if (result.status === 'suspended' && checkpointProgress && takesResume.has(k)) { + // It used the resume data and asks again: checkpoint the new request, so a restart before the + // foreach ends leaves the item waiting there instead of handing it the resume data again. + foreachProgress[k] = { + status: 'suspended', + suspendPayload: result.suspendPayload, + suspendedAt: result.suspendedAt, + ...(result.metadata ? { metadata: result.metadata } : {}), + resumeLabels: Object.keys(executionContext.resumeLabels).filter( + label => executionContext.resumeLabels[label]?.foreachIndex === k, + ), + }; + unsentItems.push(k); + scheduleCheckpoint(); + } + if (result.status !== 'success') { await handleNonSuccessResult(result, k); } else { @@ -1493,7 +1559,14 @@ export async function executeForeach( const prevItemResult = prevForeachOutput[k]; if ( prevItemResult?.status === 'success' || - (prevItemResult?.status === 'suspended' && resume?.forEachIndex !== k && resume?.forEachIndex !== undefined) + (prevItemResult?.status === 'suspended' && resume?.forEachIndex !== k && resume?.forEachIndex !== undefined) || + // On a restart, a suspended item stays suspended when the stored resume targeted another item or recorded no target, + // or when the item already used it and suspended again. + (prevItemResult?.status === 'suspended' && + restartsRunning && + (resumeTarget === undefined || + (typeof resumeTarget === 'number' && resumeTarget !== k) || + isResuspendedItem(foreachProgress[k]))) ) { if (prevItemResult?.status === 'success') { // Already succeeded in a previous run – clean up resume label @@ -1532,6 +1605,7 @@ export async function executeForeach( } } + if (resumeToUse || (restartsRunning && takesStoredForeachResume(prevPayload, k))) takesResume.add(k); inFlight++; queue.push({ item: prevOutput[k]!, k, resumeToUse }); } diff --git a/packages/core/src/workflows/handlers/foreach-progress.ts b/packages/core/src/workflows/handlers/foreach-progress.ts index ef5656e2774..e95d9e4cea0 100644 --- a/packages/core/src/workflows/handlers/foreach-progress.ts +++ b/packages/core/src/workflows/handlers/foreach-progress.ts @@ -5,7 +5,24 @@ * after a crash skips items that already ran (#26214). It is removed once the * foreach ends and never appears in a final step result. */ -export type ForeachProgress = Record; +export type ForeachProgress = Record; + +/** + * A progress entry for an item that took the stored `resumePayload` and suspended again before the + * foreach ended: a restart leaves it suspended at this new request instead of handing it the + * payload again. + */ +export type ForeachResuspendedItem = { + status: 'suspended'; + suspendPayload: unknown; + suspendedAt: number; + metadata?: Record; + resumeLabels: string[]; +}; + +export function isResuspendedItem(entry: ForeachProgress[number] | undefined): entry is ForeachResuspendedItem { + return (entry as { status?: unknown } | undefined)?.status === 'suspended'; +} function isPlainRecord(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); @@ -27,12 +44,73 @@ export function getForeachProgress(stepResult: { metadata?: unknown } | undefine return isForeachProgress(progress) ? progress : undefined; } -/** Copy of `stepInfo` without `metadata.foreachProgress`; drops `metadata` if nothing else is left. */ +/** + * Which items the `resumePayload` on a foreach step's running result belongs to, stored next to + * the progress under `metadata.foreachResumeTarget`: the index of the resumed item, `'all'` for a + * resume without an index (every item suspended at the time), `'none'` when the foreach was + * entered without a resume. A restart hands the stored payload only to those items. + */ +export type ForeachResumeTarget = number | 'all' | 'none'; + +export function getForeachResumeTarget( + stepResult: { metadata?: unknown } | undefined, +): ForeachResumeTarget | undefined { + const metadata = stepResult?.metadata; + if (!isPlainRecord(metadata)) return undefined; + const target = metadata.foreachResumeTarget; + return target === 'all' || target === 'none' || (typeof target === 'number' && Number.isInteger(target)) + ? target + : undefined; +} + +/** Whether a restart hands the stored `resumePayload` of a running foreach result to item `index`. */ +export function takesStoredForeachResume( + stepResult: { metadata?: unknown; suspendPayload?: any } | undefined, + index: number, +): boolean { + const target = getForeachResumeTarget(stepResult); + if (target === 'all') { + return stepResult?.suspendPayload?.__workflow_meta?.foreachOutput?.[index]?.status === 'suspended'; + } + return target === index; +} + +/** + * What a nested workflow step's resume targeted inside the nested run, stored under + * `metadata.nestedResumeTarget` on its running result. A restart that resumes the nested run resumes only that. + */ +export type NestedResumeTarget = { steps: string[]; label?: string; forEachIndex?: number }; + +export function getNestedResumeTarget(stepResult: { metadata?: unknown } | undefined): NestedResumeTarget | undefined { + const metadata = stepResult?.metadata; + if (!isPlainRecord(metadata)) return undefined; + const target = metadata.nestedResumeTarget; + if (!isPlainRecord(target) || !Array.isArray(target.steps)) return undefined; + if (!target.steps.every(step => typeof step === 'string')) return undefined; + if (target.label !== undefined && typeof target.label !== 'string') return undefined; + if (target.forEachIndex !== undefined && !Number.isInteger(target.forEachIndex)) return undefined; + return { + steps: target.steps, + ...(target.label !== undefined ? { label: target.label } : {}), + ...(target.forEachIndex !== undefined ? { forEachIndex: target.forEachIndex as number } : {}), + }; +} + +/** + * Copy of `stepInfo` without the keys only a running result carries (`metadata.foreachProgress`, + * `metadata.foreachResumeTarget`, `metadata.nestedResumeTarget`); drops `metadata` if nothing + * else is left. + */ export function omitForeachProgress( stepInfo: T, ): Omit & { metadata?: Record } { const { metadata, ...rest } = stepInfo; if (!isPlainRecord(metadata)) return rest; - const { foreachProgress: _foreachProgress, ...others } = metadata; + const { + foreachProgress: _foreachProgress, + foreachResumeTarget: _foreachResumeTarget, + nestedResumeTarget: _nestedResumeTarget, + ...others + } = metadata; return Object.keys(others).length > 0 ? { ...rest, metadata: others } : rest; } diff --git a/packages/core/src/workflows/handlers/step.ts b/packages/core/src/workflows/handlers/step.ts index 541d57f0bc2..bfa12d01c77 100644 --- a/packages/core/src/workflows/handlers/step.ts +++ b/packages/core/src/workflows/handlers/step.ts @@ -21,6 +21,7 @@ import { HAND_BACK_STATE_SYMBOL, PARENT_STATE_SYMBOL, PUBSUB_SYMBOL, + RESTART_RESUME_SYMBOL, STREAM_FORMAT_SYMBOL, WORKFLOW_CANCELLED_SYMBOL, } from '../constants'; @@ -51,7 +52,8 @@ import { validateStepStateData, validateStepRequestContext, } from '../utils'; -import { omitForeachProgress } from './foreach-progress'; +import { getNestedResumeTarget, omitForeachProgress, takesStoredForeachResume } from './foreach-progress'; +import type { NestedResumeTarget } from './foreach-progress'; export interface ExecuteStepParams extends ObservabilityContext { workflowId: string; @@ -120,6 +122,9 @@ export async function executeStep( const journal = engine.getRunJournal(runId); const stepCallId = globalThis.crypto.randomUUID(); + // Read before the awaits below: a foreach's items share this record, and a sibling that ends + // meanwhile replaces it with its own result. + const storedResult = stepResults[step.id]; const persistedNestedRunId = executionContext.foreachIndex !== undefined ? stepResults[step.id]?.suspendPayload?.__workflow_meta?.foreachOutput?.[executionContext.foreachIndex]?.metadata @@ -152,6 +157,7 @@ export async function executeStep( }); let resumeDataToUse: unknown; + let restartsResume = false; if (timeTravelResumeData !== undefined && !timeTravelResumeValidationError) { resumeDataToUse = timeTravelResumeData; } else if (timeTravelResumeData !== undefined && timeTravelResumeValidationError) { @@ -161,17 +167,46 @@ export async function executeStep( }); } else if (resume?.steps[0] === step.id) { resumeDataToUse = resume?.resumePayload; - } else if (restart?.activeStepsPath?.[step.id] && stepResults[step.id]?.status === 'running') { + } else if ( + restart?.activeStepsPath?.[step.id] && + storedResult?.status === 'running' && + // A foreach item's record is the foreach's own: its payload belongs only to the resumed items. + (executionContext.foreachIndex === undefined || + takesStoredForeachResume(storedResult, executionContext.foreachIndex)) + ) { // A resumed step that was in flight when the process died re-runs with its original resume data. - resumeDataToUse = (stepResults[step.id] as { resumePayload?: unknown }).resumePayload; + resumeDataToUse = (storedResult as { resumePayload?: unknown }).resumePayload; + restartsResume = true; } + // Kept on the nested step's running record: without it, a restart that resumes the nested run + // again would resume every suspended step there. A foreach item's is on the foreach's record. + let nestedResumeTarget: NestedResumeTarget | undefined; + if (step.component === 'WORKFLOW') { + if (resume?.steps[0] === step.id && executionContext.foreachIndex === undefined) { + nestedResumeTarget = { + steps: resume.steps.slice(1), + ...(resume.label ? { label: resume.label } : {}), + ...(resume.forEachIndex !== undefined ? { forEachIndex: resume.forEachIndex } : {}), + }; + } else if (restartsResume) { + nestedResumeTarget = getNestedResumeTarget(storedResult); + } + } + // The nested run's state this run took in when it last parked on it (storage v1). + const parkedMeta = storedResult?.suspendPayload?.__workflow_meta; + const receivedState: Record | undefined = ( + executionContext.foreachIndex !== undefined + ? parkedMeta?.foreachOutput?.[executionContext.foreachIndex]?.suspendPayload?.__workflow_meta + : parkedMeta + )?.startState; + // Extract suspend data if this step was previously suspended // A resumed step re-run after a crash is stored as 'running' but still carries its suspendPayload. - const priorStatus = stepResults[step.id]?.status; + const priorStatus = storedResult?.status; let suspendDataToUse = priorStatus === 'suspended' || (priorStatus === 'running' && resumeDataToUse !== undefined) - ? stepResults[step.id]?.suspendPayload + ? storedResult?.suspendPayload : undefined; // A suspended foreach step's step-level suspendPayload only carries the FIRST suspended @@ -263,7 +298,10 @@ export async function executeStep( serializedStepGraph, stepResults: { ...stepResults, - [step.id]: stepInfo, + [step.id]: + nestedResumeTarget && executionContext.foreachIndex === undefined + ? { ...stepInfo, metadata: { ...stepInfo.metadata, nestedResumeTarget } } + : stepInfo, } as Record>, executionContext, workflowStatus: 'running', @@ -471,17 +509,7 @@ export async function executeStep( storageVersion: 1, }); if (snapshot?.status === 'success') { - // The state the nested run started from, recorded when this run parked on it. - const meta = stepResults[step.id]?.suspendPayload?.__workflow_meta; - completedNestedOutput = { - found: true, - value: snapshot.result, - state: snapshot.value, - base: - executionContext.foreachIndex !== undefined - ? meta?.foreachOutput?.[executionContext.foreachIndex]?.suspendPayload?.__workflow_meta?.startState - : meta?.startState, - }; + completedNestedOutput = { found: true, value: snapshot.result, state: snapshot.value, base: receivedState }; } } } @@ -641,13 +669,15 @@ export async function executeStep( // In foreach, stepResults[step.id] is shared by all iterations, so also require that // this iteration itself suspended; otherwise it would claim a sibling's child run. resume: - stepResults[step.id]?.status === 'suspended' && (foreachIndex === undefined || nestedIterationSuspendPayload) + storedResult?.status === 'suspended' && (foreachIndex === undefined || nestedIterationSuspendPayload) ? { steps: resume?.steps?.slice(1) || [], resumePayload: resume?.resumePayload, - runId: (nestedIterationSuspendPayload ?? stepResults[step.id]?.suspendPayload)?.__workflow_meta?.runId, + runId: (nestedIterationSuspendPayload ?? storedResult?.suspendPayload)?.__workflow_meta?.runId, label: resume?.label, - forEachIndex: resume?.forEachIndex, + // Inside a foreach the index picked this item; otherwise it picks an item of a + // foreach inside the nested run. + forEachIndex: foreachIndex === undefined ? resume?.forEachIndex : undefined, } : undefined, // Only pass restart data if this step is part of activeStepsPath @@ -671,6 +701,13 @@ export async function executeStep( ? (((state: unknown) => journal.baseFor(state)) satisfies ParentStateLink) : undefined, [HAND_BACK_STATE_SYMBOL]: isNestedWorkflow ? setState : undefined, + // No target means the restarted resume's target is unknown (a record stored before targets + // were): the nested workflow then leaves the nested run suspended. A restart of a step that + // was never resumed has no target and no payload, and sends nothing. + [RESTART_RESUME_SYMBOL]: + restartsResume && step.component === 'WORKFLOW' + ? (nestedResumeTarget ?? (resumeDataToUse !== undefined ? { received: receivedState } : undefined)) + : undefined, engine: engine.getEngineContext(), abortSignal: abortController?.signal, outputWriter, diff --git a/packages/core/src/workflows/restart-mid-block.test.ts b/packages/core/src/workflows/restart-mid-block.test.ts index e48aa7e6cc3..eea86b1d720 100644 --- a/packages/core/src/workflows/restart-mid-block.test.ts +++ b/packages/core/src/workflows/restart-mid-block.test.ts @@ -21,6 +21,8 @@ import { Mastra } from '../mastra'; import { cloneRunData } from '../storage/domains/workflows/inmemory'; import { MockStore } from '../storage/mock'; import { createWorkflow } from './create'; +import { copyRunTables, interceptWrites, restoreRunTables, runTablesOf } from './test-utils/run-tables'; +import type { RunTables } from './test-utils/run-tables'; import type { WorkflowRunState } from './types'; import { createStep } from './workflow'; @@ -47,20 +49,6 @@ function progressCount(snapshot: WorkflowRunState | undefined, stepId: string) { return Object.keys((snapshot?.context?.[stepId] as any)?.metadata?.foreachProgress ?? {}).length; } -/** - * Replaces every workflows-store write with `intercept`, which makes the write by calling `write`. - * Covers the writes of both storage versions. - */ -function interceptWrites( - store: any, - intercept: (args: any, write: (args: any) => Promise) => Promise, -) { - for (const method of WRITE_METHODS) { - const write = store[method].bind(store); - vi.spyOn(store, method).mockImplementation((args: any) => intercept(args, write)); - } -} - /** The run's snapshot as storage holds it, rebuilt from the journal on storage v2. */ async function storedSnapshot(store: any, workflowName: string, runId: string) { return (await store.loadWorkflowSnapshot({ workflowName, runId })) as WorkflowRunState | null; @@ -255,9 +243,12 @@ describe.each([1, 2] as const)( expect(restarted.status).toBe('success'); expect(restarted.result).toEqual(original.result); expect(restarted.steps.item.output).toEqual(original.steps.item.output); - // Per-item progress is a crash-recovery record only; it must not survive into results. + // Per-item progress and the resume target are crash-recovery records only; they must not + // survive into results. expect(JSON.stringify(restarted)).not.toContain('foreachProgress'); expect(JSON.stringify(stored)).not.toContain('foreachProgress'); + expect(JSON.stringify(restarted)).not.toContain('foreachResumeTarget'); + expect(JSON.stringify(stored)).not.toContain('foreachResumeTarget'); }, ); @@ -535,36 +526,6 @@ describe.each([1, 2] as const)( }, ); -const RUN_TABLES = ['workflows', 'workflowRunHeaders', 'workflowRunEntries'] as const; -const WRITE_METHODS = [ - 'persistWorkflowSnapshot', - 'updateWorkflowResults', - 'updateWorkflowState', - 'updateWorkflowRunHeader', - 'writeWorkflowRunCheckpoint', - 'appendWorkflowRunEntries', - 'appendWorkflowRunStateDraft', - 'deleteWorkflowRunById', -] as const; - -type RunTables = Record<(typeof RUN_TABLES)[number], Map>; - -async function runTablesOf(storage: InstanceType): Promise { - return ((await storage.getStore('workflows')) as any).db; -} - -function copyRunTables(tables: RunTables): RunTables { - return Object.fromEntries(RUN_TABLES.map(table => [table, cloneRunData(tables[table])])) as RunTables; -} - -/** Makes `tables` hold exactly what `copy` holds. */ -function restoreRunTables(tables: RunTables, copy: RunTables) { - for (const table of RUN_TABLES) { - tables[table].clear(); - for (const [key, value] of copy[table]) tables[table].set(key, value); - } -} - function seeded(seed: number) { return () => (seed = (seed * 1103515245 + 12345) % 2 ** 31) / 2 ** 31; } diff --git a/packages/core/src/workflows/test-utils/run-tables.ts b/packages/core/src/workflows/test-utils/run-tables.ts new file mode 100644 index 00000000000..2b998b42db8 --- /dev/null +++ b/packages/core/src/workflows/test-utils/run-tables.ts @@ -0,0 +1,52 @@ +/** + * Copies of a `MockStore`'s workflow run tables, for tests that crash a run by copying what + * storage holds at one moment and restarting from that copy in fresh storage. The copy is of the + * tables, so it works for both storage versions. + */ + +import { vi } from 'vitest'; +import { cloneRunData } from '../../storage/domains/workflows/inmemory'; +import type { MockStore } from '../../storage/mock'; + +const RUN_TABLES = ['workflows', 'workflowRunHeaders', 'workflowRunEntries'] as const; + +/** Every workflows-store method that writes, across both storage versions. */ +export const WRITE_METHODS = [ + 'persistWorkflowSnapshot', + 'updateWorkflowResults', + 'updateWorkflowState', + 'updateWorkflowRunHeader', + 'writeWorkflowRunCheckpoint', + 'appendWorkflowRunEntries', + 'appendWorkflowRunStateDraft', + 'deleteWorkflowRunById', +] as const; + +export type RunTables = Record<(typeof RUN_TABLES)[number], Map>; + +export async function runTablesOf(storage: InstanceType): Promise { + return ((await storage.getStore('workflows')) as any).db; +} + +export function copyRunTables(tables: RunTables): RunTables { + return Object.fromEntries(RUN_TABLES.map(table => [table, cloneRunData(tables[table])])) as RunTables; +} + +/** Makes `tables` hold exactly what `copy` holds. */ +export function restoreRunTables(tables: RunTables, copy: RunTables) { + for (const table of RUN_TABLES) { + tables[table].clear(); + for (const [key, value] of copy[table]) tables[table].set(key, value); + } +} + +/** Replaces every workflows-store write with `intercept`, which makes the write by calling `write`. */ +export function interceptWrites( + store: any, + intercept: (args: any, write: (args: any) => Promise) => Promise, +) { + for (const method of WRITE_METHODS) { + const write = store[method].bind(store); + vi.spyOn(store, method).mockImplementation((args: any) => intercept(args, write)); + } +} diff --git a/packages/core/src/workflows/workflow.ts b/packages/core/src/workflows/workflow.ts index 10513cd8968..c61f4861b62 100644 --- a/packages/core/src/workflows/workflow.ts +++ b/packages/core/src/workflows/workflow.ts @@ -83,11 +83,18 @@ import { isMastraTool } from '../tools/toolchecks'; import type { ToolExecutionContext } from '../tools/types'; import type { DynamicArgument } from '../types'; import { resolveDynamicArgument } from '../utils/dynamic-argument'; -import { HAND_BACK_STATE_SYMBOL, PARENT_STATE_SYMBOL, PUBSUB_SYMBOL, WORKFLOW_CANCELLED_SYMBOL } from './constants'; +import { + HAND_BACK_STATE_SYMBOL, + PARENT_STATE_SYMBOL, + PUBSUB_SYMBOL, + RESTART_RESUME_SYMBOL, + WORKFLOW_CANCELLED_SYMBOL, +} from './constants'; import { DefaultExecutionEngine } from './default'; import type { ClassifierStepOutput } from './entry-executors'; import { withoutAuthToken } from './evented/helpers'; import type { ExecutionEngine, ExecutionGraph } from './execution-engine'; +import type { NestedResumeTarget } from './handlers/foreach-progress'; import { validateTemplate } from './mapping-template'; import { cancelNestedRun, @@ -3463,6 +3470,7 @@ export class Workflow< [PUBSUB_SYMBOL]: pubsub, [PARENT_STATE_SYMBOL]: parentState, [HAND_BACK_STATE_SYMBOL]: handBackState, + [RESTART_RESUME_SYMBOL]: restartResume, mastra, requestContext, abort, @@ -3504,6 +3512,11 @@ export class Workflow< [PUBSUB_SYMBOL]: PubSub; [PARENT_STATE_SYMBOL]?: ParentStateLink; [HAND_BACK_STATE_SYMBOL]?: (state: TState, base: Record) => Promise; + /** + * Without `steps`: the restarted resume didn't record its target inside the nested run; + * `received` is the nested run's state the parent last took in (storage v1). + */ + [RESTART_RESUME_SYMBOL]?: NestedResumeTarget | { received?: Record }; mastra: Mastra; requestContext?: RequestContext; engine: DefaultEngineType; @@ -3627,6 +3640,8 @@ export class Workflow< // What the nested run's hand-back is relative to. A nested run a restarting parent resumes may // have changes its parent never received, so it hands back its whole state. let handBackBase: Record | undefined; + // The state a nested run left suspended by a restart had handed back when it parked. + let parkedState: Record | undefined; try { let restartNested = !!restart; let resumeNested = isResume; @@ -3686,16 +3701,35 @@ export class Workflow< outputWriter, outputOptions: { includeState: true, includeResumeLabels: true }, }); + } else if (resumeNested && !resume && restartResume && !('steps' in restartResume)) { + // The interrupted resume didn't record its target inside the nested run: leave the nested + // run suspended as stored, and this step suspends with it, to be resumed again. + const stored = await run.runStorage.loadFull(); + res = { + status: 'suspended', + steps: (stored?.context ?? {}) as any, + resumeLabels: stored?.resumeLabels, + } as WorkflowResult; + if (!parentState) { + // The resumed nested run may have changed the state and parked again before this run + // stored that: hand its changes back against the state this run last took in from it, or + // whole without one, as a restart does. The parent then holds its stored state. + res.state = stored?.value as TState; + handBackBase = restartResume.received; + parkedState = toJsonState(stored?.value); + } } else if (resumeNested) { + const target = resume ?? (restartResume && 'steps' in restartResume ? restartResume : undefined); res = await run.resumeNestedByParent({ resumeData, - step: resume?.steps?.length ? (resume.steps as any) : undefined, + step: target?.steps?.length ? (target.steps as any) : undefined, requestContext, actor, ...observabilityContext, outputWriter, outputOptions: { includeState: true, includeResumeLabels: true }, - label: resume?.label, + label: target?.label, + forEachIndex: target?.forEachIndex, perStep, }); if (isResume) handBackBase = run.resumedFromState; @@ -3737,7 +3771,7 @@ export class Workflow< // parent's current state from here, so the changes it already handed back aren't applied twice. // Storage v2 keeps the state each drive started from, so a v2 parent reads it from there // instead of storing a copy. - const handedBack = handBackBase && !parentState ? toJsonState(res.state) : undefined; + const handedBack = parkedState ?? (handBackBase && !parentState ? toJsonState(res.state) : undefined); if (suspendedSteps?.length) { for (const [stepName, stepResult] of suspendedSteps) {