From 9999cc549ed67c7b8ad0e9a3f56d880f388f4c2c Mon Sep 17 00:00:00 2001 From: Leo <108278866+leoisadev1@users.noreply.github.com> Date: Mon, 16 Nov 2026 12:23:17 -0500 Subject: [PATCH 01/15] fix(core): let durable background tasks with memory suspend after the run finished A deferred background tool (or a sub-agent reaching a tool that needs approval) on a durable agent with memory failed with DURABLE_AGENT_EXECUTION_UNVERIFIED when it suspended after the run that dispatched it had finished. The background executor runs the origin run's suspend bookkeeping first, and its memory save used the strict ownership check, which cannot verify a claim the finished run already released. The task's own suspend never ran. The background path now saves that bookkeeping with the post-settlement check (assertExecutionNotSuperseded), the same one background-task results already use. It still stops writes once another execution has claimed the run. Foreground suspends keep the strict check. Fixes #30182 Co-Authored-By: Claude Opus 5.5 --- .changeset/twelve-papers-stand.md | 5 + .../durable/workflows/steps/tool-call.ts | 275 +++++++++--------- ...background-task-lifecycle.scenario.test.ts | 179 +++++++++++- 3 files changed, 325 insertions(+), 134 deletions(-) create mode 100644 .changeset/twelve-papers-stand.md diff --git a/.changeset/twelve-papers-stand.md b/.changeset/twelve-papers-stand.md new file mode 100644 index 00000000000..759a399d8dc --- /dev/null +++ b/.changeset/twelve-papers-stand.md @@ -0,0 +1,5 @@ +--- +'@mastra/core': patch +--- + +Fixed background tasks on durable agents with memory failing with "could not verify that this execution still owns the run" when they suspend after the agent's run has finished. The task is now marked suspended and can be resumed with `backgroundTaskManager.resume()`. This covers tools that call `suspend()` and sub-agents that reach a tool needing approval. Fixes [#30182](https://github.com/mastra-ai/mastra/issues/30182). diff --git a/packages/core/src/agent/durable/workflows/steps/tool-call.ts b/packages/core/src/agent/durable/workflows/steps/tool-call.ts index e77d31978fe..05fea0e8307 100644 --- a/packages/core/src/agent/durable/workflows/steps/tool-call.ts +++ b/packages/core/src/agent/durable/workflows/steps/tool-call.ts @@ -877,7 +877,7 @@ export function createDurableToolCallStep() { assertExecutionNotSuperseded({ runId, agentId: initData.agentId, requestContext, mastra: mastra as Mastra }); // Remote workers have no registry message list; persist updates through the recovered resume list. - const doFlush = async (messagesToFlush = messageList ?? resumeMessageList) => { + const doFlush = async (messagesToFlush = messageList ?? resumeMessageList, beforePersist = assertOwned) => { if (!messagesToFlush && processorDataParts.length) { const invocation = [...(contextMessageList?.get.all.db() ?? [])].reverse().find(hasInvocation); if (invocation) { @@ -935,7 +935,7 @@ export function createDurableToolCallStep() { onThreadCreated: () => { threadExists = true; }, - beforePersist: assertOwned, + beforePersist, }); if (!flushed) { logger?.warn?.('Failed to persist messages before durable tool suspension', { @@ -1583,143 +1583,79 @@ export function createDurableToolCallStep() { ...(toolAbortSignal ? { abortSignal: toolAbortSignal } : {}), outputWriter, // In-execution suspend callback — allows tools to suspend mid-execution - suspend: async (suspendPayload: any, suspendOptions?: SuspendOptions) => { - wasSuspended = true; - // When a delegated sub-agent requests approval, the delegation tool - // wrapper passes its inner suspended run id via `suspendOptions.runId` - // (see the agent-tool wrapper's `suspend(..., { runId, isAgentSuspend })`). - // Persist it with the approval so the resume leg targets that inner - // run instead of restarting the sub-agent from scratch. - const delegatedRunId = - typeof suspendOptions?.runId === 'string' && suspendOptions.runId !== runId - ? suspendOptions.runId - : undefined; - // Set by the agent-tool wrapper when a delegated sub-agent paused: the sub-agent's own - // tool call. Carried alongside the delegation call, which stays the resume key. Ignored - // unless the wrapper raised the pause, so a plain tool's suspend options can't set it. - const fromDelegate = suspendOptions?.isAgentSuspend === true; - const delegatedToolCallId = - fromDelegate && typeof suspendOptions?.delegatedToolCallId === 'string' - ? suspendOptions.delegatedToolCallId - : undefined; - const delegatedToolName = - fromDelegate && typeof suspendOptions?.delegatedToolName === 'string' - ? suspendOptions.delegatedToolName - : undefined; - if (suspendOptions?.requireToolApproval) { - const innerApproval = - typeof suspendOptions.requireToolApproval === 'object' && suspendOptions.requireToolApproval - ? suspendOptions.requireToolApproval - : typeof suspendPayload?.requireToolApproval === 'object' && suspendPayload?.requireToolApproval - ? suspendPayload.requireToolApproval - : null; + suspend: (suspendPayload: any, suspendOptions?: SuspendOptions) => + suspendToolCall(suspendPayload, suspendOptions, assertOwned), + }; + // `beforePersist` guards the suspension's memory writes. A background + // task can suspend after the run settled and released its claim, so it + // passes the post-settlement check instead of the strict one (#30182). + const suspendToolCall = async ( + suspendPayload: any, + suspendOptions: SuspendOptions | undefined, + beforePersist: typeof assertOwned, + ) => { + wasSuspended = true; + // When a delegated sub-agent requests approval, the delegation tool + // wrapper passes its inner suspended run id via `suspendOptions.runId` + // (see the agent-tool wrapper's `suspend(..., { runId, isAgentSuspend })`). + // Persist it with the approval so the resume leg targets that inner + // run instead of restarting the sub-agent from scratch. + const delegatedRunId = + typeof suspendOptions?.runId === 'string' && suspendOptions.runId !== runId + ? suspendOptions.runId + : undefined; + // Set by the agent-tool wrapper when a delegated sub-agent paused: the sub-agent's own + // tool call. Carried alongside the delegation call, which stays the resume key. Ignored + // unless the wrapper raised the pause, so a plain tool's suspend options can't set it. + const fromDelegate = suspendOptions?.isAgentSuspend === true; + const delegatedToolCallId = + fromDelegate && typeof suspendOptions?.delegatedToolCallId === 'string' + ? suspendOptions.delegatedToolCallId + : undefined; + const delegatedToolName = + fromDelegate && typeof suspendOptions?.delegatedToolName === 'string' + ? suspendOptions.delegatedToolName + : undefined; + if (suspendOptions?.requireToolApproval) { + const innerApproval = + typeof suspendOptions.requireToolApproval === 'object' && suspendOptions.requireToolApproval + ? suspendOptions.requireToolApproval + : typeof suspendPayload?.requireToolApproval === 'object' && suspendPayload?.requireToolApproval + ? suspendPayload.requireToolApproval + : null; - const approvalToolName = innerApproval?.toolName ?? toolName; - const approvalArgs = innerApproval?.args !== undefined ? innerApproval.args : args; + const approvalToolName = innerApproval?.toolName ?? toolName; + const approvalArgs = innerApproval?.args !== undefined ? innerApproval.args : args; - // Tool is requesting approval during execution - await stopGoalActivity({ agentId: initData.agentId, runId }); + // Tool is requesting approval during execution + await stopGoalActivity({ agentId: initData.agentId, runId }); - // Add approval metadata to message before persisting - addToolMetadata({ - type: 'approval', - resumeSchema: approvalResumeSchema, - delegatedRunId, - delegatedToolCallId, - delegatedToolName, - ...(innerApproval ? { approvalToolName, approvalArgs } : {}), - }); - - await doFlush(); - - if (pubsub) { - const approvalChunk = await applyToolPayloadTransformToChunk( - { - type: 'tool-call-approval' as const, - runId, - from: ChunkFrom.AGENT, - payload: { - toolCallId, - toolName: approvalToolName, - args: approvalArgs, - ...(delegatedToolCallId ? { delegatedToolCallId } : {}), - ...(delegatedToolName ? { delegatedToolName } : {}), - resumeSchema: approvalResumeSchema, - updatedAt: Date.now(), - usage: typedInput.stepUsage, - }, - }, - { - policy: toolPayloadTransform, - tools: toolsForTransform, - logger: logger as any, - }, - ); - await publishAfterPersist(() => emitChunkEvent(pubsub, runId, approvalChunk)); - } - - if (pubsub) { - await publishAfterPersist(() => - emitSuspendedEvent(pubsub, runId, { - toolCallId, - toolName: approvalToolName, - args: approvalArgs, - type: 'approval', - resumeSchema: approvalResumeSchema, - }), - ); - } - - endSpansAsSuspended({ toolCallId, toolName: approvalToolName, reason: 'approval' }); - - return suspend( - { - type: 'approval', - requireToolApproval: { toolCallId, toolName: approvalToolName, args: approvalArgs }, - // Persist the inner suspended run id in the workflow snapshot, - // partitioned per tool call (resumeLabel = toolCallId), so the - // resume leg can recover it even if message metadata is stale. - ...(delegatedRunId ? { suspendedToolRunId: delegatedRunId } : {}), - }, - { resumeLabel: toolCallId }, - ); - } - // General tool suspension (e.g., tool calls context.agent.suspend()) - const suspendedEventData: AgentSuspendedEventData = { - toolCallId, - toolName, - args, - suspendPayload, - type: 'suspension', - resumeSchema: suspendOptions?.resumeSchema, - }; - - // Add suspension metadata to message before persisting + // Add approval metadata to message before persisting addToolMetadata({ - type: 'suspension', - suspendPayload, - resumeSchema: suspendOptions?.resumeSchema, + type: 'approval', + resumeSchema: approvalResumeSchema, delegatedRunId, delegatedToolCallId, delegatedToolName, + ...(innerApproval ? { approvalToolName, approvalArgs } : {}), }); - await doFlush(); + await doFlush(undefined, beforePersist); if (pubsub) { - const suspensionChunk = await applyToolPayloadTransformToChunk( + const approvalChunk = await applyToolPayloadTransformToChunk( { - type: 'tool-call-suspended' as const, + type: 'tool-call-approval' as const, runId, from: ChunkFrom.AGENT, payload: { toolCallId, - toolName, + toolName: approvalToolName, + args: approvalArgs, ...(delegatedToolCallId ? { delegatedToolCallId } : {}), ...(delegatedToolName ? { delegatedToolName } : {}), - suspendPayload, - args, - resumeSchema: suspendOptions?.resumeSchema, + resumeSchema: approvalResumeSchema, + updatedAt: Date.now(), usage: typedInput.stepUsage, }, }, @@ -1729,28 +1665,101 @@ export function createDurableToolCallStep() { logger: logger as any, }, ); - await publishAfterPersist(() => emitChunkEvent(pubsub, runId, suspensionChunk)); - await publishAfterPersist(() => emitSuspendedEvent(pubsub, runId, suspendedEventData)); + await publishAfterPersist(() => emitChunkEvent(pubsub, runId, approvalChunk)); } - endSpansAsSuspended({ toolCallId, toolName, reason: 'suspension' }); + if (pubsub) { + await publishAfterPersist(() => + emitSuspendedEvent(pubsub, runId, { + toolCallId, + toolName: approvalToolName, + args: approvalArgs, + type: 'approval', + resumeSchema: approvalResumeSchema, + }), + ); + } + + endSpansAsSuspended({ toolCallId, toolName: approvalToolName, reason: 'approval' }); return suspend( { - type: 'suspension', - toolCallSuspended: suspendPayload, - toolCallId, - toolName, - resumeLabel: suspendOptions?.resumeLabel, + type: 'approval', + requireToolApproval: { toolCallId, toolName: approvalToolName, args: approvalArgs }, // Persist the inner suspended run id in the workflow snapshot, // partitioned per tool call (resumeLabel = toolCallId), so the - // resume leg continues the delegate's suspended run instead of - // restarting it (#20496; mirrors the approval branch above). + // resume leg can recover it even if message metadata is stale. ...(delegatedRunId ? { suspendedToolRunId: delegatedRunId } : {}), }, { resumeLabel: toolCallId }, ); - }, + } + // General tool suspension (e.g., tool calls context.agent.suspend()) + const suspendedEventData: AgentSuspendedEventData = { + toolCallId, + toolName, + args, + suspendPayload, + type: 'suspension', + resumeSchema: suspendOptions?.resumeSchema, + }; + + // Add suspension metadata to message before persisting + addToolMetadata({ + type: 'suspension', + suspendPayload, + resumeSchema: suspendOptions?.resumeSchema, + delegatedRunId, + delegatedToolCallId, + delegatedToolName, + }); + + await doFlush(undefined, beforePersist); + + if (pubsub) { + const suspensionChunk = await applyToolPayloadTransformToChunk( + { + type: 'tool-call-suspended' as const, + runId, + from: ChunkFrom.AGENT, + payload: { + toolCallId, + toolName, + ...(delegatedToolCallId ? { delegatedToolCallId } : {}), + ...(delegatedToolName ? { delegatedToolName } : {}), + suspendPayload, + args, + resumeSchema: suspendOptions?.resumeSchema, + usage: typedInput.stepUsage, + }, + }, + { + policy: toolPayloadTransform, + tools: toolsForTransform, + logger: logger as any, + }, + ); + await publishAfterPersist(() => emitChunkEvent(pubsub, runId, suspensionChunk)); + await publishAfterPersist(() => emitSuspendedEvent(pubsub, runId, suspendedEventData)); + } + + endSpansAsSuspended({ toolCallId, toolName, reason: 'suspension' }); + + return suspend( + { + type: 'suspension', + toolCallSuspended: suspendPayload, + toolCallId, + toolName, + resumeLabel: suspendOptions?.resumeLabel, + // Persist the inner suspended run id in the workflow snapshot, + // partitioned per tool call (resumeLabel = toolCallId), so the + // resume leg continues the delegate's suspended run instead of + // restarting it (#20496; mirrors the approval branch above). + ...(delegatedRunId ? { suspendedToolRunId: delegatedRunId } : {}), + }, + { resumeLabel: toolCallId }, + ); }; // Raw `Tool` instances resolved from the Mastra registry (the cross-process // fallback path) are not wrapped by CoreToolBuilder, so they only get a @@ -1843,7 +1852,7 @@ export function createDurableToolCallStep() { // suspension state (#23739) — never the model-authored one. suspendedToolRunId: taskContext?.suspendedToolRunId, suspend: async (data?: unknown, options?: SuspendOptions) => { - await toolOptions.suspend?.(data, options); + await suspendToolCall(data, options, assertNotSuperseded); return taskContext?.suspend?.(data, options); }, outputWriter: async (chunk: any) => { diff --git a/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts b/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts index ac3a9dcb65c..b6366bfa526 100644 --- a/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts +++ b/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts @@ -15,6 +15,8 @@ import { isDurableEngineVariant, } from '../aimock-scenario'; import { BackgroundTaskManager } from '../../../../background-tasks/manager'; +import type { Mastra } from '../../../../mastra'; +import { SCENARIO_MODEL_ID } from '../types'; /** * Regression classes: the end-to-end background-task lifecycle driven by real @@ -578,7 +580,11 @@ describeForAllEngines( // A chat that reloads the thread while a deferred background tool waits for input must // still see the open request on the tool call, and the resumed tool must know which // request it is answering. (#28986) - function makeSuspendingResearchTool(seenOnResume: unknown[], transform?: ToolPayloadTransform) { + function makeSuspendingResearchTool( + seenOnResume: unknown[], + transform?: ToolPayloadTransform, + gate?: Promise, + ) { return createTool({ id: 'research-with-approval', description: 'Research a topic after analyst approval', @@ -589,6 +595,7 @@ describeForAllEngines( execute: async ({ topic }, options) => { const resumeData = options?.agent?.resumeData as { notes?: string } | undefined; if (!resumeData) { + await gate; return options?.agent?.suspend?.({ awaiting: 'analyst-approval', topic }) as any; } seenOnResume.push(options?.agent?.suspendPayload); @@ -720,6 +727,176 @@ describeForAllEngines( expect(seenOnResume).toEqual([{ awaiting: 'analyst-approval', topic: 'solana' }]); }, 30_000); + // Durable engines release the run's claim once it finishes. Tests that need the run fully + // over before a background task acts wait for storage to show that. Runs without a claim + // have no record. + async function waitForRunToRelease(mastra: Mastra, runId: string) { + const workflows = await mastra.getStorage()!.getStore('workflows'); + await waitFor( + async () => (await workflows?.getRunOwnership?.({ runId }))?.live ?? false, + live => !live, + ); + } + + // Technical: the background executor runs the origin run's suspend bookkeeping before the + // task's own suspend. Its memory save required the finished run to still own its claim, + // which can't be verified once the claim is released, so the task failed with + // DURABLE_AGENT_EXECUTION_UNVERIFIED instead of suspending. (#30182) + it('parks a deferred tool that asks for input after the reply was sent, and resumes it (#30182)', async () => { + let release!: () => void; + const gate = new Promise(resolve => (release = resolve)); + const seenOnResume: unknown[] = []; + const memory = new MockMemory(); + const thread = { threadId: 'bg-suspend-after-finish-thread', resourceId: 'bg-suspend-after-finish-user' }; + const { chunks, mastra, output } = await runLoopScenario({ + engine, + llm: getMock(), + prompt: 'Research near', + tools: { researchWithApproval: makeSuspendingResearchTool(seenOnResume, undefined, gate) }, + agentBackgroundTasks: { tools: { researchWithApproval: true } }, + backgroundTasks: BG_CONFIG, + memory, + ...thread, + stopWhen: stepCountIs(3), + collectChunks: true, + fixtures: llm => + onReportTurns(llm, 'Research near', { + dispatch: { + toolCalls: [{ id: 'call-bg-1', name: 'researchWithApproval', arguments: { topic: 'near' } }], + }, + ack: 'Waiting for the analyst.', + report: 'The analyst answered.', + }), + }); + // The run has replied and ended before the tool asks for input. + expect(await output.text).toBe('Waiting for the analyst.'); + const manager = mastra.backgroundTaskManager!; + const started = chunks!.find(c => c.type === 'background-task-started') as Chunk & { runId: string }; + const taskId = started.payload.taskId as string; + await waitForRunToRelease(mastra, started.runId); + release(); + + const parked = (await waitFor( + () => manager.getTask(taskId), + (t: any) => t?.status !== 'pending' && t?.status !== 'running', + )) as any; + expect({ status: parked.status, error: parked.error?.message }).toEqual({ + status: 'suspended', + error: undefined, + }); + expect(parked.suspendPayload).toEqual({ awaiting: 'analyst-approval', topic: 'near' }); + const open = await waitFor( + () => storedToolCall(memory, thread), + s => s.suspended?.taskId !== undefined, + ); + expect(open.suspended).toMatchObject({ taskId, suspendPayload: { awaiting: 'analyst-approval', topic: 'near' } }); + + await manager.resume(taskId, { notes: 'looks promising' }); + const done = (await waitFor( + () => manager.getTask(taskId), + (t: any) => t?.status === 'completed', + )) as any; + expect(done.result).toEqual({ summary: 'Research on "near": looks promising' }); + expect(seenOnResume).toEqual([{ awaiting: 'analyst-approval', topic: 'near' }]); + }, 30_000); + + // Technical: same path as above, reached through a deferred sub-agent: when its tool needs + // approval, the delegation tool suspends the background task with that approval. (#30182) + it('parks a deferred sub-agent whose tool needs approval after the reply was sent, and resumes it (#30182)', async () => { + const mock = getMock(); + const openai = createOpenAI({ apiKey: 'aimock-test-key', baseURL: `${mock.url.replace(/\/+$/, '')}/v1` }); + let release!: () => void; + const gate = new Promise(resolve => (release = resolve)); + const published: string[] = []; + const publishTool = createTool({ + id: 'publish', + description: 'Publish a post', + inputSchema: z.object({ title: z.string() }), + requireApproval: true, + execute: async ({ title }) => { + published.push(title); + return { published: title }; + }, + }); + const helper = new Agent({ + id: 'helper', + name: 'Helper', + description: 'Publishes posts', + instructions: 'Publish posts with publishTool.', + model: openai(SCENARIO_MODEL_ID), + tools: { publishTool }, + // Holds the sub-agent until the supervisor's run is over. + inputProcessors: [ + { + id: 'wait-for-supervisor', + processInput: async ({ messages }) => { + await gate; + return messages; + }, + }, + ], + }); + const toolNames = (req: any): string[] => (req?.tools ?? []).map((t: any) => t.function?.name); + const isSupervisor = (req: unknown) => toolNames(req).includes('agent-helper'); + const isHelper = (req: unknown) => toolNames(req).includes('publishTool'); + const memory = new MockMemory(); + const thread = { threadId: 'bg-subagent-approval-thread', resourceId: 'bg-subagent-approval-user' }; + const { chunks, mastra, output } = await runLoopScenario({ + engine, + llm: mock, + prompt: 'Publish the launch post', + agents: { helper }, + agentBackgroundTasks: { tools: { helper: true } }, + backgroundTasks: BG_CONFIG, + memory, + ...thread, + stopWhen: stepCountIs(3), + collectChunks: true, + fixtures: llm => { + llm.on( + { endpoint: 'chat', predicate: (req: unknown) => isSupervisor(req) && !hasToolResult(req) }, + { toolCalls: [{ id: 'call-bg-1', name: 'agent-helper', arguments: { prompt: 'Publish "Launch"' } }] }, + ); + llm.on( + { endpoint: 'chat', predicate: (req: unknown) => isSupervisor(req) && hasToolResult(req) }, + { content: 'Handed to the helper.' }, + ); + llm.on( + { endpoint: 'chat', predicate: (req: unknown) => isHelper(req) && !hasToolResult(req) }, + { toolCalls: [{ id: 'call-publish-1', name: 'publishTool', arguments: { title: 'Launch' } }] }, + ); + llm.on( + { endpoint: 'chat', predicate: (req: unknown) => isHelper(req) && hasToolResult(req) }, + { content: 'Published Launch.' }, + ); + }, + }); + expect(await output.text).toBe('Handed to the helper.'); + const manager = mastra.backgroundTaskManager!; + const started = chunks!.find(c => c.type === 'background-task-started') as Chunk & { runId: string }; + const taskId = started.payload.taskId as string; + await waitForRunToRelease(mastra, started.runId); + release(); + + const parked = (await waitFor( + () => manager.getTask(taskId), + (t: any) => t?.status !== 'pending' && t?.status !== 'running', + )) as any; + expect({ status: parked.status, error: parked.error?.message }).toEqual({ + status: 'suspended', + error: undefined, + }); + expect(published).toEqual([]); + + await manager.resume(taskId, { approved: true }); + const done = (await waitFor( + () => manager.getTask(taskId), + (t: any) => t?.status !== 'suspended' && t?.status !== 'running' && t?.status !== 'pending', + )) as any; + expect({ status: done.status, error: done.error?.message }).toEqual({ status: 'completed', error: undefined }); + expect(published).toEqual(['Launch']); + }, 30_000); + it('clears the open request from memory when a suspended deferred task is cancelled (#28986)', async () => { const { memory, thread, manager, taskId } = await startSuspendingResearch([], 'bg-suspend-cancel-thread'); await waitFor( From 512adefe2d17d1360a866dbb77d2e43ce73dd64f Mon Sep 17 00:00:00 2001 From: Leo <108278866+leoisadev1@users.noreply.github.com> Date: Mon, 16 Nov 2026 12:36:57 -0500 Subject: [PATCH 02/15] test(core): prove a late background suspend can't write after a successor claims the run The #30182 fix lets a background task suspend after its run released its claim. This scenario covers the other side: once another execution claims the run, the old execution's suspension is refused with the superseded error, nothing reaches memory, and the successor's claim is unchanged. Co-Authored-By: Claude Opus 5.5 --- ...background-task-lifecycle.scenario.test.ts | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts b/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts index b6366bfa526..98d032ed959 100644 --- a/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts +++ b/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts @@ -800,6 +800,75 @@ describeForAllEngines( expect(seenOnResume).toEqual([{ awaiting: 'analyst-approval', topic: 'near' }]); }, 30_000); + // Technical: the late suspend above is only allowed while no other execution has claimed the + // run. Once a successor claims it, the old execution's suspension must not reach memory or + // park the task, and the successor's claim must stay as it was. (#30182) + it.runIf(isDurableEngineVariant(engine))( + "doesn't park a deferred tool that asks for input after another execution took over the run (#30182)", + async () => { + let release!: () => void; + const gate = new Promise(resolve => (release = resolve)); + const memory = new MockMemory(); + const thread = { threadId: 'bg-suspend-after-takeover-thread', resourceId: 'bg-suspend-after-takeover-user' }; + const { chunks, mastra, output } = await runLoopScenario({ + engine, + llm: getMock(), + prompt: 'Research near', + tools: { researchWithApproval: makeSuspendingResearchTool([], undefined, gate) }, + agentBackgroundTasks: { tools: { researchWithApproval: true } }, + backgroundTasks: BG_CONFIG, + memory, + ...thread, + stopWhen: stepCountIs(3), + collectChunks: true, + fixtures: llm => + onReportTurns(llm, 'Research near', { + dispatch: { + toolCalls: [{ id: 'call-bg-1', name: 'researchWithApproval', arguments: { topic: 'near' } }], + }, + ack: 'Waiting for the analyst.', + report: 'The analyst answered.', + }), + }); + expect(await output.text).toBe('Waiting for the analyst.'); + const manager = mastra.backgroundTaskManager!; + const started = chunks!.find(c => c.type === 'background-task-started') as Chunk & { runId: string }; + const taskId = started.payload.taskId as string; + await waitForRunToRelease(mastra, started.runId); + + // Another execution claims the run, as a recovery or a new turn would. + const workflows = (await mastra.getStorage()!.getStore('workflows'))!; + const released = await workflows.getRunOwnership({ runId: started.runId }); + const successor = await workflows.claimRunOwnership({ + runId: started.runId, + ownerId: 'successor-execution', + leaseMs: 60_000, + expectedGeneration: released!.generation, + }); + expect(successor.acquired).toBe(true); + release(); + + const settled = (await waitFor( + () => manager.getTask(taskId), + (t: any) => t?.status !== 'pending' && t?.status !== 'running', + )) as any; + expect({ status: settled.status, error: settled.error?.message }).toEqual({ + status: 'failed', + // DURABLE_AGENT_EXECUTION_SUPERSEDED + error: expect.stringContaining(`Durable run ${started.runId} lost its execution lease`), + }); + expect(settled.suspendPayload).toBeUndefined(); + expect((await storedToolCall(memory, thread)).suspended).toBeUndefined(); + const after = await workflows.getRunOwnership({ runId: started.runId }); + expect({ ownerId: after?.ownerId, generation: after?.generation, live: after?.live }).toEqual({ + ownerId: 'successor-execution', + generation: successor.record.generation, + live: true, + }); + }, + 30_000, + ); + // Technical: same path as above, reached through a deferred sub-agent: when its tool needs // approval, the delegation tool suspends the background task with that approval. (#30182) it('parks a deferred sub-agent whose tool needs approval after the reply was sent, and resumes it (#30182)', async () => { From 1fc1bf8b07114a89d4ca4efc52be42b8dccfc41c Mon Sep 17 00:00:00 2001 From: Leo <108278866+leoisadev1@users.noreply.github.com> Date: Mon, 16 Nov 2026 12:52:33 -0500 Subject: [PATCH 03/15] fix(core): let a late background suspend through while the finished run is still cleaning up A finished run releases its claim, then retires its memory fence. While that retirement was pending, the execution was still registered locally, so the late suspend's check verified the claim it had just released and failed with DURABLE_AGENT_EXECUTION_SUPERSEDED though no other execution had claimed the run. Once the claim is released, the local check now accepts the write until another execution claims the run, matching the check used after the fence is unregistered. Co-Authored-By: Claude Opus 5.5 --- .../core/src/agent/durable/execution-fence.ts | 18 ++- ...background-task-lifecycle.scenario.test.ts | 126 ++++++++++++++++++ 2 files changed, 143 insertions(+), 1 deletion(-) diff --git a/packages/core/src/agent/durable/execution-fence.ts b/packages/core/src/agent/durable/execution-fence.ts index 079b036bb6a..d0894dbea84 100644 --- a/packages/core/src/agent/durable/execution-fence.ts +++ b/packages/core/src/agent/durable/execution-fence.ts @@ -570,6 +570,8 @@ export class ExecutionFence implements RunFenceScope { #renewalInFlight = false; #settlement?: Promise; #settledAs?: ExecutionSettlement; + /** Settlement released the claim; cleanup after the release may still be running. */ + #released = false; #resolveSettled!: () => void; /** Resolves once {@link settle} has finished and the claim is released or left to its new holder. */ readonly whenSettled: Promise; @@ -925,6 +927,19 @@ export class ExecutionFence implements RunFenceScope { if (this.generation === undefined) this.#leaseExpiresAt = startedAt + EXECUTION_LEASE_TTL_MS; } + /** + * {@link verify} for writes that may land after settlement. Once settlement + * released the claim, the write is accepted until another execution claims + * the run, as it is after this fence is unregistered. + */ + async verifyNotSuperseded(): Promise { + if (!this.#released) return this.verify(); + const details = this.#details(); + if (this.#takenOver || (await retryOnce(() => this.#backend.isSuperseded(this.claim), details))) { + throw new DurableExecutionFenceError(EXECUTION_SUPERSEDED_ERROR_ID, details); + } + } + stopHeartbeat(): void { if (this.#heartbeat) clearInterval(this.#heartbeat); this.#heartbeat = undefined; @@ -1045,6 +1060,7 @@ export class ExecutionFence implements RunFenceScope { async #release(): Promise { try { await this.#backend.release(this.claim); + this.#released = true; } catch (error) { this.#logger?.warn?.(`[DurableAgent] run ${this.runId}: failed to release the execution claim: ${error}`); } @@ -1315,7 +1331,7 @@ export async function assertExecutionNotSuperseded(args: { const local = args.mastra?.__getRunScope(args.runId)?.get(DURABLE_PARTICIPANT_FENCES)?.get(claim.executionId) ?? fencesByExecutionId.get(claim.executionId); - if (local) return local.verify(); + if (local) return local.verifyNotSuperseded(); const backend = await resolveRemoteBackend(args.mastra, args.agentId, args.runId, claim); if (!backend) return; diff --git a/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts b/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts index 98d032ed959..d21d632a3f2 100644 --- a/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts +++ b/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts @@ -869,6 +869,132 @@ describeForAllEngines( 30_000, ); + // Starts the deferred research tool, lets the run finish and release its claim, and holds + // the run's memory fence retirement open, as slow storage would, until `finishCleanup`. + async function startResearchDuringCleanup(threadId: string) { + let release!: () => void; + const gate = new Promise(resolve => (release = resolve)); + let finishCleanup!: () => void; + const cleanupGate = new Promise(resolve => (finishCleanup = resolve)); + const memory = new MockMemory(); + const memoryStore = (await memory.storage.getStore('memory'))!; + const retireRunFence = memoryStore.retireRunFence.bind(memoryStore); + let retiring = false; + vi.spyOn(memoryStore, 'retireRunFence').mockImplementation(async fence => { + retiring = true; + await cleanupGate; + return retireRunFence(fence); + }); + const thread = { threadId, resourceId: `${threadId}-user` }; + const { chunks, mastra, output } = await runLoopScenario({ + engine, + llm: getMock(), + prompt: 'Research near', + tools: { researchWithApproval: makeSuspendingResearchTool([], undefined, gate) }, + agentBackgroundTasks: { tools: { researchWithApproval: true } }, + backgroundTasks: BG_CONFIG, + memory, + ...thread, + stopWhen: stepCountIs(3), + collectChunks: true, + fixtures: llm => + onReportTurns(llm, 'Research near', { + dispatch: { + toolCalls: [{ id: 'call-bg-1', name: 'researchWithApproval', arguments: { topic: 'near' } }], + }, + ack: 'Waiting for the analyst.', + report: 'The analyst answered.', + }), + }); + expect(await output.text).toBe('Waiting for the analyst.'); + const started = chunks!.find(c => c.type === 'background-task-started') as Chunk & { runId: string }; + await waitForRunToRelease(mastra, started.runId); + await waitFor( + async () => retiring, + isRetiring => isRetiring, + ); + const workflows = (await mastra.getStorage()!.getStore('workflows'))!; + const settledTask = async () => + (await waitFor( + () => mastra.backgroundTaskManager!.getTask(started.payload.taskId as string), + (t: any) => t?.status !== 'pending' && t?.status !== 'running', + )) as any; + return { memory, thread, workflows, runId: started.runId, release, finishCleanup, settledTask }; + } + + // Technical: a finished run releases its claim, then retires its memory fence. Until that + // retirement resolves, the execution stays registered in this process, and the late suspend's + // check asked it to prove it still held the claim it had just released, so the task failed + // with DURABLE_AGENT_EXECUTION_SUPERSEDED though no other execution had claimed the run. (#30182) + it.runIf(isDurableEngineVariant(engine))( + 'parks a deferred tool that asks for input while the finished run is still cleaning up (#30182)', + async () => { + const run = await startResearchDuringCleanup('bg-suspend-during-cleanup-thread'); + const released = await run.workflows.getRunOwnership({ runId: run.runId }); + run.release(); + try { + const parked = await run.settledTask(); + expect({ status: parked.status, error: parked.error?.message }).toEqual({ + status: 'suspended', + error: undefined, + }); + expect(parked.suspendPayload).toEqual({ awaiting: 'analyst-approval', topic: 'near' }); + const open = await waitFor( + () => storedToolCall(run.memory, run.thread), + s => s.suspended?.taskId !== undefined, + ); + expect(open.suspended).toMatchObject({ taskId: parked.id }); + // Accepting the late suspend left the released claim as it was. + const after = await run.workflows.getRunOwnership({ runId: run.runId }); + expect({ ownerId: after?.ownerId, generation: after?.generation, live: after?.live }).toEqual({ + ownerId: released?.ownerId, + generation: released?.generation, + live: false, + }); + } finally { + run.finishCleanup(); + } + }, + 30_000, + ); + + // Technical: the cleanup window above must not let a superseded execution through: once a + // successor claims the run, the late suspend still fails without writing. (#30182) + it.runIf(isDurableEngineVariant(engine))( + "doesn't park a deferred tool that asks for input after another execution took over a run still cleaning up (#30182)", + async () => { + const run = await startResearchDuringCleanup('bg-suspend-cleanup-takeover-thread'); + const released = await run.workflows.getRunOwnership({ runId: run.runId }); + const successor = await run.workflows.claimRunOwnership({ + runId: run.runId, + ownerId: 'successor-execution', + leaseMs: 60_000, + expectedGeneration: released!.generation, + }); + expect(successor.acquired).toBe(true); + run.release(); + try { + const settled = await run.settledTask(); + expect({ status: settled.status, error: settled.error?.message }).toEqual({ + status: 'failed', + // DURABLE_AGENT_EXECUTION_SUPERSEDED + error: expect.stringContaining(`Durable run ${run.runId} lost its execution lease`), + }); + expect(settled.suspendPayload).toBeUndefined(); + expect((await storedToolCall(run.memory, run.thread)).suspended).toBeUndefined(); + const after = await run.workflows.getRunOwnership({ runId: run.runId }); + expect({ ownerId: after?.ownerId, generation: after?.generation, live: after?.live }).toEqual({ + ownerId: 'successor-execution', + generation: successor.record.generation, + live: true, + }); + } finally { + run.finishCleanup(); + } + }, + 30_000, + ); + // Technical: same path as above, reached through a deferred sub-agent: when its tool needs // approval, the delegation tool suspends the background task with that approval. (#30182) it('parks a deferred sub-agent whose tool needs approval after the reply was sent, and resumes it (#30182)', async () => { From d1766814a4628c7c2ad77d957dece617c9a95844 Mon Sep 17 00:00:00 2001 From: Leo <108278866+leoisadev1@users.noreply.github.com> Date: Mon, 16 Nov 2026 13:23:30 -0500 Subject: [PATCH 04/15] fix(core): fence a late background suspension to the finished run's claim A late background suspend passed its ownership check and then saved to memory unfenced, because a run that settled as owner stops fencing its own writes. A successor that claimed the run and raised the memory fence while the save was in flight still found the old suspension written and the task suspended. The suspension's memory writes now carry the finished run's own claim, so storage rejects them once a newer claim raised the memory fence. On a shared evented transport, the worker's participant fence stayed registered between releasing the claim and stopping the participant, so a late suspend in that gap ran the strict check and failed as superseded. releaseSharedClaim now marks that participant fence as released. Adds unit tests for verifyNotSuperseded on the storage and lease backends. Co-Authored-By: Claude Opus 5.5 --- .../src/agent/durable/execution-fence.test.ts | 96 +++++++++++ .../core/src/agent/durable/execution-fence.ts | 44 ++++- .../durable/workflows/steps/tool-call.ts | 9 +- ...background-task-lifecycle.scenario.test.ts | 162 +++++++++++++++++- 4 files changed, 300 insertions(+), 11 deletions(-) diff --git a/packages/core/src/agent/durable/execution-fence.test.ts b/packages/core/src/agent/durable/execution-fence.test.ts index fbc10316346..40853913730 100644 --- a/packages/core/src/agent/durable/execution-fence.test.ts +++ b/packages/core/src/agent/durable/execution-fence.test.ts @@ -1194,6 +1194,102 @@ describe('storage-backed ExecutionFence', () => { }); }); +// A background task can write after its run settled: `verifyNotSuperseded` is the check for those +// writes. Before settlement releases the claim it is `verify`; after, it only refuses once another +// execution claimed the run (#30182). +describe.each([ + { + backend: 'storage', + async setup() { + const storage = new InMemoryStore(); + const workflowsStore = (await storage.getStore('workflows')) as WorkflowsStorage; + const fence = await ExecutionFence.claim({ + leaseProvider: NoopLeaseProvider, + workflowsStore, + agentId, + runId: 'run-1', + mode: 'acquire', + }); + const own = { runId: 'run-1', generation: fence.generation!, ownerId: fence.executionId }; + return { + fence, + dropClaim: async () => void (await workflowsStore.releaseRunOwnership(own)), + claimSuccessor: async () => + void (await workflowsStore.claimRunOwnership({ runId: 'run-1', ownerId: 'successor', leaseMs: 60_000 })), + takeOver: async () => + void (await workflowsStore.claimRunOwnership({ + runId: 'run-1', + ownerId: 'successor', + leaseMs: 60_000, + force: true, + })), + failNextRelease: () => + vi.spyOn(workflowsStore, 'releaseRunOwnership').mockRejectedValueOnce(new Error('store down')), + }; + }, + }, + { + backend: 'lease', + async setup() { + const pubsub = new EventEmitterPubSub(); + const fence = await claim(pubsub, 'run-1'); + return { + fence, + dropClaim: () => pubsub.releaseLease(key('run-1'), fence.executionId), + claimSuccessor: async () => void (await pubsub.acquireLease(key('run-1'), 'successor', EXECUTION_LEASE_TTL_MS)), + takeOver: async () => + void (await pubsub.transferLease(key('run-1'), fence.executionId, 'successor', EXECUTION_LEASE_TTL_MS)), + failNextRelease: () => vi.spyOn(pubsub, 'releaseLease').mockRejectedValueOnce(new Error('redis down')), + }; + }, + }, +])('ExecutionFence.verifyNotSuperseded ($backend)', ({ setup }) => { + it('before settlement, is verify: passes while owned, and refuses a claim that is gone though nobody took it', async () => { + const { fence, dropClaim } = await setup(); + await expect(fence.verifyNotSuperseded()).resolves.toBeUndefined(); + + await dropClaim(); + await expect(fence.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + expect(fence.isLost()).toBe(true); + }); + + it('after settlement released the claim, passes while no other execution claimed the run', async () => { + const { fence } = await setup(); + expect(await fence.settle(async () => {})).toBe('owned'); + + await expect(fence.verify()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + await expect(fence.verifyNotSuperseded()).resolves.toBeUndefined(); + }); + + it('after settlement released the claim, refuses once another execution claimed the run', async () => { + const { fence, claimSuccessor } = await setup(); + expect(await fence.settle(async () => {})).toBe('owned'); + + await claimSuccessor(); + await expect(fence.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + }); + + it('stays strict when settlement failed to release the claim', async () => { + const { fence, dropClaim, failNextRelease } = await setup(); + failNextRelease(); + expect(await fence.settle(async () => {})).toBe('owned'); + // The claim is still held, so the check passes as verify would. + await expect(fence.verifyNotSuperseded()).resolves.toBeUndefined(); + + // The unreleased claim lapses with nobody taking the run: verify's rule, not the post-release one. + await dropClaim(); + await expect(fence.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + }); + + it('refuses after settling as superseded', async () => { + const { fence, takeOver } = await setup(); + await takeOver(); + expect(await fence.settle(async () => {})).toBe('superseded'); + + await expect(fence.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + }); +}); + describe('execution claim carrier', () => { it('round-trips the claim and drops malformed entries', () => { const requestContext = new RequestContext(); diff --git a/packages/core/src/agent/durable/execution-fence.ts b/packages/core/src/agent/durable/execution-fence.ts index d0894dbea84..f3a4573d758 100644 --- a/packages/core/src/agent/durable/execution-fence.ts +++ b/packages/core/src/agent/durable/execution-fence.ts @@ -38,7 +38,7 @@ import { MASTRA_DURABLE_EXECUTIONS_KEY } from '../request-context-snapshot'; import { AGENT_LEASE_RENEW_INTERVAL_MS, AGENT_LEASE_TTL_MS } from '../run-ownership'; import { agentThreadStreamRuntime } from '../thread-stream-runtime'; import { ExecutionOwnershipGuard } from './execution-ownership-guard'; -import { runInRunFenceScope } from './run-fence-scope'; +import { currentRunFenceScope, runInRunFenceScope } from './run-fence-scope'; import { getRunRegistryEntry } from './run-registry'; /** Execution lease timing follows the agent thread lease (`MASTRA_AGENT_THREAD_LEASE_TTL_MS`). */ @@ -692,7 +692,12 @@ export class ExecutionFence implements RunFenceScope { static async releaseSharedClaim(mastra: Mastra, runId: string, claim: DurableExecutionClaim): Promise { if (!claim.agentId) return; const backend = await resolveRemoteBackend(mastra, claim.agentId, runId, claim); - await backend?.release(claim); + if (!backend) return; + await backend.release(claim); + // This worker's participant fence stays registered until its cleanup stops it. Late writes + // checked against it meanwhile get the post-release rule of {@link verifyNotSuperseded}. + const participant = mastra.__getRunScope(runId)?.get(DURABLE_PARTICIPANT_FENCES)?.get(claim.executionId); + if (participant) participant.#released = true; } /** Join an existing claim without acquiring ownership or advancing its generation. */ @@ -1341,6 +1346,41 @@ export async function assertExecutionNotSuperseded(args: { } } +/** + * Run `fn` with its writes to `memoryStore` carrying the memory fence of the + * claim `requestContext` holds for `runId`, also once that execution settled + * and stopped fencing its own writes. For writes that may land after the run + * finished, such as a background task's suspension: an ownership check before + * the write can't see a successor that claims the run while the write is on + * its way, but storage rejects the write once that successor raised the memory + * fence. Runs `fn` unchanged when the claim never raised memory (lease-backed + * claims, memory stores without run fencing). Other writes keep the fence of + * the enclosing scope. + */ +export async function runFencedToClaim( + args: { + requestContext: RequestContext | undefined; + runId: string; + /** Resolves the memory store the claim raised; called only when it raised one. */ + memoryStore: () => Promise; + }, + fn: () => Promise, +): Promise { + const fence = getRunFence(args.requestContext, args.runId, 'memory'); + const memoryStore = fence && (await args.memoryStore()); + if (!fence || !memoryStore) return fn(); + const outer = currentRunFenceScope(); + return runInRunFenceScope( + { + fenceFor: (store, runId) => (store === memoryStore ? fence : outer?.fenceFor(store, runId)), + onConflict: conflict => outer?.onConflict?.(conflict), + coverNestedRun: (parentRunId, nestedRunId) => outer?.coverNestedRun?.(parentRunId, nestedRunId), + generationFor: runId => outer?.generationFor?.(runId), + }, + fn, + ); +} + const OWNERSHIP_GUARD = Symbol('durable-execution-ownership-guard'); type GuardHandle = { scope: RunScope; key: RunScopeKey }; diff --git a/packages/core/src/agent/durable/workflows/steps/tool-call.ts b/packages/core/src/agent/durable/workflows/steps/tool-call.ts index 05fea0e8307..0d3134da048 100644 --- a/packages/core/src/agent/durable/workflows/steps/tool-call.ts +++ b/packages/core/src/agent/durable/workflows/steps/tool-call.ts @@ -70,6 +70,7 @@ import { assertExecutionOwned, getExecutionOwnershipGuard, isExecutionFenceError, + runFencedToClaim, } from '../../execution-fence'; import { authorizeDurableMemory, getDurableMemoryAuthorizationChecks } from '../../memory-fga'; import { clearResumeInFlight, getRunRegistryEntry, globalRunRegistry, markRunActive } from '../../run-registry'; @@ -1852,7 +1853,13 @@ export function createDurableToolCallStep() { // suspension state (#23739) — never the model-authored one. suspendedToolRunId: taskContext?.suspendedToolRunId, suspend: async (data?: unknown, options?: SuspendOptions) => { - await suspendToolCall(data, options, assertNotSuperseded); + // The run may have settled and stopped fencing its writes. The suspension's + // memory save carries the run's claim, so storage rejects it once a successor + // raised the memory fence, even when it lands after the ownership check (#30182). + await runFencedToClaim( + { requestContext, runId, memoryStore: async () => memory?.storage.getStore('memory') }, + () => suspendToolCall(data, options, assertNotSuperseded), + ); return taskContext?.suspend?.(data, options); }, outputWriter: async (chunk: any) => { diff --git a/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts b/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts index d21d632a3f2..be97e75c48d 100644 --- a/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts +++ b/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts @@ -1,8 +1,12 @@ +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import { stepCountIs } from '@internal/ai-sdk-v5'; import { createOpenAI } from '@ai-sdk/openai-v5'; -import { it, expect, vi } from 'vitest'; +import { it, expect, onTestFinished, vi } from 'vitest'; import { z } from 'zod'; import { Agent } from '../../../../agent'; +import { agentThreadStreamRuntime } from '../../../../agent/thread-stream-runtime'; import { MockMemory } from '../../../../memory/mock'; import { RequestContext } from '../../../../request-context'; import { createTool } from '../../../../tools'; @@ -15,6 +19,8 @@ import { isDurableEngineVariant, } from '../aimock-scenario'; import { BackgroundTaskManager } from '../../../../background-tasks/manager'; +import type { PubSub } from '../../../../events/pubsub'; +import { UnixSocketPubSub } from '../../../../events/unix-socket-pubsub'; import type { Mastra } from '../../../../mastra'; import { SCENARIO_MODEL_ID } from '../types'; @@ -870,21 +876,41 @@ describeForAllEngines( ); // Starts the deferred research tool, lets the run finish and release its claim, and holds - // the run's memory fence retirement open, as slow storage would, until `finishCleanup`. - async function startResearchDuringCleanup(threadId: string) { + // one cleanup step after the release open, as slow storage would, until `finishCleanup`: + // retiring the run's memory fence, or releasing the thread's lease (evented, with a shared + // `pubsub`). `cleanedUp` resolves once the held step has finished. + async function startResearchDuringCleanup( + threadId: string, + { hold = 'memory-fence', pubsub }: { hold?: 'memory-fence' | 'thread-lease'; pubsub?: PubSub } = {}, + ) { let release!: () => void; const gate = new Promise(resolve => (release = resolve)); let finishCleanup!: () => void; const cleanupGate = new Promise(resolve => (finishCleanup = resolve)); + let signalCleanedUp!: () => void; + const cleanedUp = new Promise(resolve => (signalCleanedUp = resolve)); const memory = new MockMemory(); const memoryStore = (await memory.storage.getStore('memory'))!; - const retireRunFence = memoryStore.retireRunFence.bind(memoryStore); let retiring = false; - vi.spyOn(memoryStore, 'retireRunFence').mockImplementation(async fence => { + const holdOpen = async (step: () => Promise): Promise => { retiring = true; await cleanupGate; - return retireRunFence(fence); - }); + try { + return await step(); + } finally { + signalCleanedUp(); + } + }; + if (hold === 'memory-fence') { + const retireRunFence = memoryStore.retireRunFence.bind(memoryStore); + vi.spyOn(memoryStore, 'retireRunFence').mockImplementation(fence => holdOpen(() => retireRunFence(fence))); + } else { + const releaseLease = agentThreadStreamRuntime.releaseDurableThreadLease.bind(agentThreadStreamRuntime); + const spy = vi + .spyOn(agentThreadStreamRuntime, 'releaseDurableThreadLease') + .mockImplementation((...args) => holdOpen(() => releaseLease(...args))); + onTestFinished(() => spy.mockRestore()); + } const thread = { threadId, resourceId: `${threadId}-user` }; const { chunks, mastra, output } = await runLoopScenario({ engine, @@ -895,6 +921,7 @@ describeForAllEngines( backgroundTasks: BG_CONFIG, memory, ...thread, + ...(pubsub ? { pubsub } : {}), stopWhen: stepCountIs(3), collectChunks: true, fixtures: llm => @@ -919,7 +946,17 @@ describeForAllEngines( () => mastra.backgroundTaskManager!.getTask(started.payload.taskId as string), (t: any) => t?.status !== 'pending' && t?.status !== 'running', )) as any; - return { memory, thread, workflows, runId: started.runId, release, finishCleanup, settledTask }; + return { + memory, + memoryStore, + thread, + workflows, + runId: started.runId, + release, + finishCleanup, + cleanedUp, + settledTask, + }; } // Technical: a finished run releases its claim, then retires its memory fence. Until that @@ -995,6 +1032,115 @@ describeForAllEngines( 30_000, ); + // Technical: the late suspend's ownership check passes while no successor exists, but its + // memory save can reach storage later. A finished run stops fencing its own writes once it + // settles, so the save went through unfenced: a successor that claimed the run and raised the + // memory fence in between found the old execution's suspension in memory and the task parked. + // The save now carries the finished run's claim, and storage rejects it. (#30182) + it.runIf(isDurableEngineVariant(engine))( + "doesn't park a deferred tool whose suspension reaches storage after another execution took over (#30182)", + async () => { + const run = await startResearchDuringCleanup('bg-suspend-slow-save-takeover-thread'); + let signalSaveStarted!: () => void; + const saveStarted = new Promise(resolve => (signalSaveStarted = resolve)); + let finishSave!: () => void; + const saveGate = new Promise(resolve => (finishSave = resolve)); + const saveMessages = run.memoryStore.saveMessages.bind(run.memoryStore); + vi.spyOn(run.memoryStore, 'saveMessages').mockImplementation(async args => { + signalSaveStarted(); + await saveGate; + return saveMessages(args); + }); + try { + run.release(); + // The suspension passed its ownership check; its save is on the way to storage. + await saveStarted; + // The finished run completes its cleanup. Then a successor claims the run and raises + // the memory fence, as recovery or a new turn does before it touches memory. + run.finishCleanup(); + await run.cleanedUp; + const released = await run.workflows.getRunOwnership({ runId: run.runId }); + const successor = await run.workflows.claimRunOwnership({ + runId: run.runId, + ownerId: 'successor-execution', + leaseMs: 60_000, + expectedGeneration: released!.generation, + }); + expect(successor.acquired).toBe(true); + const successorFence = { + runId: run.runId, + generation: successor.record!.generation, + ownerId: successor.record!.ownerId, + }; + expect(await run.memoryStore.raiseRunFence(successorFence)).toBe(true); + finishSave(); + + const settled = await run.settledTask(); + expect({ status: settled.status, error: settled.error?.message }).toEqual({ + status: 'failed', + // STORAGE_RUN_FENCE_CONFLICT + error: expect.stringContaining("is no longer the run's current claim"), + }); + expect(settled.suspendPayload).toBeUndefined(); + expect((await storedToolCall(run.memory, run.thread)).suspended).toBeUndefined(); + const after = await run.workflows.getRunOwnership({ runId: run.runId }); + expect({ ownerId: after?.ownerId, generation: after?.generation, live: after?.live }).toEqual({ + ownerId: 'successor-execution', + generation: successor.record!.generation, + live: true, + }); + } finally { + finishSave(); + run.finishCleanup(); + } + }, + 30_000, + ); + + // Technical: on a shared transport, an evented run's worker releases the run's claim, then + // the thread's lease, and only then stops its participant fence. Until the lease release + // returns, that fence stays registered, and the late suspend's check asked it to prove it + // still held the claim it had just released, so the task failed with + // DURABLE_AGENT_EXECUTION_SUPERSEDED though nobody took over. (#30182) + it.runIf(engine === 'evented')( + 'parks a deferred tool that asks for input while a shared evented run is still releasing its thread (#30182)', + async () => { + const directory = await mkdtemp(join(tmpdir(), 'mastra-30182-')); + const pubsub = new UnixSocketPubSub(join(directory, 'pubsub.sock')); + const run = await startResearchDuringCleanup('bg-suspend-during-lease-release-thread', { + hold: 'thread-lease', + pubsub, + }); + const released = await run.workflows.getRunOwnership({ runId: run.runId }); + run.release(); + try { + const parked = await run.settledTask(); + expect({ status: parked.status, error: parked.error?.message }).toEqual({ + status: 'suspended', + error: undefined, + }); + expect(parked.suspendPayload).toEqual({ awaiting: 'analyst-approval', topic: 'near' }); + const open = await waitFor( + () => storedToolCall(run.memory, run.thread), + s => s.suspended?.taskId !== undefined, + ); + expect(open.suspended).toMatchObject({ taskId: parked.id }); + const after = await run.workflows.getRunOwnership({ runId: run.runId }); + expect({ ownerId: after?.ownerId, generation: after?.generation, live: after?.live }).toEqual({ + ownerId: released?.ownerId, + generation: released?.generation, + live: false, + }); + } finally { + run.finishCleanup(); + await run.cleanedUp; + await pubsub.close(); + await rm(directory, { recursive: true, force: true }); + } + }, + 30_000, + ); + // Technical: same path as above, reached through a deferred sub-agent: when its tool needs // approval, the delegation tool suspends the background task with that approval. (#30182) it('parks a deferred sub-agent whose tool needs approval after the reply was sent, and resumes it (#30182)', async () => { From 697e8f3dcac708c5a27b5487b4fc00e69d301c46 Mon Sep 17 00:00:00 2001 From: Leo <108278866+leoisadev1@users.noreply.github.com> Date: Mon, 16 Nov 2026 13:51:31 -0500 Subject: [PATCH 05/15] fix(core): don't mark a late background suspend lost while the finished run's release is in flight A post-settlement ownership check that read the claim after storage cleared its lease, but before the release call returned, ran the strict check and marked the execution lost though nobody took the run. The fence now records a release when it starts; a check that fails during it waits for the release and then applies the post-release supersession rule, staying strict if the release failed. Co-Authored-By: Claude Opus 5.5 --- .../src/agent/durable/execution-fence.test.ts | 67 ++++++- .../core/src/agent/durable/execution-fence.ts | 51 +++++- ...background-task-lifecycle.scenario.test.ts | 172 +++++++++++++++++- 3 files changed, 278 insertions(+), 12 deletions(-) diff --git a/packages/core/src/agent/durable/execution-fence.test.ts b/packages/core/src/agent/durable/execution-fence.test.ts index 40853913730..b24596e3a9c 100644 --- a/packages/core/src/agent/durable/execution-fence.test.ts +++ b/packages/core/src/agent/durable/execution-fence.test.ts @@ -1194,9 +1194,30 @@ describe('storage-backed ExecutionFence', () => { }); }); +/** + * Make the next call of `method` on `target` apply (unless `fail`), then wait for `finish()` before + * returning or, with `fail`, throwing: a release storage has applied but not yet confirmed. + */ +function holdNextCall(target: T, method: K, fail: boolean) { + const original = (target[method] as (...args: unknown[]) => Promise).bind(target); + let signalApplied!: () => void; + const applied = new Promise(resolve => (signalApplied = resolve)); + let finish!: () => void; + const finished = new Promise(resolve => (finish = resolve)); + vi.spyOn(target, method as never).mockImplementationOnce((async (...args: unknown[]) => { + const result = fail ? undefined : await original(...args); + signalApplied(); + await finished; + if (fail) throw new Error('backend down'); + return result; + }) as never); + return { applied, finish }; +} + // A background task can write after its run settled: `verifyNotSuperseded` is the check for those // writes. Before settlement releases the claim it is `verify`; after, it only refuses once another -// execution claimed the run (#30182). +// execution claimed the run. While the release is in flight, a check that fails waits for the +// release before deciding (#30182). describe.each([ { backend: 'storage', @@ -1225,6 +1246,7 @@ describe.each([ })), failNextRelease: () => vi.spyOn(workflowsStore, 'releaseRunOwnership').mockRejectedValueOnce(new Error('store down')), + holdNextRelease: ({ fail = false } = {}) => holdNextCall(workflowsStore, 'releaseRunOwnership', fail), }; }, }, @@ -1240,6 +1262,7 @@ describe.each([ takeOver: async () => void (await pubsub.transferLease(key('run-1'), fence.executionId, 'successor', EXECUTION_LEASE_TTL_MS)), failNextRelease: () => vi.spyOn(pubsub, 'releaseLease').mockRejectedValueOnce(new Error('redis down')), + holdNextRelease: ({ fail = false } = {}) => holdNextCall(pubsub, 'releaseLease', fail), }; }, }, @@ -1281,6 +1304,48 @@ describe.each([ await expect(fence.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); }); + it('while settlement is releasing the claim, waits for the release and passes while no other execution claimed the run', async () => { + const { fence, holdNextRelease } = await setup(); + const release = holdNextRelease(); + const settled = fence.settle(async () => {}); + await release.applied; + + // The backend already shows the claim released; the strict check alone would mark the execution lost. + const check = fence.verifyNotSuperseded(); + release.finish(); + await expect(check).resolves.toBeUndefined(); + expect(fence.isLost()).toBe(false); + expect(await settled).toBe('owned'); + }); + + it('while settlement is releasing the claim, refuses once another execution claimed the run', async () => { + const { fence, holdNextRelease, claimSuccessor } = await setup(); + const release = holdNextRelease(); + const settled = fence.settle(async () => {}); + await release.applied; + await claimSuccessor(); + + const check = fence.verifyNotSuperseded(); + release.finish(); + await expect(check).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + await settled; + }); + + it('stays strict when the release in flight fails', async () => { + const { fence, dropClaim, holdNextRelease } = await setup(); + // The claim lapsed with nobody taking the run; settlement still tries to release it. + await dropClaim(); + const release = holdNextRelease({ fail: true }); + const settled = fence.settle(async () => {}); + await release.applied; + + const check = fence.verifyNotSuperseded(); + release.finish(); + await expect(check).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + expect(fence.isLost()).toBe(true); + await settled; + }); + it('refuses after settling as superseded', async () => { const { fence, takeOver } = await setup(); await takeOver(); diff --git a/packages/core/src/agent/durable/execution-fence.ts b/packages/core/src/agent/durable/execution-fence.ts index f3a4573d758..ccfc56b9cd4 100644 --- a/packages/core/src/agent/durable/execution-fence.ts +++ b/packages/core/src/agent/durable/execution-fence.ts @@ -572,6 +572,8 @@ export class ExecutionFence implements RunFenceScope { #settledAs?: ExecutionSettlement; /** Settlement released the claim; cleanup after the release may still be running. */ #released = false; + /** Set once settlement starts releasing the claim. Resolves to whether the release succeeded. */ + #releasing?: Promise; #resolveSettled!: () => void; /** Resolves once {@link settle} has finished and the claim is released or left to its new holder. */ readonly whenSettled: Promise; @@ -693,11 +695,12 @@ export class ExecutionFence implements RunFenceScope { if (!claim.agentId) return; const backend = await resolveRemoteBackend(mastra, claim.agentId, runId, claim); if (!backend) return; - await backend.release(claim); // This worker's participant fence stays registered until its cleanup stops it. Late writes // checked against it meanwhile get the post-release rule of {@link verifyNotSuperseded}. const participant = mastra.__getRunScope(runId)?.get(DURABLE_PARTICIPANT_FENCES)?.get(claim.executionId); - if (participant) participant.#released = true; + const release = backend.release(claim); + if (participant) void participant.#trackRelease(release); + await release; } /** Join an existing claim without acquiring ownership or advancing its generation. */ @@ -919,26 +922,43 @@ export class ExecutionFence implements RunFenceScope { * never recovers. */ async verify(): Promise { - if (this.#lossError) throw this.#lossError; + await this.#verify(false); + } + + /** + * {@link verify}. With `yieldToRelease`, a check that fails once settlement + * has started releasing the claim returns `false` instead of marking the + * execution lost: the check may have read that release, not a successor. + */ + async #verify(yieldToRelease: boolean): Promise { + if (this.#lossError) { + if (yieldToRelease && this.#releasing) return false; + throw this.#lossError; + } const startedAt = Date.now(); try { await this.#backend.verify(this.claim, this.#details()); } catch (error) { + if (yieldToRelease && this.#releasing) return false; this.#markLost(error as DurableExecutionFenceError); if ((error as DurableExecutionFenceError).id === EXECUTION_SUPERSEDED_ERROR_ID) await this.#noteTakeover(); throw error; } // A lease backend renews on verify; a storage read leaves expiry to the heartbeat. if (this.generation === undefined) this.#leaseExpiresAt = startedAt + EXECUTION_LEASE_TTL_MS; + return true; } /** * {@link verify} for writes that may land after settlement. Once settlement * released the claim, the write is accepted until another execution claims - * the run, as it is after this fence is unregistered. + * the run, as it is after this fence is unregistered. A check that fails + * while the release is in flight waits for it, then applies the same rule. */ async verifyNotSuperseded(): Promise { - if (!this.#released) return this.verify(); + if (!this.#released && (await this.#verify(true))) return; + // Settlement released the claim, or is releasing it. A release that failed left the claim held: stay strict. + if (!(await this.#releasing)) return this.verify(); const details = this.#details(); if (this.#takenOver || (await retryOnce(() => this.#backend.isSuperseded(this.claim), details))) { throw new DurableExecutionFenceError(EXECUTION_SUPERSEDED_ERROR_ID, details); @@ -1063,9 +1083,10 @@ export class ExecutionFence implements RunFenceScope { } async #release(): Promise { + const release = this.#backend.release(this.claim); + void this.#trackRelease(release); try { - await this.#backend.release(this.claim); - this.#released = true; + await release; } catch (error) { this.#logger?.warn?.(`[DurableAgent] run ${this.runId}: failed to release the execution claim: ${error}`); } @@ -1074,6 +1095,22 @@ export class ExecutionFence implements RunFenceScope { } } + /** + * Record a release of this claim for {@link verifyNotSuperseded}. A failed + * release keeps the outcome of an earlier one. + */ + #trackRelease(release: Promise): Promise { + const previous = this.#releasing; + this.#releasing = release.then( + () => { + this.#released = true; + return true; + }, + async () => (await previous) ?? false, + ); + return this.#releasing; + } + /** Mark a settled fence retired in memory so `memory.runFences` retention can prune it. */ async #retireMemoryFence(fence: RunFence): Promise { if (!this.#memoryStore) return; diff --git a/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts b/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts index be97e75c48d..4086e307cac 100644 --- a/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts +++ b/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts @@ -22,6 +22,7 @@ import { BackgroundTaskManager } from '../../../../background-tasks/manager'; import type { PubSub } from '../../../../events/pubsub'; import { UnixSocketPubSub } from '../../../../events/unix-socket-pubsub'; import type { Mastra } from '../../../../mastra'; +import { WorkflowsInMemory } from '../../../../storage/domains/workflows/inmemory'; import { SCENARIO_MODEL_ID } from '../types'; /** @@ -876,12 +877,16 @@ describeForAllEngines( ); // Starts the deferred research tool, lets the run finish and release its claim, and holds - // one cleanup step after the release open, as slow storage would, until `finishCleanup`: - // retiring the run's memory fence, or releasing the thread's lease (evented, with a shared - // `pubsub`). `cleanedUp` resolves once the held step has finished. + // one cleanup step open, as slow storage would, until `finishCleanup`: retiring the run's + // memory fence, releasing the thread's lease (evented, with a shared `pubsub`), or the claim + // release itself, which storage has applied but not yet returned to the run. `cleanedUp` + // resolves once the held step has finished. async function startResearchDuringCleanup( threadId: string, - { hold = 'memory-fence', pubsub }: { hold?: 'memory-fence' | 'thread-lease'; pubsub?: PubSub } = {}, + { + hold = 'memory-fence', + pubsub, + }: { hold?: 'memory-fence' | 'thread-lease' | 'claim-release'; pubsub?: PubSub } = {}, ) { let release!: () => void; const gate = new Promise(resolve => (release = resolve)); @@ -904,6 +909,16 @@ describeForAllEngines( if (hold === 'memory-fence') { const retireRunFence = memoryStore.retireRunFence.bind(memoryStore); vi.spyOn(memoryStore, 'retireRunFence').mockImplementation(fence => holdOpen(() => retireRunFence(fence))); + } else if (hold === 'claim-release') { + // The scenario builds its own store, so the hold goes on the in-memory workflows domain. + const releaseRunOwnership = WorkflowsInMemory.prototype.releaseRunOwnership; + const spy = vi + .spyOn(WorkflowsInMemory.prototype, 'releaseRunOwnership') + .mockImplementation(async function (this: WorkflowsInMemory, fence) { + const released = await releaseRunOwnership.call(this, fence); + return holdOpen(async () => released); + }); + onTestFinished(() => spy.mockRestore()); } else { const releaseLease = agentThreadStreamRuntime.releaseDurableThreadLease.bind(agentThreadStreamRuntime); const spy = vi @@ -1097,6 +1112,110 @@ describeForAllEngines( 30_000, ); + // Resolves once the next ownership read after this call has returned, which is the late + // suspend's check when nothing else reads the record meanwhile. + function nextOwnershipRead() { + let signalRead!: () => void; + const read = new Promise(resolve => (signalRead = resolve)); + const getRunOwnership = WorkflowsInMemory.prototype.getRunOwnership; + const spy = vi + .spyOn(WorkflowsInMemory.prototype, 'getRunOwnership') + .mockImplementation(async function (this: WorkflowsInMemory, args) { + try { + return await getRunOwnership.call(this, args); + } finally { + signalRead(); + } + }); + onTestFinished(() => spy.mockRestore()); + return read; + } + + // Technical: settlement releases the claim, and only once storage's release call returns + // does the run count as released. The late suspend checked ownership in between: storage + // already showed the claim released, so the strict check failed and marked the execution + // lost though nobody had taken over, and the task failed before it suspended. (#30182) + it.runIf(isDurableEngineVariant(engine))( + 'parks a deferred tool that asks for input while the finished run is still releasing its claim (#30182)', + async () => { + const run = await startResearchDuringCleanup('bg-suspend-during-claim-release-thread', { + hold: 'claim-release', + }); + const released = await run.workflows.getRunOwnership({ runId: run.runId }); + expect(released?.live).toBe(false); + const checked = nextOwnershipRead(); + run.release(); + try { + // The suspend read the released claim while the release was still in flight. + await checked; + run.finishCleanup(); + const parked = await run.settledTask(); + expect({ status: parked.status, error: parked.error?.message }).toEqual({ + status: 'suspended', + error: undefined, + }); + expect(parked.suspendPayload).toEqual({ awaiting: 'analyst-approval', topic: 'near' }); + const open = await waitFor( + () => storedToolCall(run.memory, run.thread), + s => s.suspended?.taskId !== undefined, + ); + expect(open.suspended).toMatchObject({ taskId: parked.id }); + const after = await run.workflows.getRunOwnership({ runId: run.runId }); + expect({ ownerId: after?.ownerId, generation: after?.generation, live: after?.live }).toEqual({ + ownerId: released?.ownerId, + generation: released?.generation, + live: false, + }); + } finally { + run.finishCleanup(); + } + }, + 30_000, + ); + + // Technical: waiting out an in-flight release must not let a superseded execution through: + // a successor that claimed the run during the release still makes the late suspend fail + // without writing. (#30182) + it.runIf(isDurableEngineVariant(engine))( + "doesn't park a deferred tool that asks for input after another execution took over a run still releasing its claim (#30182)", + async () => { + const run = await startResearchDuringCleanup('bg-suspend-claim-release-takeover-thread', { + hold: 'claim-release', + }); + const released = await run.workflows.getRunOwnership({ runId: run.runId }); + const successor = await run.workflows.claimRunOwnership({ + runId: run.runId, + ownerId: 'successor-execution', + leaseMs: 60_000, + expectedGeneration: released!.generation, + }); + expect(successor.acquired).toBe(true); + const checked = nextOwnershipRead(); + run.release(); + try { + await checked; + run.finishCleanup(); + const settled = await run.settledTask(); + expect({ status: settled.status, error: settled.error?.message }).toEqual({ + status: 'failed', + // DURABLE_AGENT_EXECUTION_SUPERSEDED + error: expect.stringContaining(`Durable run ${run.runId} lost its execution lease`), + }); + expect(settled.suspendPayload).toBeUndefined(); + expect((await storedToolCall(run.memory, run.thread)).suspended).toBeUndefined(); + const after = await run.workflows.getRunOwnership({ runId: run.runId }); + expect({ ownerId: after?.ownerId, generation: after?.generation, live: after?.live }).toEqual({ + ownerId: 'successor-execution', + generation: successor.record.generation, + live: true, + }); + } finally { + run.finishCleanup(); + } + }, + 30_000, + ); + // Technical: on a shared transport, an evented run's worker releases the run's claim, then // the thread's lease, and only then stops its participant fence. Until the lease release // returns, that fence stays registered, and the late suspend's check asked it to prove it @@ -1141,6 +1260,51 @@ describeForAllEngines( 30_000, ); + // Technical: the same in-flight release as above, on the worker's participant fence: an + // evented worker on a shared transport releases the run's claim itself, and a late suspend + // that read the released claim before that release returned marked the execution lost. (#30182) + it.runIf(engine === 'evented')( + 'parks a deferred tool that asks for input while a shared evented run is still releasing its claim (#30182)', + async () => { + const directory = await mkdtemp(join(tmpdir(), 'mastra-30182-')); + const pubsub = new UnixSocketPubSub(join(directory, 'pubsub.sock')); + const run = await startResearchDuringCleanup('bg-suspend-during-shared-claim-release-thread', { + hold: 'claim-release', + pubsub, + }); + const released = await run.workflows.getRunOwnership({ runId: run.runId }); + const checked = nextOwnershipRead(); + run.release(); + try { + await checked; + run.finishCleanup(); + const parked = await run.settledTask(); + expect({ status: parked.status, error: parked.error?.message }).toEqual({ + status: 'suspended', + error: undefined, + }); + expect(parked.suspendPayload).toEqual({ awaiting: 'analyst-approval', topic: 'near' }); + const open = await waitFor( + () => storedToolCall(run.memory, run.thread), + s => s.suspended?.taskId !== undefined, + ); + expect(open.suspended).toMatchObject({ taskId: parked.id }); + const after = await run.workflows.getRunOwnership({ runId: run.runId }); + expect({ ownerId: after?.ownerId, generation: after?.generation, live: after?.live }).toEqual({ + ownerId: released?.ownerId, + generation: released?.generation, + live: false, + }); + } finally { + run.finishCleanup(); + await run.cleanedUp; + await pubsub.close(); + await rm(directory, { recursive: true, force: true }); + } + }, + 30_000, + ); + // Technical: same path as above, reached through a deferred sub-agent: when its tool needs // approval, the delegation tool suspends the background task with that approval. (#30182) it('parks a deferred sub-agent whose tool needs approval after the reply was sent, and resumes it (#30182)', async () => { From 82d76c50833a48cfb9ca1e559efdc55e258dc5fb Mon Sep 17 00:00:00 2001 From: Leo <108278866+leoisadev1@users.noreply.github.com> Date: Mon, 16 Nov 2026 14:38:25 -0500 Subject: [PATCH 06/15] fix(core): accept a late background write after a release only storage shows A late background suspend failed as superseded when the shared claim was released by another evented worker, or when storage applied this process's release but its response failed. A post-settlement check that fails without a confirmed release now reads the ownership record: the same owner and generation with the lease cleared counts as released and gets the post-release rule. A successor is refused; a lapsed lease, a lease-backed claim or a failed read stays strict. Foreground verify() is unchanged. Adds the two-worker regression, a lost-release-ack scenario and its takeover case, and a late onResult during cleanup. (#30182) Co-Authored-By: Claude Opus 5.5 --- .changeset/twelve-papers-stand.md | 2 +- .../agent/durable/evented-execution.test.ts | 58 ++++++ .../src/agent/durable/execution-fence.test.ts | 188 ++++++++++++++++-- .../core/src/agent/durable/execution-fence.ts | 56 ++++-- ...background-task-lifecycle.scenario.test.ts | 152 +++++++++++++- 5 files changed, 420 insertions(+), 36 deletions(-) diff --git a/.changeset/twelve-papers-stand.md b/.changeset/twelve-papers-stand.md index 759a399d8dc..c0d62854195 100644 --- a/.changeset/twelve-papers-stand.md +++ b/.changeset/twelve-papers-stand.md @@ -2,4 +2,4 @@ '@mastra/core': patch --- -Fixed background tasks on durable agents with memory failing with "could not verify that this execution still owns the run" when they suspend after the agent's run has finished. The task is now marked suspended and can be resumed with `backgroundTaskManager.resume()`. This covers tools that call `suspend()` and sub-agents that reach a tool needing approval. Fixes [#30182](https://github.com/mastra-ai/mastra/issues/30182). +Fixed background tasks on durable agents with memory failing with "could not verify that this execution still owns the run" when they suspend after the agent's run has finished. The task is now marked suspended and can be resumed with `backgroundTaskManager.resume()`. This covers tools that call `suspend()` and sub-agents that reach a tool needing approval, including when another evented worker finished the run. A background task's result that arrives while the finished run is still cleaning up is now saved to memory instead of being dropped. Fixes [#30182](https://github.com/mastra-ai/mastra/issues/30182). diff --git a/packages/core/src/agent/durable/evented-execution.test.ts b/packages/core/src/agent/durable/evented-execution.test.ts index 10f445d1adf..4e839e455a4 100644 --- a/packages/core/src/agent/durable/evented-execution.test.ts +++ b/packages/core/src/agent/durable/evented-execution.test.ts @@ -31,6 +31,8 @@ import { } from './evented-execution'; import { __resetExecutionFencesForTests, + assertExecutionNotSuperseded, + DURABLE_PARTICIPANT_FENCES, EXECUTION_LEASE_TTL_MS, EXECUTION_SUPERSEDED_ERROR_ID, EXECUTION_UNVERIFIED_ERROR_ID, @@ -1287,6 +1289,62 @@ it('detaches worker participants on shutdown without invalidating a surviving pa }); }); +// Worker B ran the tool-call step; worker A handled the terminal event and released the shared claim. B's +// participant fence stays registered until its heartbeat notices, and a background task's late write on B +// was checked strictly against it, so it failed as superseded though nobody took the run. (#30182) +it("accepts a late background write on one worker after another worker released the run's claim", async () => { + const { workflowsStore, pubsub, mastra: workerA, coordinator, event } = await createExecution(); + const workerB = new Mastra({ storage: workerA.getStorage(), pubsub }); + instances.push(workerB); + await participateInDurableExecution(workerA, event, async () => {}); + await participateInDurableExecution(workerB, event, async () => {}); + const participantB = workerB.__getRunScope('run')!.get(DURABLE_PARTICIPANT_FENCES)!.get(coordinator.executionId)!; + + await settleEventedDurableExecution(workerA, { ...event, type: 'workflow.end' }); + expect(await workflowsStore.getRunOwnership({ runId: 'run' })).toMatchObject({ + ownerId: coordinator.executionId, + generation: coordinator.generation, + leaseExpiresAt: null, + }); + const late = { runId: 'run', agentId: 'agent', mastra: workerB }; + const requestContext = new RequestContext(Object.entries(event.data.requestContext)); + await expect(assertExecutionNotSuperseded({ ...late, requestContext })).resolves.toBeUndefined(); + expect(participantB.isLost()).toBe(false); + + // Once another execution claims the run, the late write on B is refused. + const successor = await workflowsStore.claimRunOwnership({ + runId: 'run', + ownerId: 'successor', + leaseMs: 60_000, + expectedGeneration: coordinator.generation, + }); + expect(successor.acquired).toBe(true); + await expect(assertExecutionNotSuperseded({ ...late, requestContext })).rejects.toMatchObject({ + id: EXECUTION_SUPERSEDED_ERROR_ID, + }); +}); + +it('refuses a late background write on one worker once a successor claimed the run another worker released', async () => { + const { workflowsStore, pubsub, mastra: workerA, coordinator, event } = await createExecution(); + const workerB = new Mastra({ storage: workerA.getStorage(), pubsub }); + instances.push(workerB); + await participateInDurableExecution(workerA, event, async () => {}); + await participateInDurableExecution(workerB, event, async () => {}); + + await settleEventedDurableExecution(workerA, { ...event, type: 'workflow.end' }); + await workflowsStore.claimRunOwnership({ + runId: 'run', + ownerId: 'successor', + leaseMs: 60_000, + expectedGeneration: coordinator.generation, + }); + const requestContext = new RequestContext(Object.entries(event.data.requestContext)); + await expect( + assertExecutionNotSuperseded({ runId: 'run', agentId: 'agent', requestContext, mastra: workerB }), + ).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + expect(await workflowsStore.getRunOwnership({ runId: 'run' })).toMatchObject({ ownerId: 'successor', live: true }); +}); + it('stops renewing the claim when the processor abandons a root terminal delivery', async () => { const directory = await mkdtemp(join(tmpdir(), 'mastra-abandoned-')); const pubsub = new UnixSocketPubSub(join(directory, 'pubsub.sock')); diff --git a/packages/core/src/agent/durable/execution-fence.test.ts b/packages/core/src/agent/durable/execution-fence.test.ts index b24596e3a9c..af982bb162c 100644 --- a/packages/core/src/agent/durable/execution-fence.test.ts +++ b/packages/core/src/agent/durable/execution-fence.test.ts @@ -13,6 +13,7 @@ import { __resetExecutionFencesForTests, assertExecutionNotSuperseded, assertExecutionOwned, + DURABLE_PARTICIPANT_FENCES, DurableExecutionFenceError, EXECUTION_ABANDONED_ERROR_ID, EXECUTION_CONFLICT_ERROR_ID, @@ -1195,17 +1196,22 @@ describe('storage-backed ExecutionFence', () => { }); /** - * Make the next call of `method` on `target` apply (unless `fail`), then wait for `finish()` before - * returning or, with `fail`, throwing: a release storage has applied but not yet confirmed. + * Make the next call of `method` on `target` apply (unless `apply` is false), then wait for `finish()` + * before returning or, with `fail`, throwing: a release storage has applied but not yet confirmed, or + * one whose confirmation is lost. */ -function holdNextCall(target: T, method: K, fail: boolean) { +function holdNextCall( + target: T, + method: K, + { apply, fail }: { apply: boolean; fail: boolean }, +) { const original = (target[method] as (...args: unknown[]) => Promise).bind(target); let signalApplied!: () => void; const applied = new Promise(resolve => (signalApplied = resolve)); let finish!: () => void; const finished = new Promise(resolve => (finish = resolve)); vi.spyOn(target, method as never).mockImplementationOnce((async (...args: unknown[]) => { - const result = fail ? undefined : await original(...args); + const result = apply ? await original(...args) : undefined; signalApplied(); await finished; if (fail) throw new Error('backend down'); @@ -1215,9 +1221,10 @@ function holdNextCall(target: T, m } // A background task can write after its run settled: `verifyNotSuperseded` is the check for those -// writes. Before settlement releases the claim it is `verify`; after, it only refuses once another +// writes. Before the claim is released it is `verify`; after, it only refuses once another // execution claimed the run. While the release is in flight, a check that fails waits for the -// release before deciding (#30182). +// release before deciding. Storage can also show a release this fence didn't make or couldn't +// confirm; a lease can't tell a release from a lapse (#30182). describe.each([ { backend: 'storage', @@ -1234,7 +1241,12 @@ describe.each([ const own = { runId: 'run-1', generation: fence.generation!, ownerId: fence.executionId }; return { fence, + showsRelease: true, dropClaim: async () => void (await workflowsStore.releaseRunOwnership(own)), + // The store lost the claim's record: nobody can say who holds the run. + loseClaim: () => workflowsStore.dangerouslyClearAll(), + failRenewals: () => + vi.spyOn(workflowsStore, 'renewRunOwnership').mockRejectedValue(new Error('store unreachable')), claimSuccessor: async () => void (await workflowsStore.claimRunOwnership({ runId: 'run-1', ownerId: 'successor', leaseMs: 60_000 })), takeOver: async () => @@ -1246,7 +1258,8 @@ describe.each([ })), failNextRelease: () => vi.spyOn(workflowsStore, 'releaseRunOwnership').mockRejectedValueOnce(new Error('store down')), - holdNextRelease: ({ fail = false } = {}) => holdNextCall(workflowsStore, 'releaseRunOwnership', fail), + holdNextRelease: ({ fail = false } = {}) => + holdNextCall(workflowsStore, 'releaseRunOwnership', { apply: !fail, fail }), }; }, }, @@ -1257,23 +1270,29 @@ describe.each([ const fence = await claim(pubsub, 'run-1'); return { fence, + showsRelease: false, dropClaim: () => pubsub.releaseLease(key('run-1'), fence.executionId), + loseClaim: () => pubsub.releaseLease(key('run-1'), fence.executionId), + failRenewals: () => vi.spyOn(pubsub, 'renewLease').mockRejectedValue(new Error('redis unreachable')), claimSuccessor: async () => void (await pubsub.acquireLease(key('run-1'), 'successor', EXECUTION_LEASE_TTL_MS)), takeOver: async () => void (await pubsub.transferLease(key('run-1'), fence.executionId, 'successor', EXECUTION_LEASE_TTL_MS)), failNextRelease: () => vi.spyOn(pubsub, 'releaseLease').mockRejectedValueOnce(new Error('redis down')), - holdNextRelease: ({ fail = false } = {}) => holdNextCall(pubsub, 'releaseLease', fail), + holdNextRelease: ({ fail = false } = {}) => holdNextCall(pubsub, 'releaseLease', { apply: !fail, fail }), }; }, }, ])('ExecutionFence.verifyNotSuperseded ($backend)', ({ setup }) => { - it('before settlement, is verify: passes while owned, and refuses a claim that is gone though nobody took it', async () => { - const { fence, dropClaim } = await setup(); + it('before settlement, is verify: passes while owned, and refuses a claim that expired though nobody took it', async () => { + vi.useFakeTimers(); + const { fence, failRenewals } = await setup(); await expect(fence.verifyNotSuperseded()).resolves.toBeUndefined(); - await dropClaim(); - await expect(fence.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + // Renewals can't reach the backend until the claim's lease runs out. + failRenewals(); + await vi.advanceTimersByTimeAsync(EXECUTION_LEASE_TTL_MS + EXECUTION_LEASE_RENEW_INTERVAL_MS); expect(fence.isLost()).toBe(true); + await expect(fence.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_UNVERIFIED_ERROR_ID }); }); it('after settlement released the claim, passes while no other execution claimed the run', async () => { @@ -1293,15 +1312,16 @@ describe.each([ }); it('stays strict when settlement failed to release the claim', async () => { - const { fence, dropClaim, failNextRelease } = await setup(); + const { fence, loseClaim, failNextRelease } = await setup(); failNextRelease(); expect(await fence.settle(async () => {})).toBe('owned'); // The claim is still held, so the check passes as verify would. await expect(fence.verifyNotSuperseded()).resolves.toBeUndefined(); - // The unreleased claim lapses with nobody taking the run: verify's rule, not the post-release one. - await dropClaim(); + // The unreleased claim is gone with nobody shown taking the run: verify's rule, not the post-release one. + await loseClaim(); await expect(fence.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + expect(fence.isLost()).toBe(true); }); it('while settlement is releasing the claim, waits for the release and passes while no other execution claimed the run', async () => { @@ -1331,9 +1351,9 @@ describe.each([ await settled; }); - it('stays strict when the release in flight fails', async () => { - const { fence, dropClaim, holdNextRelease } = await setup(); - // The claim lapsed with nobody taking the run; settlement still tries to release it. + it('when the release in flight fails, passes only if the backend shows this claim released', async () => { + const { fence, dropClaim, holdNextRelease, showsRelease } = await setup(); + // The claim was released already, as another worker would; settlement still tries to release it. await dropClaim(); const release = holdNextRelease({ fail: true }); const settled = fence.settle(async () => {}); @@ -1341,8 +1361,14 @@ describe.each([ const check = fence.verifyNotSuperseded(); release.finish(); - await expect(check).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); - expect(fence.isLost()).toBe(true); + if (showsRelease) { + // Storage still holds this exact claim, released: no other execution took the run. + await expect(check).resolves.toBeUndefined(); + } else { + // A missing lease may have lapsed: verify's rule. + await expect(check).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + expect(fence.isLost()).toBe(true); + } await settled; }); @@ -1355,6 +1381,128 @@ describe.each([ }); }); +// Storage keeps a released claim's record with its lease cleared, so it can show a release this fence +// didn't see: one made by another process, or one storage applied but whose response was lost (#30182). +describe('ExecutionFence.verifyNotSuperseded with a release only storage shows', () => { + async function setup() { + const storage = new InMemoryStore(); + const workflowsStore = (await storage.getStore('workflows')) as WorkflowsStorage; + const coordinator = await ExecutionFence.claim({ + leaseProvider: NoopLeaseProvider, + workflowsStore, + agentId, + runId: 'run-1', + mode: 'acquire', + }); + const own = { runId: 'run-1', generation: coordinator.generation!, ownerId: coordinator.executionId }; + const claimSuccessor = () => + workflowsStore.claimRunOwnership({ + runId: 'run-1', + ownerId: 'successor', + leaseMs: 60_000, + expectedGeneration: own.generation, + }); + return { storage, workflowsStore, coordinator, own, claimSuccessor }; + } + + it("passes on a worker's participant fence once another process released the shared claim", async () => { + const { storage, workflowsStore, coordinator, own, claimSuccessor } = await setup(); + coordinator.stopHeartbeat(); + const mastra = new Mastra({ storage, logger: false }); + const participant = (await ExecutionFence.follow(mastra, 'run-1', coordinator.claim))!; + onTestFinished(() => participant.stopHeartbeat()); + + // Another worker settled the run and released the claim; this process never saw it. + expect(await workflowsStore.releaseRunOwnership(own)).toBe(true); + await expect(participant.verifyNotSuperseded()).resolves.toBeUndefined(); + expect(participant.isLost()).toBe(false); + // Writes that need a live claim are still refused. + await expect(participant.verify()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + + // Once another execution claims the run, the late write is refused. + expect((await claimSuccessor()).acquired).toBe(true); + await expect(participant.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + }); + + it("passes on a participant fence whose heartbeat already saw another process's release", async () => { + vi.useFakeTimers(); + const { storage, workflowsStore, coordinator, own } = await setup(); + coordinator.stopHeartbeat(); + const mastra = new Mastra({ storage, logger: false }); + const participant = (await ExecutionFence.follow(mastra, 'run-1', coordinator.claim))!; + onTestFinished(() => participant.stopHeartbeat()); + + expect(await workflowsStore.releaseRunOwnership(own)).toBe(true); + // The next renewal finds the claim released and marks the fence lost. + await vi.advanceTimersByTimeAsync(EXECUTION_LEASE_RENEW_INTERVAL_MS + 1); + expect(participant.isLost()).toBe(true); + await expect(participant.verifyNotSuperseded()).resolves.toBeUndefined(); + }); + + it('refuses on a participant fence when another execution claimed the run, not a release', async () => { + const { storage, coordinator, workflowsStore } = await setup(); + coordinator.stopHeartbeat(); + const mastra = new Mastra({ storage, logger: false }); + const participant = (await ExecutionFence.follow(mastra, 'run-1', coordinator.claim))!; + onTestFinished(() => participant.stopHeartbeat()); + + await workflowsStore.claimRunOwnership({ runId: 'run-1', ownerId: 'successor', leaseMs: 60_000, force: true }); + await expect(participant.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + expect(participant.isLost()).toBe(true); + }); + + it("stays strict on a participant fence when its release failed and the claim's lease lapsed", async () => { + vi.useFakeTimers(); + const { storage, workflowsStore, coordinator } = await setup(); + coordinator.stopHeartbeat(); + const mastra = new Mastra({ storage, logger: false }); + const participant = (await ExecutionFence.follow(mastra, 'run-1', coordinator.claim))!; + onTestFinished(() => participant.stopHeartbeat()); + mastra.__createRunScope('run-1').set(DURABLE_PARTICIPANT_FENCES, new Map([[coordinator.executionId, participant]])); + + // The release never reaches storage, and neither do renewals, until the claim's lease runs out. + vi.spyOn(workflowsStore, 'releaseRunOwnership').mockRejectedValueOnce(new Error('store down')); + vi.spyOn(workflowsStore, 'renewRunOwnership').mockRejectedValue(new Error('store down')); + await expect(ExecutionFence.releaseSharedClaim(mastra, 'run-1', coordinator.claim)).rejects.toThrow('store down'); + await vi.advanceTimersByTimeAsync(EXECUTION_LEASE_TTL_MS + EXECUTION_LEASE_RENEW_INTERVAL_MS); + const record = await workflowsStore.getRunOwnership({ runId: 'run-1' }); + expect(record).toMatchObject({ ownerId: coordinator.executionId, live: false }); + expect(record?.leaseExpiresAt).not.toBeNull(); + + // An expired claim is not a released one. + await expect(participant.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_UNVERIFIED_ERROR_ID }); + }); + + it('counts a release storage applied but whose response failed as released', async () => { + const { workflowsStore, coordinator, claimSuccessor } = await setup(); + const release = holdNextCall(workflowsStore, 'releaseRunOwnership', { apply: true, fail: true }); + const settled = coordinator.settle(async () => {}); + await release.applied; + release.finish(); + expect(await settled).toBe('owned'); + + expect(await workflowsStore.getRunOwnership({ runId: 'run-1' })).toMatchObject({ leaseExpiresAt: null }); + await expect(coordinator.verifyNotSuperseded()).resolves.toBeUndefined(); + expect(coordinator.isLost()).toBe(false); + + expect((await claimSuccessor()).acquired).toBe(true); + await expect(coordinator.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + }); + + it('refuses after a failed release when storage shows another execution claimed the run', async () => { + const { workflowsStore, coordinator, claimSuccessor } = await setup(); + // The release applies, a successor claims the run, and only then does the release's response fail. + const release = holdNextCall(workflowsStore, 'releaseRunOwnership', { apply: true, fail: true }); + const settled = coordinator.settle(async () => {}); + await release.applied; + expect((await claimSuccessor()).acquired).toBe(true); + release.finish(); + await settled; + + await expect(coordinator.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + }); +}); + describe('execution claim carrier', () => { it('round-trips the claim and drops malformed entries', () => { const requestContext = new RequestContext(); diff --git a/packages/core/src/agent/durable/execution-fence.ts b/packages/core/src/agent/durable/execution-fence.ts index ccfc56b9cd4..e6b77811791 100644 --- a/packages/core/src/agent/durable/execution-fence.ts +++ b/packages/core/src/agent/durable/execution-fence.ts @@ -353,6 +353,11 @@ interface OwnershipBackend { renew(claim: DurableExecutionClaim): Promise<{ renewed: boolean; leaseExpiresAt?: Date | null }>; /** Whether another execution claimed the run after `claim`. Throws on backend errors. */ isSuperseded(claim: DurableExecutionClaim): Promise; + /** + * Whether the store still holds `claim` and shows it released on purpose, not lapsed or replaced. + * `false` when the backend can't tell. Throws on backend errors. + */ + isReleased(claim: DurableExecutionClaim, details: ExecutionDetails): Promise; release(claim: DurableExecutionClaim): Promise; /** * Release `claim` so the run can be recovered at once, and make storage @@ -413,6 +418,8 @@ function leaseBackend(provider: LeaseProvider, agentId: string, runId: string): const owner = await provider.getLeaseOwner(key); return owner !== undefined && owner !== claim.executionId; }, + // A missing lease may have been released or may have lapsed. + isReleased: async () => false, async release(claim) { await provider.releaseLease(key, claim.executionId); }, @@ -501,6 +508,11 @@ function storageBackend(store: WorkflowsStorage, runId: string): OwnershipBacken const record = await store.getRunOwnership({ runId }); return !record || record.generation !== claim.generation; }, + // A release clears the lease of the claim's own record; a lapsed lease keeps its expiry. + async isReleased(claim, details) { + const record = await retryOnce(() => store.getRunOwnership({ runId }), details); + return record?.leaseExpiresAt === null && matchesRunFence(record, fence(claim)); + }, async release(claim) { await store.releaseRunOwnership(fence(claim)); }, @@ -926,20 +938,20 @@ export class ExecutionFence implements RunFenceScope { } /** - * {@link verify}. With `yieldToRelease`, a check that fails once settlement - * has started releasing the claim returns `false` instead of marking the - * execution lost: the check may have read that release, not a successor. + * {@link verify}. With `yieldToRelease`, a failed check returns `false` + * instead of marking the execution lost: it may have read a release of the + * claim, not a successor. */ async #verify(yieldToRelease: boolean): Promise { if (this.#lossError) { - if (yieldToRelease && this.#releasing) return false; + if (yieldToRelease) return false; throw this.#lossError; } const startedAt = Date.now(); try { await this.#backend.verify(this.claim, this.#details()); } catch (error) { - if (yieldToRelease && this.#releasing) return false; + if (yieldToRelease) return false; this.#markLost(error as DurableExecutionFenceError); if ((error as DurableExecutionFenceError).id === EXECUTION_SUPERSEDED_ERROR_ID) await this.#noteTakeover(); throw error; @@ -950,15 +962,18 @@ export class ExecutionFence implements RunFenceScope { } /** - * {@link verify} for writes that may land after settlement. Once settlement - * released the claim, the write is accepted until another execution claims - * the run, as it is after this fence is unregistered. A check that fails - * while the release is in flight waits for it, then applies the same rule. + * {@link verify} for writes that may land after settlement. Once the claim + * was released, the write is accepted until another execution claims the + * run, as it is after this fence is unregistered. A check that fails while + * a release is in flight waits for it, then applies the same rule. + * Otherwise a failed check asks the store whether the claim was released: + * by another process, such as the evented worker that settled the run, or + * by a release here whose response failed. A lapsed claim stays strict. */ async verifyNotSuperseded(): Promise { if (!this.#released && (await this.#verify(true))) return; - // Settlement released the claim, or is releasing it. A release that failed left the claim held: stay strict. - if (!(await this.#releasing)) return this.verify(); + // Only a release excuses a failed check: one confirmed here, or one the store shows for this exact claim. + if (!(await this.#releasing) && !(await this.#confirmRelease())) return this.verify(); const details = this.#details(); if (this.#takenOver || (await retryOnce(() => this.#backend.isSuperseded(this.claim), details))) { throw new DurableExecutionFenceError(EXECUTION_SUPERSEDED_ERROR_ID, details); @@ -1097,7 +1112,9 @@ export class ExecutionFence implements RunFenceScope { /** * Record a release of this claim for {@link verifyNotSuperseded}. A failed - * release keeps the outcome of an earlier one. + * release keeps the outcome of an earlier one. It may still have been + * applied, with only its response lost: {@link verifyNotSuperseded} asks the + * store before staying strict. */ #trackRelease(release: Promise): Promise { const previous = this.#releasing; @@ -1111,6 +1128,21 @@ export class ExecutionFence implements RunFenceScope { return this.#releasing; } + /** + * Whether the store shows this exact claim released on purpose. From then + * on the claim counts as released. A lapsed or replaced claim, a lease + * backend, or a failed read: `false`. + */ + async #confirmRelease(): Promise { + try { + if (!(await this.#backend.isReleased(this.claim, this.#details()))) return false; + } catch { + return false; + } + this.#released = true; + return true; + } + /** Mark a settled fence retired in memory so `memory.runFences` retention can prune it. */ async #retireMemoryFence(fence: RunFence): Promise { if (!this.#memoryStore) return; diff --git a/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts b/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts index 4086e307cac..61b19cc9d05 100644 --- a/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts +++ b/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts @@ -880,13 +880,22 @@ describeForAllEngines( // one cleanup step open, as slow storage would, until `finishCleanup`: retiring the run's // memory fence, releasing the thread's lease (evented, with a shared `pubsub`), or the claim // release itself, which storage has applied but not yet returned to the run. `cleanedUp` - // resolves once the held step has finished. + // resolves once the held step has finished. With `releaseAckFails`, the claim release is + // applied in storage but its response fails, as a dropped connection would. With + // `outcome: 'result'`, the tool returns a result once let go instead of asking for input. async function startResearchDuringCleanup( threadId: string, { hold = 'memory-fence', pubsub, - }: { hold?: 'memory-fence' | 'thread-lease' | 'claim-release'; pubsub?: PubSub } = {}, + releaseAckFails = false, + outcome = 'suspend', + }: { + hold?: 'memory-fence' | 'thread-lease' | 'claim-release'; + pubsub?: PubSub; + releaseAckFails?: boolean; + outcome?: 'suspend' | 'result'; + } = {}, ) { let release!: () => void; const gate = new Promise(resolve => (release = resolve)); @@ -906,6 +915,16 @@ describeForAllEngines( signalCleanedUp(); } }; + if (releaseAckFails) { + const releaseRunOwnership = WorkflowsInMemory.prototype.releaseRunOwnership; + const spy = vi + .spyOn(WorkflowsInMemory.prototype, 'releaseRunOwnership') + .mockImplementationOnce(async function (this: WorkflowsInMemory, fence) { + await releaseRunOwnership.call(this, fence); + throw new Error('connection lost after the release was applied'); + }); + onTestFinished(() => spy.mockRestore()); + } if (hold === 'memory-fence') { const retireRunFence = memoryStore.retireRunFence.bind(memoryStore); vi.spyOn(memoryStore, 'retireRunFence').mockImplementation(fence => holdOpen(() => retireRunFence(fence))); @@ -927,11 +946,25 @@ describeForAllEngines( onTestFinished(() => spy.mockRestore()); } const thread = { threadId, resourceId: `${threadId}-user` }; + const researchTool = + outcome === 'result' + ? createTool({ + id: 'research-with-approval', + description: 'Research a topic', + inputSchema: z.object({ topic: z.string() }), + outputSchema: z.object({ summary: z.string() }), + background: { enabled: true }, + execute: async ({ topic }) => { + await gate; + return { summary: `Research on "${topic}"` }; + }, + }) + : makeSuspendingResearchTool([], undefined, gate); const { chunks, mastra, output } = await runLoopScenario({ engine, llm: getMock(), prompt: 'Research near', - tools: { researchWithApproval: makeSuspendingResearchTool([], undefined, gate) }, + tools: { researchWithApproval: researchTool }, agentBackgroundTasks: { tools: { researchWithApproval: true } }, backgroundTasks: BG_CONFIG, memory, @@ -1216,6 +1249,119 @@ describeForAllEngines( 30_000, ); + // Technical: storage applied the finished run's claim release, but the response failed, as a + // dropped connection would. The fence treated the release as never made and stayed strict, so + // a late suspend during the remaining cleanup read the released claim, marked the execution + // lost though nobody had taken over, and the task failed before it suspended. (#30182) + it.runIf(isDurableEngineVariant(engine))( + 'parks a deferred tool that asks for input after the finished run lost the reply to its claim release (#30182)', + async () => { + const run = await startResearchDuringCleanup('bg-suspend-after-lost-release-ack-thread', { + releaseAckFails: true, + }); + const released = await run.workflows.getRunOwnership({ runId: run.runId }); + expect(released).toMatchObject({ leaseExpiresAt: null, live: false }); + run.release(); + try { + const parked = await run.settledTask(); + expect({ status: parked.status, error: parked.error?.message }).toEqual({ + status: 'suspended', + error: undefined, + }); + expect(parked.suspendPayload).toEqual({ awaiting: 'analyst-approval', topic: 'near' }); + const open = await waitFor( + () => storedToolCall(run.memory, run.thread), + s => s.suspended?.taskId !== undefined, + ); + expect(open.suspended).toMatchObject({ taskId: parked.id }); + const after = await run.workflows.getRunOwnership({ runId: run.runId }); + expect({ ownerId: after?.ownerId, generation: after?.generation, live: after?.live }).toEqual({ + ownerId: released?.ownerId, + generation: released?.generation, + live: false, + }); + } finally { + run.finishCleanup(); + } + }, + 30_000, + ); + + // Technical: counting a release whose reply was lost as made must not let a superseded + // execution through: once a successor claims the run, the late suspend fails without + // writing. (#30182) + it.runIf(isDurableEngineVariant(engine))( + "doesn't park a deferred tool that asks for input after another execution took over a run that lost the reply to its claim release (#30182)", + async () => { + const run = await startResearchDuringCleanup('bg-suspend-lost-release-ack-takeover-thread', { + releaseAckFails: true, + }); + const released = await run.workflows.getRunOwnership({ runId: run.runId }); + const successor = await run.workflows.claimRunOwnership({ + runId: run.runId, + ownerId: 'successor-execution', + leaseMs: 60_000, + expectedGeneration: released!.generation, + }); + expect(successor.acquired).toBe(true); + run.release(); + try { + const settled = await run.settledTask(); + expect({ status: settled.status, error: settled.error?.message }).toEqual({ + status: 'failed', + // DURABLE_AGENT_EXECUTION_SUPERSEDED + error: expect.stringContaining(`Durable run ${run.runId} lost its execution lease`), + }); + expect(settled.suspendPayload).toBeUndefined(); + expect((await storedToolCall(run.memory, run.thread)).suspended).toBeUndefined(); + const after = await run.workflows.getRunOwnership({ runId: run.runId }); + expect({ ownerId: after?.ownerId, generation: after?.generation, live: after?.live }).toEqual({ + ownerId: 'successor-execution', + generation: successor.record.generation, + live: true, + }); + } finally { + run.finishCleanup(); + } + }, + 30_000, + ); + + // Technical: a background task's result is written to memory through the same post-settlement + // check as a late suspend. While the finished run is still cleaning up, that check now accepts + // the write when nobody has taken over, so the result reaches memory. The write itself carries + // no claim fence; that gap predates this fix. (#30182) + it.runIf(isDurableEngineVariant(engine))( + "records a deferred tool's result that arrives while the finished run is still cleaning up (#30182)", + async () => { + const run = await startResearchDuringCleanup('bg-result-during-cleanup-thread', { outcome: 'result' }); + const released = await run.workflows.getRunOwnership({ runId: run.runId }); + run.release(); + try { + const done = await run.settledTask(); + expect({ status: done.status, error: done.error?.message }).toEqual({ + status: 'completed', + error: undefined, + }); + expect(done.result).toEqual({ summary: 'Research on "near"' }); + // The dispatch left a placeholder result; the task's own result replaces it. + const answered = await waitFor( + () => storedToolCall(run.memory, run.thread), + s => s.invocation?.result?.summary !== undefined, + ); + expect(answered.invocation).toMatchObject({ state: 'result', result: { summary: 'Research on "near"' } }); + const after = await run.workflows.getRunOwnership({ runId: run.runId }); + expect({ ownerId: after?.ownerId, generation: after?.generation }).toEqual({ + ownerId: released?.ownerId, + generation: released?.generation, + }); + } finally { + run.finishCleanup(); + } + }, + 30_000, + ); + // Technical: on a shared transport, an evented run's worker releases the run's claim, then // the thread's lease, and only then stops its participant fence. Until the lease release // returns, that fence stays registered, and the late suspend's check asked it to prove it From d0f80f1ec06fe95ff7bdcdc46b2b98bde78e2644 Mon Sep 17 00:00:00 2001 From: Leo <108278866+leoisadev1@users.noreply.github.com> Date: Mon, 16 Nov 2026 16:14:05 -0500 Subject: [PATCH 07/15] fix(core): fence late background result writes and count only confirmed releases A background task's onResult and onExecution memory flushes now run under the finished run's claim fence, like the late suspend, so storage rejects a save that lands after a successor raised the memory fence. The final check of verifyNotSuperseded reads the claim once and classifies a matching released record as a release instead of falling back to a release-blind strict check, so another worker's release between two reads no longer marks the fence lost. Storage release now returns whether the claim was still held, and only a confirmed release counts as released. Co-Authored-By: Claude Opus 5.5 --- .../src/agent/durable/execution-fence.test.ts | 124 +++++++++++++++++ .../core/src/agent/durable/execution-fence.ts | 113 +++++++++------ .../durable/workflows/steps/tool-call.ts | 35 +++-- ...background-task-lifecycle.scenario.test.ts | 131 +++++++++++++++++- 4 files changed, 346 insertions(+), 57 deletions(-) diff --git a/packages/core/src/agent/durable/execution-fence.test.ts b/packages/core/src/agent/durable/execution-fence.test.ts index af982bb162c..150b13ecc8a 100644 --- a/packages/core/src/agent/durable/execution-fence.test.ts +++ b/packages/core/src/agent/durable/execution-fence.test.ts @@ -1381,6 +1381,43 @@ describe.each([ }); }); +// A lease can't show a release, so a worker's participant fence learns of the shared release it is +// making only from `releaseSharedClaim` itself (#30182). +describe('ExecutionFence.verifyNotSuperseded on a lease-backed participant', () => { + async function setup() { + const pubsub = new EventEmitterPubSub(); + const coordinator = await claim(pubsub, 'run-1'); + coordinator.stopHeartbeat(); + const mastra = new Mastra({ pubsub, logger: false }); + const participant = (await ExecutionFence.follow(mastra, 'run-1', coordinator.claim))!; + onTestFinished(() => participant.stopHeartbeat()); + mastra.__createRunScope('run-1').set(DURABLE_PARTICIPANT_FENCES, new Map([[coordinator.executionId, participant]])); + return { pubsub, mastra, coordinator, participant }; + } + + it("waits for this worker's shared release in flight, then passes while nobody claimed the run", async () => { + const { pubsub, mastra, coordinator, participant } = await setup(); + const release = holdNextCall(pubsub, 'releaseLease', { apply: true, fail: false }); + const releasing = ExecutionFence.releaseSharedClaim(mastra, 'run-1', coordinator.claim); + await release.applied; + expect(await pubsub.getLeaseOwner(key('run-1'))).toBeUndefined(); + + const check = participant.verifyNotSuperseded(); + release.finish(); + await expect(check).resolves.toBeUndefined(); + expect(participant.isLost()).toBe(false); + await releasing; + }); + + it("refuses once another execution took the lease after this worker's shared release", async () => { + const { pubsub, mastra, coordinator, participant } = await setup(); + await ExecutionFence.releaseSharedClaim(mastra, 'run-1', coordinator.claim); + await pubsub.acquireLease(key('run-1'), 'successor', EXECUTION_LEASE_TTL_MS); + + await expect(participant.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + }); +}); + // Storage keeps a released claim's record with its lease cleared, so it can show a release this fence // didn't see: one made by another process, or one storage applied but whose response was lost (#30182). describe('ExecutionFence.verifyNotSuperseded with a release only storage shows', () => { @@ -1489,6 +1526,93 @@ describe('ExecutionFence.verifyNotSuperseded with a release only storage shows', await expect(coordinator.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); }); + // Reads of the claim during one check: the first two fail as a dropped connection would, which + // exhausts the first read's retry. `before` and `after` run around the third read. + function failTwoReads( + workflowsStore: WorkflowsStorage, + { before, after }: { before?: () => Promise; after?: () => Promise }, + ) { + const read = workflowsStore.getRunOwnership.bind(workflowsStore); + let reads = 0; + return vi.spyOn(workflowsStore, 'getRunOwnership').mockImplementation(async args => { + reads++; + if (reads <= 2) throw new Error('connection reset'); + if (reads === 3) await before?.(); + const record = await read(args); + if (reads === 3) await after?.(); + return record; + }); + } + + it('passes when another process releases the claim between the reads of one check', async () => { + const { workflowsStore, coordinator, own } = await setup(); + coordinator.stopHeartbeat(); + // The check's next read still finds the claim live; another worker releases it just after. + const reads = failTwoReads(workflowsStore, { + after: async () => expect(await workflowsStore.releaseRunOwnership(own)).toBe(true), + }); + + await expect(coordinator.verifyNotSuperseded()).resolves.toBeUndefined(); + expect(coordinator.isLost()).toBe(false); + reads.mockRestore(); + expect(await workflowsStore.getRunOwnership({ runId: 'run-1' })).toMatchObject({ ...own, leaseExpiresAt: null }); + await expect(coordinator.verifyNotSuperseded()).resolves.toBeUndefined(); + expect(coordinator.isLost()).toBe(false); + }); + + it('refuses when another execution claims the run between the reads of one check', async () => { + const { workflowsStore, coordinator, own } = await setup(); + coordinator.stopHeartbeat(); + // Another worker releases the claim and a successor claims the run before the check's next read. + failTwoReads(workflowsStore, { + before: async () => { + expect(await workflowsStore.releaseRunOwnership(own)).toBe(true); + await workflowsStore.claimRunOwnership({ runId: 'run-1', ownerId: 'successor', leaseMs: 60_000 }); + }, + }); + + await expect(coordinator.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + expect(coordinator.isLost()).toBe(true); + }); + + it("refuses, and marks the execution lost, when storage shows a successor's claim released", async () => { + const { storage, workflowsStore, coordinator, own } = await setup(); + coordinator.stopHeartbeat(); + const mastra = new Mastra({ storage, logger: false }); + const participant = (await ExecutionFence.follow(mastra, 'run-1', coordinator.claim))!; + onTestFinished(() => participant.stopHeartbeat()); + + // A successor claimed the run and has already released it: the record's lease is cleared, but + // it names the successor, not this claim. + expect(await workflowsStore.releaseRunOwnership(own)).toBe(true); + const successor = await workflowsStore.claimRunOwnership({ runId: 'run-1', ownerId: 'successor', leaseMs: 60_000 }); + expect(await workflowsStore.releaseRunOwnership(successor.record)).toBe(true); + + await expect(participant.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + expect(participant.isLost()).toBe(true); + }); + + it('marks the execution lost when its release found the claim already replaced', async () => { + const { workflowsStore, coordinator } = await setup(); + const releaseRunOwnership = workflowsStore.releaseRunOwnership.bind(workflowsStore); + // A successor takes the run just before settlement's release, which then releases nothing. + vi.spyOn(workflowsStore, 'releaseRunOwnership').mockImplementationOnce(async fence => { + await workflowsStore.claimRunOwnership({ runId: 'run-1', ownerId: 'successor', leaseMs: 60_000, force: true }); + const released = await releaseRunOwnership(fence); + expect(released).toBe(false); + return released; + }); + expect(await coordinator.settle(async () => {})).toBe('owned'); + + // Not a release: verify's rule applies, and a superseded execution is lost. + await expect(coordinator.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + expect(coordinator.isLost()).toBe(true); + expect(await workflowsStore.getRunOwnership({ runId: 'run-1' })).toMatchObject({ + ownerId: 'successor', + live: true, + }); + }); + it('refuses after a failed release when storage shows another execution claimed the run', async () => { const { workflowsStore, coordinator, claimSuccessor } = await setup(); // The release applies, a successor claims the run, and only then does the release's response fail. diff --git a/packages/core/src/agent/durable/execution-fence.ts b/packages/core/src/agent/durable/execution-fence.ts index e6b77811791..379ca149799 100644 --- a/packages/core/src/agent/durable/execution-fence.ts +++ b/packages/core/src/agent/durable/execution-fence.ts @@ -354,11 +354,19 @@ interface OwnershipBackend { /** Whether another execution claimed the run after `claim`. Throws on backend errors. */ isSuperseded(claim: DurableExecutionClaim): Promise; /** - * Whether the store still holds `claim` and shows it released on purpose, not lapsed or replaced. - * `false` when the backend can't tell. Throws on backend errors. + * One read of `claim`, for a check that may follow its release: `'held'` when + * {@link verify} would pass, `'released'` when the store still names this + * claim's owner and generation with its lease cleared, which only a release + * does (a lapsed lease keeps its expiry). Throws like {@link verify} + * otherwise. `undefined`, without a read, when the backend can't tell a + * release from a lapse. */ - isReleased(claim: DurableExecutionClaim, details: ExecutionDetails): Promise; - release(claim: DurableExecutionClaim): Promise; + inspect(claim: DurableExecutionClaim, details: ExecutionDetails): Promise<'held' | 'released' | undefined>; + /** + * Release `claim`. Resolves `false` when the store no longer held it, so + * nothing was released. Lease backends can't tell and resolve `true`. + */ + release(claim: DurableExecutionClaim): Promise; /** * Release `claim` so the run can be recovered at once, and make storage * reject its remaining writes. Returns the claim that superseded it, when @@ -419,9 +427,10 @@ function leaseBackend(provider: LeaseProvider, agentId: string, runId: string): return owner !== undefined && owner !== claim.executionId; }, // A missing lease may have been released or may have lapsed. - isReleased: async () => false, + inspect: async () => undefined, async release(claim) { await provider.releaseLease(key, claim.executionId); + return true; }, // A lease has no generation to move past: releasing is all it can do. async abandon(claim, whileOwned) { @@ -447,6 +456,12 @@ function storageBackend(store: WorkflowsStorage, runId: string): OwnershipBacken holder: record?.ownerId ?? undefined, leaseExpiresAt: record?.leaseExpiresAt, }); + // A release clears the lease of the claim's own record; a lapsed lease keeps its expiry. + const inspect: OwnershipBackend['inspect'] = async (claim, details) => { + const record = await retryOnce(() => store.getRunOwnership({ runId }), details); + if (record && matchesRunFence(record, fence(claim))) return record.leaseExpiresAt ? 'held' : 'released'; + throw new DurableExecutionFenceError(EXECUTION_SUPERSEDED_ERROR_ID, details); + }; return { expires: true, // A release keeps the generation, so a pinned claim only misses when another @@ -486,8 +501,7 @@ function storageBackend(store: WorkflowsStorage, runId: string): OwnershipBacken // renewal, it fails once the claim was released: storage still accepts the // released claim's late writes, but the execution no longer drives the run. async verify(claim, details) { - const record = await retryOnce(() => store.getRunOwnership({ runId }), details); - if (!record?.leaseExpiresAt || !matchesRunFence(record, fence(claim))) { + if ((await inspect(claim, details)) !== 'held') { throw new DurableExecutionFenceError(EXECUTION_SUPERSEDED_ERROR_ID, details); } }, @@ -508,14 +522,8 @@ function storageBackend(store: WorkflowsStorage, runId: string): OwnershipBacken const record = await store.getRunOwnership({ runId }); return !record || record.generation !== claim.generation; }, - // A release clears the lease of the claim's own record; a lapsed lease keeps its expiry. - async isReleased(claim, details) { - const record = await retryOnce(() => store.getRunOwnership({ runId }), details); - return record?.leaseExpiresAt === null && matchesRunFence(record, fence(claim)); - }, - async release(claim) { - await store.releaseRunOwnership(fence(claim)); - }, + inspect, + release: claim => store.releaseRunOwnership(fence(claim)), // Releasing alone keeps accepting the claim's writes until the next claim. // Claiming under a fresh owner first moves the generation past them; the // pinned generation leaves a claim made in between to its claimant. @@ -952,28 +960,34 @@ export class ExecutionFence implements RunFenceScope { await this.#backend.verify(this.claim, this.#details()); } catch (error) { if (yieldToRelease) return false; - this.#markLost(error as DurableExecutionFenceError); - if ((error as DurableExecutionFenceError).id === EXECUTION_SUPERSEDED_ERROR_ID) await this.#noteTakeover(); - throw error; + return this.#fail(error as DurableExecutionFenceError); } // A lease backend renews on verify; a storage read leaves expiry to the heartbeat. if (this.generation === undefined) this.#leaseExpiresAt = startedAt + EXECUTION_LEASE_TTL_MS; return true; } + /** Mark the execution lost after a failed check, and throw. */ + async #fail(error: DurableExecutionFenceError): Promise { + this.#markLost(error); + if (error.id === EXECUTION_SUPERSEDED_ERROR_ID) await this.#noteTakeover(); + throw error; + } + /** * {@link verify} for writes that may land after settlement. Once the claim * was released, the write is accepted until another execution claims the * run, as it is after this fence is unregistered. A check that fails while * a release is in flight waits for it, then applies the same rule. - * Otherwise a failed check asks the store whether the claim was released: - * by another process, such as the evented worker that settled the run, or - * by a release here whose response failed. A lapsed claim stays strict. + * Otherwise a failed check reads the claim once more, and a store that + * shows it released counts as a release: by another process, such as the + * evented worker that settled the run, or by a release here whose response + * failed. A lapsed claim stays strict. */ async verifyNotSuperseded(): Promise { if (!this.#released && (await this.#verify(true))) return; // Only a release excuses a failed check: one confirmed here, or one the store shows for this exact claim. - if (!(await this.#releasing) && !(await this.#confirmRelease())) return this.verify(); + if (!(await this.#releasing) && !(await this.#verifyOrConfirmRelease())) return; const details = this.#details(); if (this.#takenOver || (await retryOnce(() => this.#backend.isSuperseded(this.claim), details))) { throw new DurableExecutionFenceError(EXECUTION_SUPERSEDED_ERROR_ID, details); @@ -1111,36 +1125,49 @@ export class ExecutionFence implements RunFenceScope { } /** - * Record a release of this claim for {@link verifyNotSuperseded}. A failed - * release keeps the outcome of an earlier one. It may still have been - * applied, with only its response lost: {@link verifyNotSuperseded} asks the - * store before staying strict. + * Record a release of this claim for {@link verifyNotSuperseded}. Only a + * release the store confirms counts. A failed release, or one that found the + * claim no longer held, keeps the outcome of an earlier one. A failed release + * may still have been applied, with only its response lost: + * {@link verifyNotSuperseded} asks the store before staying strict. */ - #trackRelease(release: Promise): Promise { + #trackRelease(release: Promise): Promise { const previous = this.#releasing; - this.#releasing = release.then( - () => { - this.#released = true; - return true; - }, - async () => (await previous) ?? false, - ); + const unconfirmed = async () => (await previous) ?? false; + this.#releasing = release.then(released => { + // `false`: the store no longer held this claim, so this call released nothing. + if (!released) return unconfirmed(); + this.#released = true; + return true; + }, unconfirmed); return this.#releasing; } /** - * Whether the store shows this exact claim released on purpose. From then - * on the claim counts as released. A lapsed or replaced claim, a lease - * backend, or a failed read: `false`. + * The last check of {@link verifyNotSuperseded} when no release was + * confirmed here: one read that passes while the claim is held, as + * {@link verify} does, and returns `true` when the store shows this exact + * claim released, which then counts as released. The release may have + * happened after the check that failed, even between two reads here. A + * lapsed or replaced claim, or a failed read, marks the execution lost as + * {@link verify} does. A fence already lost only reads for a release. Lease + * backends can't tell a release from a lapse and run {@link verify}. */ - async #confirmRelease(): Promise { + async #verifyOrConfirmRelease(): Promise { + let state: Awaited>; try { - if (!(await this.#backend.isReleased(this.claim, this.#details()))) return false; - } catch { - return false; + state = await this.#backend.inspect(this.claim, this.#details()); + } catch (error) { + if (this.#lossError) throw this.#lossError; + return this.#fail(error as DurableExecutionFenceError); } - this.#released = true; - return true; + if (state === 'released') { + this.#released = true; + return true; + } + if (state === undefined) await this.verify(); + else if (this.#lossError) throw this.#lossError; + return false; } /** Mark a settled fence retired in memory so `memory.runFences` retention can prune it. */ diff --git a/packages/core/src/agent/durable/workflows/steps/tool-call.ts b/packages/core/src/agent/durable/workflows/steps/tool-call.ts index 0d3134da048..1cdd3192e20 100644 --- a/packages/core/src/agent/durable/workflows/steps/tool-call.ts +++ b/packages/core/src/agent/durable/workflows/steps/tool-call.ts @@ -876,6 +876,13 @@ export function createDurableToolCallStep() { // claimed the run. const assertNotSuperseded = () => assertExecutionNotSuperseded({ runId, agentId: initData.agentId, requestContext, mastra: mastra as Mastra }); + // Those writes carry the run's claim, so storage rejects one that lands after a successor + // raised the memory fence, even though it passed the check above (#30182). + const claimFenceArgs = { + requestContext, + runId, + memoryStore: async () => memory?.storage.getStore('memory'), + }; // Remote workers have no registry message list; persist updates through the recovered resume list. const doFlush = async (messagesToFlush = messageList ?? resumeMessageList, beforePersist = assertOwned) => { @@ -1856,10 +1863,7 @@ export function createDurableToolCallStep() { // The run may have settled and stopped fencing its writes. The suspension's // memory save carries the run's claim, so storage rejects it once a successor // raised the memory fence, even when it lands after the ownership check (#30182). - await runFencedToClaim( - { requestContext, runId, memoryStore: async () => memory?.storage.getStore('memory') }, - () => suspendToolCall(data, options, assertNotSuperseded), - ); + await runFencedToClaim(claimFenceArgs, () => suspendToolCall(data, options, assertNotSuperseded)); return taskContext?.suspend?.(data, options); }, outputWriter: async (chunk: any) => { @@ -2028,10 +2032,14 @@ export function createDurableToolCallStep() { flush, }); await applyTo(messageList, async () => { - if (saveQueueManager && state?.threadId && !state?.memoryConfig?.readOnly) { - await saveQueueManager.flushMessages(messageList, state.threadId, state.memoryConfig, { - beforePersist: assertNotSuperseded, - }); + const threadId = state?.threadId; + if (saveQueueManager && threadId && !state?.memoryConfig?.readOnly) { + // Like a late suspend, the result can land after the run settled: the write carries its claim. + await runFencedToClaim(claimFenceArgs, () => + saveQueueManager.flushMessages(messageList, threadId, state.memoryConfig, { + beforePersist: assertNotSuperseded, + }), + ); } }); if (info.disposition !== 'awaited' && liveEntry) { @@ -2082,11 +2090,14 @@ export function createDurableToolCallStep() { // A superseded execution skips the write. The hook runs outside this // step (the background-task manager calls it), so the error is not // rethrown: that would fail the task instead of stopping the step. - if (saveQueueManager && state?.threadId && !state?.memoryConfig?.readOnly) { + const threadId = state?.threadId; + if (saveQueueManager && threadId && !state?.memoryConfig?.readOnly) { try { - await saveQueueManager.flushMessages(messageList, state.threadId, state.memoryConfig, { - beforePersist: assertNotSuperseded, - }); + await runFencedToClaim(claimFenceArgs, () => + saveQueueManager.flushMessages(messageList, threadId, state.memoryConfig, { + beforePersist: assertNotSuperseded, + }), + ); } catch (error) { if (!isExecutionFenceError(error)) throw error; logger?.debug?.('[DurableAgent] Skipped background-task metadata flush: execution superseded', { diff --git a/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts b/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts index 61b19cc9d05..38be5c11f88 100644 --- a/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts +++ b/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts @@ -22,7 +22,9 @@ import { BackgroundTaskManager } from '../../../../background-tasks/manager'; import type { PubSub } from '../../../../events/pubsub'; import { UnixSocketPubSub } from '../../../../events/unix-socket-pubsub'; import type { Mastra } from '../../../../mastra'; +import type { MemoryStorage } from '../../../../storage/domains/memory/base'; import { WorkflowsInMemory } from '../../../../storage/domains/workflows/inmemory'; +import { RUN_FENCE_CONFLICT_ERROR_ID } from '../../../../storage/run-fencing'; import { SCENARIO_MODEL_ID } from '../types'; /** @@ -1329,8 +1331,7 @@ describeForAllEngines( // Technical: a background task's result is written to memory through the same post-settlement // check as a late suspend. While the finished run is still cleaning up, that check now accepts - // the write when nobody has taken over, so the result reaches memory. The write itself carries - // no claim fence; that gap predates this fix. (#30182) + // the write when nobody has taken over, so the result reaches memory. (#30182) it.runIf(isDurableEngineVariant(engine))( "records a deferred tool's result that arrives while the finished run is still cleaning up (#30182)", async () => { @@ -1362,6 +1363,132 @@ describeForAllEngines( 30_000, ); + // Holds the `nth` memory save (counting from 1) until `finishSave`. `saveStarted` resolves once + // it is held; `saveOutcome` resolves once it reached storage, to its error or `undefined`. + function holdMemorySave(memoryStore: MemoryStorage, nth: number) { + let signalSaveStarted!: () => void; + const saveStarted = new Promise(resolve => (signalSaveStarted = resolve)); + let finishSave!: () => void; + const saveGate = new Promise(resolve => (finishSave = resolve)); + let settleOutcome!: (error: unknown) => void; + const saveOutcome = new Promise(resolve => (settleOutcome = resolve)); + const saveMessages = memoryStore.saveMessages.bind(memoryStore); + let saves = 0; + const spy = vi.spyOn(memoryStore, 'saveMessages').mockImplementation(async args => { + if (++saves !== nth) return saveMessages(args); + signalSaveStarted(); + await saveGate; + try { + const saved = await saveMessages(args); + settleOutcome(undefined); + return saved; + } catch (error) { + settleOutcome(error); + throw error; + } + }); + onTestFinished(() => { + finishSave(); + spy.mockRestore(); + }); + return { saveStarted, finishSave, saveOutcome, saveMessages }; + } + + // The finished run completes its cleanup. Then a successor claims the run and raises the + // memory fence, as recovery or a new turn does before it touches memory. + async function takeOverAfterCleanup(run: Awaited>) { + run.finishCleanup(); + await run.cleanedUp; + const released = await run.workflows.getRunOwnership({ runId: run.runId }); + const successor = await run.workflows.claimRunOwnership({ + runId: run.runId, + ownerId: 'successor-execution', + leaseMs: 60_000, + expectedGeneration: released!.generation, + }); + expect(successor.acquired).toBe(true); + expect( + await run.memoryStore.raiseRunFence({ + runId: run.runId, + generation: successor.record!.generation, + ownerId: successor.record!.ownerId, + }), + ).toBe(true); + return successor.record!; + } + + async function expectSuccessorUnchanged( + run: Awaited>, + successor: { generation: number }, + ) { + const after = await run.workflows.getRunOwnership({ runId: run.runId }); + expect({ ownerId: after?.ownerId, generation: after?.generation, live: after?.live }).toEqual({ + ownerId: 'successor-execution', + generation: successor.generation, + live: true, + }); + } + + // Technical: after a late suspend saves its open request, the background-task manager saves + // the task's metadata again (`onExecution`). That save passed the post-settlement check but + // reached storage only after the finished run's cleanup and a successor's claim, which raised + // the memory fence and cleared the request. It carried no claim fence, so it restored the + // superseded execution's open request. It now carries the finished run's claim. (#30182) + it.runIf(isDurableEngineVariant(engine))( + "doesn't restore a deferred tool's open request when its task metadata reaches storage after another execution took over (#30182)", + async () => { + const run = await startResearchDuringCleanup('bg-suspend-slow-metadata-save-takeover-thread'); + // The first save is the suspension's own; the second is the manager's metadata save. + const save = holdMemorySave(run.memoryStore, 2); + try { + run.release(); + await save.saveStarted; + const successor = await takeOverAfterCleanup(run); + // The successor works on the thread: here it drops the open request. + const { messages } = await run.memoryStore.listMessages({ ...run.thread, perPage: false }); + const cleared = structuredClone(messages); + for (const message of cleared) delete message.content.metadata?.suspendedTools; + await save.saveMessages({ messages: cleared }); + expect((await storedToolCall(run.memory, run.thread)).suspended).toBeUndefined(); + + save.finishSave(); + expect(await save.saveOutcome).toMatchObject({ id: RUN_FENCE_CONFLICT_ERROR_ID }); + expect((await storedToolCall(run.memory, run.thread)).suspended).toBeUndefined(); + await expectSuccessorUnchanged(run, successor); + } finally { + run.finishCleanup(); + } + }, + 30_000, + ); + + // Technical: a background task's result reaches memory through the same post-settlement check + // as a late suspend. A result save that passed it but reached storage after a successor + // claimed the run and raised the memory fence went through, since it carried no claim fence, + // so a superseded execution wrote the tool's result. It now carries the finished run's + // claim. (#30182) + it.runIf(isDurableEngineVariant(engine))( + "doesn't record a deferred tool's result that reaches storage after another execution took over (#30182)", + async () => { + const run = await startResearchDuringCleanup('bg-result-slow-save-takeover-thread', { outcome: 'result' }); + const save = holdMemorySave(run.memoryStore, 1); + try { + run.release(); + await save.saveStarted; + const successor = await takeOverAfterCleanup(run); + + save.finishSave(); + expect(await save.saveOutcome).toMatchObject({ id: RUN_FENCE_CONFLICT_ERROR_ID }); + // Memory keeps the dispatch placeholder, not the superseded execution's result. + expect((await storedToolCall(run.memory, run.thread)).invocation?.result?.summary).toBeUndefined(); + await expectSuccessorUnchanged(run, successor); + } finally { + run.finishCleanup(); + } + }, + 30_000, + ); + // Technical: on a shared transport, an evented run's worker releases the run's claim, then // the thread's lease, and only then stops its participant fence. Until the lease release // returns, that fence stays registered, and the late suspend's check asked it to prove it From 0601c44e0af7d61beeb43b6243b4323fb448ea50 Mon Sep 17 00:00:00 2001 From: Leo <108278866+leoisadev1@users.noreply.github.com> Date: Mon, 16 Nov 2026 17:11:19 -0500 Subject: [PATCH 08/15] fix(core): accept a late background write when the heartbeat sees the release mid-check After two failed ownership reads, the last read of verifyNotSuperseded could see the claim held while another worker released it and the heartbeat marked the fence lost. The check then threw that loss though storage showed this exact claim released. A loss recorded while that read is on its way now triggers one more read: the same claim released is accepted, a successor is still refused. Adds LibSQL-backed cases for a late suspension and a late result save that reach the database after a successor raised the memory fence, on durable and evented agents. Co-Authored-By: Claude Opus 5.5 --- .../src/agent/durable/execution-fence.test.ts | 72 ++++ .../core/src/agent/durable/execution-fence.ts | 27 +- ...urable-background-late-write-fence.test.ts | 315 ++++++++++++++++++ 3 files changed, 405 insertions(+), 9 deletions(-) create mode 100644 stores/libsql/src/__tests__/durable-background-late-write-fence.test.ts diff --git a/packages/core/src/agent/durable/execution-fence.test.ts b/packages/core/src/agent/durable/execution-fence.test.ts index 150b13ecc8a..fd00d489b2c 100644 --- a/packages/core/src/agent/durable/execution-fence.test.ts +++ b/packages/core/src/agent/durable/execution-fence.test.ts @@ -1560,6 +1560,78 @@ describe('ExecutionFence.verifyNotSuperseded with a release only storage shows', expect(coordinator.isLost()).toBe(false); }); + it("passes through assertExecutionNotSuperseded when the heartbeat sees another process's release while the check reads the claim as held", async () => { + vi.useFakeTimers(); + const { storage, workflowsStore, coordinator, own } = await setup(); + const mastra = new Mastra({ storage, logger: false }); + const requestContext = new RequestContext(); + setExecutionClaim(requestContext, 'run-1', coordinator.claim); + const args = { runId: 'run-1', agentId, requestContext, mastra }; + + // The check's next read finds the claim live. Before that read returns, another worker releases + // the claim and this fence's heartbeat renewal finds it released and marks the fence lost. + let signalReleased!: () => void; + const released = new Promise(resolve => (signalReleased = resolve)); + let resumeRead!: () => void; + const readResumed = new Promise(resolve => (resumeRead = resolve)); + const reads = failTwoReads(workflowsStore, { + after: async () => { + expect(await workflowsStore.releaseRunOwnership(own)).toBe(true); + signalReleased(); + await readResumed; + }, + }); + + const check = assertExecutionNotSuperseded(args); + // Past the first read's retry delay. + await vi.advanceTimersByTimeAsync(1_000); + await released; + await vi.advanceTimersByTimeAsync(EXECUTION_LEASE_RENEW_INTERVAL_MS); + expect(coordinator.isLost()).toBe(true); + resumeRead(); + + await expect(check).resolves.toBeUndefined(); + reads.mockRestore(); + expect(await workflowsStore.getRunOwnership({ runId: 'run-1' })).toMatchObject({ ...own, leaseExpiresAt: null }); + await expect(assertExecutionNotSuperseded(args)).resolves.toBeUndefined(); + }); + + it('refuses through assertExecutionNotSuperseded when the heartbeat sees a successor while the check reads the claim as held', async () => { + vi.useFakeTimers(); + const { storage, workflowsStore, coordinator } = await setup(); + const mastra = new Mastra({ storage, logger: false }); + const requestContext = new RequestContext(); + setExecutionClaim(requestContext, 'run-1', coordinator.claim); + const args = { runId: 'run-1', agentId, requestContext, mastra }; + + // As above, but the claim is taken over, not released, before the heartbeat renews. + let signalTakenOver!: () => void; + const takenOver = new Promise(resolve => (signalTakenOver = resolve)); + let resumeRead!: () => void; + const readResumed = new Promise(resolve => (resumeRead = resolve)); + failTwoReads(workflowsStore, { + after: async () => { + await workflowsStore.claimRunOwnership({ runId: 'run-1', ownerId: 'successor', leaseMs: 60_000, force: true }); + signalTakenOver(); + await readResumed; + }, + }); + + const check = assertExecutionNotSuperseded(args); + void check.catch(() => {}); + await vi.advanceTimersByTimeAsync(1_000); + await takenOver; + await vi.advanceTimersByTimeAsync(EXECUTION_LEASE_RENEW_INTERVAL_MS); + expect(coordinator.isLost()).toBe(true); + resumeRead(); + + await expect(check).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + expect(await workflowsStore.getRunOwnership({ runId: 'run-1' })).toMatchObject({ + ownerId: 'successor', + live: true, + }); + }); + it('refuses when another execution claims the run between the reads of one check', async () => { const { workflowsStore, coordinator, own } = await setup(); coordinator.stopHeartbeat(); diff --git a/packages/core/src/agent/durable/execution-fence.ts b/packages/core/src/agent/durable/execution-fence.ts index 379ca149799..9f0164f5c75 100644 --- a/packages/core/src/agent/durable/execution-fence.ts +++ b/packages/core/src/agent/durable/execution-fence.ts @@ -1150,17 +1150,16 @@ export class ExecutionFence implements RunFenceScope { * claim released, which then counts as released. The release may have * happened after the check that failed, even between two reads here. A * lapsed or replaced claim, or a failed read, marks the execution lost as - * {@link verify} does. A fence already lost only reads for a release. Lease - * backends can't tell a release from a lapse and run {@link verify}. + * {@link verify} does. A fence already lost only reads for a release. A + * loss recorded while the read was on its way, such as a heartbeat renewal + * that found the claim released after the read, reads the claim once more: + * the same claim released still counts. Lease backends can't tell a release + * from a lapse and run {@link verify}. */ async #verifyOrConfirmRelease(): Promise { - let state: Awaited>; - try { - state = await this.#backend.inspect(this.claim, this.#details()); - } catch (error) { - if (this.#lossError) throw this.#lossError; - return this.#fail(error as DurableExecutionFenceError); - } + const lostBefore = this.#lossError !== undefined; + let state = await this.#inspectForRelease(); + if (state === 'held' && this.#lossError && !lostBefore) state = await this.#inspectForRelease(); if (state === 'released') { this.#released = true; return true; @@ -1170,6 +1169,16 @@ export class ExecutionFence implements RunFenceScope { return false; } + /** One {@link OwnershipBackend.inspect} read. A failed read throws the recorded loss, or marks the execution lost. */ + async #inspectForRelease(): Promise>> { + try { + return await this.#backend.inspect(this.claim, this.#details()); + } catch (error) { + if (this.#lossError) throw this.#lossError; + return this.#fail(error as DurableExecutionFenceError); + } + } + /** Mark a settled fence retired in memory so `memory.runFences` retention can prune it. */ async #retireMemoryFence(fence: RunFence): Promise { if (!this.#memoryStore) return; diff --git a/stores/libsql/src/__tests__/durable-background-late-write-fence.test.ts b/stores/libsql/src/__tests__/durable-background-late-write-fence.test.ts new file mode 100644 index 00000000000..119b674e233 --- /dev/null +++ b/stores/libsql/src/__tests__/durable-background-late-write-fence.test.ts @@ -0,0 +1,315 @@ +/** + * A durable agent's background task can write to memory after the run that dispatched it has + * finished and released its claim: the task's suspension (it asks for input) or its result. The + * write passes an ownership check while nobody has claimed the run, but can reach storage after a + * successor claimed the run and raised the memory fence. These cases run that ordering on a real + * LibSQL file, with workflows and memory in the same store, and check that the late write carries + * the finished run's claim to the memory domain object the adapter fences, so LibSQL's write + * transaction rejects it. (#30182) + */ +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { Agent } from '@mastra/core/agent'; +import { createDurableAgent, createEventedAgent } from '@mastra/core/agent/durable'; +import type { BackgroundTask } from '@mastra/core/background-tasks'; +import { EventEmitterPubSub } from '@mastra/core/events'; +import { Mastra } from '@mastra/core/mastra'; +import { MockMemory } from '@mastra/core/memory'; +import type { MastraDBMessage } from '@mastra/core/memory'; +import { RUN_FENCE_CONFLICT_ERROR_ID } from '@mastra/core/storage'; +import type { MemoryStorage, RunFence } from '@mastra/core/storage'; +import { MastraLanguageModelV2Mock } from '@mastra/core/test-utils/llm-mock'; +import { createTool } from '@mastra/core/tools'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { z } from 'zod'; + +import { LibSQLStore } from '../storage'; +import { gate } from './cross-process/step-agent'; + +const TOOL_CALL_ID = 'call-bg-1'; +const ACK = 'Waiting for the analyst.'; + +const cleanups: Array<() => Promise> = []; +afterEach(async () => { + for (const cleanup of cleanups.splice(0).reverse()) await cleanup(); + vi.restoreAllMocks(); +}); + +async function waitFor(read: () => Promise, ok: (value: T) => boolean, label: string): Promise { + const deadline = Date.now() + 15_000; + for (;;) { + const value = await read(); + if (ok(value)) return value; + if (Date.now() > deadline) throw new Error(`Timed out waiting for ${label}`); + await new Promise(resolve => setTimeout(resolve, 20)); + } +} + +function stream(parts: unknown[]) { + return new ReadableStream({ + start(controller) { + for (const part of parts) controller.enqueue(part); + controller.close(); + }, + }); +} + +// Calls the background tool on the first turn, then acknowledges. Any later turn (a wake to +// report the task) acknowledges too. +function scriptedModel() { + return new MastraLanguageModelV2Mock({ + doStream: async ({ prompt }) => { + const head = [ + { type: 'stream-start', warnings: [] }, + { type: 'response-metadata', id: 'resp', modelId: 'mock-model-id', timestamp: new Date(0) }, + ]; + const usage = { inputTokens: 1, outputTokens: 1, totalTokens: 2 }; + const sawToolResult = prompt.some(message => message.role === 'tool'); + return { + rawCall: { rawPrompt: null, rawSettings: {} }, + stream: stream( + sawToolResult + ? [ + ...head, + { type: 'text-start', id: 'text-1' }, + { type: 'text-delta', id: 'text-1', delta: ACK }, + { type: 'text-end', id: 'text-1' }, + { type: 'finish', finishReason: 'stop', usage }, + ] + : [ + ...head, + { + type: 'tool-call', + toolCallId: TOOL_CALL_ID, + toolName: 'research', + input: JSON.stringify({ topic: 'near' }), + providerExecuted: false, + }, + { type: 'finish', finishReason: 'tool-calls', usage }, + ], + ), + }; + }, + }); +} + +/** Holds the next `saveMessages` on `memoryStore` until `finish()`, and reports how it ended. */ +function holdNextSave(memoryStore: MemoryStorage) { + const started = gate(); + const release = gate(); + const outcome = gate(); + const saveMessages = memoryStore.saveMessages.bind(memoryStore); + vi.spyOn(memoryStore, 'saveMessages').mockImplementationOnce(async args => { + started.resolve(); + await release.promise; + try { + const saved = await saveMessages(args); + outcome.resolve(undefined); + return saved; + } catch (error) { + outcome.resolve(error); + throw error; + } + }); + cleanups.push(async () => release.resolve()); + return { started: started.promise, finish: () => release.resolve(), outcome: outcome.promise }; +} + +function toolCallIn(messages: MastraDBMessage[]) { + for (const message of messages) { + for (const part of message.content.parts ?? []) { + if (part.type === 'tool-invocation' && part.toolInvocation.toolCallId === TOOL_CALL_ID) { + const suspendedTools = message.content.metadata?.suspendedTools; + const suspended = + suspendedTools && typeof suspendedTools === 'object' && TOOL_CALL_ID in suspendedTools + ? (suspendedTools as Record)[TOOL_CALL_ID] + : undefined; + return { invocation: part.toolInvocation, suspended }; + } + } + } + return undefined; +} + +async function startDeferredResearch(engine: 'durable' | 'evented', outcome: 'suspend' | 'result') { + const dir = await mkdtemp(join(tmpdir(), 'mastra-30182-libsql-')); + const storage = new LibSQLStore({ id: `late-write-${engine}-${outcome}`, url: `file:${join(dir, 'mastra.db')}` }); + cleanups.push(() => rm(dir, { recursive: true, force: true })); + // Memory on the same LibSQL file as the run's claim, as a deployment configures it. + const memory = new MockMemory(); + memory.setStorage(storage); + const memoryStore = (await memory.storage.getStore('memory'))!; + // The object the late write's claim fence is keyed to is the one LibSQL's memory domain checks. + expect(memoryStore).toBe(await storage.getStore('memory')); + + // The finished run retires its memory fence last; the takeover waits for that. + const cleanedUp = gate(); + const retireRunFence = memoryStore.retireRunFence.bind(memoryStore); + vi.spyOn(memoryStore, 'retireRunFence').mockImplementation(async (fence: RunFence) => { + try { + return await retireRunFence(fence); + } finally { + cleanedUp.resolve(); + } + }); + + const toolGate = gate(); + const research = createTool({ + id: 'research', + description: 'Research a topic after analyst approval', + inputSchema: z.object({ topic: z.string() }), + outputSchema: z.object({ summary: z.string() }), + background: { enabled: true }, + execute: async ({ topic }, options) => { + await toolGate.promise; + if (outcome === 'result') return { summary: `Research on "${topic}"` }; + return options?.agent?.suspend?.({ awaiting: 'analyst-approval', topic }) as never; + }, + }); + const agent = new Agent({ + id: `late-write-${engine}`, + name: 'Late write agent', + instructions: 'Research what the user asks.', + model: scriptedModel(), + tools: { research }, + memory, + backgroundTasks: { tools: { research: true } }, + }); + const pubsub = new EventEmitterPubSub(); + const durable = engine === 'durable' ? createDurableAgent({ agent }) : createEventedAgent({ agent, pubsub }); + const mastra = new Mastra({ + logger: false, + storage, + pubsub, + agents: { [durable.id]: durable }, + backgroundTasks: { enabled: true, globalConcurrency: 5, perAgentConcurrency: 3 }, + }); + await mastra.startWorkers(); + cleanups.push(async () => { + await mastra.backgroundTaskManager?.shutdown(); + await mastra.stopWorkers(); + }); + + const thread = { threadId: `late-write-${engine}-${outcome}`, resourceId: 'late-write-user' }; + const run = await durable.stream('Research near', { + memory: { thread: thread.threadId, resource: thread.resourceId }, + maxSteps: 3, + }); + let taskId: string | undefined; + for await (const chunk of run.fullStream) { + if (chunk.type === 'background-task-started') taskId = chunk.payload.taskId; + } + expect(await run.output.text).toBe(ACK); + expect(taskId).toBeDefined(); + + const workflows = (await storage.getStore('workflows'))!; + const released = await waitFor( + () => workflows.getRunOwnership({ runId: run.runId }), + record => record?.live === false, + 'the finished run to release its claim', + ); + await cleanedUp.promise; + + return { + memoryStore, + workflows, + thread, + runId: run.runId, + released: released!, + letToolGo: () => toolGate.resolve(), + storedToolCall: async () => toolCallIn((await memoryStore.listMessages({ ...thread, perPage: false })).messages), + settledTask: () => + waitFor( + () => mastra.backgroundTaskManager!.getTask(taskId!), + (task: BackgroundTask | null) => task?.status !== 'pending' && task?.status !== 'running', + 'the background task to settle', + ), + }; +} + +type Research = Awaited>; + +/** A successor claims the run and raises the memory fence, as recovery or a new turn does before it touches memory. */ +async function takeOver(research: Research) { + const successor = await research.workflows.claimRunOwnership({ + runId: research.runId, + ownerId: 'successor-execution', + leaseMs: 60_000, + expectedGeneration: research.released.generation, + }); + expect(successor.acquired).toBe(true); + const fence = { runId: research.runId, generation: successor.record!.generation, ownerId: 'successor-execution' }; + expect(await research.memoryStore.raiseRunFence(fence)).toBe(true); + return fence; +} + +/** LibSQL refused the write because it carried the finished run's own claim, now replaced. */ +function expectRejectedForFinishedClaim(error: unknown, research: Research) { + expect(error).toMatchObject({ + id: RUN_FENCE_CONFLICT_ERROR_ID, + details: { + runId: research.runId, + generation: research.released.generation, + ownerId: research.released.ownerId, + operation: 'saveMessages', + }, + }); +} + +async function expectSuccessorUnchanged(research: Research, successor: RunFence) { + const after = await research.workflows.getRunOwnership({ runId: research.runId }); + expect({ ownerId: after?.ownerId, generation: after?.generation, live: after?.live }).toEqual({ + ownerId: successor.ownerId, + generation: successor.generation, + live: true, + }); +} + +describe.each(['durable', 'evented'] as const)( + '%s agent on LibSQL: late background writes after a takeover', + engine => { + // Technical: the suspension's save passed `assertExecutionNotSuperseded` with no successor yet, + // then reached LibSQL after a successor claimed the run and raised the memory fence. + // `runFencedToClaim` gives the save the finished run's claim, and the write transaction rejects it. (#30182) + it("doesn't park a deferred tool whose suspension reaches the database after another execution took over (#30182)", async () => { + const research = await startDeferredResearch(engine, 'suspend'); + const save = holdNextSave(research.memoryStore); + research.letToolGo(); + await save.started; + const successor = await takeOver(research); + save.finish(); + + expectRejectedForFinishedClaim(await save.outcome, research); + const task = await research.settledTask(); + expect({ status: task?.status, error: task?.error?.message }).toEqual({ + status: 'failed', + error: expect.stringContaining("is no longer the run's current claim"), + }); + expect(task?.suspendPayload).toBeUndefined(); + const stored = await research.storedToolCall(); + expect(stored?.invocation).toBeDefined(); + expect(stored?.suspended).toBeUndefined(); + await expectSuccessorUnchanged(research, successor); + }, 30_000); + + // Technical: same ordering for the background task's result (`onResult` flush). (#30182) + it("doesn't record a deferred tool's result that reaches the database after another execution took over (#30182)", async () => { + const research = await startDeferredResearch(engine, 'result'); + const save = holdNextSave(research.memoryStore); + research.letToolGo(); + await save.started; + const successor = await takeOver(research); + save.finish(); + + expectRejectedForFinishedClaim(await save.outcome, research); + // Memory keeps the dispatch placeholder, not the superseded execution's result. + const stored = await research.storedToolCall(); + expect(stored?.invocation).toBeDefined(); + expect(JSON.stringify(stored?.invocation)).not.toContain('Research on'); + expect(stored?.suspended).toBeUndefined(); + await expectSuccessorUnchanged(research, successor); + }, 30_000); + }, +); From 843ea58f9e96ff91eae39ef89440b12222f6bbc2 Mon Sep 17 00:00:00 2001 From: Leo <108278866+leoisadev1@users.noreply.github.com> Date: Mon, 16 Nov 2026 18:35:55 -0500 Subject: [PATCH 09/15] fix(core): accept a late background write unless a successor claimed the run With ownership in storage, verifyNotSuperseded now makes one ownership read and refuses only when another execution claimed the run (a newer generation or a missing record). It no longer tracks releases, inspects released records or waits for a release's reply, so a claim release whose reply never came no longer leaves a suspending background task running. The storage fence that runFencedToClaim puts on the write still rejects it once a successor raised the memory fence. Lease-backed claims have no generation and no storage fence, so they keep the strict check until this process starts releasing the claim. Accepting a lapsed lease there would let a stale write land after a recovery claimed and released the run. Adds the held-release-reply scenarios on durable and evented, a two-sibling deferred task scenario, and the lapsed-lease recovery case; updates the unit tests that pinned an expired storage claim as refused. (#30182) Co-Authored-By: Claude Opus 5.5 --- .changeset/twelve-papers-stand.md | 2 +- .../src/agent/durable/execution-fence.test.ts | 144 +++++++++------ .../core/src/agent/durable/execution-fence.ts | 167 +++++------------- ...background-task-lifecycle.scenario.test.ts | 142 +++++++++++---- 4 files changed, 243 insertions(+), 212 deletions(-) diff --git a/.changeset/twelve-papers-stand.md b/.changeset/twelve-papers-stand.md index c0d62854195..3e73912e3c0 100644 --- a/.changeset/twelve-papers-stand.md +++ b/.changeset/twelve-papers-stand.md @@ -2,4 +2,4 @@ '@mastra/core': patch --- -Fixed background tasks on durable agents with memory failing with "could not verify that this execution still owns the run" when they suspend after the agent's run has finished. The task is now marked suspended and can be resumed with `backgroundTaskManager.resume()`. This covers tools that call `suspend()` and sub-agents that reach a tool needing approval, including when another evented worker finished the run. A background task's result that arrives while the finished run is still cleaning up is now saved to memory instead of being dropped. Fixes [#30182](https://github.com/mastra-ai/mastra/issues/30182). +Fixed background tasks on durable agents with memory failing with "could not verify that this execution still owns the run" when they suspend after the agent's run has finished. The task is now marked suspended and can be resumed with `backgroundTaskManager.resume()`. This covers tools that call `suspend()` and sub-agents that reach a tool needing approval, including when another evented worker finished the run. A background task's result that arrives while the finished run is still cleaning up is now saved to memory instead of being dropped. A background task no longer stays `running` while storage is slow to confirm that the finished run gave up its claim. Fixes [#30182](https://github.com/mastra-ai/mastra/issues/30182). diff --git a/packages/core/src/agent/durable/execution-fence.test.ts b/packages/core/src/agent/durable/execution-fence.test.ts index fd00d489b2c..50b20e52abb 100644 --- a/packages/core/src/agent/durable/execution-fence.test.ts +++ b/packages/core/src/agent/durable/execution-fence.test.ts @@ -1221,10 +1221,10 @@ function holdNextCall( } // A background task can write after its run settled: `verifyNotSuperseded` is the check for those -// writes. Before the claim is released it is `verify`; after, it only refuses once another -// execution claimed the run. While the release is in flight, a check that fails waits for the -// release before deciding. Storage can also show a release this fence didn't make or couldn't -// confirm; a lease can't tell a release from a lapse (#30182). +// writes. With ownership in storage it only refuses once another execution claimed the run, released +// or lapsed or not, and never waits for a release in flight. A lease can't tell a release from a lapse +// that another execution claimed and released since, so a lease-backed claim is `verify` until a +// release of it starts here (#30182). describe.each([ { backend: 'storage', @@ -1241,7 +1241,8 @@ describe.each([ const own = { runId: 'run-1', generation: fence.generation!, ownerId: fence.executionId }; return { fence, - showsRelease: true, + // Storage keeps the claim's generation: a lapse or a release it didn't see still accepts the write. + storageBacked: true, dropClaim: async () => void (await workflowsStore.releaseRunOwnership(own)), // The store lost the claim's record: nobody can say who holds the run. loseClaim: () => workflowsStore.dangerouslyClearAll(), @@ -1270,7 +1271,7 @@ describe.each([ const fence = await claim(pubsub, 'run-1'); return { fence, - showsRelease: false, + storageBacked: false, dropClaim: () => pubsub.releaseLease(key('run-1'), fence.executionId), loseClaim: () => pubsub.releaseLease(key('run-1'), fence.executionId), failRenewals: () => vi.spyOn(pubsub, 'renewLease').mockRejectedValue(new Error('redis unreachable')), @@ -1283,16 +1284,24 @@ describe.each([ }, }, ])('ExecutionFence.verifyNotSuperseded ($backend)', ({ setup }) => { - it('before settlement, is verify: passes while owned, and refuses a claim that expired though nobody took it', async () => { + it('before settlement, passes while owned; after the claim expired, passes only when storage holds the claim', async () => { vi.useFakeTimers(); - const { fence, failRenewals } = await setup(); + const { fence, failRenewals, claimSuccessor, storageBacked } = await setup(); await expect(fence.verifyNotSuperseded()).resolves.toBeUndefined(); // Renewals can't reach the backend until the claim's lease runs out. failRenewals(); await vi.advanceTimersByTimeAsync(EXECUTION_LEASE_TTL_MS + EXECUTION_LEASE_RENEW_INTERVAL_MS); expect(fence.isLost()).toBe(true); - await expect(fence.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_UNVERIFIED_ERROR_ID }); + if (storageBacked) { + // Nobody claimed the run, so storage would still accept the claim's write. + await expect(fence.verifyNotSuperseded()).resolves.toBeUndefined(); + await claimSuccessor(); + await expect(fence.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + } else { + // Without a release here, a missing lease may be one another execution claimed and released: verify's rule. + await expect(fence.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_UNVERIFIED_ERROR_ID }); + } }); it('after settlement released the claim, passes while no other execution claimed the run', async () => { @@ -1318,23 +1327,24 @@ describe.each([ // The claim is still held, so the check passes as verify would. await expect(fence.verifyNotSuperseded()).resolves.toBeUndefined(); - // The unreleased claim is gone with nobody shown taking the run: verify's rule, not the post-release one. + // The unreleased claim is gone and nothing shows who holds the run now: storage lost the record, + // and a lease without a release here gets verify's rule. await loseClaim(); await expect(fence.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); expect(fence.isLost()).toBe(true); }); - it('while settlement is releasing the claim, waits for the release and passes while no other execution claimed the run', async () => { + it("while settlement is releasing the claim, passes without waiting for the release's reply while no other execution claimed the run", async () => { const { fence, holdNextRelease } = await setup(); const release = holdNextRelease(); const settled = fence.settle(async () => {}); await release.applied; - // The backend already shows the claim released; the strict check alone would mark the execution lost. - const check = fence.verifyNotSuperseded(); - release.finish(); - await expect(check).resolves.toBeUndefined(); + // The backend already shows the claim released and its reply is still held, as on a stalled + // connection: the check neither marks the execution lost nor waits for the reply. + await expect(fence.verifyNotSuperseded()).resolves.toBeUndefined(); expect(fence.isLost()).toBe(false); + release.finish(); expect(await settled).toBe('owned'); }); @@ -1345,31 +1355,31 @@ describe.each([ await release.applied; await claimSuccessor(); - const check = fence.verifyNotSuperseded(); + await expect(fence.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); release.finish(); - await expect(check).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); await settled; }); - it('when the release in flight fails, passes only if the backend shows this claim released', async () => { - const { fence, dropClaim, holdNextRelease, showsRelease } = await setup(); + it('once the release in flight fails, passes only while storage holds the claim', async () => { + const { fence, dropClaim, holdNextRelease, storageBacked } = await setup(); // The claim was released already, as another worker would; settlement still tries to release it. await dropClaim(); const release = holdNextRelease({ fail: true }); const settled = fence.settle(async () => {}); await release.applied; - - const check = fence.verifyNotSuperseded(); + // While the release is in flight, it counts. + await expect(fence.verifyNotSuperseded()).resolves.toBeUndefined(); release.finish(); - if (showsRelease) { - // Storage still holds this exact claim, released: no other execution took the run. - await expect(check).resolves.toBeUndefined(); + await settled; + + if (storageBacked) { + // Storage still holds this claim's generation, released: no other execution took the run. + await expect(fence.verifyNotSuperseded()).resolves.toBeUndefined(); } else { - // A missing lease may have lapsed: verify's rule. - await expect(check).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + // The release failed, so a missing lease may be one another execution claimed and released: verify's rule. + await expect(fence.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); expect(fence.isLost()).toBe(true); } - await settled; }); it('refuses after settling as superseded', async () => { @@ -1395,17 +1405,16 @@ describe('ExecutionFence.verifyNotSuperseded on a lease-backed participant', () return { pubsub, mastra, coordinator, participant }; } - it("waits for this worker's shared release in flight, then passes while nobody claimed the run", async () => { + it("counts this worker's shared release in flight without waiting for its reply, and passes while nobody claimed the run", async () => { const { pubsub, mastra, coordinator, participant } = await setup(); const release = holdNextCall(pubsub, 'releaseLease', { apply: true, fail: false }); const releasing = ExecutionFence.releaseSharedClaim(mastra, 'run-1', coordinator.claim); await release.applied; expect(await pubsub.getLeaseOwner(key('run-1'))).toBeUndefined(); - const check = participant.verifyNotSuperseded(); - release.finish(); - await expect(check).resolves.toBeUndefined(); + await expect(participant.verifyNotSuperseded()).resolves.toBeUndefined(); expect(participant.isLost()).toBe(false); + release.finish(); await releasing; }); @@ -1418,9 +1427,33 @@ describe('ExecutionFence.verifyNotSuperseded on a lease-backed participant', () }); }); -// Storage keeps a released claim's record with its lease cleared, so it can show a release this fence -// didn't see: one made by another process, or one storage applied but whose response was lost (#30182). -describe('ExecutionFence.verifyNotSuperseded with a release only storage shows', () => { +// A lease has no generation and no storage fence behind the check: once a lapsed lease was claimed and +// released by another execution, the lease looks exactly as it would after this execution's own release. +// Only a release this process started lets a late write through (#30182). +describe('ExecutionFence.verifyNotSuperseded on a lapsed lease', () => { + it('refuses a late write after a recovery claimed the lapsed lease and released it', async () => { + vi.useFakeTimers(); + const pubsub = new EventEmitterPubSub(); + const fence = await claim(pubsub, 'run-1'); + // Renewals can't reach the lease provider until the lease runs out, as a dropped connection would. + const renewals = vi.spyOn(pubsub, 'renewLease').mockRejectedValue(new Error('redis unreachable')); + await vi.advanceTimersByTimeAsync(EXECUTION_LEASE_TTL_MS + EXECUTION_LEASE_RENEW_INTERVAL_MS); + expect(fence.isLost()).toBe(true); + renewals.mockRestore(); + + // Another process recovers the run, finishes it and releases the lease. + expect((await pubsub.acquireLease(key('run-1'), 'recovery', EXECUTION_LEASE_TTL_MS)).acquired).toBe(true); + await pubsub.releaseLease(key('run-1'), 'recovery'); + expect(await pubsub.getLeaseOwner(key('run-1'))).toBeUndefined(); + + await expect(fence.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_UNVERIFIED_ERROR_ID }); + }); +}); + +// Storage keeps a released or lapsed claim's record and generation, so a late write is checked against +// the store, not against what this fence saw: a release made by another process, one storage applied but +// whose response was lost, or a lapse. Only another execution's claim refuses it (#30182). +describe('ExecutionFence.verifyNotSuperseded with ownership in storage', () => { async function setup() { const storage = new InMemoryStore(); const workflowsStore = (await storage.getStore('workflows')) as WorkflowsStorage; @@ -1488,7 +1521,7 @@ describe('ExecutionFence.verifyNotSuperseded with a release only storage shows', expect(participant.isLost()).toBe(true); }); - it("stays strict on a participant fence when its release failed and the claim's lease lapsed", async () => { + it("passes on a participant fence whose release failed and whose claim's lease lapsed, until another execution claims the run", async () => { vi.useFakeTimers(); const { storage, workflowsStore, coordinator } = await setup(); coordinator.stopHeartbeat(); @@ -1506,8 +1539,17 @@ describe('ExecutionFence.verifyNotSuperseded with a release only storage shows', expect(record).toMatchObject({ ownerId: coordinator.executionId, live: false }); expect(record?.leaseExpiresAt).not.toBeNull(); - // An expired claim is not a released one. - await expect(participant.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_UNVERIFIED_ERROR_ID }); + // Storage would still accept the expired claim's write: nobody claimed the run. + vi.mocked(workflowsStore.renewRunOwnership).mockRestore(); + await expect(participant.verifyNotSuperseded()).resolves.toBeUndefined(); + const successor = await workflowsStore.claimRunOwnership({ + runId: 'run-1', + ownerId: 'successor', + leaseMs: 60_000, + expectedGeneration: coordinator.generation, + }); + expect(successor.acquired).toBe(true); + await expect(participant.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); }); it('counts a release storage applied but whose response failed as released', async () => { @@ -1526,9 +1568,9 @@ describe('ExecutionFence.verifyNotSuperseded with a release only storage shows', await expect(coordinator.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); }); - // Reads of the claim during one check: the first two fail as a dropped connection would, which - // exhausts the first read's retry. `before` and `after` run around the third read. - function failTwoReads( + // Reads of the claim during one check: the first fails as a dropped connection would, and the + // check retries it. `before` and `after` run around the retried read. + function failFirstRead( workflowsStore: WorkflowsStorage, { before, after }: { before?: () => Promise; after?: () => Promise }, ) { @@ -1536,19 +1578,19 @@ describe('ExecutionFence.verifyNotSuperseded with a release only storage shows', let reads = 0; return vi.spyOn(workflowsStore, 'getRunOwnership').mockImplementation(async args => { reads++; - if (reads <= 2) throw new Error('connection reset'); - if (reads === 3) await before?.(); + if (reads === 1) throw new Error('connection reset'); + if (reads === 2) await before?.(); const record = await read(args); - if (reads === 3) await after?.(); + if (reads === 2) await after?.(); return record; }); } - it('passes when another process releases the claim between the reads of one check', async () => { + it('passes when another process releases the claim while one check reads it', async () => { const { workflowsStore, coordinator, own } = await setup(); coordinator.stopHeartbeat(); - // The check's next read still finds the claim live; another worker releases it just after. - const reads = failTwoReads(workflowsStore, { + // The check's retried read still finds the claim live; another worker releases it just after. + const reads = failFirstRead(workflowsStore, { after: async () => expect(await workflowsStore.releaseRunOwnership(own)).toBe(true), }); @@ -1574,7 +1616,7 @@ describe('ExecutionFence.verifyNotSuperseded with a release only storage shows', const released = new Promise(resolve => (signalReleased = resolve)); let resumeRead!: () => void; const readResumed = new Promise(resolve => (resumeRead = resolve)); - const reads = failTwoReads(workflowsStore, { + const reads = failFirstRead(workflowsStore, { after: async () => { expect(await workflowsStore.releaseRunOwnership(own)).toBe(true); signalReleased(); @@ -1609,7 +1651,7 @@ describe('ExecutionFence.verifyNotSuperseded with a release only storage shows', const takenOver = new Promise(resolve => (signalTakenOver = resolve)); let resumeRead!: () => void; const readResumed = new Promise(resolve => (resumeRead = resolve)); - failTwoReads(workflowsStore, { + failFirstRead(workflowsStore, { after: async () => { await workflowsStore.claimRunOwnership({ runId: 'run-1', ownerId: 'successor', leaseMs: 60_000, force: true }); signalTakenOver(); @@ -1632,11 +1674,11 @@ describe('ExecutionFence.verifyNotSuperseded with a release only storage shows', }); }); - it('refuses when another execution claims the run between the reads of one check', async () => { + it('refuses when another execution claims the run while one check reads it', async () => { const { workflowsStore, coordinator, own } = await setup(); coordinator.stopHeartbeat(); - // Another worker releases the claim and a successor claims the run before the check's next read. - failTwoReads(workflowsStore, { + // Another worker releases the claim and a successor claims the run before the check's retried read. + failFirstRead(workflowsStore, { before: async () => { expect(await workflowsStore.releaseRunOwnership(own)).toBe(true); await workflowsStore.claimRunOwnership({ runId: 'run-1', ownerId: 'successor', leaseMs: 60_000 }); diff --git a/packages/core/src/agent/durable/execution-fence.ts b/packages/core/src/agent/durable/execution-fence.ts index 9f0164f5c75..190647c802a 100644 --- a/packages/core/src/agent/durable/execution-fence.ts +++ b/packages/core/src/agent/durable/execution-fence.ts @@ -353,20 +353,7 @@ interface OwnershipBackend { renew(claim: DurableExecutionClaim): Promise<{ renewed: boolean; leaseExpiresAt?: Date | null }>; /** Whether another execution claimed the run after `claim`. Throws on backend errors. */ isSuperseded(claim: DurableExecutionClaim): Promise; - /** - * One read of `claim`, for a check that may follow its release: `'held'` when - * {@link verify} would pass, `'released'` when the store still names this - * claim's owner and generation with its lease cleared, which only a release - * does (a lapsed lease keeps its expiry). Throws like {@link verify} - * otherwise. `undefined`, without a read, when the backend can't tell a - * release from a lapse. - */ - inspect(claim: DurableExecutionClaim, details: ExecutionDetails): Promise<'held' | 'released' | undefined>; - /** - * Release `claim`. Resolves `false` when the store no longer held it, so - * nothing was released. Lease backends can't tell and resolve `true`. - */ - release(claim: DurableExecutionClaim): Promise; + release(claim: DurableExecutionClaim): Promise; /** * Release `claim` so the run can be recovered at once, and make storage * reject its remaining writes. Returns the claim that superseded it, when @@ -426,11 +413,8 @@ function leaseBackend(provider: LeaseProvider, agentId: string, runId: string): const owner = await provider.getLeaseOwner(key); return owner !== undefined && owner !== claim.executionId; }, - // A missing lease may have been released or may have lapsed. - inspect: async () => undefined, async release(claim) { await provider.releaseLease(key, claim.executionId); - return true; }, // A lease has no generation to move past: releasing is all it can do. async abandon(claim, whileOwned) { @@ -456,12 +440,6 @@ function storageBackend(store: WorkflowsStorage, runId: string): OwnershipBacken holder: record?.ownerId ?? undefined, leaseExpiresAt: record?.leaseExpiresAt, }); - // A release clears the lease of the claim's own record; a lapsed lease keeps its expiry. - const inspect: OwnershipBackend['inspect'] = async (claim, details) => { - const record = await retryOnce(() => store.getRunOwnership({ runId }), details); - if (record && matchesRunFence(record, fence(claim))) return record.leaseExpiresAt ? 'held' : 'released'; - throw new DurableExecutionFenceError(EXECUTION_SUPERSEDED_ERROR_ID, details); - }; return { expires: true, // A release keeps the generation, so a pinned claim only misses when another @@ -501,7 +479,8 @@ function storageBackend(store: WorkflowsStorage, runId: string): OwnershipBacken // renewal, it fails once the claim was released: storage still accepts the // released claim's late writes, but the execution no longer drives the run. async verify(claim, details) { - if ((await inspect(claim, details)) !== 'held') { + const record = await retryOnce(() => store.getRunOwnership({ runId }), details); + if (!record?.leaseExpiresAt || !matchesRunFence(record, fence(claim))) { throw new DurableExecutionFenceError(EXECUTION_SUPERSEDED_ERROR_ID, details); } }, @@ -522,8 +501,9 @@ function storageBackend(store: WorkflowsStorage, runId: string): OwnershipBacken const record = await store.getRunOwnership({ runId }); return !record || record.generation !== claim.generation; }, - inspect, - release: claim => store.releaseRunOwnership(fence(claim)), + async release(claim) { + await store.releaseRunOwnership(fence(claim)); + }, // Releasing alone keeps accepting the claim's writes until the next claim. // Claiming under a fresh owner first moves the generation past them; the // pinned generation leaves a claim made in between to its claimant. @@ -590,10 +570,12 @@ export class ExecutionFence implements RunFenceScope { #renewalInFlight = false; #settlement?: Promise; #settledAs?: ExecutionSettlement; - /** Settlement released the claim; cleanup after the release may still be running. */ + /** + * A release of this claim started here (settlement, or this worker's shared + * release) and has not failed. Only lease-backed claims need it: see + * {@link verifyNotSuperseded}. + */ #released = false; - /** Set once settlement starts releasing the claim. Resolves to whether the release succeeded. */ - #releasing?: Promise; #resolveSettled!: () => void; /** Resolves once {@link settle} has finished and the claim is released or left to its new holder. */ readonly whenSettled: Promise; @@ -716,11 +698,10 @@ export class ExecutionFence implements RunFenceScope { const backend = await resolveRemoteBackend(mastra, claim.agentId, runId, claim); if (!backend) return; // This worker's participant fence stays registered until its cleanup stops it. Late writes - // checked against it meanwhile get the post-release rule of {@link verifyNotSuperseded}. + // checked against it meanwhile count this release (see {@link verifyNotSuperseded}). const participant = mastra.__getRunScope(runId)?.get(DURABLE_PARTICIPANT_FENCES)?.get(claim.executionId); - const release = backend.release(claim); - if (participant) void participant.#trackRelease(release); - await release; + const release = () => backend.release(claim); + await (participant ? participant.#countRelease(release) : release()); } /** Join an existing claim without acquiring ownership or advancing its generation. */ @@ -942,55 +923,43 @@ export class ExecutionFence implements RunFenceScope { * never recovers. */ async verify(): Promise { - await this.#verify(false); - } - - /** - * {@link verify}. With `yieldToRelease`, a failed check returns `false` - * instead of marking the execution lost: it may have read a release of the - * claim, not a successor. - */ - async #verify(yieldToRelease: boolean): Promise { - if (this.#lossError) { - if (yieldToRelease) return false; - throw this.#lossError; - } + if (this.#lossError) throw this.#lossError; const startedAt = Date.now(); try { await this.#backend.verify(this.claim, this.#details()); } catch (error) { - if (yieldToRelease) return false; - return this.#fail(error as DurableExecutionFenceError); + this.#markLost(error as DurableExecutionFenceError); + if ((error as DurableExecutionFenceError).id === EXECUTION_SUPERSEDED_ERROR_ID) await this.#noteTakeover(); + throw error; } // A lease backend renews on verify; a storage read leaves expiry to the heartbeat. if (this.generation === undefined) this.#leaseExpiresAt = startedAt + EXECUTION_LEASE_TTL_MS; - return true; - } - - /** Mark the execution lost after a failed check, and throw. */ - async #fail(error: DurableExecutionFenceError): Promise { - this.#markLost(error); - if (error.id === EXECUTION_SUPERSEDED_ERROR_ID) await this.#noteTakeover(); - throw error; } /** - * {@link verify} for writes that may land after settlement. Once the claim - * was released, the write is accepted until another execution claims the - * run, as it is after this fence is unregistered. A check that fails while - * a release is in flight waits for it, then applies the same rule. - * Otherwise a failed check reads the claim once more, and a store that - * shows it released counts as a release: by another process, such as the - * evented worker that settled the run, or by a release here whose response - * failed. A lapsed claim stays strict. + * {@link verify} for writes that may land after settlement, such as a + * background task's result or suspension: refuses only once another + * execution claimed the run. A released or lapsed claim still passes, as + * storage accepts the claim's writes until the next claim, and the write's + * own fence ({@link runFencedToClaim}) rejects it once a successor raised + * memory. Never waits for a release in flight. + * + * A lease has no generation and no storage fence behind this check, and a + * lapsed lease that another execution claimed and released since looks like + * a released one. A lease-backed claim gets this rule only after a release + * started here; otherwise it is {@link verify}. A successor that claimed and + * released the lease after that release still goes unnoticed. */ async verifyNotSuperseded(): Promise { - if (!this.#released && (await this.#verify(true))) return; - // Only a release excuses a failed check: one confirmed here, or one the store shows for this exact claim. - if (!(await this.#releasing) && !(await this.#verifyOrConfirmRelease())) return; + if (this.generation === undefined && !this.#released) return this.verify(); const details = this.#details(); - if (this.#takenOver || (await retryOnce(() => this.#backend.isSuperseded(this.claim), details))) { - throw new DurableExecutionFenceError(EXECUTION_SUPERSEDED_ERROR_ID, details); + const superseded = this.#takenOver || (await retryOnce(() => this.#backend.isSuperseded(this.claim), details)); + // A takeover the heartbeat saw while the read was on its way also refuses. + if (superseded || this.#takenOver) { + const error = new DurableExecutionFenceError(EXECUTION_SUPERSEDED_ERROR_ID, details); + this.#takenOver = true; + this.#markLost(error); + throw error; } } @@ -1112,10 +1081,8 @@ export class ExecutionFence implements RunFenceScope { } async #release(): Promise { - const release = this.#backend.release(this.claim); - void this.#trackRelease(release); try { - await release; + await this.#countRelease(() => this.#backend.release(this.claim)); } catch (error) { this.#logger?.warn?.(`[DurableAgent] run ${this.runId}: failed to release the execution claim: ${error}`); } @@ -1124,58 +1091,14 @@ export class ExecutionFence implements RunFenceScope { } } - /** - * Record a release of this claim for {@link verifyNotSuperseded}. Only a - * release the store confirms counts. A failed release, or one that found the - * claim no longer held, keeps the outcome of an earlier one. A failed release - * may still have been applied, with only its response lost: - * {@link verifyNotSuperseded} asks the store before staying strict. - */ - #trackRelease(release: Promise): Promise { - const previous = this.#releasing; - const unconfirmed = async () => (await previous) ?? false; - this.#releasing = release.then(released => { - // `false`: the store no longer held this claim, so this call released nothing. - if (!released) return unconfirmed(); - this.#released = true; - return true; - }, unconfirmed); - return this.#releasing; - } - - /** - * The last check of {@link verifyNotSuperseded} when no release was - * confirmed here: one read that passes while the claim is held, as - * {@link verify} does, and returns `true` when the store shows this exact - * claim released, which then counts as released. The release may have - * happened after the check that failed, even between two reads here. A - * lapsed or replaced claim, or a failed read, marks the execution lost as - * {@link verify} does. A fence already lost only reads for a release. A - * loss recorded while the read was on its way, such as a heartbeat renewal - * that found the claim released after the read, reads the claim once more: - * the same claim released still counts. Lease backends can't tell a release - * from a lapse and run {@link verify}. - */ - async #verifyOrConfirmRelease(): Promise { - const lostBefore = this.#lossError !== undefined; - let state = await this.#inspectForRelease(); - if (state === 'held' && this.#lossError && !lostBefore) state = await this.#inspectForRelease(); - if (state === 'released') { - this.#released = true; - return true; - } - if (state === undefined) await this.verify(); - else if (this.#lossError) throw this.#lossError; - return false; - } - - /** One {@link OwnershipBackend.inspect} read. A failed read throws the recorded loss, or marks the execution lost. */ - async #inspectForRelease(): Promise>> { + /** Run a release of this claim, counted as {@link #released} from its start unless it fails. */ + async #countRelease(release: () => Promise): Promise { + this.#released = true; try { - return await this.#backend.inspect(this.claim, this.#details()); + await release(); } catch (error) { - if (this.#lossError) throw this.#lossError; - return this.#fail(error as DurableExecutionFenceError); + this.#released = false; + throw error; } } diff --git a/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts b/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts index 38be5c11f88..a1710d24aee 100644 --- a/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts +++ b/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts @@ -809,6 +809,97 @@ describeForAllEngines( expect(seenOnResume).toEqual([{ awaiting: 'analyst-approval', topic: 'near' }]); }, 30_000); + // Technical: two deferred tools from one turn share the finished run's claim. Answering the + // first finishes it and wakes the agent to report it, which runs on the thread again. The + // second tool's late suspend still carries the first run's claim and must still park. (#30182) + it.runIf(isDurableEngineVariant(engine))( + 'parks the second of two deferred tools after the first was answered and reported (#30182)', + async () => { + let releaseAlpha!: () => void; + let releaseBeta!: () => void; + const gates: Record> = { + alpha: new Promise(resolve => (releaseAlpha = resolve)), + beta: new Promise(resolve => (releaseBeta = resolve)), + }; + const memory = new MockMemory(); + const thread = { threadId: 'bg-suspend-sibling-thread', resourceId: 'bg-suspend-sibling-user' }; + const prompt = 'Research alpha and beta'; + const tool = createTool({ + id: 'research-with-approval', + description: 'Research a topic after analyst approval', + inputSchema: z.object({ topic: z.string() }), + outputSchema: z.object({ summary: z.string() }), + background: { enabled: true }, + execute: async ({ topic }, options) => { + const resumeData = options?.agent?.resumeData as { notes?: string } | undefined; + if (!resumeData) { + await gates[topic]; + return options?.agent?.suspend?.({ awaiting: 'analyst-approval', topic }) as any; + } + return { summary: `Research on "${topic}": ${resumeData.notes}` }; + }, + }); + const { chunks, mastra, output } = await runLoopScenario({ + engine, + llm: getMock(), + prompt, + tools: { researchWithApproval: tool }, + agentBackgroundTasks: { tools: { researchWithApproval: true } }, + backgroundTasks: BG_CONFIG, + memory, + ...thread, + stopWhen: stepCountIs(3), + collectChunks: true, + fixtures: llm => + onReportTurns(llm, prompt, { + dispatch: { + toolCalls: [ + { id: 'call-bg-1', name: 'researchWithApproval', arguments: { topic: 'alpha' } }, + { id: 'call-bg-2', name: 'researchWithApproval', arguments: { topic: 'beta' } }, + ], + }, + ack: 'Waiting for the analyst.', + report: 'Alpha answered.', + }), + }); + expect(await output.text).toBe('Waiting for the analyst.'); + const manager = mastra.backgroundTaskManager!; + const started = chunks!.filter(c => c.type === 'background-task-started') as Array; + expect(started).toHaveLength(2); + const tasks = await Promise.all(started.map(c => manager.getTask(c.payload.taskId as string))); + const taskFor = (topic: string) => tasks.find(t => (t?.args as { topic?: string })?.topic === topic)!.id; + const alpha = taskFor('alpha'); + const beta = taskFor('beta'); + await waitForRunToRelease(mastra, started[0].runId); + + releaseAlpha(); + await waitFor( + () => manager.getTask(alpha), + (t: any) => t?.status === 'suspended', + ); + await manager.resume(alpha, { notes: 'looks promising' }); + await waitFor( + () => manager.getTask(alpha), + (t: any) => t?.status === 'completed', + ); + // The wake run reported the first tool and ended. + await waitFor(storedText(memory, thread), s => s.includes('Alpha answered.')); + await waitForRunToRelease(mastra, started[0].runId); + + releaseBeta(); + const parked = (await waitFor( + () => manager.getTask(beta), + (t: any) => t?.status !== 'pending' && t?.status !== 'running', + )) as any; + expect({ status: parked.status, error: parked.error?.message }).toEqual({ + status: 'suspended', + error: undefined, + }); + expect(parked.suspendPayload).toEqual({ awaiting: 'analyst-approval', topic: 'beta' }); + }, + 30_000, + ); + // Technical: the late suspend above is only allowed while no other execution has claimed the // run. Once a successor claims it, the old execution's suspension must not reach memory or // park the task, and the successor's claim must stay as it was. (#30182) @@ -1147,43 +1238,23 @@ describeForAllEngines( 30_000, ); - // Resolves once the next ownership read after this call has returned, which is the late - // suspend's check when nothing else reads the record meanwhile. - function nextOwnershipRead() { - let signalRead!: () => void; - const read = new Promise(resolve => (signalRead = resolve)); - const getRunOwnership = WorkflowsInMemory.prototype.getRunOwnership; - const spy = vi - .spyOn(WorkflowsInMemory.prototype, 'getRunOwnership') - .mockImplementation(async function (this: WorkflowsInMemory, args) { - try { - return await getRunOwnership.call(this, args); - } finally { - signalRead(); - } - }); - onTestFinished(() => spy.mockRestore()); - return read; - } - // Technical: settlement releases the claim, and only once storage's release call returns - // does the run count as released. The late suspend checked ownership in between: storage + // did the run count as released. The late suspend checked ownership in between: storage // already showed the claim released, so the strict check failed and marked the execution - // lost though nobody had taken over, and the task failed before it suspended. (#30182) + // lost though nobody had taken over. Waiting for the release's reply instead left the task + // running for as long as that reply took, forever on a stalled connection. Here the reply is + // held until the task has settled. (#30182) it.runIf(isDurableEngineVariant(engine))( - 'parks a deferred tool that asks for input while the finished run is still releasing its claim (#30182)', + "parks a deferred tool that asks for input while the finished run's claim release has not returned (#30182)", async () => { const run = await startResearchDuringCleanup('bg-suspend-during-claim-release-thread', { hold: 'claim-release', }); const released = await run.workflows.getRunOwnership({ runId: run.runId }); expect(released?.live).toBe(false); - const checked = nextOwnershipRead(); run.release(); try { - // The suspend read the released claim while the release was still in flight. - await checked; - run.finishCleanup(); + // The release's reply is still held: the task settles without it. const parked = await run.settledTask(); expect({ status: parked.status, error: parked.error?.message }).toEqual({ status: 'suspended', @@ -1208,11 +1279,11 @@ describeForAllEngines( 30_000, ); - // Technical: waiting out an in-flight release must not let a superseded execution through: - // a successor that claimed the run during the release still makes the late suspend fail - // without writing. (#30182) + // Technical: accepting a late suspend before the release's reply returns must not let a + // superseded execution through: a successor that claimed the run during the release still + // makes the late suspend fail without writing. (#30182) it.runIf(isDurableEngineVariant(engine))( - "doesn't park a deferred tool that asks for input after another execution took over a run still releasing its claim (#30182)", + "doesn't park a deferred tool that asks for input after another execution took over a run whose claim release has not returned (#30182)", async () => { const run = await startResearchDuringCleanup('bg-suspend-claim-release-takeover-thread', { hold: 'claim-release', @@ -1225,11 +1296,8 @@ describeForAllEngines( expectedGeneration: released!.generation, }); expect(successor.acquired).toBe(true); - const checked = nextOwnershipRead(); run.release(); try { - await checked; - run.finishCleanup(); const settled = await run.settledTask(); expect({ status: settled.status, error: settled.error?.message }).toEqual({ status: 'failed', @@ -1535,9 +1603,10 @@ describeForAllEngines( // Technical: the same in-flight release as above, on the worker's participant fence: an // evented worker on a shared transport releases the run's claim itself, and a late suspend - // that read the released claim before that release returned marked the execution lost. (#30182) + // that read the released claim before that release returned marked the execution lost, or + // waited on the release's reply. The reply is held until the task has settled. (#30182) it.runIf(engine === 'evented')( - 'parks a deferred tool that asks for input while a shared evented run is still releasing its claim (#30182)', + "parks a deferred tool that asks for input while a shared evented run's claim release has not returned (#30182)", async () => { const directory = await mkdtemp(join(tmpdir(), 'mastra-30182-')); const pubsub = new UnixSocketPubSub(join(directory, 'pubsub.sock')); @@ -1546,11 +1615,8 @@ describeForAllEngines( pubsub, }); const released = await run.workflows.getRunOwnership({ runId: run.runId }); - const checked = nextOwnershipRead(); run.release(); try { - await checked; - run.finishCleanup(); const parked = await run.settledTask(); expect({ status: parked.status, error: parked.error?.message }).toEqual({ status: 'suspended', From 4b12cdf453d2a00e71fc5ed598fc22c004db7652 Mon Sep 17 00:00:00 2001 From: Leo <108278866+leoisadev1@users.noreply.github.com> Date: Mon, 16 Nov 2026 19:57:42 -0500 Subject: [PATCH 10/15] fix(core): keep the strict check for lease-backed late background suspends A lease keeps no record once released, so an empty lease can't tell the finished run's own release from a recovery that claimed, finished and released the run since, and no storage fence stands behind the write. Lease-backed fences now always use verify(), and a late background suspend on a lease-backed claim uses the strict check once the fence is gone too. The relaxed post-settlement rule needs a storage generation. Removes the release counting that rule made dead, pins the strict check for foreground suspends, and adds the lease-only scenario. Co-Authored-By: Claude Opus 5.5 --- .changeset/twelve-papers-stand.md | 4 +- .../durable-agent-execution-fence.test.ts | 64 ++++++++++++++ .../src/agent/durable/execution-fence.test.ts | 85 +++++++++++++------ .../core/src/agent/durable/execution-fence.ts | 54 ++++++------ .../durable/workflows/steps/tool-call.ts | 19 ++++- ...background-task-lifecycle.scenario.test.ts | 74 ++++++++++++++++ 6 files changed, 239 insertions(+), 61 deletions(-) diff --git a/.changeset/twelve-papers-stand.md b/.changeset/twelve-papers-stand.md index 3e73912e3c0..ea70207e1b6 100644 --- a/.changeset/twelve-papers-stand.md +++ b/.changeset/twelve-papers-stand.md @@ -2,4 +2,6 @@ '@mastra/core': patch --- -Fixed background tasks on durable agents with memory failing with "could not verify that this execution still owns the run" when they suspend after the agent's run has finished. The task is now marked suspended and can be resumed with `backgroundTaskManager.resume()`. This covers tools that call `suspend()` and sub-agents that reach a tool needing approval, including when another evented worker finished the run. A background task's result that arrives while the finished run is still cleaning up is now saved to memory instead of being dropped. A background task no longer stays `running` while storage is slow to confirm that the finished run gave up its claim. Fixes [#30182](https://github.com/mastra-ai/mastra/issues/30182). +Fixed background tasks on durable agents with memory failing with "could not verify that this execution still owns the run" when they suspend after the agent's run has finished. The task is now marked suspended and can be resumed with `backgroundTaskManager.resume()`. This covers tools that call `suspend()` and sub-agents that reach a tool needing approval, including when another evented worker finished the run. A background task's result that arrives while the finished run is still cleaning up is now saved to memory instead of being dropped. A background task no longer stays `running` while storage is slow to confirm that the finished run gave up its claim. + +The fix applies when your storage adapter tracks run ownership, as the in-memory, LibSQL and Postgres stores do. When run ownership lives only in the pubsub lease, a background task that suspends after its run finished still fails as before, because the lease can't show whether another execution took over and finished the run in the meantime. Fixes [#30182](https://github.com/mastra-ai/mastra/issues/30182). diff --git a/packages/core/src/agent/durable/__tests__/durable-agent-execution-fence.test.ts b/packages/core/src/agent/durable/__tests__/durable-agent-execution-fence.test.ts index be48c902a20..7da9b6f5826 100644 --- a/packages/core/src/agent/durable/__tests__/durable-agent-execution-fence.test.ts +++ b/packages/core/src/agent/durable/__tests__/durable-agent-execution-fence.test.ts @@ -1462,6 +1462,70 @@ describe.each(['storage', 'lease'])( result.cleanup(); }); + // A foreground tool's suspension keeps the strict check: only a background task's suspension may land + // after the run released its claim. An execution that couldn't renew its claim past the lease, with + // nobody else claiming the run, must not record an approval or question nobody can answer (#30182). + it('a foreground tool that suspends after its execution lost the lease stops the run without recording the suspension', async () => { + vi.useFakeTimers({ toFake: ['setInterval', 'clearInterval', 'Date'] }); + const storage = createStorage(backend); + const memory = new MockMemory({ storage }); + const model = toolCallThenTextModel('askReviewer', { topic: 'refund' }, 'must not be reached'); + let enterTool!: () => void; + const toolEntered = new Promise(resolve => (enterTool = resolve)); + let releaseTool!: () => void; + const toolGate = new Promise(resolve => (releaseTool = resolve)); + const askReviewer = createTool({ + id: 'askReviewer', + description: 'Ask a reviewer to approve the refund', + inputSchema: z.object({ topic: z.string() }), + execute: async ({ topic }, options) => { + enterTool(); + await toolGate; + return options?.agent?.suspend?.({ awaiting: 'reviewer', topic }) as any; + }, + }); + const durableAgent = buildAgent({ model: model.model, storage, memory, tools: { askReviewer } }); + const result = await durableAgent.stream('Refund this order once a reviewer approves', { + memory: { thread: THREAD, resource: RESOURCE }, + }); + const { runId } = result; + const stream = collect(result.fullStream); + const errors: any[] = []; + await pubsub.subscribe(AGENT_STREAM_TOPIC(runId), (event: any) => { + if (event.type === 'error') errors.push(event); + }); + try { + await toolEntered; + const fence = ExecutionFence.getLocalActive(runId)!; + // Renewals can't reach the backend until the claim's lease runs out, as a dropped connection would. + if (backend === 'storage') { + vi.spyOn(storage.stores.workflows!, 'renewRunOwnership').mockRejectedValue(new Error('store unreachable')); + } else { + vi.spyOn(pubsub, 'renewLease').mockRejectedValue(new Error('redis unreachable')); + } + await vi.advanceTimersByTimeAsync(EXECUTION_LEASE_TTL_MS + 2 * EXECUTION_LEASE_RENEW_INTERVAL_MS); + expect(fence.isLost()).toBe(true); + expect(await foreignOwnership(backend, storage, pubsub).owner(durableAgent.id, runId)).not.toBe('successor'); + + releaseTool(); + await fence.whenSettled; + await vi.waitFor(() => expect(errors).toHaveLength(1)); + expect(errors[0].data.error.message).toContain( + `Durable run ${runId}: could not verify that this execution still owns the run`, + ); + const { messages } = await memory.recall({ threadId: THREAD, resourceId: RESOURCE }); + const metadata = messages.map(message => message.content.metadata ?? {}); + // No open request for the reviewer: nothing a later resume() could answer. + expect(metadata.some(entry => 'suspendedTools' in entry || 'pendingToolApprovals' in entry)).toBe(false); + expect(model.calls()).toBe(1); + } finally { + releaseTool(); + vi.useRealTimers(); + await stream.stop(); + result.cleanup(); + } + }); + it.each([ ['stream', 'takeOver'], ['stream', 'vanish'], diff --git a/packages/core/src/agent/durable/execution-fence.test.ts b/packages/core/src/agent/durable/execution-fence.test.ts index 50b20e52abb..0e821ac3ade 100644 --- a/packages/core/src/agent/durable/execution-fence.test.ts +++ b/packages/core/src/agent/durable/execution-fence.test.ts @@ -1222,9 +1222,9 @@ function holdNextCall( // A background task can write after its run settled: `verifyNotSuperseded` is the check for those // writes. With ownership in storage it only refuses once another execution claimed the run, released -// or lapsed or not, and never waits for a release in flight. A lease can't tell a release from a lapse -// that another execution claimed and released since, so a lease-backed claim is `verify` until a -// release of it starts here (#30182). +// or lapsed or not, and never waits for a release in flight. A lease keeps nothing once released, so it +// can't tell this claim's release from another execution that claimed and released the run since: a +// lease-backed claim always gets `verify`'s rule, as before this check existed (#30182). describe.each([ { backend: 'storage', @@ -1299,17 +1299,22 @@ describe.each([ await claimSuccessor(); await expect(fence.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); } else { - // Without a release here, a missing lease may be one another execution claimed and released: verify's rule. + // A missing lease may be one another execution claimed and released: verify's rule. await expect(fence.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_UNVERIFIED_ERROR_ID }); } }); - it('after settlement released the claim, passes while no other execution claimed the run', async () => { - const { fence } = await setup(); + it('after settlement released the claim, passes only when storage holds the claim', async () => { + const { fence, storageBacked } = await setup(); expect(await fence.settle(async () => {})).toBe('owned'); await expect(fence.verify()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); - await expect(fence.verifyNotSuperseded()).resolves.toBeUndefined(); + if (storageBacked) { + await expect(fence.verifyNotSuperseded()).resolves.toBeUndefined(); + } else { + // The released lease looks like one another execution claimed and released since: verify's rule. + await expect(fence.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + } }); it('after settlement released the claim, refuses once another execution claimed the run', async () => { @@ -1334,16 +1339,22 @@ describe.each([ expect(fence.isLost()).toBe(true); }); - it("while settlement is releasing the claim, passes without waiting for the release's reply while no other execution claimed the run", async () => { - const { fence, holdNextRelease } = await setup(); + it("while settlement is releasing the claim, answers without waiting for the release's reply", async () => { + const { fence, holdNextRelease, storageBacked } = await setup(); const release = holdNextRelease(); const settled = fence.settle(async () => {}); await release.applied; // The backend already shows the claim released and its reply is still held, as on a stalled - // connection: the check neither marks the execution lost nor waits for the reply. - await expect(fence.verifyNotSuperseded()).resolves.toBeUndefined(); - expect(fence.isLost()).toBe(false); + // connection: the check doesn't wait for the reply. + if (storageBacked) { + // No other execution claimed the run, so storage would still accept the write. + await expect(fence.verifyNotSuperseded()).resolves.toBeUndefined(); + expect(fence.isLost()).toBe(false); + } else { + // The empty lease can't show whose release emptied it: verify's rule. + await expect(fence.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + } release.finish(); expect(await settled).toBe('owned'); }); @@ -1367,19 +1378,22 @@ describe.each([ const release = holdNextRelease({ fail: true }); const settled = fence.settle(async () => {}); await release.applied; - // While the release is in flight, it counts. - await expect(fence.verifyNotSuperseded()).resolves.toBeUndefined(); - release.finish(); - await settled; - if (storageBacked) { // Storage still holds this claim's generation, released: no other execution took the run. await expect(fence.verifyNotSuperseded()).resolves.toBeUndefined(); } else { - // The release failed, so a missing lease may be one another execution claimed and released: verify's rule. + // A missing lease may be one another execution claimed and released: verify's rule. await expect(fence.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); expect(fence.isLost()).toBe(true); } + release.finish(); + await settled; + + if (storageBacked) { + await expect(fence.verifyNotSuperseded()).resolves.toBeUndefined(); + } else { + await expect(fence.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + } }); it('refuses after settling as superseded', async () => { @@ -1391,8 +1405,8 @@ describe.each([ }); }); -// A lease can't show a release, so a worker's participant fence learns of the shared release it is -// making only from `releaseSharedClaim` itself (#30182). +// A worker's participant fence on a lease gets `verify`'s rule too: its own shared release leaves the +// same empty lease another execution's claim and release would (#30182). describe('ExecutionFence.verifyNotSuperseded on a lease-backed participant', () => { async function setup() { const pubsub = new EventEmitterPubSub(); @@ -1405,15 +1419,14 @@ describe('ExecutionFence.verifyNotSuperseded on a lease-backed participant', () return { pubsub, mastra, coordinator, participant }; } - it("counts this worker's shared release in flight without waiting for its reply, and passes while nobody claimed the run", async () => { + it("refuses while this worker's shared release is in flight, without waiting for its reply", async () => { const { pubsub, mastra, coordinator, participant } = await setup(); const release = holdNextCall(pubsub, 'releaseLease', { apply: true, fail: false }); const releasing = ExecutionFence.releaseSharedClaim(mastra, 'run-1', coordinator.claim); await release.applied; expect(await pubsub.getLeaseOwner(key('run-1'))).toBeUndefined(); - await expect(participant.verifyNotSuperseded()).resolves.toBeUndefined(); - expect(participant.isLost()).toBe(false); + await expect(participant.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); release.finish(); await releasing; }); @@ -1427,10 +1440,10 @@ describe('ExecutionFence.verifyNotSuperseded on a lease-backed participant', () }); }); -// A lease has no generation and no storage fence behind the check: once a lapsed lease was claimed and -// released by another execution, the lease looks exactly as it would after this execution's own release. -// Only a release this process started lets a late write through (#30182). -describe('ExecutionFence.verifyNotSuperseded on a lapsed lease', () => { +// A lease has no generation and no storage fence behind the check: once another execution claimed and +// released the run, the lease looks exactly as it would after this execution's own release. A late write +// that passed would land on the other execution's finished run unfenced (#30182). +describe('ExecutionFence.verifyNotSuperseded on a lease another execution claimed and released', () => { it('refuses a late write after a recovery claimed the lapsed lease and released it', async () => { vi.useFakeTimers(); const pubsub = new EventEmitterPubSub(); @@ -1448,6 +1461,24 @@ describe('ExecutionFence.verifyNotSuperseded on a lapsed lease', () => { await expect(fence.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_UNVERIFIED_ERROR_ID }); }); + + it("refuses a late write after a recovery claimed and released the run while this execution's release reply was held", async () => { + const pubsub = new EventEmitterPubSub(); + const fence = await claim(pubsub, 'run-1'); + // This execution's release reaches the provider, but its reply stalls, as on a slow connection. + const release = holdNextCall(pubsub, 'releaseLease', { apply: true, fail: false }); + const settled = fence.settle(async () => {}); + await release.applied; + + // Meanwhile another process recovers the run, finishes it and releases the lease. + expect((await pubsub.acquireLease(key('run-1'), 'recovery', EXECUTION_LEASE_TTL_MS)).acquired).toBe(true); + await pubsub.releaseLease(key('run-1'), 'recovery'); + expect(await pubsub.getLeaseOwner(key('run-1'))).toBeUndefined(); + + await expect(fence.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + release.finish(); + await settled; + }); }); // Storage keeps a released or lapsed claim's record and generation, so a late write is checked against diff --git a/packages/core/src/agent/durable/execution-fence.ts b/packages/core/src/agent/durable/execution-fence.ts index 190647c802a..4265f3bce2a 100644 --- a/packages/core/src/agent/durable/execution-fence.ts +++ b/packages/core/src/agent/durable/execution-fence.ts @@ -570,12 +570,6 @@ export class ExecutionFence implements RunFenceScope { #renewalInFlight = false; #settlement?: Promise; #settledAs?: ExecutionSettlement; - /** - * A release of this claim started here (settlement, or this worker's shared - * release) and has not failed. Only lease-backed claims need it: see - * {@link verifyNotSuperseded}. - */ - #released = false; #resolveSettled!: () => void; /** Resolves once {@link settle} has finished and the claim is released or left to its new holder. */ readonly whenSettled: Promise; @@ -696,12 +690,7 @@ export class ExecutionFence implements RunFenceScope { static async releaseSharedClaim(mastra: Mastra, runId: string, claim: DurableExecutionClaim): Promise { if (!claim.agentId) return; const backend = await resolveRemoteBackend(mastra, claim.agentId, runId, claim); - if (!backend) return; - // This worker's participant fence stays registered until its cleanup stops it. Late writes - // checked against it meanwhile count this release (see {@link verifyNotSuperseded}). - const participant = mastra.__getRunScope(runId)?.get(DURABLE_PARTICIPANT_FENCES)?.get(claim.executionId); - const release = () => backend.release(claim); - await (participant ? participant.#countRelease(release) : release()); + await backend?.release(claim); } /** Join an existing claim without acquiring ownership or advancing its generation. */ @@ -944,14 +933,14 @@ export class ExecutionFence implements RunFenceScope { * own fence ({@link runFencedToClaim}) rejects it once a successor raised * memory. Never waits for a release in flight. * - * A lease has no generation and no storage fence behind this check, and a - * lapsed lease that another execution claimed and released since looks like - * a released one. A lease-backed claim gets this rule only after a release - * started here; otherwise it is {@link verify}. A successor that claimed and - * released the lease after that release still goes unnoticed. + * Only storage-backed claims get this rule: their generation tells a + * released claim from one a successor claimed, released or not. A lease + * keeps no record once released, so an empty lease can't tell this claim's + * release from a successor that claimed and released it since, and nothing + * fences the write behind it. A lease-backed claim always gets {@link verify}. */ async verifyNotSuperseded(): Promise { - if (this.generation === undefined && !this.#released) return this.verify(); + if (this.generation === undefined) return this.verify(); const details = this.#details(); const superseded = this.#takenOver || (await retryOnce(() => this.#backend.isSuperseded(this.claim), details)); // A takeover the heartbeat saw while the read was on its way also refuses. @@ -1082,7 +1071,7 @@ export class ExecutionFence implements RunFenceScope { async #release(): Promise { try { - await this.#countRelease(() => this.#backend.release(this.claim)); + await this.#backend.release(this.claim); } catch (error) { this.#logger?.warn?.(`[DurableAgent] run ${this.runId}: failed to release the execution claim: ${error}`); } @@ -1091,17 +1080,6 @@ export class ExecutionFence implements RunFenceScope { } } - /** Run a release of this claim, counted as {@link #released} from its start unless it fails. */ - async #countRelease(release: () => Promise): Promise { - this.#released = true; - try { - await release(); - } catch (error) { - this.#released = false; - throw error; - } - } - /** Mark a settled fence retired in memory so `memory.runFences` retention can prune it. */ async #retireMemoryFence(fence: RunFence): Promise { if (!this.#memoryStore) return; @@ -1374,6 +1352,22 @@ export async function assertExecutionNotSuperseded(args: { } } +/** + * {@link assertExecutionNotSuperseded} for a claim with a storage generation, + * {@link assertExecutionOwned} for one without. For writes that refused a + * released claim before late writes were accepted, such as a background + * task's suspension: only a generation tells this claim's release from + * another execution that claimed and released the run since, and a + * lease-backed write has no storage fence behind it (#30182). + */ +export async function assertExecutionNotSupersededInStorage( + args: Parameters[0], +): Promise { + const claim = getExecutionClaim(args.requestContext, args.runId); + if (claim?.generation === undefined) return assertExecutionOwned(args); + return assertExecutionNotSuperseded(args); +} + /** * Run `fn` with its writes to `memoryStore` carrying the memory fence of the * claim `requestContext` holds for `runId`, also once that execution settled diff --git a/packages/core/src/agent/durable/workflows/steps/tool-call.ts b/packages/core/src/agent/durable/workflows/steps/tool-call.ts index 1cdd3192e20..23166947c21 100644 --- a/packages/core/src/agent/durable/workflows/steps/tool-call.ts +++ b/packages/core/src/agent/durable/workflows/steps/tool-call.ts @@ -67,6 +67,7 @@ import { ensureRemoteAbortListener, readRunDeadline } from '../../abort-transpor import { DurableStepIds } from '../../constants'; import { assertExecutionNotSuperseded, + assertExecutionNotSupersededInStorage, assertExecutionOwned, getExecutionOwnershipGuard, isExecutionFenceError, @@ -876,6 +877,15 @@ export function createDurableToolCallStep() { // claimed the run. const assertNotSuperseded = () => assertExecutionNotSuperseded({ runId, agentId: initData.agentId, requestContext, mastra: mastra as Mastra }); + // A late suspension gets that rule only with ownership in storage: a released lease can't show + // whether another execution claimed and finished the run since, so a lease keeps the strict check. + const assertSuspensionNotSuperseded = () => + assertExecutionNotSupersededInStorage({ + runId, + agentId: initData.agentId, + requestContext, + mastra: mastra as Mastra, + }); // Those writes carry the run's claim, so storage rejects one that lands after a successor // raised the memory fence, even though it passed the check above (#30182). const claimFenceArgs = { @@ -1595,8 +1605,9 @@ export function createDurableToolCallStep() { suspendToolCall(suspendPayload, suspendOptions, assertOwned), }; // `beforePersist` guards the suspension's memory writes. A background - // task can suspend after the run settled and released its claim, so it - // passes the post-settlement check instead of the strict one (#30182). + // task can suspend after the run settled and released its claim, so with + // ownership in storage it passes the post-settlement check instead of the + // strict one (#30182). const suspendToolCall = async ( suspendPayload: any, suspendOptions: SuspendOptions | undefined, @@ -1863,7 +1874,9 @@ export function createDurableToolCallStep() { // The run may have settled and stopped fencing its writes. The suspension's // memory save carries the run's claim, so storage rejects it once a successor // raised the memory fence, even when it lands after the ownership check (#30182). - await runFencedToClaim(claimFenceArgs, () => suspendToolCall(data, options, assertNotSuperseded)); + await runFencedToClaim(claimFenceArgs, () => + suspendToolCall(data, options, assertSuspensionNotSuperseded), + ); return taskContext?.suspend?.(data, options); }, outputWriter: async (chunk: any) => { diff --git a/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts b/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts index a1710d24aee..8a00c73b47d 100644 --- a/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts +++ b/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts @@ -18,6 +18,7 @@ import { describeForAllEngines, isDurableEngineVariant, } from '../aimock-scenario'; +import { ExecutionFence, executionLeaseKey, resolveLeaseProvider } from '../../../../agent/durable/execution-fence'; import { BackgroundTaskManager } from '../../../../background-tasks/manager'; import type { PubSub } from '../../../../events/pubsub'; import { UnixSocketPubSub } from '../../../../events/unix-socket-pubsub'; @@ -969,6 +970,79 @@ describeForAllEngines( 30_000, ); + // Technical: with ownership only in a pubsub lease (storage without run fencing), a released + // lease keeps no record, so it can't show whether another execution claimed and finished the + // run since, and no storage fence stands behind the write. The late suspend keeps the strict + // check: the task fails as before rather than write over a run a recovery finished. (#30182) + it.runIf(isDurableEngineVariant(engine))( + "doesn't park a deferred tool that asks for input after its run finished when ownership is only a lease (#30182)", + async () => { + vi.spyOn(WorkflowsInMemory.prototype, 'supportsRunFencing').mockReturnValue(false); + onTestFinished(() => vi.restoreAllMocks()); + let release!: () => void; + const gate = new Promise(resolve => (release = resolve)); + const memory = new MockMemory(); + const thread = { threadId: 'bg-suspend-lease-only-thread', resourceId: 'bg-suspend-lease-only-user' }; + const { chunks, mastra, agent, output } = await runLoopScenario({ + engine, + llm: getMock(), + prompt: 'Research near', + tools: { researchWithApproval: makeSuspendingResearchTool([], undefined, gate) }, + agentBackgroundTasks: { tools: { researchWithApproval: true } }, + backgroundTasks: BG_CONFIG, + memory, + ...thread, + stopWhen: stepCountIs(3), + collectChunks: true, + fixtures: llm => + onReportTurns(llm, 'Research near', { + dispatch: { + toolCalls: [{ id: 'call-bg-1', name: 'researchWithApproval', arguments: { topic: 'near' } }], + }, + ack: 'Waiting for the analyst.', + report: 'The analyst answered.', + }), + }); + expect(await output.text).toBe('Waiting for the analyst.'); + const manager = mastra.backgroundTaskManager!; + const started = chunks!.find(c => c.type === 'background-task-started') as Chunk & { runId: string }; + const taskId = started.payload.taskId as string; + const agentPubsub = (mastra.getAgentById(agent.id) as { pubsub?: PubSub }).pubsub ?? mastra.pubsub; + const leases = resolveLeaseProvider(agentPubsub); + const key = executionLeaseKey(agent.id, started.runId); + // The run finished: its fence is gone and its lease released. + await waitFor( + async () => ({ fence: ExecutionFence.getLocalActive(started.runId), owner: await leases.getLeaseOwner(key) }), + ({ fence, owner }) => fence === undefined && owner === undefined, + ); + + // Another process recovers the run, finishes it and releases the lease. + expect((await leases.acquireLease(key, 'recovery-execution', 60_000)).acquired).toBe(true); + await leases.releaseLease(key, 'recovery-execution'); + release(); + + const settled = (await waitFor( + () => manager.getTask(taskId), + (t: any) => t?.status !== 'pending' && t?.status !== 'running', + )) as any; + const unverified = `Durable run ${started.runId}: could not verify that this execution still owns the run`; + expect({ status: settled.status, error: settled.error?.message }).toEqual({ + status: 'failed', + // DURABLE_AGENT_EXECUTION_UNVERIFIED + error: expect.stringContaining(unverified), + }); + expect(settled.suspendPayload).toBeUndefined(); + // Memory records the tool call as failed. The refused suspension's `suspendedTools` entry + // still reaches memory next to that error, as it does on main (not yet investigated). + const stored = await waitFor( + () => storedToolCall(memory, thread), + s => s.invocation?.state === 'output-error', + ); + expect(stored.invocation.errorText).toContain(unverified); + }, + 30_000, + ); + // Starts the deferred research tool, lets the run finish and release its claim, and holds // one cleanup step open, as slow storage would, until `finishCleanup`: retiring the run's // memory fence, releasing the thread's lease (evented, with a shared `pubsub`), or the claim From bf00cf0f511cb06113f587fc67cbc65e61a23cc0 Mon Sep 17 00:00:00 2001 From: Leo <108278866+leoisadev1@users.noreply.github.com> Date: Mon, 16 Nov 2026 20:47:03 -0500 Subject: [PATCH 11/15] test(core): pin a late background suspend after a same-turn approval resume On the durable engine the task parks; on the evented engine it fails with "lost its execution lease", as it does on main. Drop a changeset sentence about a bug only an earlier version of this PR had, and note the evented limit. Co-Authored-By: Claude Opus 5.5 --- .changeset/twelve-papers-stand.md | 4 +- ...background-task-lifecycle.scenario.test.ts | 99 +++++++++++++++++++ 2 files changed, 101 insertions(+), 2 deletions(-) diff --git a/.changeset/twelve-papers-stand.md b/.changeset/twelve-papers-stand.md index ea70207e1b6..9add3ca7fa6 100644 --- a/.changeset/twelve-papers-stand.md +++ b/.changeset/twelve-papers-stand.md @@ -2,6 +2,6 @@ '@mastra/core': patch --- -Fixed background tasks on durable agents with memory failing with "could not verify that this execution still owns the run" when they suspend after the agent's run has finished. The task is now marked suspended and can be resumed with `backgroundTaskManager.resume()`. This covers tools that call `suspend()` and sub-agents that reach a tool needing approval, including when another evented worker finished the run. A background task's result that arrives while the finished run is still cleaning up is now saved to memory instead of being dropped. A background task no longer stays `running` while storage is slow to confirm that the finished run gave up its claim. +Fixed background tasks on durable agents with memory failing with "could not verify that this execution still owns the run" or "lost its execution lease" when they suspend after the agent's run has finished. The task is now marked suspended and can be resumed with `backgroundTaskManager.resume()`. This covers tools that call `suspend()` and sub-agents that reach a tool needing approval, including when another evented worker finished the run. A background task's result that arrives while the finished run is still cleaning up is now saved to memory instead of the "Background task started" placeholder. -The fix applies when your storage adapter tracks run ownership, as the in-memory, LibSQL and Postgres stores do. When run ownership lives only in the pubsub lease, a background task that suspends after its run finished still fails as before, because the lease can't show whether another execution took over and finished the run in the meantime. Fixes [#30182](https://github.com/mastra-ai/mastra/issues/30182). +The fix applies when your storage adapter tracks run ownership, as the in-memory, LibSQL and Postgres stores do. When run ownership lives only in the pubsub lease, a background task that suspends after its run finished still fails as before, because the lease can't show whether another execution took over and finished the run in the meantime. On the evented engine, a background task that suspends after its run was resumed from an approval in the same turn still fails, because the resume's claim counts as another execution. Fixes [#30182](https://github.com/mastra-ai/mastra/issues/30182). diff --git a/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts b/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts index 8a00c73b47d..7444478b33e 100644 --- a/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts +++ b/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts @@ -901,6 +901,105 @@ describeForAllEngines( 30_000, ); + // Technical: one turn dispatches a deferred tool and also calls a tool that needs approval. + // Approving it resumes the same run under a new claim generation, and only then does the + // deferred tool call suspend(). The durable engine parks it. Known limit: the evented engine + // counts the resume's claim as another execution's, so the task fails with "lost its + // execution lease", as it does on main. Telling this resume from a real takeover would widen + // this fix, so the evented result is pinned. Inngest isn't covered here. (#30182) + it.runIf(engine === 'durable' || engine === 'evented')( + 'parks a deferred tool that asks for input after an approval in the same turn resumed the run (#30182)', + async () => { + let release!: () => void; + const gate = new Promise(resolve => (release = resolve)); + const memory = new MockMemory(); + const llm = getMock(); + const thread = { threadId: 'bg-suspend-after-resume-thread', resourceId: 'bg-suspend-after-resume-user' }; + const lookup = createTool({ + id: 'lookup', + description: 'Look up a user record', + inputSchema: z.object({ name: z.string() }), + outputSchema: z.object({ id: z.string() }), + requireApproval: true, + execute: async ({ name }) => ({ id: `user-${name}` }), + }); + const shared = await createSharedAgent(llm, { + tools: { researchWithApproval: makeSuspendingResearchTool([], undefined, gate), lookup }, + agentBackgroundTasks: { tools: { researchWithApproval: true } }, + backgroundTasks: BG_CONFIG, + memory, + engine, + }); + llm.on( + { endpoint: 'chat', sequenceIndex: 0 }, + { + toolCalls: [ + { id: 'call-bg-1', name: 'researchWithApproval', arguments: { topic: 'near' } }, + { id: 'call-lookup-1', name: 'lookup', arguments: { name: 'dero' } }, + ], + }, + ); + for (let i = 1; i < 8; i++) llm.on({ endpoint: 'chat', sequenceIndex: i }, { content: `Reply ${i}.` }); + + const first = await runLoopScenario({ + engine, + llm, + prompt: 'Research near and look up dero', + sharedAgent: shared, + fixtures: () => {}, + memory, + ...thread, + streamUntilIdle: true, + manualStreamConsumption: true, + }); + const firstChunks = await collectUntil( + first.output.fullStream as ReadableStream, + 'tool-call-approval', + ); + const started = firstChunks.find(c => c.type === 'background-task-started'); + expect(firstChunks.find(c => c.type === 'tool-call-approval')?.payload.toolCallId).toBe('call-lookup-1'); + const taskId = started!.payload.taskId as string; + const workflows = (await shared.mastra.getStorage()!.getStore('workflows'))!; + const before = await workflows.getRunOwnership({ runId: first.output.runId }); + + const resumed = await shared.agent.resumeStreamUntilIdle( + { approved: true }, + { + runId: first.output.runId, + toolCallId: 'call-lookup-1', + memory: { thread: thread.threadId, resource: thread.resourceId }, + }, + ); + // The resume claimed the same run again before the deferred tool suspends. + await waitFor( + () => workflows.getRunOwnership({ runId: first.output.runId }), + o => o !== null && o.generation !== before?.generation, + ); + release(); + const resumedChunks = collect(resumed.fullStream as AsyncIterable); + + const parked = (await waitFor( + () => shared.mastra.backgroundTaskManager!.getTask(taskId), + (t: any) => t?.status !== 'pending' && t?.status !== 'running', + )) as any; + if (engine === 'durable') { + expect({ status: parked.status, error: parked.error?.message }).toEqual({ + status: 'suspended', + error: undefined, + }); + expect(parked.suspendPayload).toEqual({ awaiting: 'analyst-approval', topic: 'near' }); + } else { + expect(parked.status).toBe('failed'); + expect(parked.error?.message).toContain('lost its execution lease'); + } + const lookupResult = (await resumedChunks).find( + c => c.type === 'tool-result' && c.payload?.toolName === 'lookup', + ); + expect(lookupResult?.payload.result).toEqual({ id: 'user-dero' }); + }, + 60_000, + ); + // Technical: the late suspend above is only allowed while no other execution has claimed the // run. Once a successor claims it, the old execution's suspension must not reach memory or // park the task, and the successor's claim must stay as it was. (#30182) From dd5ef57d9e0fad7fa82612668c12aea586b6ca99 Mon Sep 17 00:00:00 2001 From: Leo <108278866+leoisadev1@users.noreply.github.com> Date: Mon, 16 Nov 2026 21:21:30 -0500 Subject: [PATCH 12/15] test(core): fail held-save takeover scenarios fast when the save never starts The three scenarios that hold a memory save until a successor takes over waited on the held save with no bound. A regression that fails the task before it saves left them hanging until the 30 s test timeout. They now fail after 2 s with "save never reached storage". Co-Authored-By: Claude Opus 5.5 --- ...background-task-lifecycle.scenario.test.ts | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts b/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts index 7444478b33e..1acd04ad677 100644 --- a/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts +++ b/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts @@ -1273,6 +1273,20 @@ describeForAllEngines( }; } + // Waits until a held memory save is on its way to storage. A regression that fails the task + // before it saves would otherwise leave the test waiting until its 30 s timeout. + async function untilSaveHeld(saveStarted: Promise): Promise { + let timer: ReturnType | undefined; + const timedOut = new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error('save never reached storage')), 2_000); + }); + try { + await Promise.race([saveStarted, timedOut]); + } finally { + clearTimeout(timer); + } + } + // Technical: a finished run releases its claim, then retires its memory fence. Until that // retirement resolves, the execution stays registered in this process, and the late suspend's // check asked it to prove it still held the claim it had just released, so the task failed @@ -1368,7 +1382,7 @@ describeForAllEngines( try { run.release(); // The suspension passed its ownership check; its save is on the way to storage. - await saveStarted; + await untilSaveHeld(saveStarted); // The finished run completes its cleanup. Then a successor claims the run and raises // the memory fence, as recovery or a new turn does before it touches memory. run.finishCleanup(); @@ -1683,7 +1697,7 @@ describeForAllEngines( const save = holdMemorySave(run.memoryStore, 2); try { run.release(); - await save.saveStarted; + await untilSaveHeld(save.saveStarted); const successor = await takeOverAfterCleanup(run); // The successor works on the thread: here it drops the open request. const { messages } = await run.memoryStore.listMessages({ ...run.thread, perPage: false }); @@ -1715,7 +1729,7 @@ describeForAllEngines( const save = holdMemorySave(run.memoryStore, 1); try { run.release(); - await save.saveStarted; + await untilSaveHeld(save.saveStarted); const successor = await takeOverAfterCleanup(run); save.finishSave(); From d7c3f7f43cc1a470bbe3e6d9becbae227f7c5589 Mon Sep 17 00:00:00 2001 From: Leo <108278866+leoisadev1@users.noreply.github.com> Date: Mon, 16 Nov 2026 22:35:22 -0500 Subject: [PATCH 13/15] test(core): answer only the approval-resume scenario's own model requests The suite shares one mock model server. The test before this one ends as soon as its second deferred tool parks, while its agent is still waking to report that task to the model. When that request lands after the next test registered its fixtures, it took the sequenceIndex 0 reply, so this test's own first turn got "Reply 1." instead of the tool calls, finished without an approval, and failed on the evented engine in CI. Scope the fixtures to this test's prompt. aimock counts the sequence per predicate, so a stray request no longer advances it. Co-Authored-By: Claude Opus 5.5 --- .../background-task-lifecycle.scenario.test.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts b/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts index 1acd04ad677..3ecb26bab52 100644 --- a/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts +++ b/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts @@ -930,8 +930,13 @@ describeForAllEngines( memory, engine, }); + const prompt = 'Research near and look up dero'; + // The mock server is shared by the suite. The previous test's agent still reports its parked + // task to the model as that test ends, and a request landing here would take this turn's + // tool calls. Answer only this thread's requests; aimock counts the sequence per predicate. + const ownRequest = (req: unknown) => JSON.stringify(req).includes(prompt); llm.on( - { endpoint: 'chat', sequenceIndex: 0 }, + { endpoint: 'chat', predicate: ownRequest, sequenceIndex: 0 }, { toolCalls: [ { id: 'call-bg-1', name: 'researchWithApproval', arguments: { topic: 'near' } }, @@ -939,12 +944,14 @@ describeForAllEngines( ], }, ); - for (let i = 1; i < 8; i++) llm.on({ endpoint: 'chat', sequenceIndex: i }, { content: `Reply ${i}.` }); + for (let i = 1; i < 8; i++) { + llm.on({ endpoint: 'chat', predicate: ownRequest, sequenceIndex: i }, { content: `Reply ${i}.` }); + } const first = await runLoopScenario({ engine, llm, - prompt: 'Research near and look up dero', + prompt, sharedAgent: shared, fixtures: () => {}, memory, From 372c2da4aaf6b35a8489d28c7a5b6c78352b8059 Mon Sep 17 00:00:00 2001 From: Leo <108278866+leoisadev1@users.noreply.github.com> Date: Mon, 16 Nov 2026 22:51:58 -0500 Subject: [PATCH 14/15] test(libsql): resume a deferred tool that suspends after its run finished; document the limits Adds a LibSQL case for the durable and evented engines: the deferred tool suspends only after the stream drained and the run's claim was released, then the task parks with its payload, resume completes it, and the thread holds the result. Documents when a late suspend works and when it still fails. (#30182) Co-Authored-By: Claude Opus 5.5 --- .../en/docs/harness/background-tasks.mdx | 6 ++ ...urable-background-late-write-fence.test.ts | 57 +++++++++++++++++-- 2 files changed, 59 insertions(+), 4 deletions(-) diff --git a/docs/src/content/en/docs/harness/background-tasks.mdx b/docs/src/content/en/docs/harness/background-tasks.mdx index f4daa637203..9a9bc7953ac 100644 --- a/docs/src/content/en/docs/harness/background-tasks.mdx +++ b/docs/src/content/en/docs/harness/background-tasks.mdx @@ -357,6 +357,12 @@ await mastra.backgroundTaskManager?.resume(taskId, { On the resumed run, `context.agent.suspendPayload` holds the `data` the tool suspended with. Use it to tell which request an answer belongs to without encoding that in `resumeData`. +### Suspend after a durable agent's run finished + +On a [durable agent](/docs/harness/durable-agents), a background tool can call `suspend()` after the agent has replied and its run has finished. The task is marked `suspended` and resumes as usual when the storage adapter supports [run fencing](/docs/harness/durable-agents#run-ownership), for example `InMemoryStore`, `@mastra/libsql`, or `@mastra/pg`. These adapters record each claim on the run, so Mastra can check that no other execution claimed the run in the meantime. + +The task fails instead in two cases. When the storage adapter doesn't support run fencing, run ownership is only a PubSub lease, and a released lease can't show whether another execution claimed and finished the run since. On an agent from `createEventedAgent()`, the task also fails when an approval for another tool call resumed the run in the same turn, because the resume's claim counts as another execution. + ### Answer a suspended task from a client While an agent's background tool call is suspended, the assistant message that holds the call records the open request under `metadata.suspendedTools[toolCallId]`. The entry carries `toolName`, `args`, `suspendPayload`, `taskId`, `runId`, and `suspendedAt`, so a chat that reloads the thread can still show the request. The entry is removed when the task resumes or is cancelled. diff --git a/stores/libsql/src/__tests__/durable-background-late-write-fence.test.ts b/stores/libsql/src/__tests__/durable-background-late-write-fence.test.ts index 119b674e233..18675aa5453 100644 --- a/stores/libsql/src/__tests__/durable-background-late-write-fence.test.ts +++ b/stores/libsql/src/__tests__/durable-background-late-write-fence.test.ts @@ -5,7 +5,8 @@ * successor claimed the run and raised the memory fence. These cases run that ordering on a real * LibSQL file, with workflows and memory in the same store, and check that the late write carries * the finished run's claim to the memory domain object the adapter fences, so LibSQL's write - * transaction rejects it. (#30182) + * transaction rejects it. With nobody taking over, the late suspension parks the task and a resume + * completes it on the same database. (#30182) */ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; @@ -163,6 +164,8 @@ async function startDeferredResearch(engine: 'durable' | 'evented', outcome: 'su outputSchema: z.object({ summary: z.string() }), background: { enabled: true }, execute: async ({ topic }, options) => { + const resumeData = options?.agent?.resumeData as { notes?: string } | undefined; + if (resumeData) return { summary: `Research on "${topic}": ${resumeData.notes}` }; await toolGate.promise; if (outcome === 'result') return { summary: `Research on "${topic}"` }; return options?.agent?.suspend?.({ awaiting: 'analyst-approval', topic }) as never; @@ -205,16 +208,20 @@ async function startDeferredResearch(engine: 'durable' | 'evented', outcome: 'su expect(taskId).toBeDefined(); const workflows = (await storage.getStore('workflows'))!; + // A released claim keeps its row with no lease, unlike one that merely lapsed. const released = await waitFor( () => workflows.getRunOwnership({ runId: run.runId }), - record => record?.live === false, + record => record?.leaseExpiresAt === null, 'the finished run to release its claim', ); await cleanedUp.promise; + const manager = mastra.backgroundTaskManager!; return { memoryStore, workflows, + manager, + taskId: taskId!, thread, runId: run.runId, released: released!, @@ -222,7 +229,7 @@ async function startDeferredResearch(engine: 'durable' | 'evented', outcome: 'su storedToolCall: async () => toolCallIn((await memoryStore.listMessages({ ...thread, perPage: false })).messages), settledTask: () => waitFor( - () => mastra.backgroundTaskManager!.getTask(taskId!), + () => manager.getTask(taskId!), (task: BackgroundTask | null) => task?.status !== 'pending' && task?.status !== 'running', 'the background task to settle', ), @@ -268,8 +275,50 @@ async function expectSuccessorUnchanged(research: Research, successor: RunFence) } describe.each(['durable', 'evented'] as const)( - '%s agent on LibSQL: late background writes after a takeover', + '%s agent on LibSQL: late background writes after the run finished', engine => { + // Technical: the late suspension's save required the finished run to still own its claim, + // which LibSQL showed as released, so the task failed with DURABLE_AGENT_EXECUTION_UNVERIFIED. + // With nobody claiming the run since, the save now passes and storage accepts the claim. (#30182) + it('parks a deferred tool that asks for input after its run finished, and resumes it (#30182)', async () => { + const research = await startDeferredResearch(engine, 'suspend'); + research.letToolGo(); + + const parked = await research.settledTask(); + expect({ status: parked?.status, error: parked?.error?.message }).toEqual({ + status: 'suspended', + error: undefined, + }); + expect(parked?.suspendPayload).toEqual({ awaiting: 'analyst-approval', topic: 'near' }); + const open = await research.storedToolCall(); + expect(open?.suspended).toMatchObject({ + taskId: research.taskId, + suspendPayload: { awaiting: 'analyst-approval', topic: 'near' }, + }); + + await research.manager.resume(research.taskId, { notes: 'looks promising' }); + const done = await waitFor( + () => research.manager.getTask(research.taskId), + (task: BackgroundTask | null) => task?.status === 'completed' || task?.status === 'failed', + 'the resumed task to finish', + ); + expect({ status: done?.status, error: done?.error?.message, result: done?.result }).toEqual({ + status: 'completed', + error: undefined, + result: { summary: 'Research on "near": looks promising' }, + }); + // The thread on LibSQL holds the answered tool call, with the open request cleared. + const answered = await waitFor( + () => research.storedToolCall(), + stored => stored?.invocation.state === 'result' && stored.suspended === undefined, + 'the tool result in memory', + ); + expect(answered?.invocation).toMatchObject({ + state: 'result', + result: { summary: 'Research on "near": looks promising' }, + }); + }, 30_000); + // Technical: the suspension's save passed `assertExecutionNotSuperseded` with no successor yet, // then reached LibSQL after a successor claimed the run and raised the memory fence. // `runFencedToClaim` gives the save the finished run's claim, and the write transaction rejects it. (#30182) From 909b0e5e73d0fc79d1aaf6e129b3c24098ed97c0 Mon Sep 17 00:00:00 2001 From: Leo <108278866+leoisadev1@users.noreply.github.com> Date: Mon, 16 Nov 2026 22:56:17 -0500 Subject: [PATCH 15/15] docs(harness): reword the late-suspend limits to pass Vale Co-Authored-By: Claude Opus 5.5 --- docs/src/content/en/docs/harness/background-tasks.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/content/en/docs/harness/background-tasks.mdx b/docs/src/content/en/docs/harness/background-tasks.mdx index 9a9bc7953ac..64c0c4be5e7 100644 --- a/docs/src/content/en/docs/harness/background-tasks.mdx +++ b/docs/src/content/en/docs/harness/background-tasks.mdx @@ -361,7 +361,7 @@ On the resumed run, `context.agent.suspendPayload` holds the `data` the tool sus On a [durable agent](/docs/harness/durable-agents), a background tool can call `suspend()` after the agent has replied and its run has finished. The task is marked `suspended` and resumes as usual when the storage adapter supports [run fencing](/docs/harness/durable-agents#run-ownership), for example `InMemoryStore`, `@mastra/libsql`, or `@mastra/pg`. These adapters record each claim on the run, so Mastra can check that no other execution claimed the run in the meantime. -The task fails instead in two cases. When the storage adapter doesn't support run fencing, run ownership is only a PubSub lease, and a released lease can't show whether another execution claimed and finished the run since. On an agent from `createEventedAgent()`, the task also fails when an approval for another tool call resumed the run in the same turn, because the resume's claim counts as another execution. +The task still fails with a storage adapter that lacks run fencing, because run ownership is then only a PubSub lease, and a released lease can't show whether another execution claimed and finished the run since. On an agent from `createEventedAgent()`, the task also fails when an approval for another tool call resumed the run in the same turn, because the resume's claim counts as another execution. ### Answer a suspended task from a client