diff --git a/.changeset/twelve-papers-stand.md b/.changeset/twelve-papers-stand.md new file mode 100644 index 00000000000..9add3ca7fa6 --- /dev/null +++ b/.changeset/twelve-papers-stand.md @@ -0,0 +1,7 @@ +--- +'@mastra/core': patch +--- + +Fixed background tasks on durable agents with memory failing with "could not verify that this execution still owns the run" or "lost its execution lease" when they suspend after the agent's run has finished. The task is now marked suspended and can be resumed with `backgroundTaskManager.resume()`. This covers tools that call `suspend()` and sub-agents that reach a tool needing approval, including when another evented worker finished the run. A background task's result that arrives while the finished run is still cleaning up is now saved to memory instead of the "Background task started" placeholder. + +The fix applies when your storage adapter tracks run ownership, as the in-memory, LibSQL and Postgres stores do. When run ownership lives only in the pubsub lease, a background task that suspends after its run finished still fails as before, because the lease can't show whether another execution took over and finished the run in the meantime. On the evented engine, a background task that suspends after its run was resumed from an approval in the same turn still fails, because the resume's claim counts as another execution. Fixes [#30182](https://github.com/mastra-ai/mastra/issues/30182). diff --git a/docs/src/content/en/docs/harness/background-tasks.mdx b/docs/src/content/en/docs/harness/background-tasks.mdx index f4daa637203..64c0c4be5e7 100644 --- a/docs/src/content/en/docs/harness/background-tasks.mdx +++ b/docs/src/content/en/docs/harness/background-tasks.mdx @@ -357,6 +357,12 @@ await mastra.backgroundTaskManager?.resume(taskId, { On the resumed run, `context.agent.suspendPayload` holds the `data` the tool suspended with. Use it to tell which request an answer belongs to without encoding that in `resumeData`. +### Suspend after a durable agent's run finished + +On a [durable agent](/docs/harness/durable-agents), a background tool can call `suspend()` after the agent has replied and its run has finished. The task is marked `suspended` and resumes as usual when the storage adapter supports [run fencing](/docs/harness/durable-agents#run-ownership), for example `InMemoryStore`, `@mastra/libsql`, or `@mastra/pg`. These adapters record each claim on the run, so Mastra can check that no other execution claimed the run in the meantime. + +The task still fails with a storage adapter that lacks run fencing, because run ownership is then only a PubSub lease, and a released lease can't show whether another execution claimed and finished the run since. On an agent from `createEventedAgent()`, the task also fails when an approval for another tool call resumed the run in the same turn, because the resume's claim counts as another execution. + ### Answer a suspended task from a client While an agent's background tool call is suspended, the assistant message that holds the call records the open request under `metadata.suspendedTools[toolCallId]`. The entry carries `toolName`, `args`, `suspendPayload`, `taskId`, `runId`, and `suspendedAt`, so a chat that reloads the thread can still show the request. The entry is removed when the task resumes or is cancelled. diff --git a/packages/core/src/agent/durable/__tests__/durable-agent-execution-fence.test.ts b/packages/core/src/agent/durable/__tests__/durable-agent-execution-fence.test.ts index be48c902a20..7da9b6f5826 100644 --- a/packages/core/src/agent/durable/__tests__/durable-agent-execution-fence.test.ts +++ b/packages/core/src/agent/durable/__tests__/durable-agent-execution-fence.test.ts @@ -1462,6 +1462,70 @@ describe.each(['storage', 'lease'])( result.cleanup(); }); + // A foreground tool's suspension keeps the strict check: only a background task's suspension may land + // after the run released its claim. An execution that couldn't renew its claim past the lease, with + // nobody else claiming the run, must not record an approval or question nobody can answer (#30182). + it('a foreground tool that suspends after its execution lost the lease stops the run without recording the suspension', async () => { + vi.useFakeTimers({ toFake: ['setInterval', 'clearInterval', 'Date'] }); + const storage = createStorage(backend); + const memory = new MockMemory({ storage }); + const model = toolCallThenTextModel('askReviewer', { topic: 'refund' }, 'must not be reached'); + let enterTool!: () => void; + const toolEntered = new Promise(resolve => (enterTool = resolve)); + let releaseTool!: () => void; + const toolGate = new Promise(resolve => (releaseTool = resolve)); + const askReviewer = createTool({ + id: 'askReviewer', + description: 'Ask a reviewer to approve the refund', + inputSchema: z.object({ topic: z.string() }), + execute: async ({ topic }, options) => { + enterTool(); + await toolGate; + return options?.agent?.suspend?.({ awaiting: 'reviewer', topic }) as any; + }, + }); + const durableAgent = buildAgent({ model: model.model, storage, memory, tools: { askReviewer } }); + const result = await durableAgent.stream('Refund this order once a reviewer approves', { + memory: { thread: THREAD, resource: RESOURCE }, + }); + const { runId } = result; + const stream = collect(result.fullStream); + const errors: any[] = []; + await pubsub.subscribe(AGENT_STREAM_TOPIC(runId), (event: any) => { + if (event.type === 'error') errors.push(event); + }); + try { + await toolEntered; + const fence = ExecutionFence.getLocalActive(runId)!; + // Renewals can't reach the backend until the claim's lease runs out, as a dropped connection would. + if (backend === 'storage') { + vi.spyOn(storage.stores.workflows!, 'renewRunOwnership').mockRejectedValue(new Error('store unreachable')); + } else { + vi.spyOn(pubsub, 'renewLease').mockRejectedValue(new Error('redis unreachable')); + } + await vi.advanceTimersByTimeAsync(EXECUTION_LEASE_TTL_MS + 2 * EXECUTION_LEASE_RENEW_INTERVAL_MS); + expect(fence.isLost()).toBe(true); + expect(await foreignOwnership(backend, storage, pubsub).owner(durableAgent.id, runId)).not.toBe('successor'); + + releaseTool(); + await fence.whenSettled; + await vi.waitFor(() => expect(errors).toHaveLength(1)); + expect(errors[0].data.error.message).toContain( + `Durable run ${runId}: could not verify that this execution still owns the run`, + ); + const { messages } = await memory.recall({ threadId: THREAD, resourceId: RESOURCE }); + const metadata = messages.map(message => message.content.metadata ?? {}); + // No open request for the reviewer: nothing a later resume() could answer. + expect(metadata.some(entry => 'suspendedTools' in entry || 'pendingToolApprovals' in entry)).toBe(false); + expect(model.calls()).toBe(1); + } finally { + releaseTool(); + vi.useRealTimers(); + await stream.stop(); + result.cleanup(); + } + }); + it.each([ ['stream', 'takeOver'], ['stream', 'vanish'], diff --git a/packages/core/src/agent/durable/evented-execution.test.ts b/packages/core/src/agent/durable/evented-execution.test.ts index 10f445d1adf..4e839e455a4 100644 --- a/packages/core/src/agent/durable/evented-execution.test.ts +++ b/packages/core/src/agent/durable/evented-execution.test.ts @@ -31,6 +31,8 @@ import { } from './evented-execution'; import { __resetExecutionFencesForTests, + assertExecutionNotSuperseded, + DURABLE_PARTICIPANT_FENCES, EXECUTION_LEASE_TTL_MS, EXECUTION_SUPERSEDED_ERROR_ID, EXECUTION_UNVERIFIED_ERROR_ID, @@ -1287,6 +1289,62 @@ it('detaches worker participants on shutdown without invalidating a surviving pa }); }); +// Worker B ran the tool-call step; worker A handled the terminal event and released the shared claim. B's +// participant fence stays registered until its heartbeat notices, and a background task's late write on B +// was checked strictly against it, so it failed as superseded though nobody took the run. (#30182) +it("accepts a late background write on one worker after another worker released the run's claim", async () => { + const { workflowsStore, pubsub, mastra: workerA, coordinator, event } = await createExecution(); + const workerB = new Mastra({ storage: workerA.getStorage(), pubsub }); + instances.push(workerB); + await participateInDurableExecution(workerA, event, async () => {}); + await participateInDurableExecution(workerB, event, async () => {}); + const participantB = workerB.__getRunScope('run')!.get(DURABLE_PARTICIPANT_FENCES)!.get(coordinator.executionId)!; + + await settleEventedDurableExecution(workerA, { ...event, type: 'workflow.end' }); + expect(await workflowsStore.getRunOwnership({ runId: 'run' })).toMatchObject({ + ownerId: coordinator.executionId, + generation: coordinator.generation, + leaseExpiresAt: null, + }); + const late = { runId: 'run', agentId: 'agent', mastra: workerB }; + const requestContext = new RequestContext(Object.entries(event.data.requestContext)); + await expect(assertExecutionNotSuperseded({ ...late, requestContext })).resolves.toBeUndefined(); + expect(participantB.isLost()).toBe(false); + + // Once another execution claims the run, the late write on B is refused. + const successor = await workflowsStore.claimRunOwnership({ + runId: 'run', + ownerId: 'successor', + leaseMs: 60_000, + expectedGeneration: coordinator.generation, + }); + expect(successor.acquired).toBe(true); + await expect(assertExecutionNotSuperseded({ ...late, requestContext })).rejects.toMatchObject({ + id: EXECUTION_SUPERSEDED_ERROR_ID, + }); +}); + +it('refuses a late background write on one worker once a successor claimed the run another worker released', async () => { + const { workflowsStore, pubsub, mastra: workerA, coordinator, event } = await createExecution(); + const workerB = new Mastra({ storage: workerA.getStorage(), pubsub }); + instances.push(workerB); + await participateInDurableExecution(workerA, event, async () => {}); + await participateInDurableExecution(workerB, event, async () => {}); + + await settleEventedDurableExecution(workerA, { ...event, type: 'workflow.end' }); + await workflowsStore.claimRunOwnership({ + runId: 'run', + ownerId: 'successor', + leaseMs: 60_000, + expectedGeneration: coordinator.generation, + }); + const requestContext = new RequestContext(Object.entries(event.data.requestContext)); + await expect( + assertExecutionNotSuperseded({ runId: 'run', agentId: 'agent', requestContext, mastra: workerB }), + ).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + expect(await workflowsStore.getRunOwnership({ runId: 'run' })).toMatchObject({ ownerId: 'successor', live: true }); +}); + it('stops renewing the claim when the processor abandons a root terminal delivery', async () => { const directory = await mkdtemp(join(tmpdir(), 'mastra-abandoned-')); const pubsub = new UnixSocketPubSub(join(directory, 'pubsub.sock')); diff --git a/packages/core/src/agent/durable/execution-fence.test.ts b/packages/core/src/agent/durable/execution-fence.test.ts index fbc10316346..0e821ac3ade 100644 --- a/packages/core/src/agent/durable/execution-fence.test.ts +++ b/packages/core/src/agent/durable/execution-fence.test.ts @@ -13,6 +13,7 @@ import { __resetExecutionFencesForTests, assertExecutionNotSuperseded, assertExecutionOwned, + DURABLE_PARTICIPANT_FENCES, DurableExecutionFenceError, EXECUTION_ABANDONED_ERROR_ID, EXECUTION_CONFLICT_ERROR_ID, @@ -1194,6 +1195,583 @@ describe('storage-backed ExecutionFence', () => { }); }); +/** + * Make the next call of `method` on `target` apply (unless `apply` is false), then wait for `finish()` + * before returning or, with `fail`, throwing: a release storage has applied but not yet confirmed, or + * one whose confirmation is lost. + */ +function holdNextCall( + target: T, + method: K, + { apply, fail }: { apply: boolean; fail: boolean }, +) { + const original = (target[method] as (...args: unknown[]) => Promise).bind(target); + let signalApplied!: () => void; + const applied = new Promise(resolve => (signalApplied = resolve)); + let finish!: () => void; + const finished = new Promise(resolve => (finish = resolve)); + vi.spyOn(target, method as never).mockImplementationOnce((async (...args: unknown[]) => { + const result = apply ? await original(...args) : undefined; + signalApplied(); + await finished; + if (fail) throw new Error('backend down'); + return result; + }) as never); + return { applied, finish }; +} + +// A background task can write after its run settled: `verifyNotSuperseded` is the check for those +// writes. With ownership in storage it only refuses once another execution claimed the run, released +// or lapsed or not, and never waits for a release in flight. A lease keeps nothing once released, so it +// can't tell this claim's release from another execution that claimed and released the run since: a +// lease-backed claim always gets `verify`'s rule, as before this check existed (#30182). +describe.each([ + { + backend: 'storage', + async setup() { + const storage = new InMemoryStore(); + const workflowsStore = (await storage.getStore('workflows')) as WorkflowsStorage; + const fence = await ExecutionFence.claim({ + leaseProvider: NoopLeaseProvider, + workflowsStore, + agentId, + runId: 'run-1', + mode: 'acquire', + }); + const own = { runId: 'run-1', generation: fence.generation!, ownerId: fence.executionId }; + return { + fence, + // Storage keeps the claim's generation: a lapse or a release it didn't see still accepts the write. + storageBacked: true, + dropClaim: async () => void (await workflowsStore.releaseRunOwnership(own)), + // The store lost the claim's record: nobody can say who holds the run. + loseClaim: () => workflowsStore.dangerouslyClearAll(), + failRenewals: () => + vi.spyOn(workflowsStore, 'renewRunOwnership').mockRejectedValue(new Error('store unreachable')), + claimSuccessor: async () => + void (await workflowsStore.claimRunOwnership({ runId: 'run-1', ownerId: 'successor', leaseMs: 60_000 })), + takeOver: async () => + void (await workflowsStore.claimRunOwnership({ + runId: 'run-1', + ownerId: 'successor', + leaseMs: 60_000, + force: true, + })), + failNextRelease: () => + vi.spyOn(workflowsStore, 'releaseRunOwnership').mockRejectedValueOnce(new Error('store down')), + holdNextRelease: ({ fail = false } = {}) => + holdNextCall(workflowsStore, 'releaseRunOwnership', { apply: !fail, fail }), + }; + }, + }, + { + backend: 'lease', + async setup() { + const pubsub = new EventEmitterPubSub(); + const fence = await claim(pubsub, 'run-1'); + return { + fence, + storageBacked: false, + dropClaim: () => pubsub.releaseLease(key('run-1'), fence.executionId), + loseClaim: () => pubsub.releaseLease(key('run-1'), fence.executionId), + failRenewals: () => vi.spyOn(pubsub, 'renewLease').mockRejectedValue(new Error('redis unreachable')), + claimSuccessor: async () => void (await pubsub.acquireLease(key('run-1'), 'successor', EXECUTION_LEASE_TTL_MS)), + takeOver: async () => + void (await pubsub.transferLease(key('run-1'), fence.executionId, 'successor', EXECUTION_LEASE_TTL_MS)), + failNextRelease: () => vi.spyOn(pubsub, 'releaseLease').mockRejectedValueOnce(new Error('redis down')), + holdNextRelease: ({ fail = false } = {}) => holdNextCall(pubsub, 'releaseLease', { apply: !fail, fail }), + }; + }, + }, +])('ExecutionFence.verifyNotSuperseded ($backend)', ({ setup }) => { + it('before settlement, passes while owned; after the claim expired, passes only when storage holds the claim', async () => { + vi.useFakeTimers(); + const { fence, failRenewals, claimSuccessor, storageBacked } = await setup(); + await expect(fence.verifyNotSuperseded()).resolves.toBeUndefined(); + + // Renewals can't reach the backend until the claim's lease runs out. + failRenewals(); + await vi.advanceTimersByTimeAsync(EXECUTION_LEASE_TTL_MS + EXECUTION_LEASE_RENEW_INTERVAL_MS); + expect(fence.isLost()).toBe(true); + if (storageBacked) { + // Nobody claimed the run, so storage would still accept the claim's write. + await expect(fence.verifyNotSuperseded()).resolves.toBeUndefined(); + await claimSuccessor(); + await expect(fence.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + } else { + // A missing lease may be one another execution claimed and released: verify's rule. + await expect(fence.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_UNVERIFIED_ERROR_ID }); + } + }); + + it('after settlement released the claim, passes only when storage holds the claim', async () => { + const { fence, storageBacked } = await setup(); + expect(await fence.settle(async () => {})).toBe('owned'); + + await expect(fence.verify()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + if (storageBacked) { + await expect(fence.verifyNotSuperseded()).resolves.toBeUndefined(); + } else { + // The released lease looks like one another execution claimed and released since: verify's rule. + await expect(fence.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + } + }); + + it('after settlement released the claim, refuses once another execution claimed the run', async () => { + const { fence, claimSuccessor } = await setup(); + expect(await fence.settle(async () => {})).toBe('owned'); + + await claimSuccessor(); + await expect(fence.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + }); + + it('stays strict when settlement failed to release the claim', async () => { + const { fence, loseClaim, failNextRelease } = await setup(); + failNextRelease(); + expect(await fence.settle(async () => {})).toBe('owned'); + // The claim is still held, so the check passes as verify would. + await expect(fence.verifyNotSuperseded()).resolves.toBeUndefined(); + + // The unreleased claim is gone and nothing shows who holds the run now: storage lost the record, + // and a lease without a release here gets verify's rule. + await loseClaim(); + await expect(fence.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + expect(fence.isLost()).toBe(true); + }); + + it("while settlement is releasing the claim, answers without waiting for the release's reply", async () => { + const { fence, holdNextRelease, storageBacked } = await setup(); + const release = holdNextRelease(); + const settled = fence.settle(async () => {}); + await release.applied; + + // The backend already shows the claim released and its reply is still held, as on a stalled + // connection: the check doesn't wait for the reply. + if (storageBacked) { + // No other execution claimed the run, so storage would still accept the write. + await expect(fence.verifyNotSuperseded()).resolves.toBeUndefined(); + expect(fence.isLost()).toBe(false); + } else { + // The empty lease can't show whose release emptied it: verify's rule. + await expect(fence.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + } + release.finish(); + expect(await settled).toBe('owned'); + }); + + it('while settlement is releasing the claim, refuses once another execution claimed the run', async () => { + const { fence, holdNextRelease, claimSuccessor } = await setup(); + const release = holdNextRelease(); + const settled = fence.settle(async () => {}); + await release.applied; + await claimSuccessor(); + + await expect(fence.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + release.finish(); + await settled; + }); + + it('once the release in flight fails, passes only while storage holds the claim', async () => { + const { fence, dropClaim, holdNextRelease, storageBacked } = await setup(); + // The claim was released already, as another worker would; settlement still tries to release it. + await dropClaim(); + const release = holdNextRelease({ fail: true }); + const settled = fence.settle(async () => {}); + await release.applied; + if (storageBacked) { + // Storage still holds this claim's generation, released: no other execution took the run. + await expect(fence.verifyNotSuperseded()).resolves.toBeUndefined(); + } else { + // A missing lease may be one another execution claimed and released: verify's rule. + await expect(fence.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + expect(fence.isLost()).toBe(true); + } + release.finish(); + await settled; + + if (storageBacked) { + await expect(fence.verifyNotSuperseded()).resolves.toBeUndefined(); + } else { + await expect(fence.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + } + }); + + it('refuses after settling as superseded', async () => { + const { fence, takeOver } = await setup(); + await takeOver(); + expect(await fence.settle(async () => {})).toBe('superseded'); + + await expect(fence.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + }); +}); + +// A worker's participant fence on a lease gets `verify`'s rule too: its own shared release leaves the +// same empty lease another execution's claim and release would (#30182). +describe('ExecutionFence.verifyNotSuperseded on a lease-backed participant', () => { + async function setup() { + const pubsub = new EventEmitterPubSub(); + const coordinator = await claim(pubsub, 'run-1'); + coordinator.stopHeartbeat(); + const mastra = new Mastra({ pubsub, logger: false }); + const participant = (await ExecutionFence.follow(mastra, 'run-1', coordinator.claim))!; + onTestFinished(() => participant.stopHeartbeat()); + mastra.__createRunScope('run-1').set(DURABLE_PARTICIPANT_FENCES, new Map([[coordinator.executionId, participant]])); + return { pubsub, mastra, coordinator, participant }; + } + + it("refuses while this worker's shared release is in flight, without waiting for its reply", async () => { + const { pubsub, mastra, coordinator, participant } = await setup(); + const release = holdNextCall(pubsub, 'releaseLease', { apply: true, fail: false }); + const releasing = ExecutionFence.releaseSharedClaim(mastra, 'run-1', coordinator.claim); + await release.applied; + expect(await pubsub.getLeaseOwner(key('run-1'))).toBeUndefined(); + + await expect(participant.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + release.finish(); + await releasing; + }); + + it("refuses once another execution took the lease after this worker's shared release", async () => { + const { pubsub, mastra, coordinator, participant } = await setup(); + await ExecutionFence.releaseSharedClaim(mastra, 'run-1', coordinator.claim); + await pubsub.acquireLease(key('run-1'), 'successor', EXECUTION_LEASE_TTL_MS); + + await expect(participant.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + }); +}); + +// A lease has no generation and no storage fence behind the check: once another execution claimed and +// released the run, the lease looks exactly as it would after this execution's own release. A late write +// that passed would land on the other execution's finished run unfenced (#30182). +describe('ExecutionFence.verifyNotSuperseded on a lease another execution claimed and released', () => { + it('refuses a late write after a recovery claimed the lapsed lease and released it', async () => { + vi.useFakeTimers(); + const pubsub = new EventEmitterPubSub(); + const fence = await claim(pubsub, 'run-1'); + // Renewals can't reach the lease provider until the lease runs out, as a dropped connection would. + const renewals = vi.spyOn(pubsub, 'renewLease').mockRejectedValue(new Error('redis unreachable')); + await vi.advanceTimersByTimeAsync(EXECUTION_LEASE_TTL_MS + EXECUTION_LEASE_RENEW_INTERVAL_MS); + expect(fence.isLost()).toBe(true); + renewals.mockRestore(); + + // Another process recovers the run, finishes it and releases the lease. + expect((await pubsub.acquireLease(key('run-1'), 'recovery', EXECUTION_LEASE_TTL_MS)).acquired).toBe(true); + await pubsub.releaseLease(key('run-1'), 'recovery'); + expect(await pubsub.getLeaseOwner(key('run-1'))).toBeUndefined(); + + await expect(fence.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_UNVERIFIED_ERROR_ID }); + }); + + it("refuses a late write after a recovery claimed and released the run while this execution's release reply was held", async () => { + const pubsub = new EventEmitterPubSub(); + const fence = await claim(pubsub, 'run-1'); + // This execution's release reaches the provider, but its reply stalls, as on a slow connection. + const release = holdNextCall(pubsub, 'releaseLease', { apply: true, fail: false }); + const settled = fence.settle(async () => {}); + await release.applied; + + // Meanwhile another process recovers the run, finishes it and releases the lease. + expect((await pubsub.acquireLease(key('run-1'), 'recovery', EXECUTION_LEASE_TTL_MS)).acquired).toBe(true); + await pubsub.releaseLease(key('run-1'), 'recovery'); + expect(await pubsub.getLeaseOwner(key('run-1'))).toBeUndefined(); + + await expect(fence.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + release.finish(); + await settled; + }); +}); + +// Storage keeps a released or lapsed claim's record and generation, so a late write is checked against +// the store, not against what this fence saw: a release made by another process, one storage applied but +// whose response was lost, or a lapse. Only another execution's claim refuses it (#30182). +describe('ExecutionFence.verifyNotSuperseded with ownership in storage', () => { + async function setup() { + const storage = new InMemoryStore(); + const workflowsStore = (await storage.getStore('workflows')) as WorkflowsStorage; + const coordinator = await ExecutionFence.claim({ + leaseProvider: NoopLeaseProvider, + workflowsStore, + agentId, + runId: 'run-1', + mode: 'acquire', + }); + const own = { runId: 'run-1', generation: coordinator.generation!, ownerId: coordinator.executionId }; + const claimSuccessor = () => + workflowsStore.claimRunOwnership({ + runId: 'run-1', + ownerId: 'successor', + leaseMs: 60_000, + expectedGeneration: own.generation, + }); + return { storage, workflowsStore, coordinator, own, claimSuccessor }; + } + + it("passes on a worker's participant fence once another process released the shared claim", async () => { + const { storage, workflowsStore, coordinator, own, claimSuccessor } = await setup(); + coordinator.stopHeartbeat(); + const mastra = new Mastra({ storage, logger: false }); + const participant = (await ExecutionFence.follow(mastra, 'run-1', coordinator.claim))!; + onTestFinished(() => participant.stopHeartbeat()); + + // Another worker settled the run and released the claim; this process never saw it. + expect(await workflowsStore.releaseRunOwnership(own)).toBe(true); + await expect(participant.verifyNotSuperseded()).resolves.toBeUndefined(); + expect(participant.isLost()).toBe(false); + // Writes that need a live claim are still refused. + await expect(participant.verify()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + + // Once another execution claims the run, the late write is refused. + expect((await claimSuccessor()).acquired).toBe(true); + await expect(participant.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + }); + + it("passes on a participant fence whose heartbeat already saw another process's release", async () => { + vi.useFakeTimers(); + const { storage, workflowsStore, coordinator, own } = await setup(); + coordinator.stopHeartbeat(); + const mastra = new Mastra({ storage, logger: false }); + const participant = (await ExecutionFence.follow(mastra, 'run-1', coordinator.claim))!; + onTestFinished(() => participant.stopHeartbeat()); + + expect(await workflowsStore.releaseRunOwnership(own)).toBe(true); + // The next renewal finds the claim released and marks the fence lost. + await vi.advanceTimersByTimeAsync(EXECUTION_LEASE_RENEW_INTERVAL_MS + 1); + expect(participant.isLost()).toBe(true); + await expect(participant.verifyNotSuperseded()).resolves.toBeUndefined(); + }); + + it('refuses on a participant fence when another execution claimed the run, not a release', async () => { + const { storage, coordinator, workflowsStore } = await setup(); + coordinator.stopHeartbeat(); + const mastra = new Mastra({ storage, logger: false }); + const participant = (await ExecutionFence.follow(mastra, 'run-1', coordinator.claim))!; + onTestFinished(() => participant.stopHeartbeat()); + + await workflowsStore.claimRunOwnership({ runId: 'run-1', ownerId: 'successor', leaseMs: 60_000, force: true }); + await expect(participant.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + expect(participant.isLost()).toBe(true); + }); + + it("passes on a participant fence whose release failed and whose claim's lease lapsed, until another execution claims the run", async () => { + vi.useFakeTimers(); + const { storage, workflowsStore, coordinator } = await setup(); + coordinator.stopHeartbeat(); + const mastra = new Mastra({ storage, logger: false }); + const participant = (await ExecutionFence.follow(mastra, 'run-1', coordinator.claim))!; + onTestFinished(() => participant.stopHeartbeat()); + mastra.__createRunScope('run-1').set(DURABLE_PARTICIPANT_FENCES, new Map([[coordinator.executionId, participant]])); + + // The release never reaches storage, and neither do renewals, until the claim's lease runs out. + vi.spyOn(workflowsStore, 'releaseRunOwnership').mockRejectedValueOnce(new Error('store down')); + vi.spyOn(workflowsStore, 'renewRunOwnership').mockRejectedValue(new Error('store down')); + await expect(ExecutionFence.releaseSharedClaim(mastra, 'run-1', coordinator.claim)).rejects.toThrow('store down'); + await vi.advanceTimersByTimeAsync(EXECUTION_LEASE_TTL_MS + EXECUTION_LEASE_RENEW_INTERVAL_MS); + const record = await workflowsStore.getRunOwnership({ runId: 'run-1' }); + expect(record).toMatchObject({ ownerId: coordinator.executionId, live: false }); + expect(record?.leaseExpiresAt).not.toBeNull(); + + // Storage would still accept the expired claim's write: nobody claimed the run. + vi.mocked(workflowsStore.renewRunOwnership).mockRestore(); + await expect(participant.verifyNotSuperseded()).resolves.toBeUndefined(); + const successor = await workflowsStore.claimRunOwnership({ + runId: 'run-1', + ownerId: 'successor', + leaseMs: 60_000, + expectedGeneration: coordinator.generation, + }); + expect(successor.acquired).toBe(true); + await expect(participant.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + }); + + it('counts a release storage applied but whose response failed as released', async () => { + const { workflowsStore, coordinator, claimSuccessor } = await setup(); + const release = holdNextCall(workflowsStore, 'releaseRunOwnership', { apply: true, fail: true }); + const settled = coordinator.settle(async () => {}); + await release.applied; + release.finish(); + expect(await settled).toBe('owned'); + + expect(await workflowsStore.getRunOwnership({ runId: 'run-1' })).toMatchObject({ leaseExpiresAt: null }); + await expect(coordinator.verifyNotSuperseded()).resolves.toBeUndefined(); + expect(coordinator.isLost()).toBe(false); + + expect((await claimSuccessor()).acquired).toBe(true); + await expect(coordinator.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + }); + + // Reads of the claim during one check: the first fails as a dropped connection would, and the + // check retries it. `before` and `after` run around the retried read. + function failFirstRead( + workflowsStore: WorkflowsStorage, + { before, after }: { before?: () => Promise; after?: () => Promise }, + ) { + const read = workflowsStore.getRunOwnership.bind(workflowsStore); + let reads = 0; + return vi.spyOn(workflowsStore, 'getRunOwnership').mockImplementation(async args => { + reads++; + if (reads === 1) throw new Error('connection reset'); + if (reads === 2) await before?.(); + const record = await read(args); + if (reads === 2) await after?.(); + return record; + }); + } + + it('passes when another process releases the claim while one check reads it', async () => { + const { workflowsStore, coordinator, own } = await setup(); + coordinator.stopHeartbeat(); + // The check's retried read still finds the claim live; another worker releases it just after. + const reads = failFirstRead(workflowsStore, { + after: async () => expect(await workflowsStore.releaseRunOwnership(own)).toBe(true), + }); + + await expect(coordinator.verifyNotSuperseded()).resolves.toBeUndefined(); + expect(coordinator.isLost()).toBe(false); + reads.mockRestore(); + expect(await workflowsStore.getRunOwnership({ runId: 'run-1' })).toMatchObject({ ...own, leaseExpiresAt: null }); + await expect(coordinator.verifyNotSuperseded()).resolves.toBeUndefined(); + expect(coordinator.isLost()).toBe(false); + }); + + it("passes through assertExecutionNotSuperseded when the heartbeat sees another process's release while the check reads the claim as held", async () => { + vi.useFakeTimers(); + const { storage, workflowsStore, coordinator, own } = await setup(); + const mastra = new Mastra({ storage, logger: false }); + const requestContext = new RequestContext(); + setExecutionClaim(requestContext, 'run-1', coordinator.claim); + const args = { runId: 'run-1', agentId, requestContext, mastra }; + + // The check's next read finds the claim live. Before that read returns, another worker releases + // the claim and this fence's heartbeat renewal finds it released and marks the fence lost. + let signalReleased!: () => void; + const released = new Promise(resolve => (signalReleased = resolve)); + let resumeRead!: () => void; + const readResumed = new Promise(resolve => (resumeRead = resolve)); + const reads = failFirstRead(workflowsStore, { + after: async () => { + expect(await workflowsStore.releaseRunOwnership(own)).toBe(true); + signalReleased(); + await readResumed; + }, + }); + + const check = assertExecutionNotSuperseded(args); + // Past the first read's retry delay. + await vi.advanceTimersByTimeAsync(1_000); + await released; + await vi.advanceTimersByTimeAsync(EXECUTION_LEASE_RENEW_INTERVAL_MS); + expect(coordinator.isLost()).toBe(true); + resumeRead(); + + await expect(check).resolves.toBeUndefined(); + reads.mockRestore(); + expect(await workflowsStore.getRunOwnership({ runId: 'run-1' })).toMatchObject({ ...own, leaseExpiresAt: null }); + await expect(assertExecutionNotSuperseded(args)).resolves.toBeUndefined(); + }); + + it('refuses through assertExecutionNotSuperseded when the heartbeat sees a successor while the check reads the claim as held', async () => { + vi.useFakeTimers(); + const { storage, workflowsStore, coordinator } = await setup(); + const mastra = new Mastra({ storage, logger: false }); + const requestContext = new RequestContext(); + setExecutionClaim(requestContext, 'run-1', coordinator.claim); + const args = { runId: 'run-1', agentId, requestContext, mastra }; + + // As above, but the claim is taken over, not released, before the heartbeat renews. + let signalTakenOver!: () => void; + const takenOver = new Promise(resolve => (signalTakenOver = resolve)); + let resumeRead!: () => void; + const readResumed = new Promise(resolve => (resumeRead = resolve)); + failFirstRead(workflowsStore, { + after: async () => { + await workflowsStore.claimRunOwnership({ runId: 'run-1', ownerId: 'successor', leaseMs: 60_000, force: true }); + signalTakenOver(); + await readResumed; + }, + }); + + const check = assertExecutionNotSuperseded(args); + void check.catch(() => {}); + await vi.advanceTimersByTimeAsync(1_000); + await takenOver; + await vi.advanceTimersByTimeAsync(EXECUTION_LEASE_RENEW_INTERVAL_MS); + expect(coordinator.isLost()).toBe(true); + resumeRead(); + + await expect(check).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + expect(await workflowsStore.getRunOwnership({ runId: 'run-1' })).toMatchObject({ + ownerId: 'successor', + live: true, + }); + }); + + it('refuses when another execution claims the run while one check reads it', async () => { + const { workflowsStore, coordinator, own } = await setup(); + coordinator.stopHeartbeat(); + // Another worker releases the claim and a successor claims the run before the check's retried read. + failFirstRead(workflowsStore, { + before: async () => { + expect(await workflowsStore.releaseRunOwnership(own)).toBe(true); + await workflowsStore.claimRunOwnership({ runId: 'run-1', ownerId: 'successor', leaseMs: 60_000 }); + }, + }); + + await expect(coordinator.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + expect(coordinator.isLost()).toBe(true); + }); + + it("refuses, and marks the execution lost, when storage shows a successor's claim released", async () => { + const { storage, workflowsStore, coordinator, own } = await setup(); + coordinator.stopHeartbeat(); + const mastra = new Mastra({ storage, logger: false }); + const participant = (await ExecutionFence.follow(mastra, 'run-1', coordinator.claim))!; + onTestFinished(() => participant.stopHeartbeat()); + + // A successor claimed the run and has already released it: the record's lease is cleared, but + // it names the successor, not this claim. + expect(await workflowsStore.releaseRunOwnership(own)).toBe(true); + const successor = await workflowsStore.claimRunOwnership({ runId: 'run-1', ownerId: 'successor', leaseMs: 60_000 }); + expect(await workflowsStore.releaseRunOwnership(successor.record)).toBe(true); + + await expect(participant.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + expect(participant.isLost()).toBe(true); + }); + + it('marks the execution lost when its release found the claim already replaced', async () => { + const { workflowsStore, coordinator } = await setup(); + const releaseRunOwnership = workflowsStore.releaseRunOwnership.bind(workflowsStore); + // A successor takes the run just before settlement's release, which then releases nothing. + vi.spyOn(workflowsStore, 'releaseRunOwnership').mockImplementationOnce(async fence => { + await workflowsStore.claimRunOwnership({ runId: 'run-1', ownerId: 'successor', leaseMs: 60_000, force: true }); + const released = await releaseRunOwnership(fence); + expect(released).toBe(false); + return released; + }); + expect(await coordinator.settle(async () => {})).toBe('owned'); + + // Not a release: verify's rule applies, and a superseded execution is lost. + await expect(coordinator.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + expect(coordinator.isLost()).toBe(true); + expect(await workflowsStore.getRunOwnership({ runId: 'run-1' })).toMatchObject({ + ownerId: 'successor', + live: true, + }); + }); + + it('refuses after a failed release when storage shows another execution claimed the run', async () => { + const { workflowsStore, coordinator, claimSuccessor } = await setup(); + // The release applies, a successor claims the run, and only then does the release's response fail. + const release = holdNextCall(workflowsStore, 'releaseRunOwnership', { apply: true, fail: true }); + const settled = coordinator.settle(async () => {}); + await release.applied; + expect((await claimSuccessor()).acquired).toBe(true); + release.finish(); + await settled; + + await expect(coordinator.verifyNotSuperseded()).rejects.toMatchObject({ id: EXECUTION_SUPERSEDED_ERROR_ID }); + }); +}); + describe('execution claim carrier', () => { it('round-trips the claim and drops malformed entries', () => { const requestContext = new RequestContext(); diff --git a/packages/core/src/agent/durable/execution-fence.ts b/packages/core/src/agent/durable/execution-fence.ts index 079b036bb6a..4265f3bce2a 100644 --- a/packages/core/src/agent/durable/execution-fence.ts +++ b/packages/core/src/agent/durable/execution-fence.ts @@ -38,7 +38,7 @@ import { MASTRA_DURABLE_EXECUTIONS_KEY } from '../request-context-snapshot'; import { AGENT_LEASE_RENEW_INTERVAL_MS, AGENT_LEASE_TTL_MS } from '../run-ownership'; import { agentThreadStreamRuntime } from '../thread-stream-runtime'; import { ExecutionOwnershipGuard } from './execution-ownership-guard'; -import { runInRunFenceScope } from './run-fence-scope'; +import { currentRunFenceScope, runInRunFenceScope } from './run-fence-scope'; import { getRunRegistryEntry } from './run-registry'; /** Execution lease timing follows the agent thread lease (`MASTRA_AGENT_THREAD_LEASE_TTL_MS`). */ @@ -925,6 +925,33 @@ export class ExecutionFence implements RunFenceScope { if (this.generation === undefined) this.#leaseExpiresAt = startedAt + EXECUTION_LEASE_TTL_MS; } + /** + * {@link verify} for writes that may land after settlement, such as a + * background task's result or suspension: refuses only once another + * execution claimed the run. A released or lapsed claim still passes, as + * storage accepts the claim's writes until the next claim, and the write's + * own fence ({@link runFencedToClaim}) rejects it once a successor raised + * memory. Never waits for a release in flight. + * + * Only storage-backed claims get this rule: their generation tells a + * released claim from one a successor claimed, released or not. A lease + * keeps no record once released, so an empty lease can't tell this claim's + * release from a successor that claimed and released it since, and nothing + * fences the write behind it. A lease-backed claim always gets {@link verify}. + */ + async verifyNotSuperseded(): Promise { + if (this.generation === undefined) return this.verify(); + const details = this.#details(); + const superseded = this.#takenOver || (await retryOnce(() => this.#backend.isSuperseded(this.claim), details)); + // A takeover the heartbeat saw while the read was on its way also refuses. + if (superseded || this.#takenOver) { + const error = new DurableExecutionFenceError(EXECUTION_SUPERSEDED_ERROR_ID, details); + this.#takenOver = true; + this.#markLost(error); + throw error; + } + } + stopHeartbeat(): void { if (this.#heartbeat) clearInterval(this.#heartbeat); this.#heartbeat = undefined; @@ -1315,7 +1342,7 @@ export async function assertExecutionNotSuperseded(args: { const local = args.mastra?.__getRunScope(args.runId)?.get(DURABLE_PARTICIPANT_FENCES)?.get(claim.executionId) ?? fencesByExecutionId.get(claim.executionId); - if (local) return local.verify(); + if (local) return local.verifyNotSuperseded(); const backend = await resolveRemoteBackend(args.mastra, args.agentId, args.runId, claim); if (!backend) return; @@ -1325,6 +1352,57 @@ export async function assertExecutionNotSuperseded(args: { } } +/** + * {@link assertExecutionNotSuperseded} for a claim with a storage generation, + * {@link assertExecutionOwned} for one without. For writes that refused a + * released claim before late writes were accepted, such as a background + * task's suspension: only a generation tells this claim's release from + * another execution that claimed and released the run since, and a + * lease-backed write has no storage fence behind it (#30182). + */ +export async function assertExecutionNotSupersededInStorage( + args: Parameters[0], +): Promise { + const claim = getExecutionClaim(args.requestContext, args.runId); + if (claim?.generation === undefined) return assertExecutionOwned(args); + return assertExecutionNotSuperseded(args); +} + +/** + * Run `fn` with its writes to `memoryStore` carrying the memory fence of the + * claim `requestContext` holds for `runId`, also once that execution settled + * and stopped fencing its own writes. For writes that may land after the run + * finished, such as a background task's suspension: an ownership check before + * the write can't see a successor that claims the run while the write is on + * its way, but storage rejects the write once that successor raised the memory + * fence. Runs `fn` unchanged when the claim never raised memory (lease-backed + * claims, memory stores without run fencing). Other writes keep the fence of + * the enclosing scope. + */ +export async function runFencedToClaim( + args: { + requestContext: RequestContext | undefined; + runId: string; + /** Resolves the memory store the claim raised; called only when it raised one. */ + memoryStore: () => Promise; + }, + fn: () => Promise, +): Promise { + const fence = getRunFence(args.requestContext, args.runId, 'memory'); + const memoryStore = fence && (await args.memoryStore()); + if (!fence || !memoryStore) return fn(); + const outer = currentRunFenceScope(); + return runInRunFenceScope( + { + fenceFor: (store, runId) => (store === memoryStore ? fence : outer?.fenceFor(store, runId)), + onConflict: conflict => outer?.onConflict?.(conflict), + coverNestedRun: (parentRunId, nestedRunId) => outer?.coverNestedRun?.(parentRunId, nestedRunId), + generationFor: runId => outer?.generationFor?.(runId), + }, + fn, + ); +} + const OWNERSHIP_GUARD = Symbol('durable-execution-ownership-guard'); type GuardHandle = { scope: RunScope; key: RunScopeKey }; diff --git a/packages/core/src/agent/durable/workflows/steps/tool-call.ts b/packages/core/src/agent/durable/workflows/steps/tool-call.ts index e77d31978fe..23166947c21 100644 --- a/packages/core/src/agent/durable/workflows/steps/tool-call.ts +++ b/packages/core/src/agent/durable/workflows/steps/tool-call.ts @@ -67,9 +67,11 @@ import { ensureRemoteAbortListener, readRunDeadline } from '../../abort-transpor import { DurableStepIds } from '../../constants'; import { assertExecutionNotSuperseded, + assertExecutionNotSupersededInStorage, assertExecutionOwned, getExecutionOwnershipGuard, isExecutionFenceError, + runFencedToClaim, } from '../../execution-fence'; import { authorizeDurableMemory, getDurableMemoryAuthorizationChecks } from '../../memory-fga'; import { clearResumeInFlight, getRunRegistryEntry, globalRunRegistry, markRunActive } from '../../run-registry'; @@ -875,9 +877,25 @@ export function createDurableToolCallStep() { // claimed the run. const assertNotSuperseded = () => assertExecutionNotSuperseded({ runId, agentId: initData.agentId, requestContext, mastra: mastra as Mastra }); + // A late suspension gets that rule only with ownership in storage: a released lease can't show + // whether another execution claimed and finished the run since, so a lease keeps the strict check. + const assertSuspensionNotSuperseded = () => + assertExecutionNotSupersededInStorage({ + runId, + agentId: initData.agentId, + requestContext, + mastra: mastra as Mastra, + }); + // Those writes carry the run's claim, so storage rejects one that lands after a successor + // raised the memory fence, even though it passed the check above (#30182). + const claimFenceArgs = { + requestContext, + runId, + memoryStore: async () => memory?.storage.getStore('memory'), + }; // Remote workers have no registry message list; persist updates through the recovered resume list. - const doFlush = async (messagesToFlush = messageList ?? resumeMessageList) => { + const doFlush = async (messagesToFlush = messageList ?? resumeMessageList, beforePersist = assertOwned) => { if (!messagesToFlush && processorDataParts.length) { const invocation = [...(contextMessageList?.get.all.db() ?? [])].reverse().find(hasInvocation); if (invocation) { @@ -935,7 +953,7 @@ export function createDurableToolCallStep() { onThreadCreated: () => { threadExists = true; }, - beforePersist: assertOwned, + beforePersist, }); if (!flushed) { logger?.warn?.('Failed to persist messages before durable tool suspension', { @@ -1583,143 +1601,80 @@ export function createDurableToolCallStep() { ...(toolAbortSignal ? { abortSignal: toolAbortSignal } : {}), outputWriter, // In-execution suspend callback — allows tools to suspend mid-execution - suspend: async (suspendPayload: any, suspendOptions?: SuspendOptions) => { - wasSuspended = true; - // When a delegated sub-agent requests approval, the delegation tool - // wrapper passes its inner suspended run id via `suspendOptions.runId` - // (see the agent-tool wrapper's `suspend(..., { runId, isAgentSuspend })`). - // Persist it with the approval so the resume leg targets that inner - // run instead of restarting the sub-agent from scratch. - const delegatedRunId = - typeof suspendOptions?.runId === 'string' && suspendOptions.runId !== runId - ? suspendOptions.runId - : undefined; - // Set by the agent-tool wrapper when a delegated sub-agent paused: the sub-agent's own - // tool call. Carried alongside the delegation call, which stays the resume key. Ignored - // unless the wrapper raised the pause, so a plain tool's suspend options can't set it. - const fromDelegate = suspendOptions?.isAgentSuspend === true; - const delegatedToolCallId = - fromDelegate && typeof suspendOptions?.delegatedToolCallId === 'string' - ? suspendOptions.delegatedToolCallId - : undefined; - const delegatedToolName = - fromDelegate && typeof suspendOptions?.delegatedToolName === 'string' - ? suspendOptions.delegatedToolName - : undefined; - if (suspendOptions?.requireToolApproval) { - const innerApproval = - typeof suspendOptions.requireToolApproval === 'object' && suspendOptions.requireToolApproval - ? suspendOptions.requireToolApproval - : typeof suspendPayload?.requireToolApproval === 'object' && suspendPayload?.requireToolApproval - ? suspendPayload.requireToolApproval - : null; + suspend: (suspendPayload: any, suspendOptions?: SuspendOptions) => + suspendToolCall(suspendPayload, suspendOptions, assertOwned), + }; + // `beforePersist` guards the suspension's memory writes. A background + // task can suspend after the run settled and released its claim, so with + // ownership in storage it passes the post-settlement check instead of the + // strict one (#30182). + const suspendToolCall = async ( + suspendPayload: any, + suspendOptions: SuspendOptions | undefined, + beforePersist: typeof assertOwned, + ) => { + wasSuspended = true; + // When a delegated sub-agent requests approval, the delegation tool + // wrapper passes its inner suspended run id via `suspendOptions.runId` + // (see the agent-tool wrapper's `suspend(..., { runId, isAgentSuspend })`). + // Persist it with the approval so the resume leg targets that inner + // run instead of restarting the sub-agent from scratch. + const delegatedRunId = + typeof suspendOptions?.runId === 'string' && suspendOptions.runId !== runId + ? suspendOptions.runId + : undefined; + // Set by the agent-tool wrapper when a delegated sub-agent paused: the sub-agent's own + // tool call. Carried alongside the delegation call, which stays the resume key. Ignored + // unless the wrapper raised the pause, so a plain tool's suspend options can't set it. + const fromDelegate = suspendOptions?.isAgentSuspend === true; + const delegatedToolCallId = + fromDelegate && typeof suspendOptions?.delegatedToolCallId === 'string' + ? suspendOptions.delegatedToolCallId + : undefined; + const delegatedToolName = + fromDelegate && typeof suspendOptions?.delegatedToolName === 'string' + ? suspendOptions.delegatedToolName + : undefined; + if (suspendOptions?.requireToolApproval) { + const innerApproval = + typeof suspendOptions.requireToolApproval === 'object' && suspendOptions.requireToolApproval + ? suspendOptions.requireToolApproval + : typeof suspendPayload?.requireToolApproval === 'object' && suspendPayload?.requireToolApproval + ? suspendPayload.requireToolApproval + : null; - const approvalToolName = innerApproval?.toolName ?? toolName; - const approvalArgs = innerApproval?.args !== undefined ? innerApproval.args : args; + const approvalToolName = innerApproval?.toolName ?? toolName; + const approvalArgs = innerApproval?.args !== undefined ? innerApproval.args : args; - // Tool is requesting approval during execution - await stopGoalActivity({ agentId: initData.agentId, runId }); + // Tool is requesting approval during execution + await stopGoalActivity({ agentId: initData.agentId, runId }); - // Add approval metadata to message before persisting - addToolMetadata({ - type: 'approval', - resumeSchema: approvalResumeSchema, - delegatedRunId, - delegatedToolCallId, - delegatedToolName, - ...(innerApproval ? { approvalToolName, approvalArgs } : {}), - }); - - await doFlush(); - - if (pubsub) { - const approvalChunk = await applyToolPayloadTransformToChunk( - { - type: 'tool-call-approval' as const, - runId, - from: ChunkFrom.AGENT, - payload: { - toolCallId, - toolName: approvalToolName, - args: approvalArgs, - ...(delegatedToolCallId ? { delegatedToolCallId } : {}), - ...(delegatedToolName ? { delegatedToolName } : {}), - resumeSchema: approvalResumeSchema, - updatedAt: Date.now(), - usage: typedInput.stepUsage, - }, - }, - { - policy: toolPayloadTransform, - tools: toolsForTransform, - logger: logger as any, - }, - ); - await publishAfterPersist(() => emitChunkEvent(pubsub, runId, approvalChunk)); - } - - if (pubsub) { - await publishAfterPersist(() => - emitSuspendedEvent(pubsub, runId, { - toolCallId, - toolName: approvalToolName, - args: approvalArgs, - type: 'approval', - resumeSchema: approvalResumeSchema, - }), - ); - } - - endSpansAsSuspended({ toolCallId, toolName: approvalToolName, reason: 'approval' }); - - return suspend( - { - type: 'approval', - requireToolApproval: { toolCallId, toolName: approvalToolName, args: approvalArgs }, - // Persist the inner suspended run id in the workflow snapshot, - // partitioned per tool call (resumeLabel = toolCallId), so the - // resume leg can recover it even if message metadata is stale. - ...(delegatedRunId ? { suspendedToolRunId: delegatedRunId } : {}), - }, - { resumeLabel: toolCallId }, - ); - } - // General tool suspension (e.g., tool calls context.agent.suspend()) - const suspendedEventData: AgentSuspendedEventData = { - toolCallId, - toolName, - args, - suspendPayload, - type: 'suspension', - resumeSchema: suspendOptions?.resumeSchema, - }; - - // Add suspension metadata to message before persisting + // Add approval metadata to message before persisting addToolMetadata({ - type: 'suspension', - suspendPayload, - resumeSchema: suspendOptions?.resumeSchema, + type: 'approval', + resumeSchema: approvalResumeSchema, delegatedRunId, delegatedToolCallId, delegatedToolName, + ...(innerApproval ? { approvalToolName, approvalArgs } : {}), }); - await doFlush(); + await doFlush(undefined, beforePersist); if (pubsub) { - const suspensionChunk = await applyToolPayloadTransformToChunk( + const approvalChunk = await applyToolPayloadTransformToChunk( { - type: 'tool-call-suspended' as const, + type: 'tool-call-approval' as const, runId, from: ChunkFrom.AGENT, payload: { toolCallId, - toolName, + toolName: approvalToolName, + args: approvalArgs, ...(delegatedToolCallId ? { delegatedToolCallId } : {}), ...(delegatedToolName ? { delegatedToolName } : {}), - suspendPayload, - args, - resumeSchema: suspendOptions?.resumeSchema, + resumeSchema: approvalResumeSchema, + updatedAt: Date.now(), usage: typedInput.stepUsage, }, }, @@ -1729,28 +1684,101 @@ export function createDurableToolCallStep() { logger: logger as any, }, ); - await publishAfterPersist(() => emitChunkEvent(pubsub, runId, suspensionChunk)); - await publishAfterPersist(() => emitSuspendedEvent(pubsub, runId, suspendedEventData)); + await publishAfterPersist(() => emitChunkEvent(pubsub, runId, approvalChunk)); } - endSpansAsSuspended({ toolCallId, toolName, reason: 'suspension' }); + if (pubsub) { + await publishAfterPersist(() => + emitSuspendedEvent(pubsub, runId, { + toolCallId, + toolName: approvalToolName, + args: approvalArgs, + type: 'approval', + resumeSchema: approvalResumeSchema, + }), + ); + } + + endSpansAsSuspended({ toolCallId, toolName: approvalToolName, reason: 'approval' }); return suspend( { - type: 'suspension', - toolCallSuspended: suspendPayload, - toolCallId, - toolName, - resumeLabel: suspendOptions?.resumeLabel, + type: 'approval', + requireToolApproval: { toolCallId, toolName: approvalToolName, args: approvalArgs }, // Persist the inner suspended run id in the workflow snapshot, // partitioned per tool call (resumeLabel = toolCallId), so the - // resume leg continues the delegate's suspended run instead of - // restarting it (#20496; mirrors the approval branch above). + // resume leg can recover it even if message metadata is stale. ...(delegatedRunId ? { suspendedToolRunId: delegatedRunId } : {}), }, { resumeLabel: toolCallId }, ); - }, + } + // General tool suspension (e.g., tool calls context.agent.suspend()) + const suspendedEventData: AgentSuspendedEventData = { + toolCallId, + toolName, + args, + suspendPayload, + type: 'suspension', + resumeSchema: suspendOptions?.resumeSchema, + }; + + // Add suspension metadata to message before persisting + addToolMetadata({ + type: 'suspension', + suspendPayload, + resumeSchema: suspendOptions?.resumeSchema, + delegatedRunId, + delegatedToolCallId, + delegatedToolName, + }); + + await doFlush(undefined, beforePersist); + + if (pubsub) { + const suspensionChunk = await applyToolPayloadTransformToChunk( + { + type: 'tool-call-suspended' as const, + runId, + from: ChunkFrom.AGENT, + payload: { + toolCallId, + toolName, + ...(delegatedToolCallId ? { delegatedToolCallId } : {}), + ...(delegatedToolName ? { delegatedToolName } : {}), + suspendPayload, + args, + resumeSchema: suspendOptions?.resumeSchema, + usage: typedInput.stepUsage, + }, + }, + { + policy: toolPayloadTransform, + tools: toolsForTransform, + logger: logger as any, + }, + ); + await publishAfterPersist(() => emitChunkEvent(pubsub, runId, suspensionChunk)); + await publishAfterPersist(() => emitSuspendedEvent(pubsub, runId, suspendedEventData)); + } + + endSpansAsSuspended({ toolCallId, toolName, reason: 'suspension' }); + + return suspend( + { + type: 'suspension', + toolCallSuspended: suspendPayload, + toolCallId, + toolName, + resumeLabel: suspendOptions?.resumeLabel, + // Persist the inner suspended run id in the workflow snapshot, + // partitioned per tool call (resumeLabel = toolCallId), so the + // resume leg continues the delegate's suspended run instead of + // restarting it (#20496; mirrors the approval branch above). + ...(delegatedRunId ? { suspendedToolRunId: delegatedRunId } : {}), + }, + { resumeLabel: toolCallId }, + ); }; // Raw `Tool` instances resolved from the Mastra registry (the cross-process // fallback path) are not wrapped by CoreToolBuilder, so they only get a @@ -1843,7 +1871,12 @@ export function createDurableToolCallStep() { // suspension state (#23739) — never the model-authored one. suspendedToolRunId: taskContext?.suspendedToolRunId, suspend: async (data?: unknown, options?: SuspendOptions) => { - await toolOptions.suspend?.(data, options); + // The run may have settled and stopped fencing its writes. The suspension's + // memory save carries the run's claim, so storage rejects it once a successor + // raised the memory fence, even when it lands after the ownership check (#30182). + await runFencedToClaim(claimFenceArgs, () => + suspendToolCall(data, options, assertSuspensionNotSuperseded), + ); return taskContext?.suspend?.(data, options); }, outputWriter: async (chunk: any) => { @@ -2012,10 +2045,14 @@ export function createDurableToolCallStep() { flush, }); await applyTo(messageList, async () => { - if (saveQueueManager && state?.threadId && !state?.memoryConfig?.readOnly) { - await saveQueueManager.flushMessages(messageList, state.threadId, state.memoryConfig, { - beforePersist: assertNotSuperseded, - }); + const threadId = state?.threadId; + if (saveQueueManager && threadId && !state?.memoryConfig?.readOnly) { + // Like a late suspend, the result can land after the run settled: the write carries its claim. + await runFencedToClaim(claimFenceArgs, () => + saveQueueManager.flushMessages(messageList, threadId, state.memoryConfig, { + beforePersist: assertNotSuperseded, + }), + ); } }); if (info.disposition !== 'awaited' && liveEntry) { @@ -2066,11 +2103,14 @@ export function createDurableToolCallStep() { // A superseded execution skips the write. The hook runs outside this // step (the background-task manager calls it), so the error is not // rethrown: that would fail the task instead of stopping the step. - if (saveQueueManager && state?.threadId && !state?.memoryConfig?.readOnly) { + const threadId = state?.threadId; + if (saveQueueManager && threadId && !state?.memoryConfig?.readOnly) { try { - await saveQueueManager.flushMessages(messageList, state.threadId, state.memoryConfig, { - beforePersist: assertNotSuperseded, - }); + await runFencedToClaim(claimFenceArgs, () => + saveQueueManager.flushMessages(messageList, threadId, state.memoryConfig, { + beforePersist: assertNotSuperseded, + }), + ); } catch (error) { if (!isExecutionFenceError(error)) throw error; logger?.debug?.('[DurableAgent] Skipped background-task metadata flush: execution superseded', { diff --git a/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts b/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts index ac3a9dcb65c..3ecb26bab52 100644 --- a/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts +++ b/packages/core/src/loop/test-utils/aimock/scenarios/background-task-lifecycle.scenario.test.ts @@ -1,8 +1,12 @@ +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import { stepCountIs } from '@internal/ai-sdk-v5'; import { createOpenAI } from '@ai-sdk/openai-v5'; -import { it, expect, vi } from 'vitest'; +import { it, expect, onTestFinished, vi } from 'vitest'; import { z } from 'zod'; import { Agent } from '../../../../agent'; +import { agentThreadStreamRuntime } from '../../../../agent/thread-stream-runtime'; import { MockMemory } from '../../../../memory/mock'; import { RequestContext } from '../../../../request-context'; import { createTool } from '../../../../tools'; @@ -14,7 +18,15 @@ import { describeForAllEngines, isDurableEngineVariant, } from '../aimock-scenario'; +import { ExecutionFence, executionLeaseKey, resolveLeaseProvider } from '../../../../agent/durable/execution-fence'; import { BackgroundTaskManager } from '../../../../background-tasks/manager'; +import type { PubSub } from '../../../../events/pubsub'; +import { UnixSocketPubSub } from '../../../../events/unix-socket-pubsub'; +import type { Mastra } from '../../../../mastra'; +import type { MemoryStorage } from '../../../../storage/domains/memory/base'; +import { WorkflowsInMemory } from '../../../../storage/domains/workflows/inmemory'; +import { RUN_FENCE_CONFLICT_ERROR_ID } from '../../../../storage/run-fencing'; +import { SCENARIO_MODEL_ID } from '../types'; /** * Regression classes: the end-to-end background-task lifecycle driven by real @@ -578,7 +590,11 @@ describeForAllEngines( // A chat that reloads the thread while a deferred background tool waits for input must // still see the open request on the tool call, and the resumed tool must know which // request it is answering. (#28986) - function makeSuspendingResearchTool(seenOnResume: unknown[], transform?: ToolPayloadTransform) { + function makeSuspendingResearchTool( + seenOnResume: unknown[], + transform?: ToolPayloadTransform, + gate?: Promise, + ) { return createTool({ id: 'research-with-approval', description: 'Research a topic after analyst approval', @@ -589,6 +605,7 @@ describeForAllEngines( execute: async ({ topic }, options) => { const resumeData = options?.agent?.resumeData as { notes?: string } | undefined; if (!resumeData) { + await gate; return options?.agent?.suspend?.({ awaiting: 'analyst-approval', topic }) as any; } seenOnResume.push(options?.agent?.suspendPayload); @@ -720,6 +737,1204 @@ describeForAllEngines( expect(seenOnResume).toEqual([{ awaiting: 'analyst-approval', topic: 'solana' }]); }, 30_000); + // Durable engines release the run's claim once it finishes. Tests that need the run fully + // over before a background task acts wait for storage to show that. Runs without a claim + // have no record. + async function waitForRunToRelease(mastra: Mastra, runId: string) { + const workflows = await mastra.getStorage()!.getStore('workflows'); + await waitFor( + async () => (await workflows?.getRunOwnership?.({ runId }))?.live ?? false, + live => !live, + ); + } + + // Technical: the background executor runs the origin run's suspend bookkeeping before the + // task's own suspend. Its memory save required the finished run to still own its claim, + // which can't be verified once the claim is released, so the task failed with + // DURABLE_AGENT_EXECUTION_UNVERIFIED instead of suspending. (#30182) + it('parks a deferred tool that asks for input after the reply was sent, and resumes it (#30182)', async () => { + let release!: () => void; + const gate = new Promise(resolve => (release = resolve)); + const seenOnResume: unknown[] = []; + const memory = new MockMemory(); + const thread = { threadId: 'bg-suspend-after-finish-thread', resourceId: 'bg-suspend-after-finish-user' }; + const { chunks, mastra, output } = await runLoopScenario({ + engine, + llm: getMock(), + prompt: 'Research near', + tools: { researchWithApproval: makeSuspendingResearchTool(seenOnResume, undefined, gate) }, + agentBackgroundTasks: { tools: { researchWithApproval: true } }, + backgroundTasks: BG_CONFIG, + memory, + ...thread, + stopWhen: stepCountIs(3), + collectChunks: true, + fixtures: llm => + onReportTurns(llm, 'Research near', { + dispatch: { + toolCalls: [{ id: 'call-bg-1', name: 'researchWithApproval', arguments: { topic: 'near' } }], + }, + ack: 'Waiting for the analyst.', + report: 'The analyst answered.', + }), + }); + // The run has replied and ended before the tool asks for input. + expect(await output.text).toBe('Waiting for the analyst.'); + const manager = mastra.backgroundTaskManager!; + const started = chunks!.find(c => c.type === 'background-task-started') as Chunk & { runId: string }; + const taskId = started.payload.taskId as string; + await waitForRunToRelease(mastra, started.runId); + release(); + + const parked = (await waitFor( + () => manager.getTask(taskId), + (t: any) => t?.status !== 'pending' && t?.status !== 'running', + )) as any; + expect({ status: parked.status, error: parked.error?.message }).toEqual({ + status: 'suspended', + error: undefined, + }); + expect(parked.suspendPayload).toEqual({ awaiting: 'analyst-approval', topic: 'near' }); + const open = await waitFor( + () => storedToolCall(memory, thread), + s => s.suspended?.taskId !== undefined, + ); + expect(open.suspended).toMatchObject({ taskId, suspendPayload: { awaiting: 'analyst-approval', topic: 'near' } }); + + await manager.resume(taskId, { notes: 'looks promising' }); + const done = (await waitFor( + () => manager.getTask(taskId), + (t: any) => t?.status === 'completed', + )) as any; + expect(done.result).toEqual({ summary: 'Research on "near": looks promising' }); + expect(seenOnResume).toEqual([{ awaiting: 'analyst-approval', topic: 'near' }]); + }, 30_000); + + // Technical: two deferred tools from one turn share the finished run's claim. Answering the + // first finishes it and wakes the agent to report it, which runs on the thread again. The + // second tool's late suspend still carries the first run's claim and must still park. (#30182) + it.runIf(isDurableEngineVariant(engine))( + 'parks the second of two deferred tools after the first was answered and reported (#30182)', + async () => { + let releaseAlpha!: () => void; + let releaseBeta!: () => void; + const gates: Record> = { + alpha: new Promise(resolve => (releaseAlpha = resolve)), + beta: new Promise(resolve => (releaseBeta = resolve)), + }; + const memory = new MockMemory(); + const thread = { threadId: 'bg-suspend-sibling-thread', resourceId: 'bg-suspend-sibling-user' }; + const prompt = 'Research alpha and beta'; + const tool = createTool({ + id: 'research-with-approval', + description: 'Research a topic after analyst approval', + inputSchema: z.object({ topic: z.string() }), + outputSchema: z.object({ summary: z.string() }), + background: { enabled: true }, + execute: async ({ topic }, options) => { + const resumeData = options?.agent?.resumeData as { notes?: string } | undefined; + if (!resumeData) { + await gates[topic]; + return options?.agent?.suspend?.({ awaiting: 'analyst-approval', topic }) as any; + } + return { summary: `Research on "${topic}": ${resumeData.notes}` }; + }, + }); + const { chunks, mastra, output } = await runLoopScenario({ + engine, + llm: getMock(), + prompt, + tools: { researchWithApproval: tool }, + agentBackgroundTasks: { tools: { researchWithApproval: true } }, + backgroundTasks: BG_CONFIG, + memory, + ...thread, + stopWhen: stepCountIs(3), + collectChunks: true, + fixtures: llm => + onReportTurns(llm, prompt, { + dispatch: { + toolCalls: [ + { id: 'call-bg-1', name: 'researchWithApproval', arguments: { topic: 'alpha' } }, + { id: 'call-bg-2', name: 'researchWithApproval', arguments: { topic: 'beta' } }, + ], + }, + ack: 'Waiting for the analyst.', + report: 'Alpha answered.', + }), + }); + expect(await output.text).toBe('Waiting for the analyst.'); + const manager = mastra.backgroundTaskManager!; + const started = chunks!.filter(c => c.type === 'background-task-started') as Array; + expect(started).toHaveLength(2); + const tasks = await Promise.all(started.map(c => manager.getTask(c.payload.taskId as string))); + const taskFor = (topic: string) => tasks.find(t => (t?.args as { topic?: string })?.topic === topic)!.id; + const alpha = taskFor('alpha'); + const beta = taskFor('beta'); + await waitForRunToRelease(mastra, started[0].runId); + + releaseAlpha(); + await waitFor( + () => manager.getTask(alpha), + (t: any) => t?.status === 'suspended', + ); + await manager.resume(alpha, { notes: 'looks promising' }); + await waitFor( + () => manager.getTask(alpha), + (t: any) => t?.status === 'completed', + ); + // The wake run reported the first tool and ended. + await waitFor(storedText(memory, thread), s => s.includes('Alpha answered.')); + await waitForRunToRelease(mastra, started[0].runId); + + releaseBeta(); + const parked = (await waitFor( + () => manager.getTask(beta), + (t: any) => t?.status !== 'pending' && t?.status !== 'running', + )) as any; + expect({ status: parked.status, error: parked.error?.message }).toEqual({ + status: 'suspended', + error: undefined, + }); + expect(parked.suspendPayload).toEqual({ awaiting: 'analyst-approval', topic: 'beta' }); + }, + 30_000, + ); + + // Technical: one turn dispatches a deferred tool and also calls a tool that needs approval. + // Approving it resumes the same run under a new claim generation, and only then does the + // deferred tool call suspend(). The durable engine parks it. Known limit: the evented engine + // counts the resume's claim as another execution's, so the task fails with "lost its + // execution lease", as it does on main. Telling this resume from a real takeover would widen + // this fix, so the evented result is pinned. Inngest isn't covered here. (#30182) + it.runIf(engine === 'durable' || engine === 'evented')( + 'parks a deferred tool that asks for input after an approval in the same turn resumed the run (#30182)', + async () => { + let release!: () => void; + const gate = new Promise(resolve => (release = resolve)); + const memory = new MockMemory(); + const llm = getMock(); + const thread = { threadId: 'bg-suspend-after-resume-thread', resourceId: 'bg-suspend-after-resume-user' }; + const lookup = createTool({ + id: 'lookup', + description: 'Look up a user record', + inputSchema: z.object({ name: z.string() }), + outputSchema: z.object({ id: z.string() }), + requireApproval: true, + execute: async ({ name }) => ({ id: `user-${name}` }), + }); + const shared = await createSharedAgent(llm, { + tools: { researchWithApproval: makeSuspendingResearchTool([], undefined, gate), lookup }, + agentBackgroundTasks: { tools: { researchWithApproval: true } }, + backgroundTasks: BG_CONFIG, + memory, + engine, + }); + const prompt = 'Research near and look up dero'; + // The mock server is shared by the suite. The previous test's agent still reports its parked + // task to the model as that test ends, and a request landing here would take this turn's + // tool calls. Answer only this thread's requests; aimock counts the sequence per predicate. + const ownRequest = (req: unknown) => JSON.stringify(req).includes(prompt); + llm.on( + { endpoint: 'chat', predicate: ownRequest, sequenceIndex: 0 }, + { + toolCalls: [ + { id: 'call-bg-1', name: 'researchWithApproval', arguments: { topic: 'near' } }, + { id: 'call-lookup-1', name: 'lookup', arguments: { name: 'dero' } }, + ], + }, + ); + for (let i = 1; i < 8; i++) { + llm.on({ endpoint: 'chat', predicate: ownRequest, sequenceIndex: i }, { content: `Reply ${i}.` }); + } + + const first = await runLoopScenario({ + engine, + llm, + prompt, + sharedAgent: shared, + fixtures: () => {}, + memory, + ...thread, + streamUntilIdle: true, + manualStreamConsumption: true, + }); + const firstChunks = await collectUntil( + first.output.fullStream as ReadableStream, + 'tool-call-approval', + ); + const started = firstChunks.find(c => c.type === 'background-task-started'); + expect(firstChunks.find(c => c.type === 'tool-call-approval')?.payload.toolCallId).toBe('call-lookup-1'); + const taskId = started!.payload.taskId as string; + const workflows = (await shared.mastra.getStorage()!.getStore('workflows'))!; + const before = await workflows.getRunOwnership({ runId: first.output.runId }); + + const resumed = await shared.agent.resumeStreamUntilIdle( + { approved: true }, + { + runId: first.output.runId, + toolCallId: 'call-lookup-1', + memory: { thread: thread.threadId, resource: thread.resourceId }, + }, + ); + // The resume claimed the same run again before the deferred tool suspends. + await waitFor( + () => workflows.getRunOwnership({ runId: first.output.runId }), + o => o !== null && o.generation !== before?.generation, + ); + release(); + const resumedChunks = collect(resumed.fullStream as AsyncIterable); + + const parked = (await waitFor( + () => shared.mastra.backgroundTaskManager!.getTask(taskId), + (t: any) => t?.status !== 'pending' && t?.status !== 'running', + )) as any; + if (engine === 'durable') { + expect({ status: parked.status, error: parked.error?.message }).toEqual({ + status: 'suspended', + error: undefined, + }); + expect(parked.suspendPayload).toEqual({ awaiting: 'analyst-approval', topic: 'near' }); + } else { + expect(parked.status).toBe('failed'); + expect(parked.error?.message).toContain('lost its execution lease'); + } + const lookupResult = (await resumedChunks).find( + c => c.type === 'tool-result' && c.payload?.toolName === 'lookup', + ); + expect(lookupResult?.payload.result).toEqual({ id: 'user-dero' }); + }, + 60_000, + ); + + // Technical: the late suspend above is only allowed while no other execution has claimed the + // run. Once a successor claims it, the old execution's suspension must not reach memory or + // park the task, and the successor's claim must stay as it was. (#30182) + it.runIf(isDurableEngineVariant(engine))( + "doesn't park a deferred tool that asks for input after another execution took over the run (#30182)", + async () => { + let release!: () => void; + const gate = new Promise(resolve => (release = resolve)); + const memory = new MockMemory(); + const thread = { threadId: 'bg-suspend-after-takeover-thread', resourceId: 'bg-suspend-after-takeover-user' }; + const { chunks, mastra, output } = await runLoopScenario({ + engine, + llm: getMock(), + prompt: 'Research near', + tools: { researchWithApproval: makeSuspendingResearchTool([], undefined, gate) }, + agentBackgroundTasks: { tools: { researchWithApproval: true } }, + backgroundTasks: BG_CONFIG, + memory, + ...thread, + stopWhen: stepCountIs(3), + collectChunks: true, + fixtures: llm => + onReportTurns(llm, 'Research near', { + dispatch: { + toolCalls: [{ id: 'call-bg-1', name: 'researchWithApproval', arguments: { topic: 'near' } }], + }, + ack: 'Waiting for the analyst.', + report: 'The analyst answered.', + }), + }); + expect(await output.text).toBe('Waiting for the analyst.'); + const manager = mastra.backgroundTaskManager!; + const started = chunks!.find(c => c.type === 'background-task-started') as Chunk & { runId: string }; + const taskId = started.payload.taskId as string; + await waitForRunToRelease(mastra, started.runId); + + // Another execution claims the run, as a recovery or a new turn would. + const workflows = (await mastra.getStorage()!.getStore('workflows'))!; + const released = await workflows.getRunOwnership({ runId: started.runId }); + const successor = await workflows.claimRunOwnership({ + runId: started.runId, + ownerId: 'successor-execution', + leaseMs: 60_000, + expectedGeneration: released!.generation, + }); + expect(successor.acquired).toBe(true); + release(); + + const settled = (await waitFor( + () => manager.getTask(taskId), + (t: any) => t?.status !== 'pending' && t?.status !== 'running', + )) as any; + expect({ status: settled.status, error: settled.error?.message }).toEqual({ + status: 'failed', + // DURABLE_AGENT_EXECUTION_SUPERSEDED + error: expect.stringContaining(`Durable run ${started.runId} lost its execution lease`), + }); + expect(settled.suspendPayload).toBeUndefined(); + expect((await storedToolCall(memory, thread)).suspended).toBeUndefined(); + const after = await workflows.getRunOwnership({ runId: started.runId }); + expect({ ownerId: after?.ownerId, generation: after?.generation, live: after?.live }).toEqual({ + ownerId: 'successor-execution', + generation: successor.record.generation, + live: true, + }); + }, + 30_000, + ); + + // Technical: with ownership only in a pubsub lease (storage without run fencing), a released + // lease keeps no record, so it can't show whether another execution claimed and finished the + // run since, and no storage fence stands behind the write. The late suspend keeps the strict + // check: the task fails as before rather than write over a run a recovery finished. (#30182) + it.runIf(isDurableEngineVariant(engine))( + "doesn't park a deferred tool that asks for input after its run finished when ownership is only a lease (#30182)", + async () => { + vi.spyOn(WorkflowsInMemory.prototype, 'supportsRunFencing').mockReturnValue(false); + onTestFinished(() => vi.restoreAllMocks()); + let release!: () => void; + const gate = new Promise(resolve => (release = resolve)); + const memory = new MockMemory(); + const thread = { threadId: 'bg-suspend-lease-only-thread', resourceId: 'bg-suspend-lease-only-user' }; + const { chunks, mastra, agent, output } = await runLoopScenario({ + engine, + llm: getMock(), + prompt: 'Research near', + tools: { researchWithApproval: makeSuspendingResearchTool([], undefined, gate) }, + agentBackgroundTasks: { tools: { researchWithApproval: true } }, + backgroundTasks: BG_CONFIG, + memory, + ...thread, + stopWhen: stepCountIs(3), + collectChunks: true, + fixtures: llm => + onReportTurns(llm, 'Research near', { + dispatch: { + toolCalls: [{ id: 'call-bg-1', name: 'researchWithApproval', arguments: { topic: 'near' } }], + }, + ack: 'Waiting for the analyst.', + report: 'The analyst answered.', + }), + }); + expect(await output.text).toBe('Waiting for the analyst.'); + const manager = mastra.backgroundTaskManager!; + const started = chunks!.find(c => c.type === 'background-task-started') as Chunk & { runId: string }; + const taskId = started.payload.taskId as string; + const agentPubsub = (mastra.getAgentById(agent.id) as { pubsub?: PubSub }).pubsub ?? mastra.pubsub; + const leases = resolveLeaseProvider(agentPubsub); + const key = executionLeaseKey(agent.id, started.runId); + // The run finished: its fence is gone and its lease released. + await waitFor( + async () => ({ fence: ExecutionFence.getLocalActive(started.runId), owner: await leases.getLeaseOwner(key) }), + ({ fence, owner }) => fence === undefined && owner === undefined, + ); + + // Another process recovers the run, finishes it and releases the lease. + expect((await leases.acquireLease(key, 'recovery-execution', 60_000)).acquired).toBe(true); + await leases.releaseLease(key, 'recovery-execution'); + release(); + + const settled = (await waitFor( + () => manager.getTask(taskId), + (t: any) => t?.status !== 'pending' && t?.status !== 'running', + )) as any; + const unverified = `Durable run ${started.runId}: could not verify that this execution still owns the run`; + expect({ status: settled.status, error: settled.error?.message }).toEqual({ + status: 'failed', + // DURABLE_AGENT_EXECUTION_UNVERIFIED + error: expect.stringContaining(unverified), + }); + expect(settled.suspendPayload).toBeUndefined(); + // Memory records the tool call as failed. The refused suspension's `suspendedTools` entry + // still reaches memory next to that error, as it does on main (not yet investigated). + const stored = await waitFor( + () => storedToolCall(memory, thread), + s => s.invocation?.state === 'output-error', + ); + expect(stored.invocation.errorText).toContain(unverified); + }, + 30_000, + ); + + // Starts the deferred research tool, lets the run finish and release its claim, and holds + // one cleanup step open, as slow storage would, until `finishCleanup`: retiring the run's + // memory fence, releasing the thread's lease (evented, with a shared `pubsub`), or the claim + // release itself, which storage has applied but not yet returned to the run. `cleanedUp` + // resolves once the held step has finished. With `releaseAckFails`, the claim release is + // applied in storage but its response fails, as a dropped connection would. With + // `outcome: 'result'`, the tool returns a result once let go instead of asking for input. + async function startResearchDuringCleanup( + threadId: string, + { + hold = 'memory-fence', + pubsub, + releaseAckFails = false, + outcome = 'suspend', + }: { + hold?: 'memory-fence' | 'thread-lease' | 'claim-release'; + pubsub?: PubSub; + releaseAckFails?: boolean; + outcome?: 'suspend' | 'result'; + } = {}, + ) { + let release!: () => void; + const gate = new Promise(resolve => (release = resolve)); + let finishCleanup!: () => void; + const cleanupGate = new Promise(resolve => (finishCleanup = resolve)); + let signalCleanedUp!: () => void; + const cleanedUp = new Promise(resolve => (signalCleanedUp = resolve)); + const memory = new MockMemory(); + const memoryStore = (await memory.storage.getStore('memory'))!; + let retiring = false; + const holdOpen = async (step: () => Promise): Promise => { + retiring = true; + await cleanupGate; + try { + return await step(); + } finally { + signalCleanedUp(); + } + }; + if (releaseAckFails) { + const releaseRunOwnership = WorkflowsInMemory.prototype.releaseRunOwnership; + const spy = vi + .spyOn(WorkflowsInMemory.prototype, 'releaseRunOwnership') + .mockImplementationOnce(async function (this: WorkflowsInMemory, fence) { + await releaseRunOwnership.call(this, fence); + throw new Error('connection lost after the release was applied'); + }); + onTestFinished(() => spy.mockRestore()); + } + if (hold === 'memory-fence') { + const retireRunFence = memoryStore.retireRunFence.bind(memoryStore); + vi.spyOn(memoryStore, 'retireRunFence').mockImplementation(fence => holdOpen(() => retireRunFence(fence))); + } else if (hold === 'claim-release') { + // The scenario builds its own store, so the hold goes on the in-memory workflows domain. + const releaseRunOwnership = WorkflowsInMemory.prototype.releaseRunOwnership; + const spy = vi + .spyOn(WorkflowsInMemory.prototype, 'releaseRunOwnership') + .mockImplementation(async function (this: WorkflowsInMemory, fence) { + const released = await releaseRunOwnership.call(this, fence); + return holdOpen(async () => released); + }); + onTestFinished(() => spy.mockRestore()); + } else { + const releaseLease = agentThreadStreamRuntime.releaseDurableThreadLease.bind(agentThreadStreamRuntime); + const spy = vi + .spyOn(agentThreadStreamRuntime, 'releaseDurableThreadLease') + .mockImplementation((...args) => holdOpen(() => releaseLease(...args))); + onTestFinished(() => spy.mockRestore()); + } + const thread = { threadId, resourceId: `${threadId}-user` }; + const researchTool = + outcome === 'result' + ? createTool({ + id: 'research-with-approval', + description: 'Research a topic', + inputSchema: z.object({ topic: z.string() }), + outputSchema: z.object({ summary: z.string() }), + background: { enabled: true }, + execute: async ({ topic }) => { + await gate; + return { summary: `Research on "${topic}"` }; + }, + }) + : makeSuspendingResearchTool([], undefined, gate); + const { chunks, mastra, output } = await runLoopScenario({ + engine, + llm: getMock(), + prompt: 'Research near', + tools: { researchWithApproval: researchTool }, + agentBackgroundTasks: { tools: { researchWithApproval: true } }, + backgroundTasks: BG_CONFIG, + memory, + ...thread, + ...(pubsub ? { pubsub } : {}), + stopWhen: stepCountIs(3), + collectChunks: true, + fixtures: llm => + onReportTurns(llm, 'Research near', { + dispatch: { + toolCalls: [{ id: 'call-bg-1', name: 'researchWithApproval', arguments: { topic: 'near' } }], + }, + ack: 'Waiting for the analyst.', + report: 'The analyst answered.', + }), + }); + expect(await output.text).toBe('Waiting for the analyst.'); + const started = chunks!.find(c => c.type === 'background-task-started') as Chunk & { runId: string }; + await waitForRunToRelease(mastra, started.runId); + await waitFor( + async () => retiring, + isRetiring => isRetiring, + ); + const workflows = (await mastra.getStorage()!.getStore('workflows'))!; + const settledTask = async () => + (await waitFor( + () => mastra.backgroundTaskManager!.getTask(started.payload.taskId as string), + (t: any) => t?.status !== 'pending' && t?.status !== 'running', + )) as any; + return { + memory, + memoryStore, + thread, + workflows, + runId: started.runId, + release, + finishCleanup, + cleanedUp, + settledTask, + }; + } + + // Waits until a held memory save is on its way to storage. A regression that fails the task + // before it saves would otherwise leave the test waiting until its 30 s timeout. + async function untilSaveHeld(saveStarted: Promise): Promise { + let timer: ReturnType | undefined; + const timedOut = new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error('save never reached storage')), 2_000); + }); + try { + await Promise.race([saveStarted, timedOut]); + } finally { + clearTimeout(timer); + } + } + + // Technical: a finished run releases its claim, then retires its memory fence. Until that + // retirement resolves, the execution stays registered in this process, and the late suspend's + // check asked it to prove it still held the claim it had just released, so the task failed + // with DURABLE_AGENT_EXECUTION_SUPERSEDED though no other execution had claimed the run. (#30182) + it.runIf(isDurableEngineVariant(engine))( + 'parks a deferred tool that asks for input while the finished run is still cleaning up (#30182)', + async () => { + const run = await startResearchDuringCleanup('bg-suspend-during-cleanup-thread'); + const released = await run.workflows.getRunOwnership({ runId: run.runId }); + run.release(); + try { + const parked = await run.settledTask(); + expect({ status: parked.status, error: parked.error?.message }).toEqual({ + status: 'suspended', + error: undefined, + }); + expect(parked.suspendPayload).toEqual({ awaiting: 'analyst-approval', topic: 'near' }); + const open = await waitFor( + () => storedToolCall(run.memory, run.thread), + s => s.suspended?.taskId !== undefined, + ); + expect(open.suspended).toMatchObject({ taskId: parked.id }); + // Accepting the late suspend left the released claim as it was. + const after = await run.workflows.getRunOwnership({ runId: run.runId }); + expect({ ownerId: after?.ownerId, generation: after?.generation, live: after?.live }).toEqual({ + ownerId: released?.ownerId, + generation: released?.generation, + live: false, + }); + } finally { + run.finishCleanup(); + } + }, + 30_000, + ); + + // Technical: the cleanup window above must not let a superseded execution through: once a + // successor claims the run, the late suspend still fails without writing. (#30182) + it.runIf(isDurableEngineVariant(engine))( + "doesn't park a deferred tool that asks for input after another execution took over a run still cleaning up (#30182)", + async () => { + const run = await startResearchDuringCleanup('bg-suspend-cleanup-takeover-thread'); + const released = await run.workflows.getRunOwnership({ runId: run.runId }); + const successor = await run.workflows.claimRunOwnership({ + runId: run.runId, + ownerId: 'successor-execution', + leaseMs: 60_000, + expectedGeneration: released!.generation, + }); + expect(successor.acquired).toBe(true); + run.release(); + try { + const settled = await run.settledTask(); + expect({ status: settled.status, error: settled.error?.message }).toEqual({ + status: 'failed', + // DURABLE_AGENT_EXECUTION_SUPERSEDED + error: expect.stringContaining(`Durable run ${run.runId} lost its execution lease`), + }); + expect(settled.suspendPayload).toBeUndefined(); + expect((await storedToolCall(run.memory, run.thread)).suspended).toBeUndefined(); + const after = await run.workflows.getRunOwnership({ runId: run.runId }); + expect({ ownerId: after?.ownerId, generation: after?.generation, live: after?.live }).toEqual({ + ownerId: 'successor-execution', + generation: successor.record.generation, + live: true, + }); + } finally { + run.finishCleanup(); + } + }, + 30_000, + ); + + // Technical: the late suspend's ownership check passes while no successor exists, but its + // memory save can reach storage later. A finished run stops fencing its own writes once it + // settles, so the save went through unfenced: a successor that claimed the run and raised the + // memory fence in between found the old execution's suspension in memory and the task parked. + // The save now carries the finished run's claim, and storage rejects it. (#30182) + it.runIf(isDurableEngineVariant(engine))( + "doesn't park a deferred tool whose suspension reaches storage after another execution took over (#30182)", + async () => { + const run = await startResearchDuringCleanup('bg-suspend-slow-save-takeover-thread'); + let signalSaveStarted!: () => void; + const saveStarted = new Promise(resolve => (signalSaveStarted = resolve)); + let finishSave!: () => void; + const saveGate = new Promise(resolve => (finishSave = resolve)); + const saveMessages = run.memoryStore.saveMessages.bind(run.memoryStore); + vi.spyOn(run.memoryStore, 'saveMessages').mockImplementation(async args => { + signalSaveStarted(); + await saveGate; + return saveMessages(args); + }); + try { + run.release(); + // The suspension passed its ownership check; its save is on the way to storage. + await untilSaveHeld(saveStarted); + // The finished run completes its cleanup. Then a successor claims the run and raises + // the memory fence, as recovery or a new turn does before it touches memory. + run.finishCleanup(); + await run.cleanedUp; + const released = await run.workflows.getRunOwnership({ runId: run.runId }); + const successor = await run.workflows.claimRunOwnership({ + runId: run.runId, + ownerId: 'successor-execution', + leaseMs: 60_000, + expectedGeneration: released!.generation, + }); + expect(successor.acquired).toBe(true); + const successorFence = { + runId: run.runId, + generation: successor.record!.generation, + ownerId: successor.record!.ownerId, + }; + expect(await run.memoryStore.raiseRunFence(successorFence)).toBe(true); + finishSave(); + + const settled = await run.settledTask(); + expect({ status: settled.status, error: settled.error?.message }).toEqual({ + status: 'failed', + // STORAGE_RUN_FENCE_CONFLICT + error: expect.stringContaining("is no longer the run's current claim"), + }); + expect(settled.suspendPayload).toBeUndefined(); + expect((await storedToolCall(run.memory, run.thread)).suspended).toBeUndefined(); + const after = await run.workflows.getRunOwnership({ runId: run.runId }); + expect({ ownerId: after?.ownerId, generation: after?.generation, live: after?.live }).toEqual({ + ownerId: 'successor-execution', + generation: successor.record!.generation, + live: true, + }); + } finally { + finishSave(); + run.finishCleanup(); + } + }, + 30_000, + ); + + // Technical: settlement releases the claim, and only once storage's release call returns + // did the run count as released. The late suspend checked ownership in between: storage + // already showed the claim released, so the strict check failed and marked the execution + // lost though nobody had taken over. Waiting for the release's reply instead left the task + // running for as long as that reply took, forever on a stalled connection. Here the reply is + // held until the task has settled. (#30182) + it.runIf(isDurableEngineVariant(engine))( + "parks a deferred tool that asks for input while the finished run's claim release has not returned (#30182)", + async () => { + const run = await startResearchDuringCleanup('bg-suspend-during-claim-release-thread', { + hold: 'claim-release', + }); + const released = await run.workflows.getRunOwnership({ runId: run.runId }); + expect(released?.live).toBe(false); + run.release(); + try { + // The release's reply is still held: the task settles without it. + const parked = await run.settledTask(); + expect({ status: parked.status, error: parked.error?.message }).toEqual({ + status: 'suspended', + error: undefined, + }); + expect(parked.suspendPayload).toEqual({ awaiting: 'analyst-approval', topic: 'near' }); + const open = await waitFor( + () => storedToolCall(run.memory, run.thread), + s => s.suspended?.taskId !== undefined, + ); + expect(open.suspended).toMatchObject({ taskId: parked.id }); + const after = await run.workflows.getRunOwnership({ runId: run.runId }); + expect({ ownerId: after?.ownerId, generation: after?.generation, live: after?.live }).toEqual({ + ownerId: released?.ownerId, + generation: released?.generation, + live: false, + }); + } finally { + run.finishCleanup(); + } + }, + 30_000, + ); + + // Technical: accepting a late suspend before the release's reply returns must not let a + // superseded execution through: a successor that claimed the run during the release still + // makes the late suspend fail without writing. (#30182) + it.runIf(isDurableEngineVariant(engine))( + "doesn't park a deferred tool that asks for input after another execution took over a run whose claim release has not returned (#30182)", + async () => { + const run = await startResearchDuringCleanup('bg-suspend-claim-release-takeover-thread', { + hold: 'claim-release', + }); + const released = await run.workflows.getRunOwnership({ runId: run.runId }); + const successor = await run.workflows.claimRunOwnership({ + runId: run.runId, + ownerId: 'successor-execution', + leaseMs: 60_000, + expectedGeneration: released!.generation, + }); + expect(successor.acquired).toBe(true); + run.release(); + try { + const settled = await run.settledTask(); + expect({ status: settled.status, error: settled.error?.message }).toEqual({ + status: 'failed', + // DURABLE_AGENT_EXECUTION_SUPERSEDED + error: expect.stringContaining(`Durable run ${run.runId} lost its execution lease`), + }); + expect(settled.suspendPayload).toBeUndefined(); + expect((await storedToolCall(run.memory, run.thread)).suspended).toBeUndefined(); + const after = await run.workflows.getRunOwnership({ runId: run.runId }); + expect({ ownerId: after?.ownerId, generation: after?.generation, live: after?.live }).toEqual({ + ownerId: 'successor-execution', + generation: successor.record.generation, + live: true, + }); + } finally { + run.finishCleanup(); + } + }, + 30_000, + ); + + // Technical: storage applied the finished run's claim release, but the response failed, as a + // dropped connection would. The fence treated the release as never made and stayed strict, so + // a late suspend during the remaining cleanup read the released claim, marked the execution + // lost though nobody had taken over, and the task failed before it suspended. (#30182) + it.runIf(isDurableEngineVariant(engine))( + 'parks a deferred tool that asks for input after the finished run lost the reply to its claim release (#30182)', + async () => { + const run = await startResearchDuringCleanup('bg-suspend-after-lost-release-ack-thread', { + releaseAckFails: true, + }); + const released = await run.workflows.getRunOwnership({ runId: run.runId }); + expect(released).toMatchObject({ leaseExpiresAt: null, live: false }); + run.release(); + try { + const parked = await run.settledTask(); + expect({ status: parked.status, error: parked.error?.message }).toEqual({ + status: 'suspended', + error: undefined, + }); + expect(parked.suspendPayload).toEqual({ awaiting: 'analyst-approval', topic: 'near' }); + const open = await waitFor( + () => storedToolCall(run.memory, run.thread), + s => s.suspended?.taskId !== undefined, + ); + expect(open.suspended).toMatchObject({ taskId: parked.id }); + const after = await run.workflows.getRunOwnership({ runId: run.runId }); + expect({ ownerId: after?.ownerId, generation: after?.generation, live: after?.live }).toEqual({ + ownerId: released?.ownerId, + generation: released?.generation, + live: false, + }); + } finally { + run.finishCleanup(); + } + }, + 30_000, + ); + + // Technical: counting a release whose reply was lost as made must not let a superseded + // execution through: once a successor claims the run, the late suspend fails without + // writing. (#30182) + it.runIf(isDurableEngineVariant(engine))( + "doesn't park a deferred tool that asks for input after another execution took over a run that lost the reply to its claim release (#30182)", + async () => { + const run = await startResearchDuringCleanup('bg-suspend-lost-release-ack-takeover-thread', { + releaseAckFails: true, + }); + const released = await run.workflows.getRunOwnership({ runId: run.runId }); + const successor = await run.workflows.claimRunOwnership({ + runId: run.runId, + ownerId: 'successor-execution', + leaseMs: 60_000, + expectedGeneration: released!.generation, + }); + expect(successor.acquired).toBe(true); + run.release(); + try { + const settled = await run.settledTask(); + expect({ status: settled.status, error: settled.error?.message }).toEqual({ + status: 'failed', + // DURABLE_AGENT_EXECUTION_SUPERSEDED + error: expect.stringContaining(`Durable run ${run.runId} lost its execution lease`), + }); + expect(settled.suspendPayload).toBeUndefined(); + expect((await storedToolCall(run.memory, run.thread)).suspended).toBeUndefined(); + const after = await run.workflows.getRunOwnership({ runId: run.runId }); + expect({ ownerId: after?.ownerId, generation: after?.generation, live: after?.live }).toEqual({ + ownerId: 'successor-execution', + generation: successor.record.generation, + live: true, + }); + } finally { + run.finishCleanup(); + } + }, + 30_000, + ); + + // Technical: a background task's result is written to memory through the same post-settlement + // check as a late suspend. While the finished run is still cleaning up, that check now accepts + // the write when nobody has taken over, so the result reaches memory. (#30182) + it.runIf(isDurableEngineVariant(engine))( + "records a deferred tool's result that arrives while the finished run is still cleaning up (#30182)", + async () => { + const run = await startResearchDuringCleanup('bg-result-during-cleanup-thread', { outcome: 'result' }); + const released = await run.workflows.getRunOwnership({ runId: run.runId }); + run.release(); + try { + const done = await run.settledTask(); + expect({ status: done.status, error: done.error?.message }).toEqual({ + status: 'completed', + error: undefined, + }); + expect(done.result).toEqual({ summary: 'Research on "near"' }); + // The dispatch left a placeholder result; the task's own result replaces it. + const answered = await waitFor( + () => storedToolCall(run.memory, run.thread), + s => s.invocation?.result?.summary !== undefined, + ); + expect(answered.invocation).toMatchObject({ state: 'result', result: { summary: 'Research on "near"' } }); + const after = await run.workflows.getRunOwnership({ runId: run.runId }); + expect({ ownerId: after?.ownerId, generation: after?.generation }).toEqual({ + ownerId: released?.ownerId, + generation: released?.generation, + }); + } finally { + run.finishCleanup(); + } + }, + 30_000, + ); + + // Holds the `nth` memory save (counting from 1) until `finishSave`. `saveStarted` resolves once + // it is held; `saveOutcome` resolves once it reached storage, to its error or `undefined`. + function holdMemorySave(memoryStore: MemoryStorage, nth: number) { + let signalSaveStarted!: () => void; + const saveStarted = new Promise(resolve => (signalSaveStarted = resolve)); + let finishSave!: () => void; + const saveGate = new Promise(resolve => (finishSave = resolve)); + let settleOutcome!: (error: unknown) => void; + const saveOutcome = new Promise(resolve => (settleOutcome = resolve)); + const saveMessages = memoryStore.saveMessages.bind(memoryStore); + let saves = 0; + const spy = vi.spyOn(memoryStore, 'saveMessages').mockImplementation(async args => { + if (++saves !== nth) return saveMessages(args); + signalSaveStarted(); + await saveGate; + try { + const saved = await saveMessages(args); + settleOutcome(undefined); + return saved; + } catch (error) { + settleOutcome(error); + throw error; + } + }); + onTestFinished(() => { + finishSave(); + spy.mockRestore(); + }); + return { saveStarted, finishSave, saveOutcome, saveMessages }; + } + + // The finished run completes its cleanup. Then a successor claims the run and raises the + // memory fence, as recovery or a new turn does before it touches memory. + async function takeOverAfterCleanup(run: Awaited>) { + run.finishCleanup(); + await run.cleanedUp; + const released = await run.workflows.getRunOwnership({ runId: run.runId }); + const successor = await run.workflows.claimRunOwnership({ + runId: run.runId, + ownerId: 'successor-execution', + leaseMs: 60_000, + expectedGeneration: released!.generation, + }); + expect(successor.acquired).toBe(true); + expect( + await run.memoryStore.raiseRunFence({ + runId: run.runId, + generation: successor.record!.generation, + ownerId: successor.record!.ownerId, + }), + ).toBe(true); + return successor.record!; + } + + async function expectSuccessorUnchanged( + run: Awaited>, + successor: { generation: number }, + ) { + const after = await run.workflows.getRunOwnership({ runId: run.runId }); + expect({ ownerId: after?.ownerId, generation: after?.generation, live: after?.live }).toEqual({ + ownerId: 'successor-execution', + generation: successor.generation, + live: true, + }); + } + + // Technical: after a late suspend saves its open request, the background-task manager saves + // the task's metadata again (`onExecution`). That save passed the post-settlement check but + // reached storage only after the finished run's cleanup and a successor's claim, which raised + // the memory fence and cleared the request. It carried no claim fence, so it restored the + // superseded execution's open request. It now carries the finished run's claim. (#30182) + it.runIf(isDurableEngineVariant(engine))( + "doesn't restore a deferred tool's open request when its task metadata reaches storage after another execution took over (#30182)", + async () => { + const run = await startResearchDuringCleanup('bg-suspend-slow-metadata-save-takeover-thread'); + // The first save is the suspension's own; the second is the manager's metadata save. + const save = holdMemorySave(run.memoryStore, 2); + try { + run.release(); + await untilSaveHeld(save.saveStarted); + const successor = await takeOverAfterCleanup(run); + // The successor works on the thread: here it drops the open request. + const { messages } = await run.memoryStore.listMessages({ ...run.thread, perPage: false }); + const cleared = structuredClone(messages); + for (const message of cleared) delete message.content.metadata?.suspendedTools; + await save.saveMessages({ messages: cleared }); + expect((await storedToolCall(run.memory, run.thread)).suspended).toBeUndefined(); + + save.finishSave(); + expect(await save.saveOutcome).toMatchObject({ id: RUN_FENCE_CONFLICT_ERROR_ID }); + expect((await storedToolCall(run.memory, run.thread)).suspended).toBeUndefined(); + await expectSuccessorUnchanged(run, successor); + } finally { + run.finishCleanup(); + } + }, + 30_000, + ); + + // Technical: a background task's result reaches memory through the same post-settlement check + // as a late suspend. A result save that passed it but reached storage after a successor + // claimed the run and raised the memory fence went through, since it carried no claim fence, + // so a superseded execution wrote the tool's result. It now carries the finished run's + // claim. (#30182) + it.runIf(isDurableEngineVariant(engine))( + "doesn't record a deferred tool's result that reaches storage after another execution took over (#30182)", + async () => { + const run = await startResearchDuringCleanup('bg-result-slow-save-takeover-thread', { outcome: 'result' }); + const save = holdMemorySave(run.memoryStore, 1); + try { + run.release(); + await untilSaveHeld(save.saveStarted); + const successor = await takeOverAfterCleanup(run); + + save.finishSave(); + expect(await save.saveOutcome).toMatchObject({ id: RUN_FENCE_CONFLICT_ERROR_ID }); + // Memory keeps the dispatch placeholder, not the superseded execution's result. + expect((await storedToolCall(run.memory, run.thread)).invocation?.result?.summary).toBeUndefined(); + await expectSuccessorUnchanged(run, successor); + } finally { + run.finishCleanup(); + } + }, + 30_000, + ); + + // Technical: on a shared transport, an evented run's worker releases the run's claim, then + // the thread's lease, and only then stops its participant fence. Until the lease release + // returns, that fence stays registered, and the late suspend's check asked it to prove it + // still held the claim it had just released, so the task failed with + // DURABLE_AGENT_EXECUTION_SUPERSEDED though nobody took over. (#30182) + it.runIf(engine === 'evented')( + 'parks a deferred tool that asks for input while a shared evented run is still releasing its thread (#30182)', + async () => { + const directory = await mkdtemp(join(tmpdir(), 'mastra-30182-')); + const pubsub = new UnixSocketPubSub(join(directory, 'pubsub.sock')); + const run = await startResearchDuringCleanup('bg-suspend-during-lease-release-thread', { + hold: 'thread-lease', + pubsub, + }); + const released = await run.workflows.getRunOwnership({ runId: run.runId }); + run.release(); + try { + const parked = await run.settledTask(); + expect({ status: parked.status, error: parked.error?.message }).toEqual({ + status: 'suspended', + error: undefined, + }); + expect(parked.suspendPayload).toEqual({ awaiting: 'analyst-approval', topic: 'near' }); + const open = await waitFor( + () => storedToolCall(run.memory, run.thread), + s => s.suspended?.taskId !== undefined, + ); + expect(open.suspended).toMatchObject({ taskId: parked.id }); + const after = await run.workflows.getRunOwnership({ runId: run.runId }); + expect({ ownerId: after?.ownerId, generation: after?.generation, live: after?.live }).toEqual({ + ownerId: released?.ownerId, + generation: released?.generation, + live: false, + }); + } finally { + run.finishCleanup(); + await run.cleanedUp; + await pubsub.close(); + await rm(directory, { recursive: true, force: true }); + } + }, + 30_000, + ); + + // Technical: the same in-flight release as above, on the worker's participant fence: an + // evented worker on a shared transport releases the run's claim itself, and a late suspend + // that read the released claim before that release returned marked the execution lost, or + // waited on the release's reply. The reply is held until the task has settled. (#30182) + it.runIf(engine === 'evented')( + "parks a deferred tool that asks for input while a shared evented run's claim release has not returned (#30182)", + async () => { + const directory = await mkdtemp(join(tmpdir(), 'mastra-30182-')); + const pubsub = new UnixSocketPubSub(join(directory, 'pubsub.sock')); + const run = await startResearchDuringCleanup('bg-suspend-during-shared-claim-release-thread', { + hold: 'claim-release', + pubsub, + }); + const released = await run.workflows.getRunOwnership({ runId: run.runId }); + run.release(); + try { + const parked = await run.settledTask(); + expect({ status: parked.status, error: parked.error?.message }).toEqual({ + status: 'suspended', + error: undefined, + }); + expect(parked.suspendPayload).toEqual({ awaiting: 'analyst-approval', topic: 'near' }); + const open = await waitFor( + () => storedToolCall(run.memory, run.thread), + s => s.suspended?.taskId !== undefined, + ); + expect(open.suspended).toMatchObject({ taskId: parked.id }); + const after = await run.workflows.getRunOwnership({ runId: run.runId }); + expect({ ownerId: after?.ownerId, generation: after?.generation, live: after?.live }).toEqual({ + ownerId: released?.ownerId, + generation: released?.generation, + live: false, + }); + } finally { + run.finishCleanup(); + await run.cleanedUp; + await pubsub.close(); + await rm(directory, { recursive: true, force: true }); + } + }, + 30_000, + ); + + // Technical: same path as above, reached through a deferred sub-agent: when its tool needs + // approval, the delegation tool suspends the background task with that approval. (#30182) + it('parks a deferred sub-agent whose tool needs approval after the reply was sent, and resumes it (#30182)', async () => { + const mock = getMock(); + const openai = createOpenAI({ apiKey: 'aimock-test-key', baseURL: `${mock.url.replace(/\/+$/, '')}/v1` }); + let release!: () => void; + const gate = new Promise(resolve => (release = resolve)); + const published: string[] = []; + const publishTool = createTool({ + id: 'publish', + description: 'Publish a post', + inputSchema: z.object({ title: z.string() }), + requireApproval: true, + execute: async ({ title }) => { + published.push(title); + return { published: title }; + }, + }); + const helper = new Agent({ + id: 'helper', + name: 'Helper', + description: 'Publishes posts', + instructions: 'Publish posts with publishTool.', + model: openai(SCENARIO_MODEL_ID), + tools: { publishTool }, + // Holds the sub-agent until the supervisor's run is over. + inputProcessors: [ + { + id: 'wait-for-supervisor', + processInput: async ({ messages }) => { + await gate; + return messages; + }, + }, + ], + }); + const toolNames = (req: any): string[] => (req?.tools ?? []).map((t: any) => t.function?.name); + const isSupervisor = (req: unknown) => toolNames(req).includes('agent-helper'); + const isHelper = (req: unknown) => toolNames(req).includes('publishTool'); + const memory = new MockMemory(); + const thread = { threadId: 'bg-subagent-approval-thread', resourceId: 'bg-subagent-approval-user' }; + const { chunks, mastra, output } = await runLoopScenario({ + engine, + llm: mock, + prompt: 'Publish the launch post', + agents: { helper }, + agentBackgroundTasks: { tools: { helper: true } }, + backgroundTasks: BG_CONFIG, + memory, + ...thread, + stopWhen: stepCountIs(3), + collectChunks: true, + fixtures: llm => { + llm.on( + { endpoint: 'chat', predicate: (req: unknown) => isSupervisor(req) && !hasToolResult(req) }, + { toolCalls: [{ id: 'call-bg-1', name: 'agent-helper', arguments: { prompt: 'Publish "Launch"' } }] }, + ); + llm.on( + { endpoint: 'chat', predicate: (req: unknown) => isSupervisor(req) && hasToolResult(req) }, + { content: 'Handed to the helper.' }, + ); + llm.on( + { endpoint: 'chat', predicate: (req: unknown) => isHelper(req) && !hasToolResult(req) }, + { toolCalls: [{ id: 'call-publish-1', name: 'publishTool', arguments: { title: 'Launch' } }] }, + ); + llm.on( + { endpoint: 'chat', predicate: (req: unknown) => isHelper(req) && hasToolResult(req) }, + { content: 'Published Launch.' }, + ); + }, + }); + expect(await output.text).toBe('Handed to the helper.'); + const manager = mastra.backgroundTaskManager!; + const started = chunks!.find(c => c.type === 'background-task-started') as Chunk & { runId: string }; + const taskId = started.payload.taskId as string; + await waitForRunToRelease(mastra, started.runId); + release(); + + const parked = (await waitFor( + () => manager.getTask(taskId), + (t: any) => t?.status !== 'pending' && t?.status !== 'running', + )) as any; + expect({ status: parked.status, error: parked.error?.message }).toEqual({ + status: 'suspended', + error: undefined, + }); + expect(published).toEqual([]); + + await manager.resume(taskId, { approved: true }); + const done = (await waitFor( + () => manager.getTask(taskId), + (t: any) => t?.status !== 'suspended' && t?.status !== 'running' && t?.status !== 'pending', + )) as any; + expect({ status: done.status, error: done.error?.message }).toEqual({ status: 'completed', error: undefined }); + expect(published).toEqual(['Launch']); + }, 30_000); + it('clears the open request from memory when a suspended deferred task is cancelled (#28986)', async () => { const { memory, thread, manager, taskId } = await startSuspendingResearch([], 'bg-suspend-cancel-thread'); await waitFor( diff --git a/stores/libsql/src/__tests__/durable-background-late-write-fence.test.ts b/stores/libsql/src/__tests__/durable-background-late-write-fence.test.ts new file mode 100644 index 00000000000..18675aa5453 --- /dev/null +++ b/stores/libsql/src/__tests__/durable-background-late-write-fence.test.ts @@ -0,0 +1,364 @@ +/** + * A durable agent's background task can write to memory after the run that dispatched it has + * finished and released its claim: the task's suspension (it asks for input) or its result. The + * write passes an ownership check while nobody has claimed the run, but can reach storage after a + * successor claimed the run and raised the memory fence. These cases run that ordering on a real + * LibSQL file, with workflows and memory in the same store, and check that the late write carries + * the finished run's claim to the memory domain object the adapter fences, so LibSQL's write + * transaction rejects it. With nobody taking over, the late suspension parks the task and a resume + * completes it on the same database. (#30182) + */ +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { Agent } from '@mastra/core/agent'; +import { createDurableAgent, createEventedAgent } from '@mastra/core/agent/durable'; +import type { BackgroundTask } from '@mastra/core/background-tasks'; +import { EventEmitterPubSub } from '@mastra/core/events'; +import { Mastra } from '@mastra/core/mastra'; +import { MockMemory } from '@mastra/core/memory'; +import type { MastraDBMessage } from '@mastra/core/memory'; +import { RUN_FENCE_CONFLICT_ERROR_ID } from '@mastra/core/storage'; +import type { MemoryStorage, RunFence } from '@mastra/core/storage'; +import { MastraLanguageModelV2Mock } from '@mastra/core/test-utils/llm-mock'; +import { createTool } from '@mastra/core/tools'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { z } from 'zod'; + +import { LibSQLStore } from '../storage'; +import { gate } from './cross-process/step-agent'; + +const TOOL_CALL_ID = 'call-bg-1'; +const ACK = 'Waiting for the analyst.'; + +const cleanups: Array<() => Promise> = []; +afterEach(async () => { + for (const cleanup of cleanups.splice(0).reverse()) await cleanup(); + vi.restoreAllMocks(); +}); + +async function waitFor(read: () => Promise, ok: (value: T) => boolean, label: string): Promise { + const deadline = Date.now() + 15_000; + for (;;) { + const value = await read(); + if (ok(value)) return value; + if (Date.now() > deadline) throw new Error(`Timed out waiting for ${label}`); + await new Promise(resolve => setTimeout(resolve, 20)); + } +} + +function stream(parts: unknown[]) { + return new ReadableStream({ + start(controller) { + for (const part of parts) controller.enqueue(part); + controller.close(); + }, + }); +} + +// Calls the background tool on the first turn, then acknowledges. Any later turn (a wake to +// report the task) acknowledges too. +function scriptedModel() { + return new MastraLanguageModelV2Mock({ + doStream: async ({ prompt }) => { + const head = [ + { type: 'stream-start', warnings: [] }, + { type: 'response-metadata', id: 'resp', modelId: 'mock-model-id', timestamp: new Date(0) }, + ]; + const usage = { inputTokens: 1, outputTokens: 1, totalTokens: 2 }; + const sawToolResult = prompt.some(message => message.role === 'tool'); + return { + rawCall: { rawPrompt: null, rawSettings: {} }, + stream: stream( + sawToolResult + ? [ + ...head, + { type: 'text-start', id: 'text-1' }, + { type: 'text-delta', id: 'text-1', delta: ACK }, + { type: 'text-end', id: 'text-1' }, + { type: 'finish', finishReason: 'stop', usage }, + ] + : [ + ...head, + { + type: 'tool-call', + toolCallId: TOOL_CALL_ID, + toolName: 'research', + input: JSON.stringify({ topic: 'near' }), + providerExecuted: false, + }, + { type: 'finish', finishReason: 'tool-calls', usage }, + ], + ), + }; + }, + }); +} + +/** Holds the next `saveMessages` on `memoryStore` until `finish()`, and reports how it ended. */ +function holdNextSave(memoryStore: MemoryStorage) { + const started = gate(); + const release = gate(); + const outcome = gate(); + const saveMessages = memoryStore.saveMessages.bind(memoryStore); + vi.spyOn(memoryStore, 'saveMessages').mockImplementationOnce(async args => { + started.resolve(); + await release.promise; + try { + const saved = await saveMessages(args); + outcome.resolve(undefined); + return saved; + } catch (error) { + outcome.resolve(error); + throw error; + } + }); + cleanups.push(async () => release.resolve()); + return { started: started.promise, finish: () => release.resolve(), outcome: outcome.promise }; +} + +function toolCallIn(messages: MastraDBMessage[]) { + for (const message of messages) { + for (const part of message.content.parts ?? []) { + if (part.type === 'tool-invocation' && part.toolInvocation.toolCallId === TOOL_CALL_ID) { + const suspendedTools = message.content.metadata?.suspendedTools; + const suspended = + suspendedTools && typeof suspendedTools === 'object' && TOOL_CALL_ID in suspendedTools + ? (suspendedTools as Record)[TOOL_CALL_ID] + : undefined; + return { invocation: part.toolInvocation, suspended }; + } + } + } + return undefined; +} + +async function startDeferredResearch(engine: 'durable' | 'evented', outcome: 'suspend' | 'result') { + const dir = await mkdtemp(join(tmpdir(), 'mastra-30182-libsql-')); + const storage = new LibSQLStore({ id: `late-write-${engine}-${outcome}`, url: `file:${join(dir, 'mastra.db')}` }); + cleanups.push(() => rm(dir, { recursive: true, force: true })); + // Memory on the same LibSQL file as the run's claim, as a deployment configures it. + const memory = new MockMemory(); + memory.setStorage(storage); + const memoryStore = (await memory.storage.getStore('memory'))!; + // The object the late write's claim fence is keyed to is the one LibSQL's memory domain checks. + expect(memoryStore).toBe(await storage.getStore('memory')); + + // The finished run retires its memory fence last; the takeover waits for that. + const cleanedUp = gate(); + const retireRunFence = memoryStore.retireRunFence.bind(memoryStore); + vi.spyOn(memoryStore, 'retireRunFence').mockImplementation(async (fence: RunFence) => { + try { + return await retireRunFence(fence); + } finally { + cleanedUp.resolve(); + } + }); + + const toolGate = gate(); + const research = createTool({ + id: 'research', + description: 'Research a topic after analyst approval', + inputSchema: z.object({ topic: z.string() }), + outputSchema: z.object({ summary: z.string() }), + background: { enabled: true }, + execute: async ({ topic }, options) => { + const resumeData = options?.agent?.resumeData as { notes?: string } | undefined; + if (resumeData) return { summary: `Research on "${topic}": ${resumeData.notes}` }; + await toolGate.promise; + if (outcome === 'result') return { summary: `Research on "${topic}"` }; + return options?.agent?.suspend?.({ awaiting: 'analyst-approval', topic }) as never; + }, + }); + const agent = new Agent({ + id: `late-write-${engine}`, + name: 'Late write agent', + instructions: 'Research what the user asks.', + model: scriptedModel(), + tools: { research }, + memory, + backgroundTasks: { tools: { research: true } }, + }); + const pubsub = new EventEmitterPubSub(); + const durable = engine === 'durable' ? createDurableAgent({ agent }) : createEventedAgent({ agent, pubsub }); + const mastra = new Mastra({ + logger: false, + storage, + pubsub, + agents: { [durable.id]: durable }, + backgroundTasks: { enabled: true, globalConcurrency: 5, perAgentConcurrency: 3 }, + }); + await mastra.startWorkers(); + cleanups.push(async () => { + await mastra.backgroundTaskManager?.shutdown(); + await mastra.stopWorkers(); + }); + + const thread = { threadId: `late-write-${engine}-${outcome}`, resourceId: 'late-write-user' }; + const run = await durable.stream('Research near', { + memory: { thread: thread.threadId, resource: thread.resourceId }, + maxSteps: 3, + }); + let taskId: string | undefined; + for await (const chunk of run.fullStream) { + if (chunk.type === 'background-task-started') taskId = chunk.payload.taskId; + } + expect(await run.output.text).toBe(ACK); + expect(taskId).toBeDefined(); + + const workflows = (await storage.getStore('workflows'))!; + // A released claim keeps its row with no lease, unlike one that merely lapsed. + const released = await waitFor( + () => workflows.getRunOwnership({ runId: run.runId }), + record => record?.leaseExpiresAt === null, + 'the finished run to release its claim', + ); + await cleanedUp.promise; + + const manager = mastra.backgroundTaskManager!; + return { + memoryStore, + workflows, + manager, + taskId: taskId!, + thread, + runId: run.runId, + released: released!, + letToolGo: () => toolGate.resolve(), + storedToolCall: async () => toolCallIn((await memoryStore.listMessages({ ...thread, perPage: false })).messages), + settledTask: () => + waitFor( + () => manager.getTask(taskId!), + (task: BackgroundTask | null) => task?.status !== 'pending' && task?.status !== 'running', + 'the background task to settle', + ), + }; +} + +type Research = Awaited>; + +/** A successor claims the run and raises the memory fence, as recovery or a new turn does before it touches memory. */ +async function takeOver(research: Research) { + const successor = await research.workflows.claimRunOwnership({ + runId: research.runId, + ownerId: 'successor-execution', + leaseMs: 60_000, + expectedGeneration: research.released.generation, + }); + expect(successor.acquired).toBe(true); + const fence = { runId: research.runId, generation: successor.record!.generation, ownerId: 'successor-execution' }; + expect(await research.memoryStore.raiseRunFence(fence)).toBe(true); + return fence; +} + +/** LibSQL refused the write because it carried the finished run's own claim, now replaced. */ +function expectRejectedForFinishedClaim(error: unknown, research: Research) { + expect(error).toMatchObject({ + id: RUN_FENCE_CONFLICT_ERROR_ID, + details: { + runId: research.runId, + generation: research.released.generation, + ownerId: research.released.ownerId, + operation: 'saveMessages', + }, + }); +} + +async function expectSuccessorUnchanged(research: Research, successor: RunFence) { + const after = await research.workflows.getRunOwnership({ runId: research.runId }); + expect({ ownerId: after?.ownerId, generation: after?.generation, live: after?.live }).toEqual({ + ownerId: successor.ownerId, + generation: successor.generation, + live: true, + }); +} + +describe.each(['durable', 'evented'] as const)( + '%s agent on LibSQL: late background writes after the run finished', + engine => { + // Technical: the late suspension's save required the finished run to still own its claim, + // which LibSQL showed as released, so the task failed with DURABLE_AGENT_EXECUTION_UNVERIFIED. + // With nobody claiming the run since, the save now passes and storage accepts the claim. (#30182) + it('parks a deferred tool that asks for input after its run finished, and resumes it (#30182)', async () => { + const research = await startDeferredResearch(engine, 'suspend'); + research.letToolGo(); + + const parked = await research.settledTask(); + expect({ status: parked?.status, error: parked?.error?.message }).toEqual({ + status: 'suspended', + error: undefined, + }); + expect(parked?.suspendPayload).toEqual({ awaiting: 'analyst-approval', topic: 'near' }); + const open = await research.storedToolCall(); + expect(open?.suspended).toMatchObject({ + taskId: research.taskId, + suspendPayload: { awaiting: 'analyst-approval', topic: 'near' }, + }); + + await research.manager.resume(research.taskId, { notes: 'looks promising' }); + const done = await waitFor( + () => research.manager.getTask(research.taskId), + (task: BackgroundTask | null) => task?.status === 'completed' || task?.status === 'failed', + 'the resumed task to finish', + ); + expect({ status: done?.status, error: done?.error?.message, result: done?.result }).toEqual({ + status: 'completed', + error: undefined, + result: { summary: 'Research on "near": looks promising' }, + }); + // The thread on LibSQL holds the answered tool call, with the open request cleared. + const answered = await waitFor( + () => research.storedToolCall(), + stored => stored?.invocation.state === 'result' && stored.suspended === undefined, + 'the tool result in memory', + ); + expect(answered?.invocation).toMatchObject({ + state: 'result', + result: { summary: 'Research on "near": looks promising' }, + }); + }, 30_000); + + // Technical: the suspension's save passed `assertExecutionNotSuperseded` with no successor yet, + // then reached LibSQL after a successor claimed the run and raised the memory fence. + // `runFencedToClaim` gives the save the finished run's claim, and the write transaction rejects it. (#30182) + it("doesn't park a deferred tool whose suspension reaches the database after another execution took over (#30182)", async () => { + const research = await startDeferredResearch(engine, 'suspend'); + const save = holdNextSave(research.memoryStore); + research.letToolGo(); + await save.started; + const successor = await takeOver(research); + save.finish(); + + expectRejectedForFinishedClaim(await save.outcome, research); + const task = await research.settledTask(); + expect({ status: task?.status, error: task?.error?.message }).toEqual({ + status: 'failed', + error: expect.stringContaining("is no longer the run's current claim"), + }); + expect(task?.suspendPayload).toBeUndefined(); + const stored = await research.storedToolCall(); + expect(stored?.invocation).toBeDefined(); + expect(stored?.suspended).toBeUndefined(); + await expectSuccessorUnchanged(research, successor); + }, 30_000); + + // Technical: same ordering for the background task's result (`onResult` flush). (#30182) + it("doesn't record a deferred tool's result that reaches the database after another execution took over (#30182)", async () => { + const research = await startDeferredResearch(engine, 'result'); + const save = holdNextSave(research.memoryStore); + research.letToolGo(); + await save.started; + const successor = await takeOver(research); + save.finish(); + + expectRejectedForFinishedClaim(await save.outcome, research); + // Memory keeps the dispatch placeholder, not the superseded execution's result. + const stored = await research.storedToolCall(); + expect(stored?.invocation).toBeDefined(); + expect(JSON.stringify(stored?.invocation)).not.toContain('Research on'); + expect(stored?.suspended).toBeUndefined(); + await expectSuccessorUnchanged(research, successor); + }, 30_000); + }, +);