From 7f24c3fc205648791e69428d8d7704ee61ff79e7 Mon Sep 17 00:00:00 2001 From: Anurag Ojha Date: Sun, 15 Nov 2026 13:55:56 +0530 Subject: [PATCH 1/3] test(observability): cover model spans when onStepFinish is slow The agent loop starts the next model call before the output stream reads the last step-finish. Both tests fail on main: the third call is recorded under the second step with its tokens counted twice, and a failed call gets an extra step. --- .../src/slow-step-callback-tracing.test.ts | 107 ++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 observability/mastra/src/slow-step-callback-tracing.test.ts diff --git a/observability/mastra/src/slow-step-callback-tracing.test.ts b/observability/mastra/src/slow-step-callback-tracing.test.ts new file mode 100644 index 00000000000..b0317400877 --- /dev/null +++ b/observability/mastra/src/slow-step-callback-tracing.test.ts @@ -0,0 +1,107 @@ +import { setTimeout as sleep } from 'node:timers/promises'; +import { MockLanguageModelV2, convertArrayToReadableStream } from '@internal/ai-sdk-v5/test'; +import { Agent } from '@mastra/core/agent'; +import { Mastra } from '@mastra/core/mastra'; +import { SpanType } from '@mastra/core/observability'; +import { createTool } from '@mastra/core/tools'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { z } from 'zod/v4'; +import { Observability } from './default'; +import { TestExporter } from './exporters'; + +type Reply = 'tool-call' | 'text' | 'error'; + +function createModel(replies: Reply[]) { + let call = 0; + return new MockLanguageModelV2({ + doStream: async () => { + const reply = replies[call++]; + if (reply === 'error') throw new Error('provider unavailable'); + const usage = { inputTokens: call * 10, outputTokens: 1, totalTokens: call * 10 + 1 }; + return { + rawCall: { rawPrompt: null, rawSettings: {} }, + warnings: [], + stream: convertArrayToReadableStream( + reply === 'tool-call' + ? [ + { type: 'stream-start', warnings: [] }, + { type: 'tool-call', toolCallId: `call-${call}`, toolName: 'lookup', input: '{}' }, + { type: 'finish', finishReason: 'tool-calls', usage }, + ] + : [ + { type: 'stream-start', warnings: [] }, + { type: 'text-start', id: 'text-1' }, + { type: 'text-delta', id: 'text-1', delta: 'done' }, + { type: 'text-end', id: 'text-1' }, + { type: 'finish', finishReason: 'stop', usage }, + ], + ), + }; + }, + }); +} + +describe('model spans when onStepFinish is slow', () => { + let exporter: TestExporter; + let observability: Observability; + + beforeEach(() => { + exporter = new TestExporter(); + }); + + afterEach(async () => { + await observability?.shutdown(); + }); + + async function run(replies: Reply[]) { + const lookup = createTool({ + id: 'lookup', + description: 'Look something up', + inputSchema: z.object({}), + execute: async () => 'found', + }); + observability = new Observability({ + configs: { default: { serviceName: 'slow-step-callback', exporters: [exporter] } }, + }); + const agent = new Agent({ + id: 'slow-callback-agent', + name: 'Slow Callback Agent', + instructions: 'Test', + model: createModel(replies), + tools: { lookup }, + }); + const mastra = new Mastra({ logger: false, agents: { agent }, observability }); + + const output = await mastra.getAgent('agent').stream('question', { + maxSteps: 5, + errorProcessors: [], + onStepFinish: () => sleep(50), + }); + for await (const _ of output.fullStream) { + } + + const steps = exporter.getSpansByType(SpanType.MODEL_STEP); + const inferences = exporter.getSpansByType(SpanType.MODEL_INFERENCE); + return steps.map(step => ({ + step, + inferences: inferences.filter(inference => inference.parentSpanId === step.id), + })); + } + + it('records one step and one inference with its own tokens for each model call', async () => { + const steps = await run(['tool-call', 'tool-call', 'text']); + + expect( + steps.map(({ inferences }) => inferences.map(inference => inference.attributes?.usage?.inputTokens)), + ).toEqual([[10], [20], [30]]); + }); + + it('records a failed model call once', async () => { + const steps = await run(['tool-call', 'error']); + + expect(steps.map(({ step, inferences }) => [Boolean(step.errorInfo), inferences.length])).toEqual([ + [false, 1], + [true, 1], + ]); + }); +}); From 8f92b3dec803213bca035c2e93697b6065459c93 Mon Sep 17 00:00:00 2001 From: Anurag Ojha Date: Sun, 15 Nov 2026 13:55:58 +0530 Subject: [PATCH 2/3] fix(core): trace agent stream chunks when the loop writes them The span tracker got step events from the agent loop and from the output stream. The stream reads chunks later when onStepFinish is slow, so the two sources ran out of order. Record each chunk at the loop's enqueue point instead, like durable agents do. Trackers without recordChunk keep the old path. --- .changeset/yellow-berries-cut.md | 5 +++++ packages/core/src/loop/loop.ts | 6 ++++-- packages/core/src/loop/workflows/stream.ts | 3 ++- packages/core/src/observability/types/tracing.ts | 2 +- 4 files changed, 12 insertions(+), 4 deletions(-) create mode 100644 .changeset/yellow-berries-cut.md diff --git a/.changeset/yellow-berries-cut.md b/.changeset/yellow-berries-cut.md new file mode 100644 index 00000000000..7b2b9e7e398 --- /dev/null +++ b/.changeset/yellow-berries-cut.md @@ -0,0 +1,5 @@ +--- +'@mastra/core': patch +--- + +Fixed extra model step and model inference spans in traces when `onStepFinish` takes time to finish. Each model call now has one step span and one inference span with its own token usage, so exporters such as PostHog and Langfuse no longer count a model call twice. diff --git a/packages/core/src/loop/loop.ts b/packages/core/src/loop/loop.ts index 1417df64995..e2d639f1d0f 100644 --- a/packages/core/src/loop/loop.ts +++ b/packages/core/src/loop/loop.ts @@ -152,8 +152,10 @@ export function loop({ }, }); - // Apply chunk tracing transform to track MODEL_STEP and MODEL_CHUNK spans - const stream = rest.modelSpanTracker?.wrapStream(baseStream) ?? baseStream; + // Trackers without recordChunk trace chunks as the consumer reads them. + const stream = rest.modelSpanTracker?.recordChunk + ? baseStream + : (rest.modelSpanTracker?.wrapStream(baseStream) ?? baseStream); // Build observability context from modelSpanTracker if tracing context is available const observabilityContext = createObservabilityContext(rest.modelSpanTracker?.getTracingContext()); diff --git a/packages/core/src/loop/workflows/stream.ts b/packages/core/src/loop/workflows/stream.ts index 0e3b2594d32..811c81b24fa 100644 --- a/packages/core/src/loop/workflows/stream.ts +++ b/packages/core/src/loop/workflows/stream.ts @@ -57,7 +57,7 @@ export function workflowLoopStream>) => { let totalTimeoutSignal: AbortSignal | undefined; - // Stamp chunks when the loop produces them; consumers may read them much later. + // Stamp and trace chunks when the loop produces them; consumers may read them much later. const controller: ReadableStreamDefaultController> = { enqueue: chunk => { // The total deadline ends the run with a timeout error after the tool grace window. @@ -71,6 +71,7 @@ export function workflowLoopStream(stream: T): T; /** @internal Record a chunk after the runtime has applied output processors. */ - recordChunk?(chunk: ChunkType): void; + recordChunk?(chunk: ChunkType): void; startStep(payload?: StepStartPayload): void; updateStep?(payload?: StepStartPayload): void; From cf10ebdf71831208f43ec3640ca43e419570e51a Mon Sep 17 00:00:00 2001 From: Anurag Ojha Date: Tue, 17 Nov 2026 13:05:06 +0530 Subject: [PATCH 3/3] fix(core): trace a chunk only after the stream accepts it A background tool result can arrive after the run's stream has closed. The loop traced that chunk before the closed stream rejected it, so the trace got a model step and a model inference span that never ended. Enqueue the chunk first, so a rejected chunk is not traced. --- packages/core/src/loop/workflows/stream.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/src/loop/workflows/stream.ts b/packages/core/src/loop/workflows/stream.ts index 811c81b24fa..551a3829a31 100644 --- a/packages/core/src/loop/workflows/stream.ts +++ b/packages/core/src/loop/workflows/stream.ts @@ -71,8 +71,8 @@ export function workflowLoopStream {