diff --git a/.changeset/yellow-berries-cut.md b/.changeset/yellow-berries-cut.md new file mode 100644 index 00000000000..7b2b9e7e398 --- /dev/null +++ b/.changeset/yellow-berries-cut.md @@ -0,0 +1,5 @@ +--- +'@mastra/core': patch +--- + +Fixed extra model step and model inference spans in traces when `onStepFinish` takes time to finish. Each model call now has one step span and one inference span with its own token usage, so exporters such as PostHog and Langfuse no longer count a model call twice. diff --git a/observability/mastra/src/slow-step-callback-tracing.test.ts b/observability/mastra/src/slow-step-callback-tracing.test.ts new file mode 100644 index 00000000000..b0317400877 --- /dev/null +++ b/observability/mastra/src/slow-step-callback-tracing.test.ts @@ -0,0 +1,107 @@ +import { setTimeout as sleep } from 'node:timers/promises'; +import { MockLanguageModelV2, convertArrayToReadableStream } from '@internal/ai-sdk-v5/test'; +import { Agent } from '@mastra/core/agent'; +import { Mastra } from '@mastra/core/mastra'; +import { SpanType } from '@mastra/core/observability'; +import { createTool } from '@mastra/core/tools'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { z } from 'zod/v4'; +import { Observability } from './default'; +import { TestExporter } from './exporters'; + +type Reply = 'tool-call' | 'text' | 'error'; + +function createModel(replies: Reply[]) { + let call = 0; + return new MockLanguageModelV2({ + doStream: async () => { + const reply = replies[call++]; + if (reply === 'error') throw new Error('provider unavailable'); + const usage = { inputTokens: call * 10, outputTokens: 1, totalTokens: call * 10 + 1 }; + return { + rawCall: { rawPrompt: null, rawSettings: {} }, + warnings: [], + stream: convertArrayToReadableStream( + reply === 'tool-call' + ? [ + { type: 'stream-start', warnings: [] }, + { type: 'tool-call', toolCallId: `call-${call}`, toolName: 'lookup', input: '{}' }, + { type: 'finish', finishReason: 'tool-calls', usage }, + ] + : [ + { type: 'stream-start', warnings: [] }, + { type: 'text-start', id: 'text-1' }, + { type: 'text-delta', id: 'text-1', delta: 'done' }, + { type: 'text-end', id: 'text-1' }, + { type: 'finish', finishReason: 'stop', usage }, + ], + ), + }; + }, + }); +} + +describe('model spans when onStepFinish is slow', () => { + let exporter: TestExporter; + let observability: Observability; + + beforeEach(() => { + exporter = new TestExporter(); + }); + + afterEach(async () => { + await observability?.shutdown(); + }); + + async function run(replies: Reply[]) { + const lookup = createTool({ + id: 'lookup', + description: 'Look something up', + inputSchema: z.object({}), + execute: async () => 'found', + }); + observability = new Observability({ + configs: { default: { serviceName: 'slow-step-callback', exporters: [exporter] } }, + }); + const agent = new Agent({ + id: 'slow-callback-agent', + name: 'Slow Callback Agent', + instructions: 'Test', + model: createModel(replies), + tools: { lookup }, + }); + const mastra = new Mastra({ logger: false, agents: { agent }, observability }); + + const output = await mastra.getAgent('agent').stream('question', { + maxSteps: 5, + errorProcessors: [], + onStepFinish: () => sleep(50), + }); + for await (const _ of output.fullStream) { + } + + const steps = exporter.getSpansByType(SpanType.MODEL_STEP); + const inferences = exporter.getSpansByType(SpanType.MODEL_INFERENCE); + return steps.map(step => ({ + step, + inferences: inferences.filter(inference => inference.parentSpanId === step.id), + })); + } + + it('records one step and one inference with its own tokens for each model call', async () => { + const steps = await run(['tool-call', 'tool-call', 'text']); + + expect( + steps.map(({ inferences }) => inferences.map(inference => inference.attributes?.usage?.inputTokens)), + ).toEqual([[10], [20], [30]]); + }); + + it('records a failed model call once', async () => { + const steps = await run(['tool-call', 'error']); + + expect(steps.map(({ step, inferences }) => [Boolean(step.errorInfo), inferences.length])).toEqual([ + [false, 1], + [true, 1], + ]); + }); +}); diff --git a/packages/core/src/loop/loop.ts b/packages/core/src/loop/loop.ts index 1417df64995..e2d639f1d0f 100644 --- a/packages/core/src/loop/loop.ts +++ b/packages/core/src/loop/loop.ts @@ -152,8 +152,10 @@ export function loop({ }, }); - // Apply chunk tracing transform to track MODEL_STEP and MODEL_CHUNK spans - const stream = rest.modelSpanTracker?.wrapStream(baseStream) ?? baseStream; + // Trackers without recordChunk trace chunks as the consumer reads them. + const stream = rest.modelSpanTracker?.recordChunk + ? baseStream + : (rest.modelSpanTracker?.wrapStream(baseStream) ?? baseStream); // Build observability context from modelSpanTracker if tracing context is available const observabilityContext = createObservabilityContext(rest.modelSpanTracker?.getTracingContext()); diff --git a/packages/core/src/loop/workflows/stream.ts b/packages/core/src/loop/workflows/stream.ts index 0e3b2594d32..551a3829a31 100644 --- a/packages/core/src/loop/workflows/stream.ts +++ b/packages/core/src/loop/workflows/stream.ts @@ -57,7 +57,7 @@ export function workflowLoopStream>) => { let totalTimeoutSignal: AbortSignal | undefined; - // Stamp chunks when the loop produces them; consumers may read them much later. + // Stamp and trace chunks when the loop produces them; consumers may read them much later. const controller: ReadableStreamDefaultController> = { enqueue: chunk => { // The total deadline ends the run with a timeout error after the tool grace window. @@ -72,6 +72,7 @@ export function workflowLoopStream { diff --git a/packages/core/src/observability/types/tracing.ts b/packages/core/src/observability/types/tracing.ts index 0793f2be21e..70352671ebb 100644 --- a/packages/core/src/observability/types/tracing.ts +++ b/packages/core/src/observability/types/tracing.ts @@ -1769,7 +1769,7 @@ export interface IModelSpanTracker { getTimeToFirstTokenMs?(): number | undefined; wrapStream(stream: T): T; /** @internal Record a chunk after the runtime has applied output processors. */ - recordChunk?(chunk: ChunkType): void; + recordChunk?(chunk: ChunkType): void; startStep(payload?: StepStartPayload): void; updateStep?(payload?: StepStartPayload): void;