From 87da733f1f70b73f8e886549371ea54a2e2abcfa Mon Sep 17 00:00:00 2001 From: Eric Pinzur Date: Mon, 16 Nov 2026 13:03:19 +0100 Subject: [PATCH 01/23] feat(observability): add bounded trace navigation API --- .changeset/eager-meteors-wear.md | 12 + .changeset/four-foxes-pick.md | 8 + .changeset/whole-streets-flow.md | 8 + client-sdks/client-js/src/client.ts | 8 +- .../resources/observability-route-types.ts | 5 + .../observability.trace-page.test.ts | 61 ++++ .../client-js/src/resources/observability.ts | 10 +- .../client-js/src/route-types.generated.ts | 158 +++++++-- client-sdks/client-js/src/types.ts | 7 + .../en/reference/client-js/observability.mdx | 38 ++ .../commands/api/route-metadata.generated.ts | 25 ++ .../src/storage/domains/observability/base.ts | 32 +- .../storage/domains/observability/index.ts | 2 + .../domains/observability/trace-page.test.ts | 138 ++++++++ .../domains/observability/trace-page.ts | 328 ++++++++++++++++++ .../handlers/observability-new-endpoints.ts | 7 +- .../server/handlers/observability-shared.ts | 9 + ...ervability-text-search-core-compat.test.ts | 19 + .../src/server/handlers/observability.ts | 73 ++++ .../server/src/server/handlers/system.test.ts | 2 + .../src/server/handlers/trace-page.test.ts | 74 ++++ packages/server/src/server/schemas/system.ts | 2 + .../server-adapter/routes/observability.ts | 2 + 23 files changed, 981 insertions(+), 47 deletions(-) create mode 100644 .changeset/eager-meteors-wear.md create mode 100644 .changeset/four-foxes-pick.md create mode 100644 .changeset/whole-streets-flow.md create mode 100644 client-sdks/client-js/src/resources/observability.trace-page.test.ts create mode 100644 packages/core/src/storage/domains/observability/trace-page.test.ts create mode 100644 packages/core/src/storage/domains/observability/trace-page.ts create mode 100644 packages/server/src/server/handlers/trace-page.test.ts diff --git a/.changeset/eager-meteors-wear.md b/.changeset/eager-meteors-wear.md new file mode 100644 index 00000000000..eb70c3f13b9 --- /dev/null +++ b/.changeset/eager-meteors-wear.md @@ -0,0 +1,12 @@ +--- +'@mastra/client-js': minor +'@mastra/server': patch +'@mastra/core': patch +'mastra': patch +--- + +Add bounded trace browsing with optional span details: + +```typescript +await client.getTrace({ version: 2, traceId, depth: 2, limit: 100, include: 'full' }); +``` diff --git a/.changeset/four-foxes-pick.md b/.changeset/four-foxes-pick.md new file mode 100644 index 00000000000..13c6dafb36d --- /dev/null +++ b/.changeset/four-foxes-pick.md @@ -0,0 +1,8 @@ +--- +'@mastra/core': minor +'@mastra/client-js': patch +'@mastra/server': patch +'mastra': patch +--- + +Add bounded trace browsing and explicit span hydration through the version 2 trace contract. diff --git a/.changeset/whole-streets-flow.md b/.changeset/whole-streets-flow.md new file mode 100644 index 00000000000..c11512a69fd --- /dev/null +++ b/.changeset/whole-streets-flow.md @@ -0,0 +1,8 @@ +--- +'@mastra/server': minor +'@mastra/client-js': patch +'@mastra/core': patch +'mastra': patch +--- + +Add a version 2 trace read endpoint with bounded pages and optional span details. diff --git a/client-sdks/client-js/src/client.ts b/client-sdks/client-js/src/client.ts index 85ee7de066f..40340868363 100644 --- a/client-sdks/client-js/src/client.ts +++ b/client-sdks/client-js/src/client.ts @@ -45,6 +45,8 @@ import type { } from './resources/observability'; import type { TraceRecord, + GetTracePageArgs, + TracePageResponse, GetTraceLightResponse, GetSpanResponse, ListTracesArgs, @@ -1279,8 +1281,10 @@ export class MastraClient extends BaseResource { } /** Retrieves a specific trace by ID. */ - getTrace(traceId: string): Promise { - return this.observability.getTrace(traceId); + getTrace(traceId: string): Promise; + getTrace(request: GetTracePageArgs): Promise; + getTrace(request: string | GetTracePageArgs): Promise { + return typeof request === 'string' ? this.observability.getTrace(request) : this.observability.getTrace(request); } /** Retrieves a lightweight trace by ID (timeline fields only, excludes heavy fields). */ diff --git a/client-sdks/client-js/src/resources/observability-route-types.ts b/client-sdks/client-js/src/resources/observability-route-types.ts index 5aea5e9f357..66e436f9a62 100644 --- a/client-sdks/client-js/src/resources/observability-route-types.ts +++ b/client-sdks/client-js/src/resources/observability-route-types.ts @@ -13,6 +13,11 @@ type ListArgs = Simplify< } >; +export type TracePageRequest = Body<'POST /observability/traces/:traceId'>; +export type GetTracePageArgs = TracePageRequest & { traceId: string }; +export type TracePageResponse = Response<'POST /observability/traces/:traceId'>; +export type TracePageNode = TracePageResponse['nodes'][number]; + export type TraceRecord = Response<'GET /observability/traces/:traceId'>; export type GetTraceLightResponse = Response<'GET /observability/traces/:traceId/light'>; export type GetSpanResponse = Response<'GET /observability/traces/:traceId/spans/:spanId'>; diff --git a/client-sdks/client-js/src/resources/observability.trace-page.test.ts b/client-sdks/client-js/src/resources/observability.trace-page.test.ts new file mode 100644 index 00000000000..b49c8f4aa30 --- /dev/null +++ b/client-sdks/client-js/src/resources/observability.trace-page.test.ts @@ -0,0 +1,61 @@ +import { createServer } from 'node:http'; +import type { AddressInfo } from 'node:net'; +import { SpanType } from '@mastra/core/observability'; +import { InMemoryStore } from '@mastra/core/storage'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { MastraClient } from '../client'; + +// Real HTTP transport and real storage: proves the overload does not serialize IDs or cursors as GET parameters. +describe('getTrace overload over HTTP', () => { + const storage = new InMemoryStore(); + let client: MastraClient; + const server = createServer(async (req, res) => { + try { + expect(req.url).toBe('/api/observability/traces/trace'); + const store = (await storage.getStore('observability'))!; + const chunks: Buffer[] = []; + for await (const chunk of req) chunks.push(Buffer.from(chunk)); + const result = + req.method === 'GET' + ? await store.getTrace({ traceId: 'trace' }) + : await store.getTracePage({ traceId: 'trace', ...JSON.parse(Buffer.concat(chunks).toString()) }); + res.setHeader('content-type', 'application/json'); + res.end(JSON.stringify(result)); + } catch (error) { + res.writeHead(500).end(JSON.stringify({ error: String(error) })); + } + }); + beforeAll(async () => { + const store = (await storage.getStore('observability'))!; + await store.batchCreateSpans({ + records: ['a', 'b'].map(spanId => ({ + traceId: 'trace', + spanId, + name: spanId, + spanType: SpanType.AGENT_RUN, + isEvent: false, + startedAt: new Date('2026-11-01'), + input: { value: spanId }, + })), + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + client = new MastraClient({ baseUrl: `http://127.0.0.1:${(server.address() as AddressInfo).port}`, retries: 0 }); + }); + afterAll(async () => { + server.closeAllConnections(); + await new Promise((resolve, reject) => server.close(error => (error ? reject(error) : resolve()))); + }); + it('keeps legacy GET responses and reads bounded pages through POST', async () => { + expect((await client.getTrace('trace')).spans.map(span => span.spanId)).toEqual(['a', 'b']); + const first = await client.getTrace({ version: 2, traceId: 'trace', limit: 1 }); + expect(first.nodes.map(node => node.spanId)).toEqual(['a']); + const next = await client.getTrace({ version: 2, traceId: 'trace', cursor: first.nextCursor! }); + expect(next.nodes.map(node => node.spanId)).toEqual(['b']); + expect(next.nextCursor).toBeUndefined(); + }); + it('hydrates exactly the IDs absent from the caller cache', async () => { + const page = await client.getTrace({ version: 2, traceId: 'trace', spanIds: ['b'], include: 'full' }); + expect(page.nodes).toHaveLength(1); + expect(page.nodes[0]!.details?.input).toEqual({ value: 'b' }); + }); +}); diff --git a/client-sdks/client-js/src/resources/observability.ts b/client-sdks/client-js/src/resources/observability.ts index ccd33e983b8..30955a14200 100644 --- a/client-sdks/client-js/src/resources/observability.ts +++ b/client-sdks/client-js/src/resources/observability.ts @@ -20,6 +20,8 @@ import { toQueryParams } from '../utils'; import { BaseResource } from './base'; import type { TraceRecord, + GetTracePageArgs, + TracePageResponse, GetTraceLightResponse, GetSpanResponse, ListTracesArgs, @@ -229,8 +231,12 @@ export class Observability extends BaseResource { * @param traceId - ID of the trace to retrieve * @returns Promise containing the trace with all its spans */ - getTrace(traceId: string): Promise { - return this.request(`/observability/traces/${traceId}`); + getTrace(traceId: string): Promise; + getTrace(request: GetTracePageArgs): Promise; + getTrace(request: string | GetTracePageArgs): Promise { + if (typeof request === 'string') return this.request(`/observability/traces/${encodeURIComponent(request)}`); + const { traceId, ...body } = request; + return this.request(`/observability/traces/${encodeURIComponent(traceId)}`, { method: 'POST', body }); } /** diff --git a/client-sdks/client-js/src/route-types.generated.ts b/client-sdks/client-js/src/route-types.generated.ts index 0d98083b8b3..6a252cb0f7d 100644 --- a/client-sdks/client-js/src/route-types.generated.ts +++ b/client-sdks/client-js/src/route-types.generated.ts @@ -24,7 +24,7 @@ type InputShared_Auxiliary_116 = { parentWorkflow?: InputShared_Auxiliary_116 | undefined; }; -type InputShared_Auxiliary_208 = +type InputShared_Auxiliary_210 = | { op: 'eq' | 'ne' | 'lt' | 'lte' | 'gt' | 'gte'; left: @@ -73,41 +73,41 @@ type InputShared_Auxiliary_208 = } | { op: 'and' | 'or'; - args: InputShared_Auxiliary_208[]; + args: InputShared_Auxiliary_210[]; } | { op: 'not'; - arg: InputShared_Auxiliary_208; + arg: InputShared_Auxiliary_210; } | { spans: | { - some: InputShared_Auxiliary_231; + some: InputShared_Auxiliary_233; } | { - none: InputShared_Auxiliary_231; + none: InputShared_Auxiliary_233; }; } | { scores: | { - some: InputShared_Auxiliary_231; + some: InputShared_Auxiliary_233; } | { - none: InputShared_Auxiliary_231; + none: InputShared_Auxiliary_233; }; } | { feedback: | { - some: InputShared_Auxiliary_231; + some: InputShared_Auxiliary_233; } | { - none: InputShared_Auxiliary_231; + none: InputShared_Auxiliary_233; }; }; -type InputShared_Auxiliary_231 = +type InputShared_Auxiliary_233 = | { op: 'eq' | 'ne' | 'lt' | 'lte' | 'gt' | 'gte'; left: @@ -156,33 +156,33 @@ type InputShared_Auxiliary_231 = } | { op: 'and' | 'or'; - args: InputShared_Auxiliary_231[]; + args: InputShared_Auxiliary_233[]; } | { op: 'not'; - arg: InputShared_Auxiliary_231; + arg: InputShared_Auxiliary_233; }; -type InputShared_Auxiliary_278 = +type InputShared_Auxiliary_280 = | { op: 'and' | 'or'; - args: InputShared_Auxiliary_278[]; + args: InputShared_Auxiliary_280[]; } | { op: 'not'; - arg: InputShared_Auxiliary_278; + arg: InputShared_Auxiliary_280; } | { traces: | { - some: InputShared_Auxiliary_208; + some: InputShared_Auxiliary_210; } | { - none: InputShared_Auxiliary_208; + none: InputShared_Auxiliary_210; }; }; -type InputShared_Auxiliary_735 = +type InputShared_Auxiliary_737 = | { op: 'eq' | 'ne' | 'lt' | 'lte' | 'gt' | 'gte'; left: @@ -227,27 +227,27 @@ type InputShared_Auxiliary_735 = } | { op: 'and' | 'or'; - args: InputShared_Auxiliary_735[]; + args: InputShared_Auxiliary_737[]; } | { op: 'not'; - arg: InputShared_Auxiliary_735; + arg: InputShared_Auxiliary_737; }; -type InputShared_Auxiliary_809 = { +type InputShared_Auxiliary_811 = { id?: string | undefined; name: string; type: 'file' | 'folder'; content?: string | undefined; - children?: InputShared_Auxiliary_809[] | undefined; + children?: InputShared_Auxiliary_811[] | undefined; }; -type Shared_Auxiliary_799 = { +type Shared_Auxiliary_801 = { id?: string | undefined; name: string; type: 'file' | 'folder'; content?: string | undefined; - children?: Shared_Auxiliary_799[] | undefined; + children?: Shared_Auxiliary_801[] | undefined; }; type InputShared_Type_0 = { @@ -1849,7 +1849,7 @@ type InputShared_Type_89 = { } | undefined; steps: InputShared_Type_84[]; - predicates: InputShared_Auxiliary_735[]; + predicates: InputShared_Auxiliary_737[]; }; type InputShared_Type_90 = { @@ -1863,7 +1863,7 @@ type InputShared_Type_90 = { | undefined; step: InputShared_Type_84; loopType: 'dowhile' | 'dountil'; - predicate: InputShared_Auxiliary_735; + predicate: InputShared_Auxiliary_737; }; type InputShared_Type_91 = @@ -3897,6 +3897,10 @@ type Shared_Type_69 = { traceQuery: boolean; /** Supports explicit `rootSource: 'aggregated'` trace queries */ traceQueryAggregatedRoots: boolean; + /** Bounded version 2 getTrace browsing and hydration */ + traceNavigation: boolean; + /** Indexed in-trace search with ancestor context */ + traceNavigationSearch: boolean; /** Literal substring search across any span in a trace */ traceQueryTextSearch: boolean; /** Literal substring search within individual spans */ @@ -4512,7 +4516,7 @@ type Shared_Type_88 = { /** List of asset file paths */ assets?: string[] | undefined; /** Full file tree structure for the skill */ - files?: Shared_Auxiliary_799[] | undefined; + files?: Shared_Auxiliary_801[] | undefined; /** Additional metadata for the skill */ metadata?: | { @@ -10515,6 +10519,82 @@ export interface GetObservabilityTracesTraceId_RouteContract { responseType: 'json'; } +// ============================================================================ +// Route: POST /observability/traces/:traceId +// ============================================================================ +export type PostObservabilityTracesTraceId_PathParams = GetObservabilityTracesTraceId_PathParams; + +export type PostObservabilityTracesTraceId_Body = + | { + version: 2; + anchorSpanId?: string | undefined; + depth?: number | undefined; + spanIds?: string[] | undefined; + search?: string | undefined; + ancestors?: boolean | undefined; + include?: 'structure' | 'full'; + limit?: number; + maxBytes?: number; + } + | { + version: 2; + cursor: string; + }; + +export type PostObservabilityTracesTraceId_Response = { + version: 2; + traceId: string; + nodes: { + /** Unique trace identifier */ + traceId: string; + /** Unique span identifier within a trace */ + spanId: string; + parentSpanId?: (string | null) | undefined; + /** Human-readable span name */ + name: string; + /** Span type (e.g., WORKFLOW_RUN, AGENT_RUN, TOOL_CALL, etc.) */ + spanType: Shared_Type_60; + /** Whether this is an event (point-in-time) vs a span (duration) */ + isEvent: boolean; + /** When the span started */ + startedAt: Date; + endedAt?: (Date | null) | undefined; + entityType?: (Shared_Type_61 | null) | undefined; + entityId?: (string | null) | undefined; + entityName?: (string | null) | undefined; + status: 'success' | 'error' | 'running'; + childCount: number | null; + match: boolean; + parentState: 'root' | 'available' | 'unavailable' | 'not-requested'; + /** Span record data */ + details?: Shared_Type_64 | undefined; + }[]; + missingSpanIds: string[]; + nextCursor?: string | undefined; + ancestorPaths: 'not-requested' | 'continued' | 'complete' | 'unavailable'; +}; + +export type PostObservabilityTracesTraceId_Request = Simplify< + (PostObservabilityTracesTraceId_PathParams extends never + ? {} + : { params: PostObservabilityTracesTraceId_PathParams }) & + (never extends never ? {} : {} extends never ? { query?: never } : { query: never }) & + (PostObservabilityTracesTraceId_Body extends never + ? {} + : {} extends PostObservabilityTracesTraceId_Body + ? { body?: PostObservabilityTracesTraceId_Body } + : { body: PostObservabilityTracesTraceId_Body }) +>; + +export interface PostObservabilityTracesTraceId_RouteContract { + pathParams: PostObservabilityTracesTraceId_PathParams; + queryParams: never; + body: PostObservabilityTracesTraceId_Body; + request: PostObservabilityTracesTraceId_Request; + response: PostObservabilityTracesTraceId_Response; + responseType: 'json'; +} + // ============================================================================ // Route: GET /observability/traces/:traceId/light // ============================================================================ @@ -10829,7 +10909,7 @@ export type PostObservabilityTracesQuery_Body = { from: string; to: string; }; - where?: InputShared_Auxiliary_208 | undefined; + where?: InputShared_Auxiliary_210 | undefined; group?: | { by: ['threadId']; @@ -10919,11 +10999,11 @@ export type PostObservabilityTracesAggregate_Body = { from: string; to: string; }; - where?: InputShared_Auxiliary_208 | undefined; + where?: InputShared_Auxiliary_210 | undefined; groupBy?: string[]; interval?: ('1m' | '5m' | '15m' | '1h' | '1d') | undefined; measures: (InputShared_Type_16 | `countDistinct.${string}`)[]; - having?: InputShared_Auxiliary_231 | undefined; + having?: InputShared_Auxiliary_233 | undefined; orderBy?: { field: string; direction: 'asc' | 'desc'; @@ -11020,11 +11100,11 @@ export type PostObservabilitySpansAggregate_Body = { from: string; to: string; }; - where?: InputShared_Auxiliary_231 | undefined; + where?: InputShared_Auxiliary_233 | undefined; groupBy?: string[]; interval?: ('1m' | '5m' | '15m' | '1h' | '1d') | undefined; measures: (InputShared_Type_16 | `countDistinct.${string}`)[]; - having?: InputShared_Auxiliary_231 | undefined; + having?: InputShared_Auxiliary_233 | undefined; orderBy?: { field: string; direction: 'asc' | 'desc'; @@ -11072,7 +11152,7 @@ export type PostObservabilityThreadsQuery_Body = field: 'startedAt' | 'endedAt'; }; }; - where?: InputShared_Auxiliary_231 | undefined; + where?: InputShared_Auxiliary_233 | undefined; orderBy?: { field: 'startedAt' | 'endedAt'; direction: 'asc' | 'desc'; @@ -11088,9 +11168,9 @@ export type PostObservabilityThreadsQuery_Body = from: string; to: string; }; - where?: InputShared_Auxiliary_208 | undefined; + where?: InputShared_Auxiliary_210 | undefined; }; - where?: InputShared_Auxiliary_278 | undefined; + where?: InputShared_Auxiliary_280 | undefined; page?: { limit?: number; after?: (string | null) | undefined; @@ -11167,7 +11247,7 @@ export type PostObservabilitySpansQuery_Body = { from: string; to: string; }; - where?: InputShared_Auxiliary_231 | undefined; + where?: InputShared_Auxiliary_233 | undefined; textSearch?: string | undefined; orderBy?: { field: 'startedAt' | 'endedAt' | 'timeToFirstChunkMs' | 'timeToFirstTokenMs' | 'durationMs' | 'model' | 'provider'; @@ -19952,7 +20032,7 @@ export type PostStoredSkills_Body = { /** List of asset file paths */ assets?: string[] | undefined; /** Full file tree structure for the skill */ - files?: InputShared_Auxiliary_809[] | undefined; + files?: InputShared_Auxiliary_811[] | undefined; /** Additional metadata for the skill */ metadata?: | { @@ -20010,7 +20090,7 @@ export type PatchStoredSkillsStoredSkillId_Body = { /** List of asset file paths */ assets?: (string[] | undefined) | undefined; /** Full file tree structure for the skill */ - files?: (InputShared_Auxiliary_809[] | undefined) | undefined; + files?: (InputShared_Auxiliary_811[] | undefined) | undefined; /** Additional metadata for the skill */ metadata?: | ( @@ -25713,6 +25793,7 @@ export interface RouteTypes { 'GET /observability/branches': GetObservabilityBranches_RouteContract; 'GET /observability/traces/:traceId/branches/:spanId': GetObservabilityTracesTraceIdBranchesSpanId_RouteContract; 'GET /observability/traces/:traceId': GetObservabilityTracesTraceId_RouteContract; + 'POST /observability/traces/:traceId': PostObservabilityTracesTraceId_RouteContract; 'GET /observability/traces/:traceId/light': GetObservabilityTracesTraceIdLight_RouteContract; 'GET /observability/traces/:traceId/spans/:spanId': GetObservabilityTracesTraceIdSpansSpanId_RouteContract; 'GET /observability/traces/:traceId/trajectory': GetObservabilityTracesTraceIdTrajectory_RouteContract; @@ -26780,6 +26861,7 @@ export interface Client { }; '/observability/traces/:traceId': { GET: GetObservabilityTracesTraceId_RouteContract; + POST: PostObservabilityTracesTraceId_RouteContract; }; '/observability/traces/:traceId/:spanId/scores': { GET: GetObservabilityTracesTraceIdSpanIdScores_RouteContract; diff --git a/client-sdks/client-js/src/types.ts b/client-sdks/client-js/src/types.ts index 7d8eb251aeb..7fce08c1412 100644 --- a/client-sdks/client-js/src/types.ts +++ b/client-sdks/client-js/src/types.ts @@ -2607,3 +2607,10 @@ export type { SavedViewSort, SavedViewState, } from '@mastra/core/storage'; + +export type { + GetTracePageArgs, + TracePageRequest, + TracePageResponse, + TracePageNode, +} from './resources/observability-route-types'; diff --git a/docs/src/content/en/reference/client-js/observability.mdx b/docs/src/content/en/reference/client-js/observability.mdx index e2d9ab0ce00..2e1d39a715c 100644 --- a/docs/src/content/en/reference/client-js/observability.mdx +++ b/docs/src/content/en/reference/client-js/observability.mdx @@ -17,6 +17,44 @@ Retrieve a specific trace by its ID, including all its spans and details: const trace = await mastraClient.getTrace('trace-id-123') ``` +### Read a bounded trace page + +Use the object overload when the server advertises `capabilities.traceNavigation`: + +```typescript +const page = await mastraClient.getTrace({ + version: 2, + traceId: 'trace-id-123', + depth: 2, + limit: 100, + include: 'full', +}) + +if (page.nextCursor) { + const nextPage = await mastraClient.getTrace({ + version: 2, + traceId: 'trace-id-123', + cursor: page.nextCursor, + }) +} +``` + +The response contains `nodes`, `missingSpanIds`, `ancestorPaths`, and an optional `nextCursor`. Each node includes its `childCount` and `parentState`. Explicit-ID requests leave child counts unknown (`null`) and parent context unrequested (`'not-requested'`). Full span payloads appear in `node.details` when `include` is `'full'`. The default is `'structure'`. + +| Option | Behavior | +| - | - | +| `anchorSpanId` | Browse a branch, including its anchor. Omit to start at the trace roots. | +| `depth` | Descendant levels below the roots or anchor. Defaults to 2. Accepts 0 to 32. | +| `limit` | Total nodes per page, including ancestors. Defaults to 100. Accepts 1 to 200. | +| `maxBytes` | Maximum serialized response bytes. Defaults to 1 MiB. Accepts 64 KiB to 4 MiB. | +| `spanIds` | Fetch up to 100 explicit IDs in request order, without expanding the tree. Can't be combined with browse or search options. | +| `search` | Find matching spans throughout the trace when `capabilities.traceNavigationSearch` is available. Matching rules depend on the server. | +| `ancestors` | Include ancestor paths for search results. Defaults to `true`. | + +Browsing visits each level before descending. Search results distinguish matches through `node.match`. Contextual ancestors also count toward `limit`. `ancestorPaths` indicates whether search paths are complete, continued on another page, or contain unavailable parents. + +Pass only `version`, `traceId`, and `cursor` to continue. Restart the selection after a `409 TRACE_CHANGED` response. A payload that can't fit by itself returns `413 PAYLOAD_TOO_LARGE`. Request structure or retrieve that span with `getSpan()`. Reuse cached immutable details and request only missing IDs with `spanIds`. + ## Getting traces with filtering Retrieve a paginated list of trace root spans with optional filtering: diff --git a/packages/cli/src/commands/api/route-metadata.generated.ts b/packages/cli/src/commands/api/route-metadata.generated.ts index 691420ba038..d068750e29f 100644 --- a/packages/cli/src/commands/api/route-metadata.generated.ts +++ b/packages/cli/src/commands/api/route-metadata.generated.ts @@ -2008,6 +2008,31 @@ export const API_ROUTE_METADATA = { "listProperty": "spans" } }, + "POST /observability/traces/:traceId": { + "method": "POST", + "path": "/observability/traces/:traceId", + "pathParams": [ + "traceId" + ], + "queryParams": [], + "bodyParams": [ + "ancestors", + "anchorSpanId", + "cursor", + "depth", + "include", + "limit", + "maxBytes", + "search", + "spanIds", + "version" + ], + "hasQuery": false, + "hasBody": true, + "responseShape": { + "kind": "single" + } + }, "GET /observability/traces/:traceId/light": { "method": "GET", "path": "/observability/traces/:traceId/light", diff --git a/packages/core/src/storage/domains/observability/base.ts b/packages/core/src/storage/domains/observability/base.ts index 5b30413b9ca..06670e2349e 100644 --- a/packages/core/src/storage/domains/observability/base.ts +++ b/packages/core/src/storage/domains/observability/base.ts @@ -70,6 +70,8 @@ import type { SpanQueryResponse, TrustedSpanQueryPlan } from './span-query'; import type { ThreadQueryResponse, TrustedThreadRowsQueryPlan } from './thread-query'; import type { TraceAggregateResponse } from './trace-aggregate'; import type { TrustedTraceAggregatePlan } from './trace-aggregate-planner'; +import { readTracePage, TracePageError } from './trace-page'; +import type { GetTracePageArgs, TracePageResponse } from './trace-page'; import type { GetTraceQueryValuesResponse, QueryThreadsResult, @@ -317,9 +319,33 @@ export class ObservabilityStorage extends StorageDomain { }); } - /** - * Retrieves a single trace with all its associated spans. - */ + /** Bounded navigation requires structural and batch reads; never falls back to full-trace payloads. */ + supportsTraceNavigation(): boolean { + return ( + (this.getStructure !== ObservabilityStorage.prototype.getStructure || + this.getTraceLight !== ObservabilityStorage.prototype.getTraceLight) && + this.getSpans !== ObservabilityStorage.prototype.getSpans + ); + } + + async getTracePage( + { traceId, ...request }: GetTracePageArgs, + options?: { scopeKey: string }, + ): Promise { + if (!this.supportsTraceNavigation()) + throw new TracePageError('UNSUPPORTED', 'Store does not support bounded trace navigation'); + return readTracePage( + { + scopeKey: options?.scopeKey ?? this.constructor.name, + structure: async id => (await this.getStructure({ traceId: id }))?.spans ?? null, + spans: async (id, spanIds) => (await this.getSpans({ traceId: id, spanIds })).spans, + }, + traceId, + request, + ); + } + + /** Retrieves a single trace with all its associated spans. */ async getTrace(_args: GetTraceArgs): Promise { throw new MastraError({ id: 'OBSERVABILITY_STORAGE_GET_TRACE_NOT_IMPLEMENTED', diff --git a/packages/core/src/storage/domains/observability/index.ts b/packages/core/src/storage/domains/observability/index.ts index b091416bcf5..df70b5e521e 100644 --- a/packages/core/src/storage/domains/observability/index.ts +++ b/packages/core/src/storage/domains/observability/index.ts @@ -165,3 +165,5 @@ export * from './tracing'; export * from './types'; export * from './span-query'; export * from './thread-query'; + +export * from './trace-page'; diff --git a/packages/core/src/storage/domains/observability/trace-page.test.ts b/packages/core/src/storage/domains/observability/trace-page.test.ts new file mode 100644 index 00000000000..8011bdacbbf --- /dev/null +++ b/packages/core/src/storage/domains/observability/trace-page.test.ts @@ -0,0 +1,138 @@ +import { describe, expect, it } from 'vitest'; +import { SpanType } from '../../../observability/types'; +import { readTracePage, tracePageRequestSchema } from './trace-page'; +import type { TracePageSource, TracePageResponse } from './trace-page'; +import type { SpanRecord } from './tracing'; + +const span = (spanId: string, parentSpanId: string | null = null, input?: unknown): SpanRecord => ({ + traceId: 'trace', + spanId, + parentSpanId, + name: spanId, + spanType: SpanType.AGENT_RUN, + isEvent: false, + startedAt: new Date('2026-11-01'), + input, +}); +function source(rows: SpanRecord[], matches: string[] = []): TracePageSource { + return { + scopeKey: 'org/project/user', + structure: async () => rows, + spans: async (_, ids) => rows.filter(row => ids.includes(row.spanId)), + search: async () => matches, + }; +} +async function drain(src: TracePageSource, first: TracePageResponse): Promise { + const pages = [first]; + while (pages.at(-1)!.nextCursor) { + if (pages.length > 100) throw new Error('Continuation failed to progress'); + pages.push((await readTracePage(src, 'trace', { version: 2, cursor: pages.at(-1)!.nextCursor! }))!); + } + return pages; +} + +describe('bounded trace exploration', () => { + it('loads a shallow breadth-first tree with deterministic sibling ordering and discoverable children', async () => { + const src = source([span('z', 'root'), span('root'), span('deep', 'a'), span('a', 'root')]); + const pages = await drain(src, (await readTracePage(src, 'trace', { version: 2, depth: 1, limit: 2 }))!); + expect(pages.flatMap(page => page.nodes.map(node => node.spanId))).toEqual(['root', 'a', 'z']); + expect(pages[0]!.nodes[1]).toMatchObject({ childCount: 1 }); + expect(pages.at(-1)!.nextCursor).toBeUndefined(); + }); + + it('finds an unloaded deep match and continues its ancestor path across strict total node limits', async () => { + const src = source( + [span('root'), span('a', 'root'), span('b', 'a'), span('match', 'b'), span('other', 'root')], + ['match'], + ); + const pages = await drain(src, (await readTracePage(src, 'trace', { version: 2, search: 'timeout', limit: 1 }))!); + expect(pages.flatMap(page => page.nodes.map(node => [node.spanId, node.match]))).toEqual([ + ['root', false], + ['a', false], + ['b', false], + ['match', true], + ]); + expect(pages.map(page => page.ancestorPaths)).toEqual(['continued', 'continued', 'continued', 'complete']); + }); + + it('deduplicates shared ancestors and distinguishes an unavailable parent from a root', async () => { + const src = source([span('root', 'missing'), span('a', 'root'), span('b', 'root')], ['a', 'b']); + const pages = await drain(src, (await readTracePage(src, 'trace', { version: 2, search: 'error', limit: 2 }))!); + expect(pages.flatMap(page => page.nodes.map(node => node.spanId))).toEqual(['root', 'a', 'b']); + expect(pages[0]!.nodes[0]!.parentState).toBe('unavailable'); + expect(pages.at(-1)!.ancestorPaths).toBe('unavailable'); + }); + + it('hydrates only explicit IDs, reports missing IDs, and preserves requested order across pages', async () => { + const src = source([span('root'), span('a', 'root', { answer: 42 }), span('b', 'a')]); + src.structure = async () => { + throw new Error('Explicit IDs must not load topology'); + }; + + const pages = await drain( + src, + (await readTracePage(src, 'trace', { + version: 2, + spanIds: ['b', 'absent', 'a', 'b'], + include: 'full', + limit: 1, + }))!, + ); + expect(pages.flatMap(page => page.nodes.map(node => node.spanId))).toEqual(['b', 'a']); + expect(pages[0]!.missingSpanIds).toEqual(['absent']); + expect(pages[1]!.nodes[0]!.details?.input).toEqual({ answer: 42 }); + expect(pages[0]!.nodes[0]!.details?.spanId).toBe('b'); + }); + + it('continues at the first unreturned payload when the byte budget fills', async () => { + const src = source([span('a', null, 'x'.repeat(40000)), span('b', null, 'y'.repeat(40000))]); + const pages = await drain( + src, + (await readTracePage(src, 'trace', { version: 2, include: 'full', maxBytes: 65536 }))!, + ); + expect(pages.map(page => page.nodes.map(node => node.spanId))).toEqual([['a'], ['b']]); + for (const page of pages) expect(Buffer.byteLength(JSON.stringify(page))).toBeLessThanOrEqual(65536); + }); + + it('gives an actionable error for one oversized payload rather than a cursor that never advances', async () => { + const src = source([span('a', null, 'x'.repeat(70000))]); + await expect(readTracePage(src, 'trace', { version: 2, include: 'full', maxBytes: 65536 })).rejects.toMatchObject({ + code: 'PAYLOAD_TOO_LARGE', + }); + const structure = await readTracePage(src, 'trace', { version: 2, maxBytes: 65536 }); + expect(structure!.nodes[0]!.details).toBeUndefined(); + expect(JSON.stringify(structure)).not.toContain('xxxxx'); + }); + + it('rejects scope changes and late arrivals even when root activity is unchanged', async () => { + const rows = [span('a'), span('b')]; + const src = source(rows); + const first = (await readTracePage(src, 'trace', { version: 2, limit: 1 }))!; + const request = { version: 2 as const, cursor: first.nextCursor! }; + await expect(readTracePage({ ...src, scopeKey: 'another/org' }, 'trace', request)).rejects.toMatchObject({ + code: 'TRACE_CHANGED', + }); + rows.push(span('aa')); + await expect(readTracePage(src, 'trace', request)).rejects.toMatchObject({ code: 'TRACE_CHANGED' }); + }); + + it('rejects cyclic topology and malformed or conflicting continuation requests', async () => { + await expect( + readTracePage(source([span('a', 'b'), span('b', 'a')]), 'trace', { version: 2 }), + ).rejects.toMatchObject({ code: 'INVALID_TOPOLOGY' }); + await expect(readTracePage(source([]), 'trace', { version: 2, cursor: 'bad' })).rejects.toMatchObject({ + code: 'INVALID_CURSOR', + }); + expect(tracePageRequestSchema.safeParse({ version: 2, cursor: 'cursor', include: 'full' }).success).toBe(false); + expect(tracePageRequestSchema.safeParse({ version: 2, spanIds: ['a'], depth: 2 }).success).toBe(false); + expect(tracePageRequestSchema.safeParse({ version: 2, search: 'foo', depth: 2 }).success).toBe(false); + }); + it('can continue a maximum-size batch of multibyte identifiers', async () => { + const ids = Array.from({ length: 100 }, (_, i) => '界'.repeat(125) + String(i)); + const src = source(ids.map(id => span(id))); + const first = (await readTracePage(src, 'trace', { version: 2, spanIds: ids, limit: 1 }))!; + expect(Buffer.byteLength(JSON.stringify({ version: 2, cursor: first.nextCursor }))).toBeLessThan(65536); + const second = await readTracePage(src, 'trace', { version: 2, cursor: first.nextCursor! }); + expect(second!.nodes[0]!.spanId).toBe(ids[1]); + }); +}); diff --git a/packages/core/src/storage/domains/observability/trace-page.ts b/packages/core/src/storage/domains/observability/trace-page.ts new file mode 100644 index 00000000000..85931c74ff4 --- /dev/null +++ b/packages/core/src/storage/domains/observability/trace-page.ts @@ -0,0 +1,328 @@ +import { createHash } from 'node:crypto'; +import { z } from 'zod/v4'; +import { spanRecordSchema, computeTraceStatus, TraceStatus } from './tracing'; +import type { SpanRecord } from './tracing'; + +// Bound serialized identities so every accepted ID batch fits in a continuation request. +const id = z + .string() + .min(1) + .max(128) + .refine(value => Buffer.byteLength(JSON.stringify(value)) <= 384, 'Identifier exceeds the serialized byte limit'); +const selectionSchema = z + .object({ + version: z.literal(2), + anchorSpanId: id.optional(), + depth: z.number().int().min(0).max(32).optional(), + spanIds: z.array(id).min(1).max(100).optional(), + search: z.string().trim().min(1).max(256).optional(), + ancestors: z.boolean().optional(), + include: z.enum(['structure', 'full']).default('structure'), + limit: z.number().int().min(1).max(200).default(100), + maxBytes: z.number().int().min(65536).max(4194304).default(1048576), + }) + .strict(); + +/** POST body for the explicitly versioned getTrace overload. Continuations cannot change selection. */ +export const tracePageRequestSchema = z.union([ + selectionSchema.superRefine((value, ctx) => { + if ( + value.spanIds && + (value.anchorSpanId || value.search || value.depth !== undefined || value.ancestors !== undefined) + ) { + ctx.addIssue({ code: 'custom', message: 'spanIds cannot be combined with browse or search selection' }); + } + if (value.search && (value.anchorSpanId || value.depth !== undefined)) { + ctx.addIssue({ code: 'custom', message: 'Search covers the entire trace; anchorSpanId is browse-only' }); + } + }), + z.object({ version: z.literal(2), cursor: z.string().min(1).max(60000) }).strict(), +]); +export type TracePageRequest = z.input; +export type GetTracePageArgs = TracePageRequest & { traceId: string }; + +/** Deliberately excludes unbounded payload, preview, metadata, and error fields. */ +export const tracePageNodeSchema = spanRecordSchema + .pick({ + traceId: true, + spanId: true, + parentSpanId: true, + name: true, + spanType: true, + isEvent: true, + startedAt: true, + endedAt: true, + entityType: true, + entityId: true, + entityName: true, + }) + .extend({ + status: z.nativeEnum(TraceStatus), + childCount: z.number().int().nonnegative().nullable(), + match: z.boolean(), + parentState: z.enum(['root', 'available', 'unavailable', 'not-requested']), + details: spanRecordSchema.optional(), + }); +export const tracePageResponseSchema = z.object({ + version: z.literal(2), + traceId: z.string(), + nodes: z.array(tracePageNodeSchema), + missingSpanIds: z.array(z.string()), + nextCursor: z.string().optional(), + ancestorPaths: z.enum(['not-requested', 'continued', 'complete', 'unavailable']), +}); +export type TracePageResponse = z.infer; +export type TracePageNode = z.infer; +export type TracePageStructure = Pick< + SpanRecord, + | 'traceId' + | 'spanId' + | 'parentSpanId' + | 'name' + | 'spanType' + | 'isEvent' + | 'startedAt' + | 'endedAt' + | 'entityType' + | 'entityId' + | 'entityName' +> & { error?: unknown }; + +export class TracePageError extends Error { + constructor( + public readonly code: + | 'INVALID_CURSOR' + | 'TRACE_CHANGED' + | 'UNSUPPORTED' + | 'LIMIT_EXCEEDED' + | 'INVALID_TOPOLOGY' + | 'PAYLOAD_TOO_LARGE', + message: string, + ) { + super(message); + this.name = 'TracePageError'; + } +} + +/** All reads must independently enforce trusted scope. A cursor is never authorization. */ +export interface TracePageSource { + scopeKey: string; + structure(traceId: string): Promise; + spans(traceId: string, spanIds: string[]): Promise; + /** Complete matching identities within this trace; fail rather than truncate. */ + search?(traceId: string, text: string): Promise; +} + +const hash = (value: unknown) => createHash('sha256').update(JSON.stringify(value)).digest('hex'); +const cursorSchema = z + .object({ + version: z.literal(2), + selection: selectionSchema, + fingerprint: z.string().regex(/^[a-f0-9]{64}$/), + offset: z.number().int().min(1).max(50000), + }) + .strict(); + +/** Bounded wire pages over a capped topology read; this is not O(page size) database work. */ +export async function readTracePage( + source: TracePageSource, + traceId: string, + request: TracePageRequest, +): Promise { + id.parse(traceId); + const parsed = tracePageRequestSchema.parse(request); + let continuation: z.infer | undefined; + if ('cursor' in parsed) { + try { + if (!/^[A-Za-z0-9_-]+$/.test(parsed.cursor)) throw new Error(); + continuation = cursorSchema.parse(JSON.parse(Buffer.from(parsed.cursor, 'base64url').toString('utf8'))); + tracePageRequestSchema.parse(continuation.selection); + } catch { + throw new TracePageError('INVALID_CURSOR', 'Invalid trace continuation'); + } + } + const selection = continuation?.selection ?? ('cursor' in parsed ? undefined : parsed); + if (!selection) throw new TracePageError('INVALID_CURSOR', 'Invalid trace continuation'); + if (selection.search && !source.search) + throw new TracePageError('UNSUPPORTED', 'Indexed in-trace search is unavailable'); + // Explicit-ID hydration does not discover or expand the trace topology. + const explicitSpans = selection.spanIds ? await source.spans(traceId, [...new Set(selection.spanIds)]) : undefined; + const structure = explicitSpans ?? (await source.structure(traceId)); + if (!structure) return null; + if (structure.length > 50000) throw new TracePageError('LIMIT_EXCEEDED', 'Trace topology exceeds 50000 spans'); + const nodes = new Map(); + for (const node of structure) { + if ( + node.traceId !== traceId || + nodes.has(node.spanId) || + !Number.isFinite(node.startedAt.getTime()) || + (selection.spanIds && !selection.spanIds.includes(node.spanId)) + ) { + throw new TracePageError('INVALID_TOPOLOGY', 'Trace contains conflicting identities or invalid timestamps'); + } + nodes.set(node.spanId, node); + } + const visited = new Set(); + for (const node of selection.spanIds ? [] : structure) { + const path = new Set(); + let current: TracePageStructure | undefined = node; + while (current && !visited.has(current.spanId)) { + if (path.has(current.spanId)) throw new TracePageError('INVALID_TOPOLOGY', 'Trace contains a parent cycle'); + path.add(current.spanId); + current = current.parentSpanId ? nodes.get(current.parentSpanId) : undefined; + } + for (const spanId of path) visited.add(spanId); + } + const sorted = [...nodes.values()].sort( + (a, b) => a.startedAt.getTime() - b.startedAt.getTime() || (a.spanId < b.spanId ? -1 : a.spanId > b.spanId ? 1 : 0), + ); + const children = new Map(); + const roots: TracePageStructure[] = []; + for (const node of sorted) { + if (node.parentSpanId && nodes.has(node.parentSpanId)) { + const siblings = children.get(node.parentSpanId) ?? []; + siblings.push(node); + children.set(node.parentSpanId, siblings); + } else roots.push(node); + } + const matches = new Set(selection.search ? await source.search!(traceId, selection.search) : []); + if ([...matches].some(spanId => !nodes.has(spanId))) + throw new TracePageError('TRACE_CHANGED', 'Search membership changed during the read; restart this selection'); + const selected: TracePageStructure[] = []; + const missingSpanIds: string[] = []; + if (selection.spanIds) { + for (const spanId of new Set(selection.spanIds)) { + const node = nodes.get(spanId); + if (node) selected.push(node); + else missingSpanIds.push(spanId); + } + } else if (selection.search) { + const emitted = new Set(); + for (const match of sorted.filter(node => matches.has(node.spanId))) { + const path: TracePageStructure[] = []; + let current: TracePageStructure | undefined = match; + while (current && !emitted.has(current.spanId)) { + path.push(current); + current = selection.ancestors !== false && current.parentSpanId ? nodes.get(current.parentSpanId) : undefined; + } + for (const node of path.reverse()) { + emitted.add(node.spanId); + selected.push(node); + } + } + } else { + const anchors = selection.anchorSpanId + ? [nodes.get(selection.anchorSpanId)].filter((node): node is TracePageStructure => !!node) + : roots; + if (selection.anchorSpanId && !anchors.length) missingSpanIds.push(selection.anchorSpanId); + const queue = anchors.map(node => ({ node, depth: 0 })); + for (let i = 0; i < queue.length; i++) { + const entry = queue[i]!; + selected.push(entry.node); + if (entry.depth < (selection.depth ?? 2)) { + for (const child of children.get(entry.node.spanId) ?? []) queue.push({ node: child, depth: entry.depth + 1 }); + } + } + } + // A topology fingerprint detects delayed arrivals, unlike root lastActivityAt. + // It does not retain a database snapshot: callers restart when membership changes. + const fingerprint = hash([ + source.scopeKey, + traceId, + selection, + sorted.map(node => [node.spanId, node.parentSpanId ?? null, node.startedAt.toISOString()]), + [...matches].sort(), + ]); + if (continuation && continuation.fingerprint !== fingerprint) { + throw new TracePageError('TRACE_CHANGED', 'Trace or request scope changed; restart this selection'); + } + const offset = continuation?.offset ?? 0; + if (continuation && offset >= selected.length) + throw new TracePageError('INVALID_CURSOR', 'Continuation is outside the selection'); + const page = selected.slice(offset, offset + selection.limit); + const details = new Map(); + if (selection.include === 'full' && page.length) { + const pageIds = new Set(page.map(node => node.spanId)); + const hydrated = explicitSpans + ? explicitSpans.filter(span => pageIds.has(span.spanId)) + : await source.spans(traceId, [...pageIds]); + for (const span of hydrated) { + if (span.traceId !== traceId || !page.some(node => node.spanId === span.spanId) || details.has(span.spanId)) { + throw new TracePageError('INVALID_TOPOLOGY', 'Detail read returned conflicting identities'); + } + details.set(span.spanId, span); + } + if (details.size !== page.length) + throw new TracePageError('TRACE_CHANGED', 'Selected spans disappeared; restart this selection'); + } + const result: TracePageResponse = { version: 2, traceId, nodes: [], missingSpanIds, ancestorPaths: 'not-requested' }; + const updateContinuation = () => { + const nextOffset = offset + result.nodes.length; + if (nextOffset < selected.length) { + result.nextCursor = Buffer.from( + JSON.stringify({ version: 2, selection, fingerprint, offset: nextOffset }), + ).toString('base64url'); + } else delete result.nextCursor; + if (selection.search && selection.ancestors !== false) { + result.ancestorPaths = result.nextCursor + ? 'continued' + : selected.some(node => node.parentSpanId && !nodes.has(node.parentSpanId)) + ? 'unavailable' + : 'complete'; + } + }; + for (const node of page) { + const { + traceId, + spanId, + parentSpanId, + name, + spanType, + isEvent, + startedAt, + endedAt, + entityType, + entityId, + entityName, + } = node; + result.nodes.push({ + traceId, + spanId, + parentSpanId, + name, + spanType, + isEvent, + startedAt, + endedAt, + entityType, + entityId, + entityName, + status: computeTraceStatus(node), + childCount: selection.spanIds ? null : (children.get(spanId)?.length ?? 0), + match: matches.has(spanId), + parentState: !parentSpanId + ? 'root' + : selection.spanIds + ? 'not-requested' + : nodes.has(parentSpanId) + ? 'available' + : 'unavailable', + ...(selection.include === 'full' ? { details: details.get(spanId)! } : {}), + }); + updateContinuation(); + if (Buffer.byteLength(JSON.stringify(result)) > selection.maxBytes) { + result.nodes.pop(); + updateContinuation(); + if (!result.nodes.length) + throw new TracePageError( + 'PAYLOAD_TOO_LARGE', + `Span '${spanId}' exceeds maxBytes; request structure or use the single-span endpoint`, + ); + break; + } + } + updateContinuation(); + if (Buffer.byteLength(JSON.stringify(result)) > selection.maxBytes) + throw new TracePageError('LIMIT_EXCEEDED', 'Trace response metadata exceeds maxBytes'); + return result; +} diff --git a/packages/server/src/server/handlers/observability-new-endpoints.ts b/packages/server/src/server/handlers/observability-new-endpoints.ts index 8be96d6c2ec..be88171867a 100644 --- a/packages/server/src/server/handlers/observability-new-endpoints.ts +++ b/packages/server/src/server/handlers/observability-new-endpoints.ts @@ -1923,7 +1923,7 @@ export const GET_CAPABILITIES = createRoute({ responseSchema: getObservabilityCapabilitiesResponseSchema, tags: ['Observability'], requiresAuth: true, - handler: async ({ mastra }) => { + handler: async ({ mastra, requestContext }) => { try { const observabilityStore = await mastra.getStorage()?.getStore('observability'); if (!observabilityStore) { @@ -1931,7 +1931,10 @@ export const GET_CAPABILITIES = createRoute({ } return { observabilityStorageType: observabilityStore.constructor.name, - capabilities: getObservabilityStorageCapabilities(observabilityStore), + capabilities: { + ...getObservabilityStorageCapabilities(observabilityStore), + ...(requestContext.get('organizationId') ? { traceNavigation: false, traceNavigationSearch: false } : {}), + }, }; } catch (error) { return handleError(error, "Error calling: 'get observability capabilities'"); diff --git a/packages/server/src/server/handlers/observability-shared.ts b/packages/server/src/server/handlers/observability-shared.ts index 6da81bb3310..56da425ed7d 100644 --- a/packages/server/src/server/handlers/observability-shared.ts +++ b/packages/server/src/server/handlers/observability-shared.ts @@ -695,6 +695,8 @@ export type ObservabilityStorageCapabilities = { deltaPolling: boolean; traceQuery: boolean; traceQueryAggregatedRoots: boolean; + traceNavigation: boolean; + traceNavigationSearch: boolean; traceQueryTextSearch: boolean; traceQueryRootDuration: boolean; traceQueryContextIds: boolean; @@ -741,6 +743,8 @@ export const NO_OBSERVABILITY_STORAGE_CAPABILITIES: ObservabilityStorageCapabili deltaPolling: false, traceQuery: false, traceQueryAggregatedRoots: false, + traceNavigation: false, + traceNavigationSearch: false, traceQueryTextSearch: false, traceQueryRootDuration: false, traceQueryContextIds: false, @@ -835,6 +839,11 @@ export function getObservabilityStorageCapabilities( coreFeatures.has(OBSERVABILITY_DELTA_POLLING_FEATURE) && declares(OBSERVABILITY_DELTA_POLLING_STORAGE_FEATURE), traceQuery, traceQueryAggregatedRoots: false, + traceNavigation: + !!coreStorage.tracePageRequestSchema && + typeof observabilityStore.supportsTraceNavigation === 'function' && + observabilityStore.supportsTraceNavigation(), + traceNavigationSearch: false, traceQueryTextSearch: traceQuery && coreStorage.observabilityTextSearchSchema !== undefined && declares('trace-query-text-search'), traceQueryRootDuration: diff --git a/packages/server/src/server/handlers/observability-text-search-core-compat.test.ts b/packages/server/src/server/handlers/observability-text-search-core-compat.test.ts index 828c4cabbe1..b9d6e0da688 100644 --- a/packages/server/src/server/handlers/observability-text-search-core-compat.test.ts +++ b/packages/server/src/server/handlers/observability-text-search-core-compat.test.ts @@ -12,11 +12,15 @@ vi.mock('@mastra/core/storage', async importOriginal => { return { ...actual, observabilityTextSearchSchema: undefined, + tracePageRequestSchema: undefined, + tracePageResponseSchema: undefined, + TracePageError: undefined, planSpanQuery: (input: unknown, options: Parameters[1]) => actual.planSpanQuery(legacyRequestSchema.parse(input), options), }; }); +const { GET_TRACE_PAGE_ROUTE, GET_TRACE_ROUTE } = await import('./observability'); const { QUERY_SPANS } = await import('./observability-new-endpoints'); const { getObservabilityStorageCapabilities } = await import('./observability-shared'); @@ -49,3 +53,18 @@ describe('text search with pre-search Core', () => { }); }); }); + +it('keeps legacy trace reads available when the installed Core predates bounded navigation', async () => { + const storage = new InMemoryStore(); + const store = (await storage.getStore('observability'))!; + const context = createTestServerContext({ mastra: new Mastra({ storage }) }); + expect(getObservabilityStorageCapabilities(store)).toMatchObject({ + traceNavigation: false, + traceNavigationSearch: false, + }); + await expect(GET_TRACE_PAGE_ROUTE.handler({ ...context, traceId: 'absent', version: 2 })).rejects.toMatchObject({ + status: 501, + }); + // A normal legacy not-found response, rather than startup/import failure from missing exports. + await expect(GET_TRACE_ROUTE.handler({ ...context, traceId: 'absent' })).rejects.toMatchObject({ status: 404 }); +}); diff --git a/packages/server/src/server/handlers/observability.ts b/packages/server/src/server/handlers/observability.ts index 8922f648f3c..d53e58813f1 100644 --- a/packages/server/src/server/handlers/observability.ts +++ b/packages/server/src/server/handlers/observability.ts @@ -1,5 +1,6 @@ import { extractTrajectoryFromTrace, listScoresResponseSchema } from '@mastra/core/evals'; import { scoreTraces } from '@mastra/core/evals/scoreTraces'; +import * as tracePageCore from '@mastra/core/storage'; import { tracesFilterSchema, tracesOrderBySchema, @@ -20,6 +21,7 @@ import { // @mastra/core@1.32.0; route them through a shim that tolerates older cores // (see ./observability-storage-schemas.ts for full rationale). import { z } from 'zod/v4'; +import { MASTRA_USER_KEY, MASTRA_RESOURCE_ID_KEY, MASTRA_AUTH_MODE_KEY, MASTRA_USER_ROLES_KEY } from '../constants'; import { HTTPException } from '../http-exception'; import type { ServerRoute } from '../server-adapter/routes'; import { createRoute, pickParams, wrapSchemaForQueryParams } from '../server-adapter/routes/route-builder'; @@ -279,6 +281,77 @@ export const GET_BRANCH_ROUTE = createRoute({ } }, }); +/** Same trace resource, explicit bounded contract. Namespace guards preserve older core compatibility. */ +export const GET_TRACE_PAGE_ROUTE: ServerRoute = createRoute({ + method: 'POST', + path: '/observability/traces/:traceId', + responseType: 'json', + pathParamSchema: getTraceArgsSchema, + bodySchema: tracePageCore.tracePageRequestSchema ?? z.never(), + responseSchema: tracePageCore.tracePageResponseSchema ?? z.never(), + maxBodySize: 65536, + requiresPermission: 'observability:read', + summary: 'Browse or search a bounded trace page', + description: 'Version 2 trace navigation with bounded structure, optional details, and request-bound continuation.', + tags: ['Observability'], + requiresAuth: true, + handler: async ({ mastra, traceId, requestContext, ...request }) => { + try { + const store = await getObservabilityStore(mastra); + if ( + !tracePageCore.tracePageRequestSchema || + typeof store.getTracePage !== 'function' || + typeof store.supportsTraceNavigation !== 'function' || + !store.supportsTraceNavigation() + ) { + throw new HTTPException(501, { message: 'Bounded trace navigation is unavailable' }); + } + // Existing structure/batch primitives cannot filter an organization. Never widen a scoped read. + if (requestContext.get('organizationId')) { + throw new HTTPException(501, { message: 'Tenant-scoped trace navigation requires a scoped backend' }); + } + const user = requestContext.get(MASTRA_USER_KEY); + const userId = user && typeof user === 'object' && 'id' in user ? user.id : null; + const roles = requestContext.get(MASTRA_USER_ROLES_KEY); + const scopeKey = JSON.stringify({ + storageId: (await getStorage(mastra)).id, + userId: typeof userId === 'string' || typeof userId === 'number' ? userId : null, + resourceId: requestContext.get(MASTRA_RESOURCE_ID_KEY) ?? null, + authMode: requestContext.get(MASTRA_AUTH_MODE_KEY) ?? null, + roles: Array.isArray(roles) + ? [...new Set(roles.filter((role): role is string => typeof role === 'string'))].sort() + : [], + }); + // Route context contains additional server-owned fields; pick only the body contract. + const fields: Record = request; + const body = Object.fromEntries( + ['version', 'cursor', 'anchorSpanId', 'depth', 'spanIds', 'search', 'ancestors', 'include', 'limit', 'maxBytes'] + .filter(key => fields[key] !== undefined) + .map(key => [key, fields[key]]), + ); + const page = await store.getTracePage( + { traceId, ...tracePageCore.tracePageRequestSchema.parse(body) }, + { scopeKey }, + ); + if (!page) throw new HTTPException(404, { message: 'Trace not found' }); + return page; + } catch (error) { + if (tracePageCore.TracePageError && error instanceof tracePageCore.TracePageError) { + const status = + error.code === 'UNSUPPORTED' + ? 501 + : error.code === 'TRACE_CHANGED' + ? 409 + : error.code === 'PAYLOAD_TOO_LARGE' || error.code === 'LIMIT_EXCEEDED' + ? 413 + : 400; + throw new HTTPException(status, { message: `${error.code}: ${error.message}` }); + } + return handleError(error, 'Error reading trace page'); + } + }, +}); + /** Route: GET /observability/traces/:traceId - retrieve a single trace with all spans. */ export const GET_TRACE_ROUTE: ServerRoute = createRoute({ method: 'GET', diff --git a/packages/server/src/server/handlers/system.test.ts b/packages/server/src/server/handlers/system.test.ts index b30aa49503f..301773df5d8 100644 --- a/packages/server/src/server/handlers/system.test.ts +++ b/packages/server/src/server/handlers/system.test.ts @@ -19,6 +19,8 @@ const NO_OBSERVABILITY_CAPABILITIES = { deltaPolling: false, traceQuery: false, traceQueryAggregatedRoots: false, + traceNavigation: false, + traceNavigationSearch: false, traceQueryTextSearch: false, spanQueryTextSearch: false, traceQueryRootDuration: false, diff --git a/packages/server/src/server/handlers/trace-page.test.ts b/packages/server/src/server/handlers/trace-page.test.ts new file mode 100644 index 00000000000..798a32064b7 --- /dev/null +++ b/packages/server/src/server/handlers/trace-page.test.ts @@ -0,0 +1,74 @@ +import { Mastra } from '@mastra/core'; +import { SpanType } from '@mastra/core/observability'; +import { InMemoryStore } from '@mastra/core/storage'; +import { describe, expect, it } from 'vitest'; +import { GET_TRACE_PAGE_ROUTE, GET_TRACE_ROUTE } from './observability'; +import { getObservabilityStorageCapabilities } from './observability-shared'; +import { createTestServerContext } from './test-utils'; + +async function setup() { + const storage = new InMemoryStore(); + const store = (await storage.getStore('observability'))!; + await store.batchCreateSpans({ + records: ['root', 'child', 'nested'].map((spanId, index) => ({ + traceId: 'trace', + spanId, + parentSpanId: index ? ['root', 'child'][index - 1] : null, + name: spanId, + spanType: SpanType.AGENT_RUN, + isEvent: false, + startedAt: new Date('2026-11-01'), + input: { payload: spanId }, + })), + }); + return { store, context: createTestServerContext({ mastra: new Mastra({ storage }) }) }; +} + +describe('versioned trace route with real storage', () => { + it('retains legacy full reads while bounded requests hydrate only the page', async () => { + const { store, context } = await setup(); + expect(getObservabilityStorageCapabilities(store)).toMatchObject({ + traceNavigation: true, + traceNavigationSearch: false, + }); + const legacy = await GET_TRACE_ROUTE.handler({ ...context, traceId: 'trace' }); + expect(legacy.spans.map((span: { spanId: string }) => span.spanId)).toEqual(['root', 'child', 'nested']); + const first = await GET_TRACE_PAGE_ROUTE.handler({ + ...context, + traceId: 'trace', + version: 2, + depth: 2, + limit: 1, + include: 'full', + }); + expect(first.nodes).toHaveLength(1); + expect(first.nodes[0]).toMatchObject({ spanId: 'root', childCount: 1, details: { input: { payload: 'root' } } }); + const next = await GET_TRACE_PAGE_ROUTE.handler({ + ...context, + traceId: 'trace', + version: 2, + cursor: first.nextCursor, + }); + expect(next.nodes[0]).toMatchObject({ spanId: 'child', details: { input: { payload: 'child' } } }); + }); + + it('reports unsupported indexed search instead of scanning payloads in memory', async () => { + const { context } = await setup(); + await expect( + GET_TRACE_PAGE_ROUTE.handler({ ...context, traceId: 'trace', version: 2, search: 'payload' }), + ).rejects.toMatchObject({ status: 501 }); + }); + it('does not reuse cursors across authenticated users or silently widen tenant scope', async () => { + const { context } = await setup(); + context.requestContext.set('mastra__user', { id: 'alice' }); + const first = await GET_TRACE_PAGE_ROUTE.handler({ ...context, traceId: 'trace', version: 2, limit: 1 }); + context.requestContext.set('mastra__user', { id: 'bob' }); + await expect( + GET_TRACE_PAGE_ROUTE.handler({ ...context, traceId: 'trace', version: 2, cursor: first.nextCursor }), + ).rejects.toMatchObject({ status: 409 }); + context.requestContext.set('organizationId', 'other-org'); + await expect(GET_TRACE_PAGE_ROUTE.handler({ ...context, traceId: 'trace', version: 2 })).rejects.toMatchObject({ + status: 501, + }); + }); +}); diff --git a/packages/server/src/server/schemas/system.ts b/packages/server/src/server/schemas/system.ts index db40d68bd06..1ab8359e7a8 100644 --- a/packages/server/src/server/schemas/system.ts +++ b/packages/server/src/server/schemas/system.ts @@ -41,6 +41,8 @@ export const observabilityStorageCapabilitiesSchema = z.object({ .boolean() .default(false) .describe("Supports explicit `rootSource: 'aggregated'` trace queries"), + traceNavigation: z.boolean().default(false).describe('Bounded version 2 getTrace browsing and hydration'), + traceNavigationSearch: z.boolean().default(false).describe('Indexed in-trace search with ancestor context'), traceQueryTextSearch: z.boolean().default(false).describe('Literal substring search across any span in a trace'), spanQueryTextSearch: z.boolean().default(false).describe('Literal substring search within individual spans'), traceQueryRootDuration: z.boolean().describe('`durationMs` predicates in trace and thread queries'), diff --git a/packages/server/src/server/server-adapter/routes/observability.ts b/packages/server/src/server/server-adapter/routes/observability.ts index ff9df6c0c1d..81d97d7f8c6 100644 --- a/packages/server/src/server/server-adapter/routes/observability.ts +++ b/packages/server/src/server/server-adapter/routes/observability.ts @@ -4,6 +4,7 @@ import { LIST_BRANCHES_ROUTE, GET_BRANCH_ROUTE, GET_TRACE_ROUTE, + GET_TRACE_PAGE_ROUTE, GET_TRACE_LIGHT_ROUTE, GET_SPAN_ROUTE, GET_TRACE_TRAJECTORY_ROUTE, @@ -61,6 +62,7 @@ export const OBSERVABILITY_ROUTES = [ LIST_BRANCHES_ROUTE, GET_BRANCH_ROUTE, GET_TRACE_ROUTE, + GET_TRACE_PAGE_ROUTE, GET_TRACE_LIGHT_ROUTE, GET_SPAN_ROUTE, GET_TRACE_TRAJECTORY_ROUTE, From 87ab76f9501b2a90589578802b6f80f219979efb Mon Sep 17 00:00:00 2001 From: Eric Pinzur Date: Mon, 16 Nov 2026 14:15:17 +0100 Subject: [PATCH 02/23] chore: split trace navigation changesets by package --- .changeset/eager-meteors-wear.md | 3 --- .changeset/four-foxes-pick.md | 3 --- .changeset/pink-eagles-repair.md | 5 +++++ .changeset/whole-streets-flow.md | 3 --- 4 files changed, 5 insertions(+), 9 deletions(-) create mode 100644 .changeset/pink-eagles-repair.md diff --git a/.changeset/eager-meteors-wear.md b/.changeset/eager-meteors-wear.md index eb70c3f13b9..2ab146cc6b8 100644 --- a/.changeset/eager-meteors-wear.md +++ b/.changeset/eager-meteors-wear.md @@ -1,8 +1,5 @@ --- '@mastra/client-js': minor -'@mastra/server': patch -'@mastra/core': patch -'mastra': patch --- Add bounded trace browsing with optional span details: diff --git a/.changeset/four-foxes-pick.md b/.changeset/four-foxes-pick.md index 13c6dafb36d..af22b27d457 100644 --- a/.changeset/four-foxes-pick.md +++ b/.changeset/four-foxes-pick.md @@ -1,8 +1,5 @@ --- '@mastra/core': minor -'@mastra/client-js': patch -'@mastra/server': patch -'mastra': patch --- Add bounded trace browsing and explicit span hydration through the version 2 trace contract. diff --git a/.changeset/pink-eagles-repair.md b/.changeset/pink-eagles-repair.md new file mode 100644 index 00000000000..a67802bc5c0 --- /dev/null +++ b/.changeset/pink-eagles-repair.md @@ -0,0 +1,5 @@ +--- +'mastra': patch +--- + +Add the bounded trace read endpoint to CLI API route metadata. diff --git a/.changeset/whole-streets-flow.md b/.changeset/whole-streets-flow.md index c11512a69fd..2cac5d10d2d 100644 --- a/.changeset/whole-streets-flow.md +++ b/.changeset/whole-streets-flow.md @@ -1,8 +1,5 @@ --- '@mastra/server': minor -'@mastra/client-js': patch -'@mastra/core': patch -'mastra': patch --- Add a version 2 trace read endpoint with bounded pages and optional span details. From e661823f2406b2389547951e9bdab64593a9c033 Mon Sep 17 00:00:00 2001 From: Eric Pinzur Date: Mon, 16 Nov 2026 14:22:35 +0100 Subject: [PATCH 03/23] fix(server): preserve unsupported trace navigation responses Allow requests through validation when older Core lacks the schema, so the handler returns 501. Reproduced with the adapter body parser. Review: https://github.com/mastra-ai/mastra/pull/30512#discussion_r4585617220 --- .../handlers/observability-text-search-core-compat.test.ts | 6 +++++- packages/server/src/server/handlers/observability.ts | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/server/src/server/handlers/observability-text-search-core-compat.test.ts b/packages/server/src/server/handlers/observability-text-search-core-compat.test.ts index b9d6e0da688..792d123f51e 100644 --- a/packages/server/src/server/handlers/observability-text-search-core-compat.test.ts +++ b/packages/server/src/server/handlers/observability-text-search-core-compat.test.ts @@ -3,6 +3,7 @@ import { InMemoryStore } from '@mastra/core/storage'; import { describe, expect, it, vi } from 'vitest'; import { HTTPException } from '../http-exception'; +import { parseRouteBody } from '../server-adapter'; import { createTestServerContext } from './test-utils'; // Model the pre-search Core contract, retaining real schema validation and planning. @@ -62,7 +63,10 @@ it('keeps legacy trace reads available when the installed Core predates bounded traceNavigation: false, traceNavigationSearch: false, }); - await expect(GET_TRACE_PAGE_ROUTE.handler({ ...context, traceId: 'absent', version: 2 })).rejects.toMatchObject({ + const body = await parseRouteBody(GET_TRACE_PAGE_ROUTE, { version: 2 }); + await expect( + GET_TRACE_PAGE_ROUTE.handler({ ...context, traceId: 'absent', ...(body as object) }), + ).rejects.toMatchObject({ status: 501, }); // A normal legacy not-found response, rather than startup/import failure from missing exports. diff --git a/packages/server/src/server/handlers/observability.ts b/packages/server/src/server/handlers/observability.ts index d53e58813f1..2b75066b4ee 100644 --- a/packages/server/src/server/handlers/observability.ts +++ b/packages/server/src/server/handlers/observability.ts @@ -287,7 +287,7 @@ export const GET_TRACE_PAGE_ROUTE: ServerRoute = createRoute({ path: '/observability/traces/:traceId', responseType: 'json', pathParamSchema: getTraceArgsSchema, - bodySchema: tracePageCore.tracePageRequestSchema ?? z.never(), + bodySchema: tracePageCore.tracePageRequestSchema ?? z.unknown().optional(), responseSchema: tracePageCore.tracePageResponseSchema ?? z.never(), maxBodySize: 65536, requiresPermission: 'observability:read', From 3728e230c76d7dc59c9dfea005e5b243628b10a8 Mon Sep 17 00:00:00 2001 From: Eric Pinzur Date: Mon, 16 Nov 2026 14:22:47 +0100 Subject: [PATCH 04/23] fix(observability): separate versioned trace page API Use /observability/v2/traces/:traceId and getTracePage() to preserve legacy routing and inferred getTrace return types. Reviews: https://github.com/mastra-ai/mastra/pull/30512#discussion_r4585617255 https://github.com/mastra-ai/mastra/pull/30512#discussion_r4585617185 --- client-sdks/client-js/src/client.ts | 11 ++++-- .../resources/observability-route-types.ts | 4 +- .../observability.trace-page.test.ts | 14 ++++--- .../client-js/src/resources/observability.ts | 12 +++--- .../client-js/src/route-types.generated.ts | 38 ++++++++++--------- .../commands/api/route-metadata.generated.ts | 4 +- .../src/server/handlers/observability.ts | 4 +- .../src/server/handlers/trace-page.test.ts | 17 +++++++++ 8 files changed, 65 insertions(+), 39 deletions(-) diff --git a/client-sdks/client-js/src/client.ts b/client-sdks/client-js/src/client.ts index 40340868363..c94c773581f 100644 --- a/client-sdks/client-js/src/client.ts +++ b/client-sdks/client-js/src/client.ts @@ -1281,10 +1281,13 @@ export class MastraClient extends BaseResource { } /** Retrieves a specific trace by ID. */ - getTrace(traceId: string): Promise; - getTrace(request: GetTracePageArgs): Promise; - getTrace(request: string | GetTracePageArgs): Promise { - return typeof request === 'string' ? this.observability.getTrace(request) : this.observability.getTrace(request); + getTrace(traceId: string): Promise { + return this.observability.getTrace(traceId); + } + + /** Retrieves a bounded trace page from a supporting backend. */ + getTracePage(request: GetTracePageArgs): Promise { + return this.observability.getTracePage(request); } /** Retrieves a lightweight trace by ID (timeline fields only, excludes heavy fields). */ diff --git a/client-sdks/client-js/src/resources/observability-route-types.ts b/client-sdks/client-js/src/resources/observability-route-types.ts index 66e436f9a62..c6b7b31e0e8 100644 --- a/client-sdks/client-js/src/resources/observability-route-types.ts +++ b/client-sdks/client-js/src/resources/observability-route-types.ts @@ -13,9 +13,9 @@ type ListArgs = Simplify< } >; -export type TracePageRequest = Body<'POST /observability/traces/:traceId'>; +export type TracePageRequest = Body<'POST /observability/v2/traces/:traceId'>; export type GetTracePageArgs = TracePageRequest & { traceId: string }; -export type TracePageResponse = Response<'POST /observability/traces/:traceId'>; +export type TracePageResponse = Response<'POST /observability/v2/traces/:traceId'>; export type TracePageNode = TracePageResponse['nodes'][number]; export type TraceRecord = Response<'GET /observability/traces/:traceId'>; diff --git a/client-sdks/client-js/src/resources/observability.trace-page.test.ts b/client-sdks/client-js/src/resources/observability.trace-page.test.ts index b49c8f4aa30..bbc692e4b3b 100644 --- a/client-sdks/client-js/src/resources/observability.trace-page.test.ts +++ b/client-sdks/client-js/src/resources/observability.trace-page.test.ts @@ -5,13 +5,15 @@ import { InMemoryStore } from '@mastra/core/storage'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { MastraClient } from '../client'; -// Real HTTP transport and real storage: proves the overload does not serialize IDs or cursors as GET parameters. -describe('getTrace overload over HTTP', () => { +// Real HTTP transport and real storage: proves the paged API does not serialize IDs or cursors as GET parameters. +describe('legacy and paged trace reads over HTTP', () => { const storage = new InMemoryStore(); let client: MastraClient; const server = createServer(async (req, res) => { try { - expect(req.url).toBe('/api/observability/traces/trace'); + expect(req.url).toBe( + req.method === 'GET' ? '/api/observability/traces/trace' : '/api/observability/v2/traces/trace', + ); const store = (await storage.getStore('observability'))!; const chunks: Buffer[] = []; for await (const chunk of req) chunks.push(Buffer.from(chunk)); @@ -47,14 +49,14 @@ describe('getTrace overload over HTTP', () => { }); it('keeps legacy GET responses and reads bounded pages through POST', async () => { expect((await client.getTrace('trace')).spans.map(span => span.spanId)).toEqual(['a', 'b']); - const first = await client.getTrace({ version: 2, traceId: 'trace', limit: 1 }); + const first = await client.getTracePage({ version: 2, traceId: 'trace', limit: 1 }); expect(first.nodes.map(node => node.spanId)).toEqual(['a']); - const next = await client.getTrace({ version: 2, traceId: 'trace', cursor: first.nextCursor! }); + const next = await client.getTracePage({ version: 2, traceId: 'trace', cursor: first.nextCursor! }); expect(next.nodes.map(node => node.spanId)).toEqual(['b']); expect(next.nextCursor).toBeUndefined(); }); it('hydrates exactly the IDs absent from the caller cache', async () => { - const page = await client.getTrace({ version: 2, traceId: 'trace', spanIds: ['b'], include: 'full' }); + const page = await client.getTracePage({ version: 2, traceId: 'trace', spanIds: ['b'], include: 'full' }); expect(page.nodes).toHaveLength(1); expect(page.nodes[0]!.details?.input).toEqual({ value: 'b' }); }); diff --git a/client-sdks/client-js/src/resources/observability.ts b/client-sdks/client-js/src/resources/observability.ts index 30955a14200..ceb3902fbe2 100644 --- a/client-sdks/client-js/src/resources/observability.ts +++ b/client-sdks/client-js/src/resources/observability.ts @@ -231,12 +231,14 @@ export class Observability extends BaseResource { * @param traceId - ID of the trace to retrieve * @returns Promise containing the trace with all its spans */ - getTrace(traceId: string): Promise; - getTrace(request: GetTracePageArgs): Promise; - getTrace(request: string | GetTracePageArgs): Promise { - if (typeof request === 'string') return this.request(`/observability/traces/${encodeURIComponent(request)}`); + getTrace(traceId: string): Promise { + return this.request(`/observability/traces/${encodeURIComponent(traceId)}`); + } + + /** Retrieves a bounded trace page from a supporting backend. */ + getTracePage(request: GetTracePageArgs): Promise { const { traceId, ...body } = request; - return this.request(`/observability/traces/${encodeURIComponent(traceId)}`, { method: 'POST', body }); + return this.request(`/observability/v2/traces/${encodeURIComponent(traceId)}`, { method: 'POST', body }); } /** diff --git a/client-sdks/client-js/src/route-types.generated.ts b/client-sdks/client-js/src/route-types.generated.ts index 6a252cb0f7d..67871e60d67 100644 --- a/client-sdks/client-js/src/route-types.generated.ts +++ b/client-sdks/client-js/src/route-types.generated.ts @@ -10520,11 +10520,11 @@ export interface GetObservabilityTracesTraceId_RouteContract { } // ============================================================================ -// Route: POST /observability/traces/:traceId +// Route: POST /observability/v2/traces/:traceId // ============================================================================ -export type PostObservabilityTracesTraceId_PathParams = GetObservabilityTracesTraceId_PathParams; +export type PostObservabilityV2TracesTraceId_PathParams = GetObservabilityTracesTraceId_PathParams; -export type PostObservabilityTracesTraceId_Body = +export type PostObservabilityV2TracesTraceId_Body = | { version: 2; anchorSpanId?: string | undefined; @@ -10541,7 +10541,7 @@ export type PostObservabilityTracesTraceId_Body = cursor: string; }; -export type PostObservabilityTracesTraceId_Response = { +export type PostObservabilityV2TracesTraceId_Response = { version: 2; traceId: string; nodes: { @@ -10574,24 +10574,24 @@ export type PostObservabilityTracesTraceId_Response = { ancestorPaths: 'not-requested' | 'continued' | 'complete' | 'unavailable'; }; -export type PostObservabilityTracesTraceId_Request = Simplify< - (PostObservabilityTracesTraceId_PathParams extends never +export type PostObservabilityV2TracesTraceId_Request = Simplify< + (PostObservabilityV2TracesTraceId_PathParams extends never ? {} - : { params: PostObservabilityTracesTraceId_PathParams }) & + : { params: PostObservabilityV2TracesTraceId_PathParams }) & (never extends never ? {} : {} extends never ? { query?: never } : { query: never }) & - (PostObservabilityTracesTraceId_Body extends never + (PostObservabilityV2TracesTraceId_Body extends never ? {} - : {} extends PostObservabilityTracesTraceId_Body - ? { body?: PostObservabilityTracesTraceId_Body } - : { body: PostObservabilityTracesTraceId_Body }) + : {} extends PostObservabilityV2TracesTraceId_Body + ? { body?: PostObservabilityV2TracesTraceId_Body } + : { body: PostObservabilityV2TracesTraceId_Body }) >; -export interface PostObservabilityTracesTraceId_RouteContract { - pathParams: PostObservabilityTracesTraceId_PathParams; +export interface PostObservabilityV2TracesTraceId_RouteContract { + pathParams: PostObservabilityV2TracesTraceId_PathParams; queryParams: never; - body: PostObservabilityTracesTraceId_Body; - request: PostObservabilityTracesTraceId_Request; - response: PostObservabilityTracesTraceId_Response; + body: PostObservabilityV2TracesTraceId_Body; + request: PostObservabilityV2TracesTraceId_Request; + response: PostObservabilityV2TracesTraceId_Response; responseType: 'json'; } @@ -25793,7 +25793,7 @@ export interface RouteTypes { 'GET /observability/branches': GetObservabilityBranches_RouteContract; 'GET /observability/traces/:traceId/branches/:spanId': GetObservabilityTracesTraceIdBranchesSpanId_RouteContract; 'GET /observability/traces/:traceId': GetObservabilityTracesTraceId_RouteContract; - 'POST /observability/traces/:traceId': PostObservabilityTracesTraceId_RouteContract; + 'POST /observability/v2/traces/:traceId': PostObservabilityV2TracesTraceId_RouteContract; 'GET /observability/traces/:traceId/light': GetObservabilityTracesTraceIdLight_RouteContract; 'GET /observability/traces/:traceId/spans/:spanId': GetObservabilityTracesTraceIdSpansSpanId_RouteContract; 'GET /observability/traces/:traceId/trajectory': GetObservabilityTracesTraceIdTrajectory_RouteContract; @@ -26861,7 +26861,6 @@ export interface Client { }; '/observability/traces/:traceId': { GET: GetObservabilityTracesTraceId_RouteContract; - POST: PostObservabilityTracesTraceId_RouteContract; }; '/observability/traces/:traceId/:spanId/scores': { GET: GetObservabilityTracesTraceIdSpanIdScores_RouteContract; @@ -26899,6 +26898,9 @@ export interface Client { '/observability/traces/score': { POST: PostObservabilityTracesScore_RouteContract; }; + '/observability/v2/traces/:traceId': { + POST: PostObservabilityV2TracesTraceId_RouteContract; + }; '/processor-providers': { GET: GetProcessorProviders_RouteContract; }; diff --git a/packages/cli/src/commands/api/route-metadata.generated.ts b/packages/cli/src/commands/api/route-metadata.generated.ts index d068750e29f..c1004aa9c9b 100644 --- a/packages/cli/src/commands/api/route-metadata.generated.ts +++ b/packages/cli/src/commands/api/route-metadata.generated.ts @@ -2008,9 +2008,9 @@ export const API_ROUTE_METADATA = { "listProperty": "spans" } }, - "POST /observability/traces/:traceId": { + "POST /observability/v2/traces/:traceId": { "method": "POST", - "path": "/observability/traces/:traceId", + "path": "/observability/v2/traces/:traceId", "pathParams": [ "traceId" ], diff --git a/packages/server/src/server/handlers/observability.ts b/packages/server/src/server/handlers/observability.ts index 2b75066b4ee..aa4e643bacb 100644 --- a/packages/server/src/server/handlers/observability.ts +++ b/packages/server/src/server/handlers/observability.ts @@ -281,10 +281,10 @@ export const GET_BRANCH_ROUTE = createRoute({ } }, }); -/** Same trace resource, explicit bounded contract. Namespace guards preserve older core compatibility. */ +/** Versioned bounded trace contract. Namespace guards preserve older core compatibility. */ export const GET_TRACE_PAGE_ROUTE: ServerRoute = createRoute({ method: 'POST', - path: '/observability/traces/:traceId', + path: '/observability/v2/traces/:traceId', responseType: 'json', pathParamSchema: getTraceArgsSchema, bodySchema: tracePageCore.tracePageRequestSchema ?? z.unknown().optional(), diff --git a/packages/server/src/server/handlers/trace-page.test.ts b/packages/server/src/server/handlers/trace-page.test.ts index 798a32064b7..03e953f274d 100644 --- a/packages/server/src/server/handlers/trace-page.test.ts +++ b/packages/server/src/server/handlers/trace-page.test.ts @@ -1,7 +1,9 @@ import { Mastra } from '@mastra/core'; import { SpanType } from '@mastra/core/observability'; import { InMemoryStore } from '@mastra/core/storage'; +import { Hono } from 'hono'; import { describe, expect, it } from 'vitest'; +import { OBSERVABILITY_ROUTES } from '../server-adapter/routes/observability'; import { GET_TRACE_PAGE_ROUTE, GET_TRACE_ROUTE } from './observability'; import { getObservabilityStorageCapabilities } from './observability-shared'; import { createTestServerContext } from './test-utils'; @@ -72,3 +74,18 @@ describe('versioned trace route with real storage', () => { }); }); }); + +// Dispatch the registered routes in their real order: the paged route must not +// capture the existing static query/aggregate POSTs (OBS-956). +it('keeps trace queries and aggregates separate from versioned trace IDs', async () => { + const app = new Hono(); + for (const route of OBSERVABILITY_ROUTES) { + if (route.method === 'POST') app.post(route.path, c => c.text(route.path)); + } + for (const name of ['query', 'aggregate']) { + const legacy = await app.request(`/observability/traces/${name}`, { method: 'POST' }); + expect(await legacy.text()).toBe(`/observability/traces/${name}`); + const page = await app.request(`/observability/v2/traces/${name}`, { method: 'POST' }); + expect(await page.text()).toBe('/observability/v2/traces/:traceId'); + } +}); From caf8fbee621254b92aaf296f36756f7c7174c611 Mon Sep 17 00:00:00 2001 From: Eric Pinzur Date: Mon, 16 Nov 2026 14:22:47 +0100 Subject: [PATCH 05/23] chore: remove trace navigation docs and trim changesets --- .changeset/eager-meteors-wear.md | 6 +-- .changeset/four-foxes-pick.md | 2 +- .changeset/pink-eagles-repair.md | 2 +- .changeset/whole-streets-flow.md | 2 +- .../en/reference/client-js/observability.mdx | 38 ------------------- 5 files changed, 4 insertions(+), 46 deletions(-) diff --git a/.changeset/eager-meteors-wear.md b/.changeset/eager-meteors-wear.md index 2ab146cc6b8..faf1d41e711 100644 --- a/.changeset/eager-meteors-wear.md +++ b/.changeset/eager-meteors-wear.md @@ -2,8 +2,4 @@ '@mastra/client-js': minor --- -Add bounded trace browsing with optional span details: - -```typescript -await client.getTrace({ version: 2, traceId, depth: 2, limit: 100, include: 'full' }); -``` +Add getTracePage() for supporting backends. diff --git a/.changeset/four-foxes-pick.md b/.changeset/four-foxes-pick.md index af22b27d457..9f168506c8d 100644 --- a/.changeset/four-foxes-pick.md +++ b/.changeset/four-foxes-pick.md @@ -2,4 +2,4 @@ '@mastra/core': minor --- -Add bounded trace browsing and explicit span hydration through the version 2 trace contract. +Add shared contracts for bounded trace reads. diff --git a/.changeset/pink-eagles-repair.md b/.changeset/pink-eagles-repair.md index a67802bc5c0..9351ec4c944 100644 --- a/.changeset/pink-eagles-repair.md +++ b/.changeset/pink-eagles-repair.md @@ -2,4 +2,4 @@ 'mastra': patch --- -Add the bounded trace read endpoint to CLI API route metadata. +Update trace API route metadata. diff --git a/.changeset/whole-streets-flow.md b/.changeset/whole-streets-flow.md index 2cac5d10d2d..c636153adb0 100644 --- a/.changeset/whole-streets-flow.md +++ b/.changeset/whole-streets-flow.md @@ -2,4 +2,4 @@ '@mastra/server': minor --- -Add a version 2 trace read endpoint with bounded pages and optional span details. +Add a versioned route for bounded trace reads on supporting backends. diff --git a/docs/src/content/en/reference/client-js/observability.mdx b/docs/src/content/en/reference/client-js/observability.mdx index 2e1d39a715c..e2d9ab0ce00 100644 --- a/docs/src/content/en/reference/client-js/observability.mdx +++ b/docs/src/content/en/reference/client-js/observability.mdx @@ -17,44 +17,6 @@ Retrieve a specific trace by its ID, including all its spans and details: const trace = await mastraClient.getTrace('trace-id-123') ``` -### Read a bounded trace page - -Use the object overload when the server advertises `capabilities.traceNavigation`: - -```typescript -const page = await mastraClient.getTrace({ - version: 2, - traceId: 'trace-id-123', - depth: 2, - limit: 100, - include: 'full', -}) - -if (page.nextCursor) { - const nextPage = await mastraClient.getTrace({ - version: 2, - traceId: 'trace-id-123', - cursor: page.nextCursor, - }) -} -``` - -The response contains `nodes`, `missingSpanIds`, `ancestorPaths`, and an optional `nextCursor`. Each node includes its `childCount` and `parentState`. Explicit-ID requests leave child counts unknown (`null`) and parent context unrequested (`'not-requested'`). Full span payloads appear in `node.details` when `include` is `'full'`. The default is `'structure'`. - -| Option | Behavior | -| - | - | -| `anchorSpanId` | Browse a branch, including its anchor. Omit to start at the trace roots. | -| `depth` | Descendant levels below the roots or anchor. Defaults to 2. Accepts 0 to 32. | -| `limit` | Total nodes per page, including ancestors. Defaults to 100. Accepts 1 to 200. | -| `maxBytes` | Maximum serialized response bytes. Defaults to 1 MiB. Accepts 64 KiB to 4 MiB. | -| `spanIds` | Fetch up to 100 explicit IDs in request order, without expanding the tree. Can't be combined with browse or search options. | -| `search` | Find matching spans throughout the trace when `capabilities.traceNavigationSearch` is available. Matching rules depend on the server. | -| `ancestors` | Include ancestor paths for search results. Defaults to `true`. | - -Browsing visits each level before descending. Search results distinguish matches through `node.match`. Contextual ancestors also count toward `limit`. `ancestorPaths` indicates whether search paths are complete, continued on another page, or contain unavailable parents. - -Pass only `version`, `traceId`, and `cursor` to continue. Restart the selection after a `409 TRACE_CHANGED` response. A payload that can't fit by itself returns `413 PAYLOAD_TOO_LARGE`. Request structure or retrieve that span with `getSpan()`. Reuse cached immutable details and request only missing IDs with `spanIds`. - ## Getting traces with filtering Retrieve a paginated list of trace root spans with optional filtering: From dccb714a48dff0bde2f0f176d5895b004667669d Mon Sep 17 00:00:00 2001 From: Eric Pinzur Date: Mon, 16 Nov 2026 14:28:35 +0100 Subject: [PATCH 06/23] fix(observability): remove redundant trace request version --- .../observability.trace-page.test.ts | 6 +-- .../client-js/src/route-types.generated.ts | 2 - .../commands/api/route-metadata.generated.ts | 3 +- .../domains/observability/trace-page.test.ts | 42 +++++++++---------- .../domains/observability/trace-page.ts | 5 +-- ...ervability-text-search-core-compat.test.ts | 2 +- .../src/server/handlers/trace-page.test.ts | 12 +++--- 7 files changed, 32 insertions(+), 40 deletions(-) diff --git a/client-sdks/client-js/src/resources/observability.trace-page.test.ts b/client-sdks/client-js/src/resources/observability.trace-page.test.ts index bbc692e4b3b..8a12f10c007 100644 --- a/client-sdks/client-js/src/resources/observability.trace-page.test.ts +++ b/client-sdks/client-js/src/resources/observability.trace-page.test.ts @@ -49,14 +49,14 @@ describe('legacy and paged trace reads over HTTP', () => { }); it('keeps legacy GET responses and reads bounded pages through POST', async () => { expect((await client.getTrace('trace')).spans.map(span => span.spanId)).toEqual(['a', 'b']); - const first = await client.getTracePage({ version: 2, traceId: 'trace', limit: 1 }); + const first = await client.getTracePage({ traceId: 'trace', limit: 1 }); expect(first.nodes.map(node => node.spanId)).toEqual(['a']); - const next = await client.getTracePage({ version: 2, traceId: 'trace', cursor: first.nextCursor! }); + const next = await client.getTracePage({ traceId: 'trace', cursor: first.nextCursor! }); expect(next.nodes.map(node => node.spanId)).toEqual(['b']); expect(next.nextCursor).toBeUndefined(); }); it('hydrates exactly the IDs absent from the caller cache', async () => { - const page = await client.getTracePage({ version: 2, traceId: 'trace', spanIds: ['b'], include: 'full' }); + const page = await client.getTracePage({ traceId: 'trace', spanIds: ['b'], include: 'full' }); expect(page.nodes).toHaveLength(1); expect(page.nodes[0]!.details?.input).toEqual({ value: 'b' }); }); diff --git a/client-sdks/client-js/src/route-types.generated.ts b/client-sdks/client-js/src/route-types.generated.ts index 67871e60d67..b8d81e56ff9 100644 --- a/client-sdks/client-js/src/route-types.generated.ts +++ b/client-sdks/client-js/src/route-types.generated.ts @@ -10526,7 +10526,6 @@ export type PostObservabilityV2TracesTraceId_PathParams = GetObservabilityTraces export type PostObservabilityV2TracesTraceId_Body = | { - version: 2; anchorSpanId?: string | undefined; depth?: number | undefined; spanIds?: string[] | undefined; @@ -10537,7 +10536,6 @@ export type PostObservabilityV2TracesTraceId_Body = maxBytes?: number; } | { - version: 2; cursor: string; }; diff --git a/packages/cli/src/commands/api/route-metadata.generated.ts b/packages/cli/src/commands/api/route-metadata.generated.ts index c1004aa9c9b..e7f4635381b 100644 --- a/packages/cli/src/commands/api/route-metadata.generated.ts +++ b/packages/cli/src/commands/api/route-metadata.generated.ts @@ -2024,8 +2024,7 @@ export const API_ROUTE_METADATA = { "limit", "maxBytes", "search", - "spanIds", - "version" + "spanIds" ], "hasQuery": false, "hasBody": true, diff --git a/packages/core/src/storage/domains/observability/trace-page.test.ts b/packages/core/src/storage/domains/observability/trace-page.test.ts index 8011bdacbbf..e892063b3cc 100644 --- a/packages/core/src/storage/domains/observability/trace-page.test.ts +++ b/packages/core/src/storage/domains/observability/trace-page.test.ts @@ -26,7 +26,7 @@ async function drain(src: TracePageSource, first: TracePageResponse): Promise 100) throw new Error('Continuation failed to progress'); - pages.push((await readTracePage(src, 'trace', { version: 2, cursor: pages.at(-1)!.nextCursor! }))!); + pages.push((await readTracePage(src, 'trace', { cursor: pages.at(-1)!.nextCursor! }))!); } return pages; } @@ -34,7 +34,7 @@ async function drain(src: TracePageSource, first: TracePageResponse): Promise { it('loads a shallow breadth-first tree with deterministic sibling ordering and discoverable children', async () => { const src = source([span('z', 'root'), span('root'), span('deep', 'a'), span('a', 'root')]); - const pages = await drain(src, (await readTracePage(src, 'trace', { version: 2, depth: 1, limit: 2 }))!); + const pages = await drain(src, (await readTracePage(src, 'trace', { depth: 1, limit: 2 }))!); expect(pages.flatMap(page => page.nodes.map(node => node.spanId))).toEqual(['root', 'a', 'z']); expect(pages[0]!.nodes[1]).toMatchObject({ childCount: 1 }); expect(pages.at(-1)!.nextCursor).toBeUndefined(); @@ -45,7 +45,7 @@ describe('bounded trace exploration', () => { [span('root'), span('a', 'root'), span('b', 'a'), span('match', 'b'), span('other', 'root')], ['match'], ); - const pages = await drain(src, (await readTracePage(src, 'trace', { version: 2, search: 'timeout', limit: 1 }))!); + const pages = await drain(src, (await readTracePage(src, 'trace', { search: 'timeout', limit: 1 }))!); expect(pages.flatMap(page => page.nodes.map(node => [node.spanId, node.match]))).toEqual([ ['root', false], ['a', false], @@ -57,7 +57,7 @@ describe('bounded trace exploration', () => { it('deduplicates shared ancestors and distinguishes an unavailable parent from a root', async () => { const src = source([span('root', 'missing'), span('a', 'root'), span('b', 'root')], ['a', 'b']); - const pages = await drain(src, (await readTracePage(src, 'trace', { version: 2, search: 'error', limit: 2 }))!); + const pages = await drain(src, (await readTracePage(src, 'trace', { search: 'error', limit: 2 }))!); expect(pages.flatMap(page => page.nodes.map(node => node.spanId))).toEqual(['root', 'a', 'b']); expect(pages[0]!.nodes[0]!.parentState).toBe('unavailable'); expect(pages.at(-1)!.ancestorPaths).toBe('unavailable'); @@ -72,7 +72,6 @@ describe('bounded trace exploration', () => { const pages = await drain( src, (await readTracePage(src, 'trace', { - version: 2, spanIds: ['b', 'absent', 'a', 'b'], include: 'full', limit: 1, @@ -86,20 +85,17 @@ describe('bounded trace exploration', () => { it('continues at the first unreturned payload when the byte budget fills', async () => { const src = source([span('a', null, 'x'.repeat(40000)), span('b', null, 'y'.repeat(40000))]); - const pages = await drain( - src, - (await readTracePage(src, 'trace', { version: 2, include: 'full', maxBytes: 65536 }))!, - ); + const pages = await drain(src, (await readTracePage(src, 'trace', { include: 'full', maxBytes: 65536 }))!); expect(pages.map(page => page.nodes.map(node => node.spanId))).toEqual([['a'], ['b']]); for (const page of pages) expect(Buffer.byteLength(JSON.stringify(page))).toBeLessThanOrEqual(65536); }); it('gives an actionable error for one oversized payload rather than a cursor that never advances', async () => { const src = source([span('a', null, 'x'.repeat(70000))]); - await expect(readTracePage(src, 'trace', { version: 2, include: 'full', maxBytes: 65536 })).rejects.toMatchObject({ + await expect(readTracePage(src, 'trace', { include: 'full', maxBytes: 65536 })).rejects.toMatchObject({ code: 'PAYLOAD_TOO_LARGE', }); - const structure = await readTracePage(src, 'trace', { version: 2, maxBytes: 65536 }); + const structure = await readTracePage(src, 'trace', { maxBytes: 65536 }); expect(structure!.nodes[0]!.details).toBeUndefined(); expect(JSON.stringify(structure)).not.toContain('xxxxx'); }); @@ -107,8 +103,8 @@ describe('bounded trace exploration', () => { it('rejects scope changes and late arrivals even when root activity is unchanged', async () => { const rows = [span('a'), span('b')]; const src = source(rows); - const first = (await readTracePage(src, 'trace', { version: 2, limit: 1 }))!; - const request = { version: 2 as const, cursor: first.nextCursor! }; + const first = (await readTracePage(src, 'trace', { limit: 1 }))!; + const request = { cursor: first.nextCursor! }; await expect(readTracePage({ ...src, scopeKey: 'another/org' }, 'trace', request)).rejects.toMatchObject({ code: 'TRACE_CHANGED', }); @@ -117,22 +113,22 @@ describe('bounded trace exploration', () => { }); it('rejects cyclic topology and malformed or conflicting continuation requests', async () => { - await expect( - readTracePage(source([span('a', 'b'), span('b', 'a')]), 'trace', { version: 2 }), - ).rejects.toMatchObject({ code: 'INVALID_TOPOLOGY' }); - await expect(readTracePage(source([]), 'trace', { version: 2, cursor: 'bad' })).rejects.toMatchObject({ + await expect(readTracePage(source([span('a', 'b'), span('b', 'a')]), 'trace', {})).rejects.toMatchObject({ + code: 'INVALID_TOPOLOGY', + }); + await expect(readTracePage(source([]), 'trace', { cursor: 'bad' })).rejects.toMatchObject({ code: 'INVALID_CURSOR', }); - expect(tracePageRequestSchema.safeParse({ version: 2, cursor: 'cursor', include: 'full' }).success).toBe(false); - expect(tracePageRequestSchema.safeParse({ version: 2, spanIds: ['a'], depth: 2 }).success).toBe(false); - expect(tracePageRequestSchema.safeParse({ version: 2, search: 'foo', depth: 2 }).success).toBe(false); + expect(tracePageRequestSchema.safeParse({ cursor: 'cursor', include: 'full' }).success).toBe(false); + expect(tracePageRequestSchema.safeParse({ spanIds: ['a'], depth: 2 }).success).toBe(false); + expect(tracePageRequestSchema.safeParse({ search: 'foo', depth: 2 }).success).toBe(false); }); it('can continue a maximum-size batch of multibyte identifiers', async () => { const ids = Array.from({ length: 100 }, (_, i) => '界'.repeat(125) + String(i)); const src = source(ids.map(id => span(id))); - const first = (await readTracePage(src, 'trace', { version: 2, spanIds: ids, limit: 1 }))!; - expect(Buffer.byteLength(JSON.stringify({ version: 2, cursor: first.nextCursor }))).toBeLessThan(65536); - const second = await readTracePage(src, 'trace', { version: 2, cursor: first.nextCursor! }); + const first = (await readTracePage(src, 'trace', { spanIds: ids, limit: 1 }))!; + expect(Buffer.byteLength(JSON.stringify({ cursor: first.nextCursor }))).toBeLessThan(65536); + const second = await readTracePage(src, 'trace', { cursor: first.nextCursor! }); expect(second!.nodes[0]!.spanId).toBe(ids[1]); }); }); diff --git a/packages/core/src/storage/domains/observability/trace-page.ts b/packages/core/src/storage/domains/observability/trace-page.ts index 85931c74ff4..361b84b5141 100644 --- a/packages/core/src/storage/domains/observability/trace-page.ts +++ b/packages/core/src/storage/domains/observability/trace-page.ts @@ -11,7 +11,6 @@ const id = z .refine(value => Buffer.byteLength(JSON.stringify(value)) <= 384, 'Identifier exceeds the serialized byte limit'); const selectionSchema = z .object({ - version: z.literal(2), anchorSpanId: id.optional(), depth: z.number().int().min(0).max(32).optional(), spanIds: z.array(id).min(1).max(100).optional(), @@ -23,7 +22,7 @@ const selectionSchema = z }) .strict(); -/** POST body for the explicitly versioned getTrace overload. Continuations cannot change selection. */ +/** POST body for the versioned getTracePage endpoint. Continuations cannot change selection. */ export const tracePageRequestSchema = z.union([ selectionSchema.superRefine((value, ctx) => { if ( @@ -36,7 +35,7 @@ export const tracePageRequestSchema = z.union([ ctx.addIssue({ code: 'custom', message: 'Search covers the entire trace; anchorSpanId is browse-only' }); } }), - z.object({ version: z.literal(2), cursor: z.string().min(1).max(60000) }).strict(), + z.object({ cursor: z.string().min(1).max(60000) }).strict(), ]); export type TracePageRequest = z.input; export type GetTracePageArgs = TracePageRequest & { traceId: string }; diff --git a/packages/server/src/server/handlers/observability-text-search-core-compat.test.ts b/packages/server/src/server/handlers/observability-text-search-core-compat.test.ts index 792d123f51e..7994c1a8370 100644 --- a/packages/server/src/server/handlers/observability-text-search-core-compat.test.ts +++ b/packages/server/src/server/handlers/observability-text-search-core-compat.test.ts @@ -63,7 +63,7 @@ it('keeps legacy trace reads available when the installed Core predates bounded traceNavigation: false, traceNavigationSearch: false, }); - const body = await parseRouteBody(GET_TRACE_PAGE_ROUTE, { version: 2 }); + const body = await parseRouteBody(GET_TRACE_PAGE_ROUTE, {}); await expect( GET_TRACE_PAGE_ROUTE.handler({ ...context, traceId: 'absent', ...(body as object) }), ).rejects.toMatchObject({ diff --git a/packages/server/src/server/handlers/trace-page.test.ts b/packages/server/src/server/handlers/trace-page.test.ts index 03e953f274d..6606ae8bbf6 100644 --- a/packages/server/src/server/handlers/trace-page.test.ts +++ b/packages/server/src/server/handlers/trace-page.test.ts @@ -38,7 +38,7 @@ describe('versioned trace route with real storage', () => { const first = await GET_TRACE_PAGE_ROUTE.handler({ ...context, traceId: 'trace', - version: 2, + depth: 2, limit: 1, include: 'full', @@ -48,7 +48,7 @@ describe('versioned trace route with real storage', () => { const next = await GET_TRACE_PAGE_ROUTE.handler({ ...context, traceId: 'trace', - version: 2, + cursor: first.nextCursor, }); expect(next.nodes[0]).toMatchObject({ spanId: 'child', details: { input: { payload: 'child' } } }); @@ -57,19 +57,19 @@ describe('versioned trace route with real storage', () => { it('reports unsupported indexed search instead of scanning payloads in memory', async () => { const { context } = await setup(); await expect( - GET_TRACE_PAGE_ROUTE.handler({ ...context, traceId: 'trace', version: 2, search: 'payload' }), + GET_TRACE_PAGE_ROUTE.handler({ ...context, traceId: 'trace', search: 'payload' }), ).rejects.toMatchObject({ status: 501 }); }); it('does not reuse cursors across authenticated users or silently widen tenant scope', async () => { const { context } = await setup(); context.requestContext.set('mastra__user', { id: 'alice' }); - const first = await GET_TRACE_PAGE_ROUTE.handler({ ...context, traceId: 'trace', version: 2, limit: 1 }); + const first = await GET_TRACE_PAGE_ROUTE.handler({ ...context, traceId: 'trace', limit: 1 }); context.requestContext.set('mastra__user', { id: 'bob' }); await expect( - GET_TRACE_PAGE_ROUTE.handler({ ...context, traceId: 'trace', version: 2, cursor: first.nextCursor }), + GET_TRACE_PAGE_ROUTE.handler({ ...context, traceId: 'trace', cursor: first.nextCursor }), ).rejects.toMatchObject({ status: 409 }); context.requestContext.set('organizationId', 'other-org'); - await expect(GET_TRACE_PAGE_ROUTE.handler({ ...context, traceId: 'trace', version: 2 })).rejects.toMatchObject({ + await expect(GET_TRACE_PAGE_ROUTE.handler({ ...context, traceId: 'trace' })).rejects.toMatchObject({ status: 501, }); }); From 0123f28fb8aca792a374354ff3c19fb72993ccf1 Mon Sep 17 00:00:00 2001 From: Eric Pinzur Date: Mon, 16 Nov 2026 15:32:32 +0100 Subject: [PATCH 07/23] fix(observability): bound trace reads to selected pages Require explicit backend support, separate topology from page projection, and pre-page requested IDs. Restrict continuation invalidation to selected context and align identifier validation, tenant capabilities, and error mapping. --- .../observability.trace-page.test.ts | 12 +- .../client-js/src/route-types.generated.ts | 76 ++--- .../src/storage/domains/observability/base.ts | 27 +- .../domains/observability/trace-page.test.ts | 64 ++++- .../domains/observability/trace-page.ts | 269 ++++++++++-------- .../server/handlers/observability-shared.ts | 2 + .../src/server/handlers/observability.ts | 16 +- packages/server/src/server/handlers/system.ts | 6 +- .../src/server/handlers/trace-page.test.ts | 90 +++++- .../server/src/server/mcp/capabilities.ts | 16 +- packages/server/src/server/mcp/catalogue.ts | 4 +- packages/server/src/server/schemas/system.ts | 2 +- 12 files changed, 392 insertions(+), 192 deletions(-) diff --git a/client-sdks/client-js/src/resources/observability.trace-page.test.ts b/client-sdks/client-js/src/resources/observability.trace-page.test.ts index 8a12f10c007..7fdc68ced5c 100644 --- a/client-sdks/client-js/src/resources/observability.trace-page.test.ts +++ b/client-sdks/client-js/src/resources/observability.trace-page.test.ts @@ -1,7 +1,7 @@ import { createServer } from 'node:http'; import type { AddressInfo } from 'node:net'; import { SpanType } from '@mastra/core/observability'; -import { InMemoryStore } from '@mastra/core/storage'; +import { InMemoryStore, readTracePage } from '@mastra/core/storage'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { MastraClient } from '../client'; @@ -20,7 +20,15 @@ describe('legacy and paged trace reads over HTTP', () => { const result = req.method === 'GET' ? await store.getTrace({ traceId: 'trace' }) - : await store.getTracePage({ traceId: 'trace', ...JSON.parse(Buffer.concat(chunks).toString()) }); + : await readTracePage( + { + scopeKey: 'test-backend', + structure: async () => (await store.getStructure({ traceId: 'trace' }))?.spans ?? null, + spans: async (_, spanIds) => (await store.getSpans({ traceId: 'trace', spanIds })).spans, + }, + 'trace', + JSON.parse(Buffer.concat(chunks).toString()), + ); res.setHeader('content-type', 'application/json'); res.end(JSON.stringify(result)); } catch (error) { diff --git a/client-sdks/client-js/src/route-types.generated.ts b/client-sdks/client-js/src/route-types.generated.ts index b8d81e56ff9..c25c38cfcd5 100644 --- a/client-sdks/client-js/src/route-types.generated.ts +++ b/client-sdks/client-js/src/route-types.generated.ts @@ -24,7 +24,7 @@ type InputShared_Auxiliary_116 = { parentWorkflow?: InputShared_Auxiliary_116 | undefined; }; -type InputShared_Auxiliary_210 = +type InputShared_Auxiliary_211 = | { op: 'eq' | 'ne' | 'lt' | 'lte' | 'gt' | 'gte'; left: @@ -73,41 +73,41 @@ type InputShared_Auxiliary_210 = } | { op: 'and' | 'or'; - args: InputShared_Auxiliary_210[]; + args: InputShared_Auxiliary_211[]; } | { op: 'not'; - arg: InputShared_Auxiliary_210; + arg: InputShared_Auxiliary_211; } | { spans: | { - some: InputShared_Auxiliary_233; + some: InputShared_Auxiliary_234; } | { - none: InputShared_Auxiliary_233; + none: InputShared_Auxiliary_234; }; } | { scores: | { - some: InputShared_Auxiliary_233; + some: InputShared_Auxiliary_234; } | { - none: InputShared_Auxiliary_233; + none: InputShared_Auxiliary_234; }; } | { feedback: | { - some: InputShared_Auxiliary_233; + some: InputShared_Auxiliary_234; } | { - none: InputShared_Auxiliary_233; + none: InputShared_Auxiliary_234; }; }; -type InputShared_Auxiliary_233 = +type InputShared_Auxiliary_234 = | { op: 'eq' | 'ne' | 'lt' | 'lte' | 'gt' | 'gte'; left: @@ -156,33 +156,33 @@ type InputShared_Auxiliary_233 = } | { op: 'and' | 'or'; - args: InputShared_Auxiliary_233[]; + args: InputShared_Auxiliary_234[]; } | { op: 'not'; - arg: InputShared_Auxiliary_233; + arg: InputShared_Auxiliary_234; }; -type InputShared_Auxiliary_280 = +type InputShared_Auxiliary_281 = | { op: 'and' | 'or'; - args: InputShared_Auxiliary_280[]; + args: InputShared_Auxiliary_281[]; } | { op: 'not'; - arg: InputShared_Auxiliary_280; + arg: InputShared_Auxiliary_281; } | { traces: | { - some: InputShared_Auxiliary_210; + some: InputShared_Auxiliary_211; } | { - none: InputShared_Auxiliary_210; + none: InputShared_Auxiliary_211; }; }; -type InputShared_Auxiliary_737 = +type InputShared_Auxiliary_738 = | { op: 'eq' | 'ne' | 'lt' | 'lte' | 'gt' | 'gte'; left: @@ -227,19 +227,19 @@ type InputShared_Auxiliary_737 = } | { op: 'and' | 'or'; - args: InputShared_Auxiliary_737[]; + args: InputShared_Auxiliary_738[]; } | { op: 'not'; - arg: InputShared_Auxiliary_737; + arg: InputShared_Auxiliary_738; }; -type InputShared_Auxiliary_811 = { +type InputShared_Auxiliary_812 = { id?: string | undefined; name: string; type: 'file' | 'folder'; content?: string | undefined; - children?: InputShared_Auxiliary_811[] | undefined; + children?: InputShared_Auxiliary_812[] | undefined; }; type Shared_Auxiliary_801 = { @@ -1849,7 +1849,7 @@ type InputShared_Type_89 = { } | undefined; steps: InputShared_Type_84[]; - predicates: InputShared_Auxiliary_737[]; + predicates: InputShared_Auxiliary_738[]; }; type InputShared_Type_90 = { @@ -1863,7 +1863,7 @@ type InputShared_Type_90 = { | undefined; step: InputShared_Type_84; loopType: 'dowhile' | 'dountil'; - predicate: InputShared_Auxiliary_737; + predicate: InputShared_Auxiliary_738; }; type InputShared_Type_91 = @@ -3897,7 +3897,7 @@ type Shared_Type_69 = { traceQuery: boolean; /** Supports explicit `rootSource: 'aggregated'` trace queries */ traceQueryAggregatedRoots: boolean; - /** Bounded version 2 getTrace browsing and hydration */ + /** Bounded getTracePage browsing and hydration */ traceNavigation: boolean; /** Indexed in-trace search with ancestor context */ traceNavigationSearch: boolean; @@ -10522,7 +10522,9 @@ export interface GetObservabilityTracesTraceId_RouteContract { // ============================================================================ // Route: POST /observability/v2/traces/:traceId // ============================================================================ -export type PostObservabilityV2TracesTraceId_PathParams = GetObservabilityTracesTraceId_PathParams; +export type PostObservabilityV2TracesTraceId_PathParams = { + traceId: string; +}; export type PostObservabilityV2TracesTraceId_Body = | { @@ -10907,7 +10909,7 @@ export type PostObservabilityTracesQuery_Body = { from: string; to: string; }; - where?: InputShared_Auxiliary_210 | undefined; + where?: InputShared_Auxiliary_211 | undefined; group?: | { by: ['threadId']; @@ -10997,11 +10999,11 @@ export type PostObservabilityTracesAggregate_Body = { from: string; to: string; }; - where?: InputShared_Auxiliary_210 | undefined; + where?: InputShared_Auxiliary_211 | undefined; groupBy?: string[]; interval?: ('1m' | '5m' | '15m' | '1h' | '1d') | undefined; measures: (InputShared_Type_16 | `countDistinct.${string}`)[]; - having?: InputShared_Auxiliary_233 | undefined; + having?: InputShared_Auxiliary_234 | undefined; orderBy?: { field: string; direction: 'asc' | 'desc'; @@ -11098,11 +11100,11 @@ export type PostObservabilitySpansAggregate_Body = { from: string; to: string; }; - where?: InputShared_Auxiliary_233 | undefined; + where?: InputShared_Auxiliary_234 | undefined; groupBy?: string[]; interval?: ('1m' | '5m' | '15m' | '1h' | '1d') | undefined; measures: (InputShared_Type_16 | `countDistinct.${string}`)[]; - having?: InputShared_Auxiliary_233 | undefined; + having?: InputShared_Auxiliary_234 | undefined; orderBy?: { field: string; direction: 'asc' | 'desc'; @@ -11150,7 +11152,7 @@ export type PostObservabilityThreadsQuery_Body = field: 'startedAt' | 'endedAt'; }; }; - where?: InputShared_Auxiliary_233 | undefined; + where?: InputShared_Auxiliary_234 | undefined; orderBy?: { field: 'startedAt' | 'endedAt'; direction: 'asc' | 'desc'; @@ -11166,9 +11168,9 @@ export type PostObservabilityThreadsQuery_Body = from: string; to: string; }; - where?: InputShared_Auxiliary_210 | undefined; + where?: InputShared_Auxiliary_211 | undefined; }; - where?: InputShared_Auxiliary_280 | undefined; + where?: InputShared_Auxiliary_281 | undefined; page?: { limit?: number; after?: (string | null) | undefined; @@ -11245,7 +11247,7 @@ export type PostObservabilitySpansQuery_Body = { from: string; to: string; }; - where?: InputShared_Auxiliary_233 | undefined; + where?: InputShared_Auxiliary_234 | undefined; textSearch?: string | undefined; orderBy?: { field: 'startedAt' | 'endedAt' | 'timeToFirstChunkMs' | 'timeToFirstTokenMs' | 'durationMs' | 'model' | 'provider'; @@ -20030,7 +20032,7 @@ export type PostStoredSkills_Body = { /** List of asset file paths */ assets?: string[] | undefined; /** Full file tree structure for the skill */ - files?: InputShared_Auxiliary_811[] | undefined; + files?: InputShared_Auxiliary_812[] | undefined; /** Additional metadata for the skill */ metadata?: | { @@ -20088,7 +20090,7 @@ export type PatchStoredSkillsStoredSkillId_Body = { /** List of asset file paths */ assets?: (string[] | undefined) | undefined; /** Full file tree structure for the skill */ - files?: (InputShared_Auxiliary_811[] | undefined) | undefined; + files?: (InputShared_Auxiliary_812[] | undefined) | undefined; /** Additional metadata for the skill */ metadata?: | ( diff --git a/packages/core/src/storage/domains/observability/base.ts b/packages/core/src/storage/domains/observability/base.ts index 06670e2349e..9dd93315a58 100644 --- a/packages/core/src/storage/domains/observability/base.ts +++ b/packages/core/src/storage/domains/observability/base.ts @@ -70,7 +70,7 @@ import type { SpanQueryResponse, TrustedSpanQueryPlan } from './span-query'; import type { ThreadQueryResponse, TrustedThreadRowsQueryPlan } from './thread-query'; import type { TraceAggregateResponse } from './trace-aggregate'; import type { TrustedTraceAggregatePlan } from './trace-aggregate-planner'; -import { readTracePage, TracePageError } from './trace-page'; +import { TracePageError } from './trace-page'; import type { GetTracePageArgs, TracePageResponse } from './trace-page'; import type { GetTraceQueryValuesResponse, @@ -319,30 +319,13 @@ export class ObservabilityStorage extends StorageDomain { }); } - /** Bounded navigation requires structural and batch reads; never falls back to full-trace payloads. */ + /** Backends must explicitly implement bounded reads; existing structure helpers do not imply support. */ supportsTraceNavigation(): boolean { - return ( - (this.getStructure !== ObservabilityStorage.prototype.getStructure || - this.getTraceLight !== ObservabilityStorage.prototype.getTraceLight) && - this.getSpans !== ObservabilityStorage.prototype.getSpans - ); + return false; } - async getTracePage( - { traceId, ...request }: GetTracePageArgs, - options?: { scopeKey: string }, - ): Promise { - if (!this.supportsTraceNavigation()) - throw new TracePageError('UNSUPPORTED', 'Store does not support bounded trace navigation'); - return readTracePage( - { - scopeKey: options?.scopeKey ?? this.constructor.name, - structure: async id => (await this.getStructure({ traceId: id }))?.spans ?? null, - spans: async (id, spanIds) => (await this.getSpans({ traceId: id, spanIds })).spans, - }, - traceId, - request, - ); + async getTracePage(_args: GetTracePageArgs, _options?: { scopeKey: string }): Promise { + throw new TracePageError('UNSUPPORTED', 'Store does not support bounded trace navigation'); } /** Retrieves a single trace with all its associated spans. */ diff --git a/packages/core/src/storage/domains/observability/trace-page.test.ts b/packages/core/src/storage/domains/observability/trace-page.test.ts index e892063b3cc..8810e8f3f5c 100644 --- a/packages/core/src/storage/domains/observability/trace-page.test.ts +++ b/packages/core/src/storage/domains/observability/trace-page.test.ts @@ -78,8 +78,10 @@ describe('bounded trace exploration', () => { }))!, ); expect(pages.flatMap(page => page.nodes.map(node => node.spanId))).toEqual(['b', 'a']); - expect(pages[0]!.missingSpanIds).toEqual(['absent']); - expect(pages[1]!.nodes[0]!.details?.input).toEqual({ answer: 42 }); + expect(pages[0]!.missingSpanIds).toEqual([]); + expect(pages[1]!.missingSpanIds).toEqual(['absent']); + expect(pages[1]!.nodes).toEqual([]); + expect(pages[2]!.nodes[0]!.details?.input).toEqual({ answer: 42 }); expect(pages[0]!.nodes[0]!.details?.spanId).toBe('b'); }); @@ -132,3 +134,61 @@ describe('bounded trace exploration', () => { expect(second!.nodes[0]!.spanId).toBe(ids[1]); }); }); + +it('reads each explicit ID only on its own page and survives unrelated arrivals', async () => { + const rows = Array.from({ length: 100 }, (_, i) => span(String(i), null, 'large payload')); + let reads = 0; + const src = source(rows); + src.spans = async (_, ids, projection) => { + expect(ids).toHaveLength(1); + expect(projection).toBe('full'); + reads += ids.length; + return rows.filter(row => ids.includes(row.spanId)); + }; + const first = (await readTracePage(src, 'trace', { + spanIds: rows.map(row => row.spanId), + limit: 1, + include: 'full', + }))!; + rows.push(span('unrelated')); + const pages = await drain(src, first); + expect(reads).toBe(100); + expect(pages.flatMap(page => page.nodes)).toHaveLength(100); +}); + +it('continues an anchored subtree through unrelated arrivals but rejects relevant changes', async () => { + const rows = [span('root'), span('a', 'root'), span('b', 'a'), span('other', 'root')]; + const src = source(rows); + const first = (await readTracePage(src, 'trace', { anchorSpanId: 'a', limit: 1 }))!; + rows.push(span('unrelated', 'other')); + expect((await readTracePage(src, 'trace', { cursor: first.nextCursor! }))!.nodes[0]!.spanId).toBe('b'); + rows.push(span('new-child', 'a')); + await expect(readTracePage(src, 'trace', { cursor: first.nextCursor! })).rejects.toMatchObject({ + code: 'TRACE_CHANGED', + }); +}); + +it('requests only topology columns then page summaries for structure browsing', async () => { + const rows = [span('a'), span('b')]; + const src = source(rows); + src.structure = async (_, limit) => { + expect(limit).toBe(50001); + return rows.map(({ traceId, spanId, parentSpanId, startedAt }) => ({ traceId, spanId, parentSpanId, startedAt })); + }; + src.spans = async (_, ids, include) => { + expect(ids).toEqual(['a']); + expect(include).toBe('structure'); + return rows.filter(row => ids.includes(row.spanId)); + }; + expect((await readTracePage(src, 'trace', { limit: 1 }))!.nodes[0]!.name).toBe('a'); +}); + +it('continues explicit IDs across a response byte boundary without skipping unread payloads', async () => { + const src = source([span('a', null, 'x'.repeat(40000)), span('b', null, 'y'.repeat(40000))]); + const pages = await drain( + src, + (await readTracePage(src, 'trace', { spanIds: ['absent', 'a', 'b'], include: 'full', maxBytes: 65536 }))!, + ); + expect(pages.flatMap(page => page.nodes.map(node => node.spanId))).toEqual(['a', 'b']); + expect(pages.flatMap(page => page.missingSpanIds)).toEqual(['absent']); +}); diff --git a/packages/core/src/storage/domains/observability/trace-page.ts b/packages/core/src/storage/domains/observability/trace-page.ts index 361b84b5141..5b284bc59b7 100644 --- a/packages/core/src/storage/domains/observability/trace-page.ts +++ b/packages/core/src/storage/domains/observability/trace-page.ts @@ -4,11 +4,12 @@ import { spanRecordSchema, computeTraceStatus, TraceStatus } from './tracing'; import type { SpanRecord } from './tracing'; // Bound serialized identities so every accepted ID batch fits in a continuation request. -const id = z +export const tracePageIdSchema = z .string() .min(1) .max(128) .refine(value => Buffer.byteLength(JSON.stringify(value)) <= 384, 'Identifier exceeds the serialized byte limit'); +const id = tracePageIdSchema; const selectionSchema = z .object({ anchorSpanId: id.optional(), @@ -72,15 +73,16 @@ export const tracePageResponseSchema = z.object({ }); export type TracePageResponse = z.infer; export type TracePageNode = z.infer; -export type TracePageStructure = Pick< +export type TracePageStructure = Pick; +export type TracePageSummary = Pick< SpanRecord, | 'traceId' | 'spanId' | 'parentSpanId' + | 'startedAt' | 'name' | 'spanType' | 'isEvent' - | 'startedAt' | 'endedAt' | 'entityType' | 'entityId' @@ -90,6 +92,7 @@ export type TracePageStructure = Pick< export class TracePageError extends Error { constructor( public readonly code: + | 'INVALID_REQUEST' | 'INVALID_CURSOR' | 'TRACE_CHANGED' | 'UNSUPPORTED' @@ -106,8 +109,10 @@ export class TracePageError extends Error { /** All reads must independently enforce trusted scope. A cursor is never authorization. */ export interface TracePageSource { scopeKey: string; - structure(traceId: string): Promise; - spans(traceId: string, spanIds: string[]): Promise; + /** Return at most limit topology rows; the extra row detects overflow. */ + structure(traceId: string, limit: number): Promise; + /** Read only the selected IDs, excluding payload fields for structure projection. */ + spans(traceId: string, spanIds: string[], include: 'structure' | 'full'): Promise; /** Complete matching identities within this trace; fail rather than truncate. */ search?(traceId: string, text: string): Promise; } @@ -128,7 +133,7 @@ export async function readTracePage( traceId: string, request: TracePageRequest, ): Promise { - id.parse(traceId); + if (!id.safeParse(traceId).success) throw new TracePageError('INVALID_REQUEST', 'Invalid trace identifier'); const parsed = tracePageRequestSchema.parse(request); let continuation: z.infer | undefined; if ('cursor' in parsed) { @@ -144,25 +149,24 @@ export async function readTracePage( if (!selection) throw new TracePageError('INVALID_CURSOR', 'Invalid trace continuation'); if (selection.search && !source.search) throw new TracePageError('UNSUPPORTED', 'Indexed in-trace search is unavailable'); - // Explicit-ID hydration does not discover or expand the trace topology. - const explicitSpans = selection.spanIds ? await source.spans(traceId, [...new Set(selection.spanIds)]) : undefined; - const structure = explicitSpans ?? (await source.structure(traceId)); + const offset = continuation?.offset ?? 0; + if (selection.spanIds) { + const ids = [...new Set(selection.spanIds)]; + const fingerprint = hash([source.scopeKey, traceId, selection]); + return renderPage(ids, fingerprint); + } + const structure = await source.structure(traceId, 50001); if (!structure) return null; if (structure.length > 50000) throw new TracePageError('LIMIT_EXCEEDED', 'Trace topology exceeds 50000 spans'); const nodes = new Map(); for (const node of structure) { - if ( - node.traceId !== traceId || - nodes.has(node.spanId) || - !Number.isFinite(node.startedAt.getTime()) || - (selection.spanIds && !selection.spanIds.includes(node.spanId)) - ) { + if (node.traceId !== traceId || nodes.has(node.spanId) || !Number.isFinite(node.startedAt.getTime())) { throw new TracePageError('INVALID_TOPOLOGY', 'Trace contains conflicting identities or invalid timestamps'); } nodes.set(node.spanId, node); } const visited = new Set(); - for (const node of selection.spanIds ? [] : structure) { + for (const node of structure) { const path = new Set(); let current: TracePageStructure | undefined = node; while (current && !visited.has(current.spanId)) { @@ -189,13 +193,7 @@ export async function readTracePage( throw new TracePageError('TRACE_CHANGED', 'Search membership changed during the read; restart this selection'); const selected: TracePageStructure[] = []; const missingSpanIds: string[] = []; - if (selection.spanIds) { - for (const spanId of new Set(selection.spanIds)) { - const node = nodes.get(spanId); - if (node) selected.push(node); - else missingSpanIds.push(spanId); - } - } else if (selection.search) { + if (selection.search) { const emitted = new Set(); for (const match of sorted.filter(node => matches.has(node.spanId))) { const path: TracePageStructure[] = []; @@ -223,105 +221,146 @@ export async function readTracePage( } } } - // A topology fingerprint detects delayed arrivals, unlike root lastActivityAt. - // It does not retain a database snapshot: callers restart when membership changes. + // Only changes affecting this selection invalidate continuation, not unrelated branches. const fingerprint = hash([ source.scopeKey, traceId, selection, - sorted.map(node => [node.spanId, node.parentSpanId ?? null, node.startedAt.toISOString()]), - [...matches].sort(), + selected.map(node => [ + node.spanId, + node.parentSpanId ?? null, + node.startedAt.toISOString(), + children.get(node.spanId)?.length ?? 0, + !!node.parentSpanId && nodes.has(node.parentSpanId), + matches.has(node.spanId), + ]), ]); - if (continuation && continuation.fingerprint !== fingerprint) { - throw new TracePageError('TRACE_CHANGED', 'Trace or request scope changed; restart this selection'); - } - const offset = continuation?.offset ?? 0; - if (continuation && offset >= selected.length) - throw new TracePageError('INVALID_CURSOR', 'Continuation is outside the selection'); - const page = selected.slice(offset, offset + selection.limit); - const details = new Map(); - if (selection.include === 'full' && page.length) { - const pageIds = new Set(page.map(node => node.spanId)); - const hydrated = explicitSpans - ? explicitSpans.filter(span => pageIds.has(span.spanId)) - : await source.spans(traceId, [...pageIds]); - for (const span of hydrated) { - if (span.traceId !== traceId || !page.some(node => node.spanId === span.spanId) || details.has(span.spanId)) { - throw new TracePageError('INVALID_TOPOLOGY', 'Detail read returned conflicting identities'); + return renderPage( + selected.map(node => node.spanId), + fingerprint, + { + nodes, + children, + matches, + missingSpanIds, + unavailableAncestors: selected.some(node => node.parentSpanId && !nodes.has(node.parentSpanId)), + }, + ); + + async function renderPage( + ids: string[], + fingerprint: string, + topology?: { + nodes: Map; + children: Map; + matches: Set; + missingSpanIds: string[]; + unavailableAncestors: boolean; + }, + ): Promise { + // Explicit-ID cursors depend on the requested positions, never on fetched membership. + if (continuation && continuation.fingerprint !== fingerprint) + throw new TracePageError('TRACE_CHANGED', 'Selection or request scope changed; restart this selection'); + if (continuation && offset >= ids.length) + throw new TracePageError('INVALID_CURSOR', 'Continuation is outside the selection'); + const pageIds = ids.slice(offset, offset + selection!.limit); + const records = pageIds.length ? await source.spans(traceId, pageIds, selection!.include) : []; + const recordsById = new Map(); + for (const record of records) { + if ( + record.traceId !== traceId || + !pageIds.includes(record.spanId) || + recordsById.has(record.spanId) || + !Number.isFinite(record.startedAt.getTime()) + ) + throw new TracePageError('INVALID_TOPOLOGY', 'Page read returned conflicting identities or timestamps'); + const node = topology?.nodes.get(record.spanId); + if ( + node && + ((node.parentSpanId ?? null) !== (record.parentSpanId ?? null) || + node.startedAt.getTime() !== record.startedAt.getTime()) + ) + throw new TracePageError('TRACE_CHANGED', 'Selected structure changed during the read'); + recordsById.set(record.spanId, record); + } + const result: TracePageResponse = { + version: 2, + traceId, + nodes: [], + missingSpanIds: [...(topology?.missingSpanIds ?? [])], + ancestorPaths: 'not-requested', + }; + let processed = 0; + const updateContinuation = () => { + const nextOffset = offset + processed; + if (nextOffset < ids.length) + result.nextCursor = Buffer.from( + JSON.stringify({ + version: 2, + selection, + fingerprint, + offset: nextOffset, + }), + ).toString('base64url'); + else delete result.nextCursor; + if (selection!.search && selection!.ancestors !== false) + result.ancestorPaths = result.nextCursor + ? 'continued' + : topology?.unavailableAncestors + ? 'unavailable' + : 'complete'; + }; + for (const spanId of pageIds) { + const record = recordsById.get(spanId); + if (!record) { + if (topology) throw new TracePageError('TRACE_CHANGED', 'Selected spans disappeared; restart this selection'); + result.missingSpanIds.push(spanId); + } else { + const { traceId, parentSpanId, name, spanType, isEvent, startedAt, endedAt, entityType, entityId, entityName } = + record; + result.nodes.push({ + traceId, + spanId, + parentSpanId, + name, + spanType, + isEvent, + startedAt, + endedAt, + entityType, + entityId, + entityName, + status: computeTraceStatus(record), + childCount: topology ? (topology.children.get(spanId)?.length ?? 0) : null, + match: topology?.matches.has(spanId) ?? false, + parentState: !parentSpanId + ? 'root' + : !topology + ? 'not-requested' + : topology.nodes.has(parentSpanId) + ? 'available' + : 'unavailable', + ...(selection!.include === 'full' ? { details: record as SpanRecord } : {}), + }); } - details.set(span.spanId, span); - } - if (details.size !== page.length) - throw new TracePageError('TRACE_CHANGED', 'Selected spans disappeared; restart this selection'); - } - const result: TracePageResponse = { version: 2, traceId, nodes: [], missingSpanIds, ancestorPaths: 'not-requested' }; - const updateContinuation = () => { - const nextOffset = offset + result.nodes.length; - if (nextOffset < selected.length) { - result.nextCursor = Buffer.from( - JSON.stringify({ version: 2, selection, fingerprint, offset: nextOffset }), - ).toString('base64url'); - } else delete result.nextCursor; - if (selection.search && selection.ancestors !== false) { - result.ancestorPaths = result.nextCursor - ? 'continued' - : selected.some(node => node.parentSpanId && !nodes.has(node.parentSpanId)) - ? 'unavailable' - : 'complete'; - } - }; - for (const node of page) { - const { - traceId, - spanId, - parentSpanId, - name, - spanType, - isEvent, - startedAt, - endedAt, - entityType, - entityId, - entityName, - } = node; - result.nodes.push({ - traceId, - spanId, - parentSpanId, - name, - spanType, - isEvent, - startedAt, - endedAt, - entityType, - entityId, - entityName, - status: computeTraceStatus(node), - childCount: selection.spanIds ? null : (children.get(spanId)?.length ?? 0), - match: matches.has(spanId), - parentState: !parentSpanId - ? 'root' - : selection.spanIds - ? 'not-requested' - : nodes.has(parentSpanId) - ? 'available' - : 'unavailable', - ...(selection.include === 'full' ? { details: details.get(spanId)! } : {}), - }); - updateContinuation(); - if (Buffer.byteLength(JSON.stringify(result)) > selection.maxBytes) { - result.nodes.pop(); + processed++; updateContinuation(); - if (!result.nodes.length) - throw new TracePageError( - 'PAYLOAD_TOO_LARGE', - `Span '${spanId}' exceeds maxBytes; request structure or use the single-span endpoint`, - ); - break; + if (Buffer.byteLength(JSON.stringify(result)) > selection!.maxBytes) { + if (record) result.nodes.pop(); + else result.missingSpanIds.pop(); + processed--; + updateContinuation(); + if (!processed) + throw new TracePageError( + 'PAYLOAD_TOO_LARGE', + `Span '${spanId}' exceeds maxBytes; request structure or use the single-span endpoint`, + ); + break; + } } + updateContinuation(); + if (Buffer.byteLength(JSON.stringify(result)) > selection!.maxBytes) + throw new TracePageError('LIMIT_EXCEEDED', 'Trace response metadata exceeds maxBytes'); + return result; } - updateContinuation(); - if (Buffer.byteLength(JSON.stringify(result)) > selection.maxBytes) - throw new TracePageError('LIMIT_EXCEEDED', 'Trace response metadata exceeds maxBytes'); - return result; } diff --git a/packages/server/src/server/handlers/observability-shared.ts b/packages/server/src/server/handlers/observability-shared.ts index 56da425ed7d..fa824ab24d3 100644 --- a/packages/server/src/server/handlers/observability-shared.ts +++ b/packages/server/src/server/handlers/observability-shared.ts @@ -790,6 +790,7 @@ export const NO_OBSERVABILITY_STORAGE_CAPABILITIES: ObservabilityStorageCapabili */ export function getObservabilityStorageCapabilities( observabilityStore: ObservabilityStorage, + scope?: { organizationId?: unknown }, ): ObservabilityStorageCapabilities { const declaredFeatures = getFeatures(observabilityStore); const features = declaredFeatures ?? []; @@ -840,6 +841,7 @@ export function getObservabilityStorageCapabilities( traceQuery, traceQueryAggregatedRoots: false, traceNavigation: + !scope?.organizationId && !!coreStorage.tracePageRequestSchema && typeof observabilityStore.supportsTraceNavigation === 'function' && observabilityStore.supportsTraceNavigation(), diff --git a/packages/server/src/server/handlers/observability.ts b/packages/server/src/server/handlers/observability.ts index aa4e643bacb..d43ea400027 100644 --- a/packages/server/src/server/handlers/observability.ts +++ b/packages/server/src/server/handlers/observability.ts @@ -286,7 +286,9 @@ export const GET_TRACE_PAGE_ROUTE: ServerRoute = createRoute({ method: 'POST', path: '/observability/v2/traces/:traceId', responseType: 'json', - pathParamSchema: getTraceArgsSchema, + pathParamSchema: tracePageCore.tracePageIdSchema + ? z.object({ traceId: tracePageCore.tracePageIdSchema }) + : getTraceArgsSchema, bodySchema: tracePageCore.tracePageRequestSchema ?? z.unknown().optional(), responseSchema: tracePageCore.tracePageResponseSchema ?? z.never(), maxBodySize: 65536, @@ -325,7 +327,7 @@ export const GET_TRACE_PAGE_ROUTE: ServerRoute = createRoute({ // Route context contains additional server-owned fields; pick only the body contract. const fields: Record = request; const body = Object.fromEntries( - ['version', 'cursor', 'anchorSpanId', 'depth', 'spanIds', 'search', 'ancestors', 'include', 'limit', 'maxBytes'] + ['cursor', 'anchorSpanId', 'depth', 'spanIds', 'search', 'ancestors', 'include', 'limit', 'maxBytes'] .filter(key => fields[key] !== undefined) .map(key => [key, fields[key]]), ); @@ -342,9 +344,13 @@ export const GET_TRACE_PAGE_ROUTE: ServerRoute = createRoute({ ? 501 : error.code === 'TRACE_CHANGED' ? 409 - : error.code === 'PAYLOAD_TOO_LARGE' || error.code === 'LIMIT_EXCEEDED' - ? 413 - : 400; + : error.code === 'INVALID_TOPOLOGY' + ? 500 + : error.code === 'LIMIT_EXCEEDED' + ? 422 + : error.code === 'PAYLOAD_TOO_LARGE' + ? 413 + : 400; throw new HTTPException(status, { message: `${error.code}: ${error.message}` }); } return handleError(error, 'Error reading trace page'); diff --git a/packages/server/src/server/handlers/system.ts b/packages/server/src/server/handlers/system.ts index 2ec68e1c7c0..ee8dc890e83 100644 --- a/packages/server/src/server/handlers/system.ts +++ b/packages/server/src/server/handlers/system.ts @@ -150,7 +150,7 @@ export const GET_SYSTEM_PACKAGES_ROUTE = createRoute({ description: 'Returns installed Mastra packages and runtime capabilities used by Studio', tags: ['System'], requiresAuth: true, - handler: async ({ mastra }) => { + handler: async ({ mastra, requestContext }) => { try { const packagesFilePath = process.env.MASTRA_PACKAGES_FILE; @@ -170,7 +170,9 @@ export const GET_SYSTEM_PACKAGES_ROUTE = createRoute({ const observabilityStorage = storage?.stores?.observability; const observabilityStorageType = observabilityStorage?.constructor.name; const observabilityStorageCapabilities = observabilityStorage - ? getObservabilityStorageCapabilities(observabilityStorage) + ? getObservabilityStorageCapabilities(observabilityStorage, { + organizationId: requestContext?.get('organizationId'), + }) : undefined; const observabilityRuntimeStrategy = observabilityStorage?.runtimeTracingStrategy; const observabilityEnabled = !!mastra.observability.getDefaultInstance(); diff --git a/packages/server/src/server/handlers/trace-page.test.ts b/packages/server/src/server/handlers/trace-page.test.ts index 6606ae8bbf6..780e2a7291c 100644 --- a/packages/server/src/server/handlers/trace-page.test.ts +++ b/packages/server/src/server/handlers/trace-page.test.ts @@ -1,11 +1,14 @@ import { Mastra } from '@mastra/core'; import { SpanType } from '@mastra/core/observability'; -import { InMemoryStore } from '@mastra/core/storage'; +import { InMemoryStore, readTracePage } from '@mastra/core/storage'; import { Hono } from 'hono'; import { describe, expect, it } from 'vitest'; +import { getBuiltinMCPCapabilities } from '../mcp/capabilities'; +import { dispatchRoute } from '../server-adapter'; import { OBSERVABILITY_ROUTES } from '../server-adapter/routes/observability'; import { GET_TRACE_PAGE_ROUTE, GET_TRACE_ROUTE } from './observability'; import { getObservabilityStorageCapabilities } from './observability-shared'; +import { GET_SYSTEM_PACKAGES_ROUTE } from './system'; import { createTestServerContext } from './test-utils'; async function setup() { @@ -23,6 +26,18 @@ async function setup() { input: { payload: spanId }, })), }); + // A deliberately opted-in backend over real storage; production OSS providers remain unsupported. + store.supportsTraceNavigation = () => true; + store.getTracePage = async ({ traceId, ...request }, options) => + readTracePage( + { + scopeKey: options!.scopeKey, + structure: async () => (await store.getStructure({ traceId }))?.spans ?? null, + spans: async (_, spanIds) => (await store.getSpans({ traceId, spanIds })).spans, + }, + traceId, + request, + ); return { store, context: createTestServerContext({ mastra: new Mastra({ storage }) }) }; } @@ -38,7 +53,6 @@ describe('versioned trace route with real storage', () => { const first = await GET_TRACE_PAGE_ROUTE.handler({ ...context, traceId: 'trace', - depth: 2, limit: 1, include: 'full', @@ -48,7 +62,6 @@ describe('versioned trace route with real storage', () => { const next = await GET_TRACE_PAGE_ROUTE.handler({ ...context, traceId: 'trace', - cursor: first.nextCursor, }); expect(next.nodes[0]).toMatchObject({ spanId: 'child', details: { input: { payload: 'child' } } }); @@ -89,3 +102,74 @@ it('keeps trace queries and aggregates separate from versioned trace IDs', async expect(await page.text()).toBe('/observability/v2/traces/:traceId'); } }); + +it('does not infer bounded navigation from existing OSS storage primitives', async () => { + const store = (await new InMemoryStore().getStore('observability'))!; + expect(getObservabilityStorageCapabilities(store).traceNavigation).toBe(false); + await expect(store.getTracePage({ traceId: 'trace' })).rejects.toMatchObject({ code: 'UNSUPPORTED' }); +}); + +it('applies request scope after cached MCP support and to system capabilities', async () => { + const { context } = await setup(); + expect((await getBuiltinMCPCapabilities(context.mastra)).observability?.traceNavigation).toBe(true); + expect( + (await getBuiltinMCPCapabilities(context.mastra, { organizationId: 'org' })).observability?.traceNavigation, + ).toBe(false); + expect((await getBuiltinMCPCapabilities(context.mastra)).observability?.traceNavigation).toBe(true); + context.requestContext.set('organizationId', 'org'); + const result = await GET_SYSTEM_PACKAGES_ROUTE.handler(context); + expect(JSON.stringify(result)).toContain('"traceNavigation":false'); +}); + +it('validates v2 path and body through HTTP dispatch and preserves server error mappings', async () => { + const { store, context } = await setup(); + const app = new Hono(); + app.post(GET_TRACE_PAGE_ROUTE.path, async c => { + const text = await c.req.text(); + const result = await dispatchRoute( + GET_TRACE_PAGE_ROUTE, + { pathParams: { traceId: c.req.param('traceId') }, body: text ? JSON.parse(text) : undefined }, + { mastra: context.mastra, requestContext: context.requestContext, abortSignal: c.req.raw.signal }, + ); + return new Response(JSON.stringify(result.ok ? result.result : result.body), { + status: result.ok ? 200 : result.status, + }); + }); + const post = (id: string, body?: unknown) => + app.request(`/observability/v2/traces/${id}`, { + method: 'POST', + ...(body === undefined ? {} : { body: JSON.stringify(body), headers: { 'content-type': 'application/json' } }), + }); + expect((await post('a'.repeat(129), {})).status).toBe(400); + expect((await post(encodeURIComponent('界'.repeat(128)), {})).status).toBe(400); + expect((await post('trace')).status).toBe(400); + expect((await post('trace', { limit: 1 })).status).toBe(200); + context.requestContext.set('organizationId', 'org'); + expect((await post('trace', {})).status).toBe(501); + context.requestContext.delete('organizationId'); + await store.batchCreateSpans({ + records: [ + { + traceId: 'cycle', + spanId: 'loop', + parentSpanId: 'loop', + name: 'loop', + spanType: SpanType.AGENT_RUN, + isEvent: false, + startedAt: new Date('2026-11-01'), + }, + ], + }); + expect((await post('cycle', {})).status).toBe(500); + await store.batchCreateSpans({ + records: Array.from({ length: 50001 }, (_, i) => ({ + traceId: 'over-budget', + spanId: String(i), + name: 'span', + spanType: SpanType.AGENT_RUN, + isEvent: false, + startedAt: new Date('2026-11-01'), + })), + }); + expect((await post('over-budget', {})).status).toBe(422); +}); diff --git a/packages/server/src/server/mcp/capabilities.ts b/packages/server/src/server/mcp/capabilities.ts index 9346a1516f8..c327425814f 100644 --- a/packages/server/src/server/mcp/capabilities.ts +++ b/packages/server/src/server/mcp/capabilities.ts @@ -63,7 +63,10 @@ async function resolveStoreCapabilities(mastra: Mastra): Promise { +export async function getBuiltinMCPCapabilities( + mastra: Mastra, + scope?: { organizationId?: unknown }, +): Promise { let pending = storeCapabilities.get(mastra); if (!pending) { pending = resolveStoreCapabilities(mastra); @@ -75,5 +78,14 @@ export async function getBuiltinMCPCapabilities(mastra: Mastra): Promise agent.hasOwnMemory?.()); const store = await pending; if (!store) storeCapabilities.delete(mastra); - return { ...(store ?? NO_STORE), storedAgents, memory: agentMemory || storedAgents }; + const resolved = store ?? NO_STORE; + return { + ...resolved, + observability: + resolved.observability && scope?.organizationId + ? { ...resolved.observability, traceNavigation: false, traceNavigationSearch: false } + : resolved.observability, + storedAgents, + memory: agentMemory || storedAgents, + }; } diff --git a/packages/server/src/server/mcp/catalogue.ts b/packages/server/src/server/mcp/catalogue.ts index 05618c3de9c..225d8de7452 100644 --- a/packages/server/src/server/mcp/catalogue.ts +++ b/packages/server/src/server/mcp/catalogue.ts @@ -183,7 +183,9 @@ export async function buildBuiltinMCPTools( config: ResolvedBuiltinMCPConfig, ): Promise> { const { mastra, requestContext } = ctx; - const capabilities = await getBuiltinMCPCapabilities(mastra); + const capabilities = await getBuiltinMCPCapabilities(mastra, { + organizationId: requestContext.get('organizationId'), + }); const routeContext: ServerContext = { mastra, requestContext, diff --git a/packages/server/src/server/schemas/system.ts b/packages/server/src/server/schemas/system.ts index 1ab8359e7a8..6e392b97cbd 100644 --- a/packages/server/src/server/schemas/system.ts +++ b/packages/server/src/server/schemas/system.ts @@ -41,7 +41,7 @@ export const observabilityStorageCapabilitiesSchema = z.object({ .boolean() .default(false) .describe("Supports explicit `rootSource: 'aggregated'` trace queries"), - traceNavigation: z.boolean().default(false).describe('Bounded version 2 getTrace browsing and hydration'), + traceNavigation: z.boolean().default(false).describe('Bounded getTracePage browsing and hydration'), traceNavigationSearch: z.boolean().default(false).describe('Indexed in-trace search with ancestor context'), traceQueryTextSearch: z.boolean().default(false).describe('Literal substring search across any span in a trace'), spanQueryTextSearch: z.boolean().default(false).describe('Literal substring search within individual spans'), From 57bb835fc870f5eea3eea870524d7d39b47a430b Mon Sep 17 00:00:00 2001 From: Eric Pinzur Date: Mon, 16 Nov 2026 15:49:16 +0100 Subject: [PATCH 08/23] refactor(observability): use trace page subresource --- .changeset/whole-streets-flow.md | 2 +- .../resources/observability-route-types.ts | 4 +- .../observability.trace-page.test.ts | 2 +- .../client-js/src/resources/observability.ts | 2 +- .../client-js/src/route-types.generated.ts | 40 +++++++++---------- .../commands/api/route-metadata.generated.ts | 4 +- .../domains/observability/trace-page.ts | 2 +- .../src/server/handlers/observability.ts | 6 +-- .../src/server/handlers/trace-page.test.ts | 12 +++--- 9 files changed, 37 insertions(+), 37 deletions(-) diff --git a/.changeset/whole-streets-flow.md b/.changeset/whole-streets-flow.md index c636153adb0..cbcda1bfa35 100644 --- a/.changeset/whole-streets-flow.md +++ b/.changeset/whole-streets-flow.md @@ -2,4 +2,4 @@ '@mastra/server': minor --- -Add a versioned route for bounded trace reads on supporting backends. +Add a page route for bounded trace reads on supporting backends. diff --git a/client-sdks/client-js/src/resources/observability-route-types.ts b/client-sdks/client-js/src/resources/observability-route-types.ts index c6b7b31e0e8..b6857b7b312 100644 --- a/client-sdks/client-js/src/resources/observability-route-types.ts +++ b/client-sdks/client-js/src/resources/observability-route-types.ts @@ -13,9 +13,9 @@ type ListArgs = Simplify< } >; -export type TracePageRequest = Body<'POST /observability/v2/traces/:traceId'>; +export type TracePageRequest = Body<'POST /observability/traces/:traceId/page'>; export type GetTracePageArgs = TracePageRequest & { traceId: string }; -export type TracePageResponse = Response<'POST /observability/v2/traces/:traceId'>; +export type TracePageResponse = Response<'POST /observability/traces/:traceId/page'>; export type TracePageNode = TracePageResponse['nodes'][number]; export type TraceRecord = Response<'GET /observability/traces/:traceId'>; diff --git a/client-sdks/client-js/src/resources/observability.trace-page.test.ts b/client-sdks/client-js/src/resources/observability.trace-page.test.ts index 7fdc68ced5c..e23f634140a 100644 --- a/client-sdks/client-js/src/resources/observability.trace-page.test.ts +++ b/client-sdks/client-js/src/resources/observability.trace-page.test.ts @@ -12,7 +12,7 @@ describe('legacy and paged trace reads over HTTP', () => { const server = createServer(async (req, res) => { try { expect(req.url).toBe( - req.method === 'GET' ? '/api/observability/traces/trace' : '/api/observability/v2/traces/trace', + req.method === 'GET' ? '/api/observability/traces/trace' : '/api/observability/traces/trace/page', ); const store = (await storage.getStore('observability'))!; const chunks: Buffer[] = []; diff --git a/client-sdks/client-js/src/resources/observability.ts b/client-sdks/client-js/src/resources/observability.ts index ceb3902fbe2..df384eba714 100644 --- a/client-sdks/client-js/src/resources/observability.ts +++ b/client-sdks/client-js/src/resources/observability.ts @@ -238,7 +238,7 @@ export class Observability extends BaseResource { /** Retrieves a bounded trace page from a supporting backend. */ getTracePage(request: GetTracePageArgs): Promise { const { traceId, ...body } = request; - return this.request(`/observability/v2/traces/${encodeURIComponent(traceId)}`, { method: 'POST', body }); + return this.request(`/observability/traces/${encodeURIComponent(traceId)}/page`, { method: 'POST', body }); } /** diff --git a/client-sdks/client-js/src/route-types.generated.ts b/client-sdks/client-js/src/route-types.generated.ts index c25c38cfcd5..80fb75f5e46 100644 --- a/client-sdks/client-js/src/route-types.generated.ts +++ b/client-sdks/client-js/src/route-types.generated.ts @@ -10520,13 +10520,13 @@ export interface GetObservabilityTracesTraceId_RouteContract { } // ============================================================================ -// Route: POST /observability/v2/traces/:traceId +// Route: POST /observability/traces/:traceId/page // ============================================================================ -export type PostObservabilityV2TracesTraceId_PathParams = { +export type PostObservabilityTracesTraceIdPage_PathParams = { traceId: string; }; -export type PostObservabilityV2TracesTraceId_Body = +export type PostObservabilityTracesTraceIdPage_Body = | { anchorSpanId?: string | undefined; depth?: number | undefined; @@ -10541,7 +10541,7 @@ export type PostObservabilityV2TracesTraceId_Body = cursor: string; }; -export type PostObservabilityV2TracesTraceId_Response = { +export type PostObservabilityTracesTraceIdPage_Response = { version: 2; traceId: string; nodes: { @@ -10574,24 +10574,24 @@ export type PostObservabilityV2TracesTraceId_Response = { ancestorPaths: 'not-requested' | 'continued' | 'complete' | 'unavailable'; }; -export type PostObservabilityV2TracesTraceId_Request = Simplify< - (PostObservabilityV2TracesTraceId_PathParams extends never +export type PostObservabilityTracesTraceIdPage_Request = Simplify< + (PostObservabilityTracesTraceIdPage_PathParams extends never ? {} - : { params: PostObservabilityV2TracesTraceId_PathParams }) & + : { params: PostObservabilityTracesTraceIdPage_PathParams }) & (never extends never ? {} : {} extends never ? { query?: never } : { query: never }) & - (PostObservabilityV2TracesTraceId_Body extends never + (PostObservabilityTracesTraceIdPage_Body extends never ? {} - : {} extends PostObservabilityV2TracesTraceId_Body - ? { body?: PostObservabilityV2TracesTraceId_Body } - : { body: PostObservabilityV2TracesTraceId_Body }) + : {} extends PostObservabilityTracesTraceIdPage_Body + ? { body?: PostObservabilityTracesTraceIdPage_Body } + : { body: PostObservabilityTracesTraceIdPage_Body }) >; -export interface PostObservabilityV2TracesTraceId_RouteContract { - pathParams: PostObservabilityV2TracesTraceId_PathParams; +export interface PostObservabilityTracesTraceIdPage_RouteContract { + pathParams: PostObservabilityTracesTraceIdPage_PathParams; queryParams: never; - body: PostObservabilityV2TracesTraceId_Body; - request: PostObservabilityV2TracesTraceId_Request; - response: PostObservabilityV2TracesTraceId_Response; + body: PostObservabilityTracesTraceIdPage_Body; + request: PostObservabilityTracesTraceIdPage_Request; + response: PostObservabilityTracesTraceIdPage_Response; responseType: 'json'; } @@ -25793,7 +25793,7 @@ export interface RouteTypes { 'GET /observability/branches': GetObservabilityBranches_RouteContract; 'GET /observability/traces/:traceId/branches/:spanId': GetObservabilityTracesTraceIdBranchesSpanId_RouteContract; 'GET /observability/traces/:traceId': GetObservabilityTracesTraceId_RouteContract; - 'POST /observability/v2/traces/:traceId': PostObservabilityV2TracesTraceId_RouteContract; + 'POST /observability/traces/:traceId/page': PostObservabilityTracesTraceIdPage_RouteContract; 'GET /observability/traces/:traceId/light': GetObservabilityTracesTraceIdLight_RouteContract; 'GET /observability/traces/:traceId/spans/:spanId': GetObservabilityTracesTraceIdSpansSpanId_RouteContract; 'GET /observability/traces/:traceId/trajectory': GetObservabilityTracesTraceIdTrajectory_RouteContract; @@ -26871,6 +26871,9 @@ export interface Client { '/observability/traces/:traceId/light': { GET: GetObservabilityTracesTraceIdLight_RouteContract; }; + '/observability/traces/:traceId/page': { + POST: PostObservabilityTracesTraceIdPage_RouteContract; + }; '/observability/traces/:traceId/spans/:spanId': { GET: GetObservabilityTracesTraceIdSpansSpanId_RouteContract; }; @@ -26898,9 +26901,6 @@ export interface Client { '/observability/traces/score': { POST: PostObservabilityTracesScore_RouteContract; }; - '/observability/v2/traces/:traceId': { - POST: PostObservabilityV2TracesTraceId_RouteContract; - }; '/processor-providers': { GET: GetProcessorProviders_RouteContract; }; diff --git a/packages/cli/src/commands/api/route-metadata.generated.ts b/packages/cli/src/commands/api/route-metadata.generated.ts index e7f4635381b..f9ba00dc56b 100644 --- a/packages/cli/src/commands/api/route-metadata.generated.ts +++ b/packages/cli/src/commands/api/route-metadata.generated.ts @@ -2008,9 +2008,9 @@ export const API_ROUTE_METADATA = { "listProperty": "spans" } }, - "POST /observability/v2/traces/:traceId": { + "POST /observability/traces/:traceId/page": { "method": "POST", - "path": "/observability/v2/traces/:traceId", + "path": "/observability/traces/:traceId/page", "pathParams": [ "traceId" ], diff --git a/packages/core/src/storage/domains/observability/trace-page.ts b/packages/core/src/storage/domains/observability/trace-page.ts index 5b284bc59b7..c40b8e42dbf 100644 --- a/packages/core/src/storage/domains/observability/trace-page.ts +++ b/packages/core/src/storage/domains/observability/trace-page.ts @@ -23,7 +23,7 @@ const selectionSchema = z }) .strict(); -/** POST body for the versioned getTracePage endpoint. Continuations cannot change selection. */ +/** POST body for the getTracePage endpoint. Continuations cannot change selection. */ export const tracePageRequestSchema = z.union([ selectionSchema.superRefine((value, ctx) => { if ( diff --git a/packages/server/src/server/handlers/observability.ts b/packages/server/src/server/handlers/observability.ts index d43ea400027..f1f86d509f8 100644 --- a/packages/server/src/server/handlers/observability.ts +++ b/packages/server/src/server/handlers/observability.ts @@ -281,10 +281,10 @@ export const GET_BRANCH_ROUTE = createRoute({ } }, }); -/** Versioned bounded trace contract. Namespace guards preserve older core compatibility. */ +/** Bounded trace page contract. Namespace guards preserve older core compatibility. */ export const GET_TRACE_PAGE_ROUTE: ServerRoute = createRoute({ method: 'POST', - path: '/observability/v2/traces/:traceId', + path: '/observability/traces/:traceId/page', responseType: 'json', pathParamSchema: tracePageCore.tracePageIdSchema ? z.object({ traceId: tracePageCore.tracePageIdSchema }) @@ -294,7 +294,7 @@ export const GET_TRACE_PAGE_ROUTE: ServerRoute = createRoute({ maxBodySize: 65536, requiresPermission: 'observability:read', summary: 'Browse or search a bounded trace page', - description: 'Version 2 trace navigation with bounded structure, optional details, and request-bound continuation.', + description: 'Trace page navigation with bounded structure, optional details, and request-bound continuation.', tags: ['Observability'], requiresAuth: true, handler: async ({ mastra, traceId, requestContext, ...request }) => { diff --git a/packages/server/src/server/handlers/trace-page.test.ts b/packages/server/src/server/handlers/trace-page.test.ts index 780e2a7291c..d89709e34ff 100644 --- a/packages/server/src/server/handlers/trace-page.test.ts +++ b/packages/server/src/server/handlers/trace-page.test.ts @@ -41,7 +41,7 @@ async function setup() { return { store, context: createTestServerContext({ mastra: new Mastra({ storage }) }) }; } -describe('versioned trace route with real storage', () => { +describe('trace page route with real storage', () => { it('retains legacy full reads while bounded requests hydrate only the page', async () => { const { store, context } = await setup(); expect(getObservabilityStorageCapabilities(store)).toMatchObject({ @@ -90,7 +90,7 @@ describe('versioned trace route with real storage', () => { // Dispatch the registered routes in their real order: the paged route must not // capture the existing static query/aggregate POSTs (OBS-956). -it('keeps trace queries and aggregates separate from versioned trace IDs', async () => { +it('keeps trace queries and aggregates separate from trace page reads', async () => { const app = new Hono(); for (const route of OBSERVABILITY_ROUTES) { if (route.method === 'POST') app.post(route.path, c => c.text(route.path)); @@ -98,8 +98,8 @@ it('keeps trace queries and aggregates separate from versioned trace IDs', async for (const name of ['query', 'aggregate']) { const legacy = await app.request(`/observability/traces/${name}`, { method: 'POST' }); expect(await legacy.text()).toBe(`/observability/traces/${name}`); - const page = await app.request(`/observability/v2/traces/${name}`, { method: 'POST' }); - expect(await page.text()).toBe('/observability/v2/traces/:traceId'); + const page = await app.request(`/observability/traces/${name}/page`, { method: 'POST' }); + expect(await page.text()).toBe('/observability/traces/:traceId/page'); } }); @@ -121,7 +121,7 @@ it('applies request scope after cached MCP support and to system capabilities', expect(JSON.stringify(result)).toContain('"traceNavigation":false'); }); -it('validates v2 path and body through HTTP dispatch and preserves server error mappings', async () => { +it('validates trace page path and body through HTTP dispatch and preserves server error mappings', async () => { const { store, context } = await setup(); const app = new Hono(); app.post(GET_TRACE_PAGE_ROUTE.path, async c => { @@ -136,7 +136,7 @@ it('validates v2 path and body through HTTP dispatch and preserves server error }); }); const post = (id: string, body?: unknown) => - app.request(`/observability/v2/traces/${id}`, { + app.request(`/observability/traces/${id}/page`, { method: 'POST', ...(body === undefined ? {} : { body: JSON.stringify(body), headers: { 'content-type': 'application/json' } }), }); From 9d3e0be7cbffaa1ede811839b057f5f8301ba29b Mon Sep 17 00:00:00 2001 From: Eric Pinzur Date: Mon, 16 Nov 2026 16:06:44 +0100 Subject: [PATCH 09/23] refactor(observability): keep trace page endpoint in Platform only --- .changeset/pink-eagles-repair.md | 5 - .changeset/whole-streets-flow.md | 5 - .../resources/observability-route-types.ts | 5 +- .../src/resources/trace-page-types.ts | 47 +++++ .../client-js/src/route-types.generated.ts | 160 ++++------------ client-sdks/client-js/src/types.ts | 9 +- .../client-js/type-tests/trace-page.test-d.ts | 24 +++ .../type-tests/tsconfig.trace-page.json | 4 + .../commands/api/route-metadata.generated.ts | 24 --- .../src/storage/domains/observability/base.ts | 15 +- .../handlers/observability-new-endpoints.ts | 7 +- .../server/handlers/observability-shared.ts | 11 -- ...ervability-text-search-core-compat.test.ts | 23 --- .../src/server/handlers/observability.ts | 79 -------- .../server/src/server/handlers/system.test.ts | 2 - packages/server/src/server/handlers/system.ts | 6 +- .../src/server/handlers/trace-page.test.ts | 175 ------------------ .../server/src/server/mcp/capabilities.ts | 16 +- packages/server/src/server/mcp/catalogue.ts | 4 +- packages/server/src/server/schemas/system.ts | 2 - .../server-adapter/routes/observability.ts | 2 - 21 files changed, 132 insertions(+), 493 deletions(-) delete mode 100644 .changeset/pink-eagles-repair.md delete mode 100644 .changeset/whole-streets-flow.md create mode 100644 client-sdks/client-js/src/resources/trace-page-types.ts create mode 100644 client-sdks/client-js/type-tests/trace-page.test-d.ts create mode 100644 client-sdks/client-js/type-tests/tsconfig.trace-page.json delete mode 100644 packages/server/src/server/handlers/trace-page.test.ts diff --git a/.changeset/pink-eagles-repair.md b/.changeset/pink-eagles-repair.md deleted file mode 100644 index 9351ec4c944..00000000000 --- a/.changeset/pink-eagles-repair.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'mastra': patch ---- - -Update trace API route metadata. diff --git a/.changeset/whole-streets-flow.md b/.changeset/whole-streets-flow.md deleted file mode 100644 index cbcda1bfa35..00000000000 --- a/.changeset/whole-streets-flow.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@mastra/server': minor ---- - -Add a page route for bounded trace reads on supporting backends. diff --git a/client-sdks/client-js/src/resources/observability-route-types.ts b/client-sdks/client-js/src/resources/observability-route-types.ts index b6857b7b312..244999c97e7 100644 --- a/client-sdks/client-js/src/resources/observability-route-types.ts +++ b/client-sdks/client-js/src/resources/observability-route-types.ts @@ -13,10 +13,7 @@ type ListArgs = Simplify< } >; -export type TracePageRequest = Body<'POST /observability/traces/:traceId/page'>; -export type GetTracePageArgs = TracePageRequest & { traceId: string }; -export type TracePageResponse = Response<'POST /observability/traces/:traceId/page'>; -export type TracePageNode = TracePageResponse['nodes'][number]; +export type { TracePageRequest, GetTracePageArgs, TracePageResponse, TracePageNode } from './trace-page-types'; export type TraceRecord = Response<'GET /observability/traces/:traceId'>; export type GetTraceLightResponse = Response<'GET /observability/traces/:traceId/light'>; diff --git a/client-sdks/client-js/src/resources/trace-page-types.ts b/client-sdks/client-js/src/resources/trace-page-types.ts new file mode 100644 index 00000000000..d375cc35e34 --- /dev/null +++ b/client-sdks/client-js/src/resources/trace-page-types.ts @@ -0,0 +1,47 @@ +import type { SpanRecord } from '@mastra/core/storage'; +import type { Serialized } from '../types'; + +// Platform serves this contract; it is not part of the @mastra/server route registry. +// Keep declarations independent of new core exports so older supported core packages can resolve them. +// type-tests/trace-page.test-d.ts verifies parity with the shared core schemas. +export type TracePageRequest = + | { + anchorSpanId?: string; + depth?: number; + spanIds?: string[]; + search?: string; + ancestors?: boolean; + include?: 'structure' | 'full'; + limit?: number; + maxBytes?: number; + } + | { cursor: string }; +export type GetTracePageArgs = TracePageRequest & { traceId: string }; +export type TracePageNode = Pick< + Serialized, + | 'traceId' + | 'spanId' + | 'parentSpanId' + | 'name' + | 'spanType' + | 'isEvent' + | 'startedAt' + | 'endedAt' + | 'entityType' + | 'entityId' + | 'entityName' +> & { + status: 'success' | 'error' | 'running'; + childCount: number | null; + match: boolean; + parentState: 'root' | 'available' | 'unavailable' | 'not-requested'; + details?: Serialized; +}; +export interface TracePageResponse { + version: 2; + traceId: string; + nodes: TracePageNode[]; + missingSpanIds: string[]; + nextCursor?: string; + ancestorPaths: 'not-requested' | 'continued' | 'complete' | 'unavailable'; +} diff --git a/client-sdks/client-js/src/route-types.generated.ts b/client-sdks/client-js/src/route-types.generated.ts index 80fb75f5e46..0d98083b8b3 100644 --- a/client-sdks/client-js/src/route-types.generated.ts +++ b/client-sdks/client-js/src/route-types.generated.ts @@ -24,7 +24,7 @@ type InputShared_Auxiliary_116 = { parentWorkflow?: InputShared_Auxiliary_116 | undefined; }; -type InputShared_Auxiliary_211 = +type InputShared_Auxiliary_208 = | { op: 'eq' | 'ne' | 'lt' | 'lte' | 'gt' | 'gte'; left: @@ -73,41 +73,41 @@ type InputShared_Auxiliary_211 = } | { op: 'and' | 'or'; - args: InputShared_Auxiliary_211[]; + args: InputShared_Auxiliary_208[]; } | { op: 'not'; - arg: InputShared_Auxiliary_211; + arg: InputShared_Auxiliary_208; } | { spans: | { - some: InputShared_Auxiliary_234; + some: InputShared_Auxiliary_231; } | { - none: InputShared_Auxiliary_234; + none: InputShared_Auxiliary_231; }; } | { scores: | { - some: InputShared_Auxiliary_234; + some: InputShared_Auxiliary_231; } | { - none: InputShared_Auxiliary_234; + none: InputShared_Auxiliary_231; }; } | { feedback: | { - some: InputShared_Auxiliary_234; + some: InputShared_Auxiliary_231; } | { - none: InputShared_Auxiliary_234; + none: InputShared_Auxiliary_231; }; }; -type InputShared_Auxiliary_234 = +type InputShared_Auxiliary_231 = | { op: 'eq' | 'ne' | 'lt' | 'lte' | 'gt' | 'gte'; left: @@ -156,33 +156,33 @@ type InputShared_Auxiliary_234 = } | { op: 'and' | 'or'; - args: InputShared_Auxiliary_234[]; + args: InputShared_Auxiliary_231[]; } | { op: 'not'; - arg: InputShared_Auxiliary_234; + arg: InputShared_Auxiliary_231; }; -type InputShared_Auxiliary_281 = +type InputShared_Auxiliary_278 = | { op: 'and' | 'or'; - args: InputShared_Auxiliary_281[]; + args: InputShared_Auxiliary_278[]; } | { op: 'not'; - arg: InputShared_Auxiliary_281; + arg: InputShared_Auxiliary_278; } | { traces: | { - some: InputShared_Auxiliary_211; + some: InputShared_Auxiliary_208; } | { - none: InputShared_Auxiliary_211; + none: InputShared_Auxiliary_208; }; }; -type InputShared_Auxiliary_738 = +type InputShared_Auxiliary_735 = | { op: 'eq' | 'ne' | 'lt' | 'lte' | 'gt' | 'gte'; left: @@ -227,27 +227,27 @@ type InputShared_Auxiliary_738 = } | { op: 'and' | 'or'; - args: InputShared_Auxiliary_738[]; + args: InputShared_Auxiliary_735[]; } | { op: 'not'; - arg: InputShared_Auxiliary_738; + arg: InputShared_Auxiliary_735; }; -type InputShared_Auxiliary_812 = { +type InputShared_Auxiliary_809 = { id?: string | undefined; name: string; type: 'file' | 'folder'; content?: string | undefined; - children?: InputShared_Auxiliary_812[] | undefined; + children?: InputShared_Auxiliary_809[] | undefined; }; -type Shared_Auxiliary_801 = { +type Shared_Auxiliary_799 = { id?: string | undefined; name: string; type: 'file' | 'folder'; content?: string | undefined; - children?: Shared_Auxiliary_801[] | undefined; + children?: Shared_Auxiliary_799[] | undefined; }; type InputShared_Type_0 = { @@ -1849,7 +1849,7 @@ type InputShared_Type_89 = { } | undefined; steps: InputShared_Type_84[]; - predicates: InputShared_Auxiliary_738[]; + predicates: InputShared_Auxiliary_735[]; }; type InputShared_Type_90 = { @@ -1863,7 +1863,7 @@ type InputShared_Type_90 = { | undefined; step: InputShared_Type_84; loopType: 'dowhile' | 'dountil'; - predicate: InputShared_Auxiliary_738; + predicate: InputShared_Auxiliary_735; }; type InputShared_Type_91 = @@ -3897,10 +3897,6 @@ type Shared_Type_69 = { traceQuery: boolean; /** Supports explicit `rootSource: 'aggregated'` trace queries */ traceQueryAggregatedRoots: boolean; - /** Bounded getTracePage browsing and hydration */ - traceNavigation: boolean; - /** Indexed in-trace search with ancestor context */ - traceNavigationSearch: boolean; /** Literal substring search across any span in a trace */ traceQueryTextSearch: boolean; /** Literal substring search within individual spans */ @@ -4516,7 +4512,7 @@ type Shared_Type_88 = { /** List of asset file paths */ assets?: string[] | undefined; /** Full file tree structure for the skill */ - files?: Shared_Auxiliary_801[] | undefined; + files?: Shared_Auxiliary_799[] | undefined; /** Additional metadata for the skill */ metadata?: | { @@ -10519,82 +10515,6 @@ export interface GetObservabilityTracesTraceId_RouteContract { responseType: 'json'; } -// ============================================================================ -// Route: POST /observability/traces/:traceId/page -// ============================================================================ -export type PostObservabilityTracesTraceIdPage_PathParams = { - traceId: string; -}; - -export type PostObservabilityTracesTraceIdPage_Body = - | { - anchorSpanId?: string | undefined; - depth?: number | undefined; - spanIds?: string[] | undefined; - search?: string | undefined; - ancestors?: boolean | undefined; - include?: 'structure' | 'full'; - limit?: number; - maxBytes?: number; - } - | { - cursor: string; - }; - -export type PostObservabilityTracesTraceIdPage_Response = { - version: 2; - traceId: string; - nodes: { - /** Unique trace identifier */ - traceId: string; - /** Unique span identifier within a trace */ - spanId: string; - parentSpanId?: (string | null) | undefined; - /** Human-readable span name */ - name: string; - /** Span type (e.g., WORKFLOW_RUN, AGENT_RUN, TOOL_CALL, etc.) */ - spanType: Shared_Type_60; - /** Whether this is an event (point-in-time) vs a span (duration) */ - isEvent: boolean; - /** When the span started */ - startedAt: Date; - endedAt?: (Date | null) | undefined; - entityType?: (Shared_Type_61 | null) | undefined; - entityId?: (string | null) | undefined; - entityName?: (string | null) | undefined; - status: 'success' | 'error' | 'running'; - childCount: number | null; - match: boolean; - parentState: 'root' | 'available' | 'unavailable' | 'not-requested'; - /** Span record data */ - details?: Shared_Type_64 | undefined; - }[]; - missingSpanIds: string[]; - nextCursor?: string | undefined; - ancestorPaths: 'not-requested' | 'continued' | 'complete' | 'unavailable'; -}; - -export type PostObservabilityTracesTraceIdPage_Request = Simplify< - (PostObservabilityTracesTraceIdPage_PathParams extends never - ? {} - : { params: PostObservabilityTracesTraceIdPage_PathParams }) & - (never extends never ? {} : {} extends never ? { query?: never } : { query: never }) & - (PostObservabilityTracesTraceIdPage_Body extends never - ? {} - : {} extends PostObservabilityTracesTraceIdPage_Body - ? { body?: PostObservabilityTracesTraceIdPage_Body } - : { body: PostObservabilityTracesTraceIdPage_Body }) ->; - -export interface PostObservabilityTracesTraceIdPage_RouteContract { - pathParams: PostObservabilityTracesTraceIdPage_PathParams; - queryParams: never; - body: PostObservabilityTracesTraceIdPage_Body; - request: PostObservabilityTracesTraceIdPage_Request; - response: PostObservabilityTracesTraceIdPage_Response; - responseType: 'json'; -} - // ============================================================================ // Route: GET /observability/traces/:traceId/light // ============================================================================ @@ -10909,7 +10829,7 @@ export type PostObservabilityTracesQuery_Body = { from: string; to: string; }; - where?: InputShared_Auxiliary_211 | undefined; + where?: InputShared_Auxiliary_208 | undefined; group?: | { by: ['threadId']; @@ -10999,11 +10919,11 @@ export type PostObservabilityTracesAggregate_Body = { from: string; to: string; }; - where?: InputShared_Auxiliary_211 | undefined; + where?: InputShared_Auxiliary_208 | undefined; groupBy?: string[]; interval?: ('1m' | '5m' | '15m' | '1h' | '1d') | undefined; measures: (InputShared_Type_16 | `countDistinct.${string}`)[]; - having?: InputShared_Auxiliary_234 | undefined; + having?: InputShared_Auxiliary_231 | undefined; orderBy?: { field: string; direction: 'asc' | 'desc'; @@ -11100,11 +11020,11 @@ export type PostObservabilitySpansAggregate_Body = { from: string; to: string; }; - where?: InputShared_Auxiliary_234 | undefined; + where?: InputShared_Auxiliary_231 | undefined; groupBy?: string[]; interval?: ('1m' | '5m' | '15m' | '1h' | '1d') | undefined; measures: (InputShared_Type_16 | `countDistinct.${string}`)[]; - having?: InputShared_Auxiliary_234 | undefined; + having?: InputShared_Auxiliary_231 | undefined; orderBy?: { field: string; direction: 'asc' | 'desc'; @@ -11152,7 +11072,7 @@ export type PostObservabilityThreadsQuery_Body = field: 'startedAt' | 'endedAt'; }; }; - where?: InputShared_Auxiliary_234 | undefined; + where?: InputShared_Auxiliary_231 | undefined; orderBy?: { field: 'startedAt' | 'endedAt'; direction: 'asc' | 'desc'; @@ -11168,9 +11088,9 @@ export type PostObservabilityThreadsQuery_Body = from: string; to: string; }; - where?: InputShared_Auxiliary_211 | undefined; + where?: InputShared_Auxiliary_208 | undefined; }; - where?: InputShared_Auxiliary_281 | undefined; + where?: InputShared_Auxiliary_278 | undefined; page?: { limit?: number; after?: (string | null) | undefined; @@ -11247,7 +11167,7 @@ export type PostObservabilitySpansQuery_Body = { from: string; to: string; }; - where?: InputShared_Auxiliary_234 | undefined; + where?: InputShared_Auxiliary_231 | undefined; textSearch?: string | undefined; orderBy?: { field: 'startedAt' | 'endedAt' | 'timeToFirstChunkMs' | 'timeToFirstTokenMs' | 'durationMs' | 'model' | 'provider'; @@ -20032,7 +19952,7 @@ export type PostStoredSkills_Body = { /** List of asset file paths */ assets?: string[] | undefined; /** Full file tree structure for the skill */ - files?: InputShared_Auxiliary_812[] | undefined; + files?: InputShared_Auxiliary_809[] | undefined; /** Additional metadata for the skill */ metadata?: | { @@ -20090,7 +20010,7 @@ export type PatchStoredSkillsStoredSkillId_Body = { /** List of asset file paths */ assets?: (string[] | undefined) | undefined; /** Full file tree structure for the skill */ - files?: (InputShared_Auxiliary_812[] | undefined) | undefined; + files?: (InputShared_Auxiliary_809[] | undefined) | undefined; /** Additional metadata for the skill */ metadata?: | ( @@ -25793,7 +25713,6 @@ export interface RouteTypes { 'GET /observability/branches': GetObservabilityBranches_RouteContract; 'GET /observability/traces/:traceId/branches/:spanId': GetObservabilityTracesTraceIdBranchesSpanId_RouteContract; 'GET /observability/traces/:traceId': GetObservabilityTracesTraceId_RouteContract; - 'POST /observability/traces/:traceId/page': PostObservabilityTracesTraceIdPage_RouteContract; 'GET /observability/traces/:traceId/light': GetObservabilityTracesTraceIdLight_RouteContract; 'GET /observability/traces/:traceId/spans/:spanId': GetObservabilityTracesTraceIdSpansSpanId_RouteContract; 'GET /observability/traces/:traceId/trajectory': GetObservabilityTracesTraceIdTrajectory_RouteContract; @@ -26871,9 +26790,6 @@ export interface Client { '/observability/traces/:traceId/light': { GET: GetObservabilityTracesTraceIdLight_RouteContract; }; - '/observability/traces/:traceId/page': { - POST: PostObservabilityTracesTraceIdPage_RouteContract; - }; '/observability/traces/:traceId/spans/:spanId': { GET: GetObservabilityTracesTraceIdSpansSpanId_RouteContract; }; diff --git a/client-sdks/client-js/src/types.ts b/client-sdks/client-js/src/types.ts index 7fce08c1412..13a56a36a96 100644 --- a/client-sdks/client-js/src/types.ts +++ b/client-sdks/client-js/src/types.ts @@ -1492,7 +1492,14 @@ export interface MastraPackage { export type GetSystemPackagesResponse = GeneratedResponse<'GET /system/packages'>; -export type GetObservabilityCapabilitiesResponse = GeneratedResponse<'GET /observability/capabilities'>; +type BaseObservabilityCapabilitiesResponse = GeneratedResponse<'GET /observability/capabilities'>; +export type GetObservabilityCapabilitiesResponse = Omit & { + capabilities: BaseObservabilityCapabilitiesResponse['capabilities'] & { + /** Optional capabilities advertised by Platform's trace page endpoint. */ + traceNavigation?: boolean; + traceNavigationSearch?: boolean; + }; +}; // ============================================================================ // Workspace Types diff --git a/client-sdks/client-js/type-tests/trace-page.test-d.ts b/client-sdks/client-js/type-tests/trace-page.test-d.ts new file mode 100644 index 00000000000..9b5f5b89b7c --- /dev/null +++ b/client-sdks/client-js/type-tests/trace-page.test-d.ts @@ -0,0 +1,24 @@ +import type { TracePageRequest as CoreRequest, TracePageResponse as CoreResponse } from '@mastra/core/storage'; +import { expectTypeOf, test } from 'vitest'; +import type { MastraClient } from '../src/client'; +import type { TracePageRequest, TracePageResponse } from '../src/resources/trace-page-types'; +import type { RouteKey } from '../src/route-types.generated'; +import type { GetObservabilityCapabilitiesResponse, Serialized } from '../src/types'; + +type CoreWireResponse = Serialized; +type CoreWireNode = CoreWireResponse['nodes'][number]; +// HTTP serializes the internal status enum as its string values. +type ExpectedResponse = Omit & { + nodes: (Omit & { status: `${CoreWireNode['status']}` })[]; +}; + +test('keeps Platform trace page types aligned without exposing an OSS server route', () => { + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toMatchTypeOf(); + expectTypeOf().toMatchTypeOf(); + expectTypeOf>().toEqualTypeOf>(); + expectTypeOf>().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf< + boolean | undefined + >(); +}); diff --git a/client-sdks/client-js/type-tests/tsconfig.trace-page.json b/client-sdks/client-js/type-tests/tsconfig.trace-page.json new file mode 100644 index 00000000000..83b4bfeeb6d --- /dev/null +++ b/client-sdks/client-js/type-tests/tsconfig.trace-page.json @@ -0,0 +1,4 @@ +{ + "extends": "./tsconfig.json", + "include": ["../src/**/*.ts", "./trace-page.test-d.ts"] +} diff --git a/packages/cli/src/commands/api/route-metadata.generated.ts b/packages/cli/src/commands/api/route-metadata.generated.ts index f9ba00dc56b..691420ba038 100644 --- a/packages/cli/src/commands/api/route-metadata.generated.ts +++ b/packages/cli/src/commands/api/route-metadata.generated.ts @@ -2008,30 +2008,6 @@ export const API_ROUTE_METADATA = { "listProperty": "spans" } }, - "POST /observability/traces/:traceId/page": { - "method": "POST", - "path": "/observability/traces/:traceId/page", - "pathParams": [ - "traceId" - ], - "queryParams": [], - "bodyParams": [ - "ancestors", - "anchorSpanId", - "cursor", - "depth", - "include", - "limit", - "maxBytes", - "search", - "spanIds" - ], - "hasQuery": false, - "hasBody": true, - "responseShape": { - "kind": "single" - } - }, "GET /observability/traces/:traceId/light": { "method": "GET", "path": "/observability/traces/:traceId/light", diff --git a/packages/core/src/storage/domains/observability/base.ts b/packages/core/src/storage/domains/observability/base.ts index 9dd93315a58..5b30413b9ca 100644 --- a/packages/core/src/storage/domains/observability/base.ts +++ b/packages/core/src/storage/domains/observability/base.ts @@ -70,8 +70,6 @@ import type { SpanQueryResponse, TrustedSpanQueryPlan } from './span-query'; import type { ThreadQueryResponse, TrustedThreadRowsQueryPlan } from './thread-query'; import type { TraceAggregateResponse } from './trace-aggregate'; import type { TrustedTraceAggregatePlan } from './trace-aggregate-planner'; -import { TracePageError } from './trace-page'; -import type { GetTracePageArgs, TracePageResponse } from './trace-page'; import type { GetTraceQueryValuesResponse, QueryThreadsResult, @@ -319,16 +317,9 @@ export class ObservabilityStorage extends StorageDomain { }); } - /** Backends must explicitly implement bounded reads; existing structure helpers do not imply support. */ - supportsTraceNavigation(): boolean { - return false; - } - - async getTracePage(_args: GetTracePageArgs, _options?: { scopeKey: string }): Promise { - throw new TracePageError('UNSUPPORTED', 'Store does not support bounded trace navigation'); - } - - /** Retrieves a single trace with all its associated spans. */ + /** + * Retrieves a single trace with all its associated spans. + */ async getTrace(_args: GetTraceArgs): Promise { throw new MastraError({ id: 'OBSERVABILITY_STORAGE_GET_TRACE_NOT_IMPLEMENTED', diff --git a/packages/server/src/server/handlers/observability-new-endpoints.ts b/packages/server/src/server/handlers/observability-new-endpoints.ts index be88171867a..8be96d6c2ec 100644 --- a/packages/server/src/server/handlers/observability-new-endpoints.ts +++ b/packages/server/src/server/handlers/observability-new-endpoints.ts @@ -1923,7 +1923,7 @@ export const GET_CAPABILITIES = createRoute({ responseSchema: getObservabilityCapabilitiesResponseSchema, tags: ['Observability'], requiresAuth: true, - handler: async ({ mastra, requestContext }) => { + handler: async ({ mastra }) => { try { const observabilityStore = await mastra.getStorage()?.getStore('observability'); if (!observabilityStore) { @@ -1931,10 +1931,7 @@ export const GET_CAPABILITIES = createRoute({ } return { observabilityStorageType: observabilityStore.constructor.name, - capabilities: { - ...getObservabilityStorageCapabilities(observabilityStore), - ...(requestContext.get('organizationId') ? { traceNavigation: false, traceNavigationSearch: false } : {}), - }, + capabilities: getObservabilityStorageCapabilities(observabilityStore), }; } catch (error) { return handleError(error, "Error calling: 'get observability capabilities'"); diff --git a/packages/server/src/server/handlers/observability-shared.ts b/packages/server/src/server/handlers/observability-shared.ts index fa824ab24d3..6da81bb3310 100644 --- a/packages/server/src/server/handlers/observability-shared.ts +++ b/packages/server/src/server/handlers/observability-shared.ts @@ -695,8 +695,6 @@ export type ObservabilityStorageCapabilities = { deltaPolling: boolean; traceQuery: boolean; traceQueryAggregatedRoots: boolean; - traceNavigation: boolean; - traceNavigationSearch: boolean; traceQueryTextSearch: boolean; traceQueryRootDuration: boolean; traceQueryContextIds: boolean; @@ -743,8 +741,6 @@ export const NO_OBSERVABILITY_STORAGE_CAPABILITIES: ObservabilityStorageCapabili deltaPolling: false, traceQuery: false, traceQueryAggregatedRoots: false, - traceNavigation: false, - traceNavigationSearch: false, traceQueryTextSearch: false, traceQueryRootDuration: false, traceQueryContextIds: false, @@ -790,7 +786,6 @@ export const NO_OBSERVABILITY_STORAGE_CAPABILITIES: ObservabilityStorageCapabili */ export function getObservabilityStorageCapabilities( observabilityStore: ObservabilityStorage, - scope?: { organizationId?: unknown }, ): ObservabilityStorageCapabilities { const declaredFeatures = getFeatures(observabilityStore); const features = declaredFeatures ?? []; @@ -840,12 +835,6 @@ export function getObservabilityStorageCapabilities( coreFeatures.has(OBSERVABILITY_DELTA_POLLING_FEATURE) && declares(OBSERVABILITY_DELTA_POLLING_STORAGE_FEATURE), traceQuery, traceQueryAggregatedRoots: false, - traceNavigation: - !scope?.organizationId && - !!coreStorage.tracePageRequestSchema && - typeof observabilityStore.supportsTraceNavigation === 'function' && - observabilityStore.supportsTraceNavigation(), - traceNavigationSearch: false, traceQueryTextSearch: traceQuery && coreStorage.observabilityTextSearchSchema !== undefined && declares('trace-query-text-search'), traceQueryRootDuration: diff --git a/packages/server/src/server/handlers/observability-text-search-core-compat.test.ts b/packages/server/src/server/handlers/observability-text-search-core-compat.test.ts index 7994c1a8370..828c4cabbe1 100644 --- a/packages/server/src/server/handlers/observability-text-search-core-compat.test.ts +++ b/packages/server/src/server/handlers/observability-text-search-core-compat.test.ts @@ -3,7 +3,6 @@ import { InMemoryStore } from '@mastra/core/storage'; import { describe, expect, it, vi } from 'vitest'; import { HTTPException } from '../http-exception'; -import { parseRouteBody } from '../server-adapter'; import { createTestServerContext } from './test-utils'; // Model the pre-search Core contract, retaining real schema validation and planning. @@ -13,15 +12,11 @@ vi.mock('@mastra/core/storage', async importOriginal => { return { ...actual, observabilityTextSearchSchema: undefined, - tracePageRequestSchema: undefined, - tracePageResponseSchema: undefined, - TracePageError: undefined, planSpanQuery: (input: unknown, options: Parameters[1]) => actual.planSpanQuery(legacyRequestSchema.parse(input), options), }; }); -const { GET_TRACE_PAGE_ROUTE, GET_TRACE_ROUTE } = await import('./observability'); const { QUERY_SPANS } = await import('./observability-new-endpoints'); const { getObservabilityStorageCapabilities } = await import('./observability-shared'); @@ -54,21 +49,3 @@ describe('text search with pre-search Core', () => { }); }); }); - -it('keeps legacy trace reads available when the installed Core predates bounded navigation', async () => { - const storage = new InMemoryStore(); - const store = (await storage.getStore('observability'))!; - const context = createTestServerContext({ mastra: new Mastra({ storage }) }); - expect(getObservabilityStorageCapabilities(store)).toMatchObject({ - traceNavigation: false, - traceNavigationSearch: false, - }); - const body = await parseRouteBody(GET_TRACE_PAGE_ROUTE, {}); - await expect( - GET_TRACE_PAGE_ROUTE.handler({ ...context, traceId: 'absent', ...(body as object) }), - ).rejects.toMatchObject({ - status: 501, - }); - // A normal legacy not-found response, rather than startup/import failure from missing exports. - await expect(GET_TRACE_ROUTE.handler({ ...context, traceId: 'absent' })).rejects.toMatchObject({ status: 404 }); -}); diff --git a/packages/server/src/server/handlers/observability.ts b/packages/server/src/server/handlers/observability.ts index f1f86d509f8..8922f648f3c 100644 --- a/packages/server/src/server/handlers/observability.ts +++ b/packages/server/src/server/handlers/observability.ts @@ -1,6 +1,5 @@ import { extractTrajectoryFromTrace, listScoresResponseSchema } from '@mastra/core/evals'; import { scoreTraces } from '@mastra/core/evals/scoreTraces'; -import * as tracePageCore from '@mastra/core/storage'; import { tracesFilterSchema, tracesOrderBySchema, @@ -21,7 +20,6 @@ import { // @mastra/core@1.32.0; route them through a shim that tolerates older cores // (see ./observability-storage-schemas.ts for full rationale). import { z } from 'zod/v4'; -import { MASTRA_USER_KEY, MASTRA_RESOURCE_ID_KEY, MASTRA_AUTH_MODE_KEY, MASTRA_USER_ROLES_KEY } from '../constants'; import { HTTPException } from '../http-exception'; import type { ServerRoute } from '../server-adapter/routes'; import { createRoute, pickParams, wrapSchemaForQueryParams } from '../server-adapter/routes/route-builder'; @@ -281,83 +279,6 @@ export const GET_BRANCH_ROUTE = createRoute({ } }, }); -/** Bounded trace page contract. Namespace guards preserve older core compatibility. */ -export const GET_TRACE_PAGE_ROUTE: ServerRoute = createRoute({ - method: 'POST', - path: '/observability/traces/:traceId/page', - responseType: 'json', - pathParamSchema: tracePageCore.tracePageIdSchema - ? z.object({ traceId: tracePageCore.tracePageIdSchema }) - : getTraceArgsSchema, - bodySchema: tracePageCore.tracePageRequestSchema ?? z.unknown().optional(), - responseSchema: tracePageCore.tracePageResponseSchema ?? z.never(), - maxBodySize: 65536, - requiresPermission: 'observability:read', - summary: 'Browse or search a bounded trace page', - description: 'Trace page navigation with bounded structure, optional details, and request-bound continuation.', - tags: ['Observability'], - requiresAuth: true, - handler: async ({ mastra, traceId, requestContext, ...request }) => { - try { - const store = await getObservabilityStore(mastra); - if ( - !tracePageCore.tracePageRequestSchema || - typeof store.getTracePage !== 'function' || - typeof store.supportsTraceNavigation !== 'function' || - !store.supportsTraceNavigation() - ) { - throw new HTTPException(501, { message: 'Bounded trace navigation is unavailable' }); - } - // Existing structure/batch primitives cannot filter an organization. Never widen a scoped read. - if (requestContext.get('organizationId')) { - throw new HTTPException(501, { message: 'Tenant-scoped trace navigation requires a scoped backend' }); - } - const user = requestContext.get(MASTRA_USER_KEY); - const userId = user && typeof user === 'object' && 'id' in user ? user.id : null; - const roles = requestContext.get(MASTRA_USER_ROLES_KEY); - const scopeKey = JSON.stringify({ - storageId: (await getStorage(mastra)).id, - userId: typeof userId === 'string' || typeof userId === 'number' ? userId : null, - resourceId: requestContext.get(MASTRA_RESOURCE_ID_KEY) ?? null, - authMode: requestContext.get(MASTRA_AUTH_MODE_KEY) ?? null, - roles: Array.isArray(roles) - ? [...new Set(roles.filter((role): role is string => typeof role === 'string'))].sort() - : [], - }); - // Route context contains additional server-owned fields; pick only the body contract. - const fields: Record = request; - const body = Object.fromEntries( - ['cursor', 'anchorSpanId', 'depth', 'spanIds', 'search', 'ancestors', 'include', 'limit', 'maxBytes'] - .filter(key => fields[key] !== undefined) - .map(key => [key, fields[key]]), - ); - const page = await store.getTracePage( - { traceId, ...tracePageCore.tracePageRequestSchema.parse(body) }, - { scopeKey }, - ); - if (!page) throw new HTTPException(404, { message: 'Trace not found' }); - return page; - } catch (error) { - if (tracePageCore.TracePageError && error instanceof tracePageCore.TracePageError) { - const status = - error.code === 'UNSUPPORTED' - ? 501 - : error.code === 'TRACE_CHANGED' - ? 409 - : error.code === 'INVALID_TOPOLOGY' - ? 500 - : error.code === 'LIMIT_EXCEEDED' - ? 422 - : error.code === 'PAYLOAD_TOO_LARGE' - ? 413 - : 400; - throw new HTTPException(status, { message: `${error.code}: ${error.message}` }); - } - return handleError(error, 'Error reading trace page'); - } - }, -}); - /** Route: GET /observability/traces/:traceId - retrieve a single trace with all spans. */ export const GET_TRACE_ROUTE: ServerRoute = createRoute({ method: 'GET', diff --git a/packages/server/src/server/handlers/system.test.ts b/packages/server/src/server/handlers/system.test.ts index 301773df5d8..b30aa49503f 100644 --- a/packages/server/src/server/handlers/system.test.ts +++ b/packages/server/src/server/handlers/system.test.ts @@ -19,8 +19,6 @@ const NO_OBSERVABILITY_CAPABILITIES = { deltaPolling: false, traceQuery: false, traceQueryAggregatedRoots: false, - traceNavigation: false, - traceNavigationSearch: false, traceQueryTextSearch: false, spanQueryTextSearch: false, traceQueryRootDuration: false, diff --git a/packages/server/src/server/handlers/system.ts b/packages/server/src/server/handlers/system.ts index ee8dc890e83..2ec68e1c7c0 100644 --- a/packages/server/src/server/handlers/system.ts +++ b/packages/server/src/server/handlers/system.ts @@ -150,7 +150,7 @@ export const GET_SYSTEM_PACKAGES_ROUTE = createRoute({ description: 'Returns installed Mastra packages and runtime capabilities used by Studio', tags: ['System'], requiresAuth: true, - handler: async ({ mastra, requestContext }) => { + handler: async ({ mastra }) => { try { const packagesFilePath = process.env.MASTRA_PACKAGES_FILE; @@ -170,9 +170,7 @@ export const GET_SYSTEM_PACKAGES_ROUTE = createRoute({ const observabilityStorage = storage?.stores?.observability; const observabilityStorageType = observabilityStorage?.constructor.name; const observabilityStorageCapabilities = observabilityStorage - ? getObservabilityStorageCapabilities(observabilityStorage, { - organizationId: requestContext?.get('organizationId'), - }) + ? getObservabilityStorageCapabilities(observabilityStorage) : undefined; const observabilityRuntimeStrategy = observabilityStorage?.runtimeTracingStrategy; const observabilityEnabled = !!mastra.observability.getDefaultInstance(); diff --git a/packages/server/src/server/handlers/trace-page.test.ts b/packages/server/src/server/handlers/trace-page.test.ts deleted file mode 100644 index d89709e34ff..00000000000 --- a/packages/server/src/server/handlers/trace-page.test.ts +++ /dev/null @@ -1,175 +0,0 @@ -import { Mastra } from '@mastra/core'; -import { SpanType } from '@mastra/core/observability'; -import { InMemoryStore, readTracePage } from '@mastra/core/storage'; -import { Hono } from 'hono'; -import { describe, expect, it } from 'vitest'; -import { getBuiltinMCPCapabilities } from '../mcp/capabilities'; -import { dispatchRoute } from '../server-adapter'; -import { OBSERVABILITY_ROUTES } from '../server-adapter/routes/observability'; -import { GET_TRACE_PAGE_ROUTE, GET_TRACE_ROUTE } from './observability'; -import { getObservabilityStorageCapabilities } from './observability-shared'; -import { GET_SYSTEM_PACKAGES_ROUTE } from './system'; -import { createTestServerContext } from './test-utils'; - -async function setup() { - const storage = new InMemoryStore(); - const store = (await storage.getStore('observability'))!; - await store.batchCreateSpans({ - records: ['root', 'child', 'nested'].map((spanId, index) => ({ - traceId: 'trace', - spanId, - parentSpanId: index ? ['root', 'child'][index - 1] : null, - name: spanId, - spanType: SpanType.AGENT_RUN, - isEvent: false, - startedAt: new Date('2026-11-01'), - input: { payload: spanId }, - })), - }); - // A deliberately opted-in backend over real storage; production OSS providers remain unsupported. - store.supportsTraceNavigation = () => true; - store.getTracePage = async ({ traceId, ...request }, options) => - readTracePage( - { - scopeKey: options!.scopeKey, - structure: async () => (await store.getStructure({ traceId }))?.spans ?? null, - spans: async (_, spanIds) => (await store.getSpans({ traceId, spanIds })).spans, - }, - traceId, - request, - ); - return { store, context: createTestServerContext({ mastra: new Mastra({ storage }) }) }; -} - -describe('trace page route with real storage', () => { - it('retains legacy full reads while bounded requests hydrate only the page', async () => { - const { store, context } = await setup(); - expect(getObservabilityStorageCapabilities(store)).toMatchObject({ - traceNavigation: true, - traceNavigationSearch: false, - }); - const legacy = await GET_TRACE_ROUTE.handler({ ...context, traceId: 'trace' }); - expect(legacy.spans.map((span: { spanId: string }) => span.spanId)).toEqual(['root', 'child', 'nested']); - const first = await GET_TRACE_PAGE_ROUTE.handler({ - ...context, - traceId: 'trace', - depth: 2, - limit: 1, - include: 'full', - }); - expect(first.nodes).toHaveLength(1); - expect(first.nodes[0]).toMatchObject({ spanId: 'root', childCount: 1, details: { input: { payload: 'root' } } }); - const next = await GET_TRACE_PAGE_ROUTE.handler({ - ...context, - traceId: 'trace', - cursor: first.nextCursor, - }); - expect(next.nodes[0]).toMatchObject({ spanId: 'child', details: { input: { payload: 'child' } } }); - }); - - it('reports unsupported indexed search instead of scanning payloads in memory', async () => { - const { context } = await setup(); - await expect( - GET_TRACE_PAGE_ROUTE.handler({ ...context, traceId: 'trace', search: 'payload' }), - ).rejects.toMatchObject({ status: 501 }); - }); - it('does not reuse cursors across authenticated users or silently widen tenant scope', async () => { - const { context } = await setup(); - context.requestContext.set('mastra__user', { id: 'alice' }); - const first = await GET_TRACE_PAGE_ROUTE.handler({ ...context, traceId: 'trace', limit: 1 }); - context.requestContext.set('mastra__user', { id: 'bob' }); - await expect( - GET_TRACE_PAGE_ROUTE.handler({ ...context, traceId: 'trace', cursor: first.nextCursor }), - ).rejects.toMatchObject({ status: 409 }); - context.requestContext.set('organizationId', 'other-org'); - await expect(GET_TRACE_PAGE_ROUTE.handler({ ...context, traceId: 'trace' })).rejects.toMatchObject({ - status: 501, - }); - }); -}); - -// Dispatch the registered routes in their real order: the paged route must not -// capture the existing static query/aggregate POSTs (OBS-956). -it('keeps trace queries and aggregates separate from trace page reads', async () => { - const app = new Hono(); - for (const route of OBSERVABILITY_ROUTES) { - if (route.method === 'POST') app.post(route.path, c => c.text(route.path)); - } - for (const name of ['query', 'aggregate']) { - const legacy = await app.request(`/observability/traces/${name}`, { method: 'POST' }); - expect(await legacy.text()).toBe(`/observability/traces/${name}`); - const page = await app.request(`/observability/traces/${name}/page`, { method: 'POST' }); - expect(await page.text()).toBe('/observability/traces/:traceId/page'); - } -}); - -it('does not infer bounded navigation from existing OSS storage primitives', async () => { - const store = (await new InMemoryStore().getStore('observability'))!; - expect(getObservabilityStorageCapabilities(store).traceNavigation).toBe(false); - await expect(store.getTracePage({ traceId: 'trace' })).rejects.toMatchObject({ code: 'UNSUPPORTED' }); -}); - -it('applies request scope after cached MCP support and to system capabilities', async () => { - const { context } = await setup(); - expect((await getBuiltinMCPCapabilities(context.mastra)).observability?.traceNavigation).toBe(true); - expect( - (await getBuiltinMCPCapabilities(context.mastra, { organizationId: 'org' })).observability?.traceNavigation, - ).toBe(false); - expect((await getBuiltinMCPCapabilities(context.mastra)).observability?.traceNavigation).toBe(true); - context.requestContext.set('organizationId', 'org'); - const result = await GET_SYSTEM_PACKAGES_ROUTE.handler(context); - expect(JSON.stringify(result)).toContain('"traceNavigation":false'); -}); - -it('validates trace page path and body through HTTP dispatch and preserves server error mappings', async () => { - const { store, context } = await setup(); - const app = new Hono(); - app.post(GET_TRACE_PAGE_ROUTE.path, async c => { - const text = await c.req.text(); - const result = await dispatchRoute( - GET_TRACE_PAGE_ROUTE, - { pathParams: { traceId: c.req.param('traceId') }, body: text ? JSON.parse(text) : undefined }, - { mastra: context.mastra, requestContext: context.requestContext, abortSignal: c.req.raw.signal }, - ); - return new Response(JSON.stringify(result.ok ? result.result : result.body), { - status: result.ok ? 200 : result.status, - }); - }); - const post = (id: string, body?: unknown) => - app.request(`/observability/traces/${id}/page`, { - method: 'POST', - ...(body === undefined ? {} : { body: JSON.stringify(body), headers: { 'content-type': 'application/json' } }), - }); - expect((await post('a'.repeat(129), {})).status).toBe(400); - expect((await post(encodeURIComponent('界'.repeat(128)), {})).status).toBe(400); - expect((await post('trace')).status).toBe(400); - expect((await post('trace', { limit: 1 })).status).toBe(200); - context.requestContext.set('organizationId', 'org'); - expect((await post('trace', {})).status).toBe(501); - context.requestContext.delete('organizationId'); - await store.batchCreateSpans({ - records: [ - { - traceId: 'cycle', - spanId: 'loop', - parentSpanId: 'loop', - name: 'loop', - spanType: SpanType.AGENT_RUN, - isEvent: false, - startedAt: new Date('2026-11-01'), - }, - ], - }); - expect((await post('cycle', {})).status).toBe(500); - await store.batchCreateSpans({ - records: Array.from({ length: 50001 }, (_, i) => ({ - traceId: 'over-budget', - spanId: String(i), - name: 'span', - spanType: SpanType.AGENT_RUN, - isEvent: false, - startedAt: new Date('2026-11-01'), - })), - }); - expect((await post('over-budget', {})).status).toBe(422); -}); diff --git a/packages/server/src/server/mcp/capabilities.ts b/packages/server/src/server/mcp/capabilities.ts index c327425814f..9346a1516f8 100644 --- a/packages/server/src/server/mcp/capabilities.ts +++ b/packages/server/src/server/mcp/capabilities.ts @@ -63,10 +63,7 @@ async function resolveStoreCapabilities(mastra: Mastra): Promise { +export async function getBuiltinMCPCapabilities(mastra: Mastra): Promise { let pending = storeCapabilities.get(mastra); if (!pending) { pending = resolveStoreCapabilities(mastra); @@ -78,14 +75,5 @@ export async function getBuiltinMCPCapabilities( const agentMemory = Object.values(mastra.listAgents?.() ?? {}).some(agent => agent.hasOwnMemory?.()); const store = await pending; if (!store) storeCapabilities.delete(mastra); - const resolved = store ?? NO_STORE; - return { - ...resolved, - observability: - resolved.observability && scope?.organizationId - ? { ...resolved.observability, traceNavigation: false, traceNavigationSearch: false } - : resolved.observability, - storedAgents, - memory: agentMemory || storedAgents, - }; + return { ...(store ?? NO_STORE), storedAgents, memory: agentMemory || storedAgents }; } diff --git a/packages/server/src/server/mcp/catalogue.ts b/packages/server/src/server/mcp/catalogue.ts index 225d8de7452..05618c3de9c 100644 --- a/packages/server/src/server/mcp/catalogue.ts +++ b/packages/server/src/server/mcp/catalogue.ts @@ -183,9 +183,7 @@ export async function buildBuiltinMCPTools( config: ResolvedBuiltinMCPConfig, ): Promise> { const { mastra, requestContext } = ctx; - const capabilities = await getBuiltinMCPCapabilities(mastra, { - organizationId: requestContext.get('organizationId'), - }); + const capabilities = await getBuiltinMCPCapabilities(mastra); const routeContext: ServerContext = { mastra, requestContext, diff --git a/packages/server/src/server/schemas/system.ts b/packages/server/src/server/schemas/system.ts index 6e392b97cbd..db40d68bd06 100644 --- a/packages/server/src/server/schemas/system.ts +++ b/packages/server/src/server/schemas/system.ts @@ -41,8 +41,6 @@ export const observabilityStorageCapabilitiesSchema = z.object({ .boolean() .default(false) .describe("Supports explicit `rootSource: 'aggregated'` trace queries"), - traceNavigation: z.boolean().default(false).describe('Bounded getTracePage browsing and hydration'), - traceNavigationSearch: z.boolean().default(false).describe('Indexed in-trace search with ancestor context'), traceQueryTextSearch: z.boolean().default(false).describe('Literal substring search across any span in a trace'), spanQueryTextSearch: z.boolean().default(false).describe('Literal substring search within individual spans'), traceQueryRootDuration: z.boolean().describe('`durationMs` predicates in trace and thread queries'), diff --git a/packages/server/src/server/server-adapter/routes/observability.ts b/packages/server/src/server/server-adapter/routes/observability.ts index 81d97d7f8c6..ff9df6c0c1d 100644 --- a/packages/server/src/server/server-adapter/routes/observability.ts +++ b/packages/server/src/server/server-adapter/routes/observability.ts @@ -4,7 +4,6 @@ import { LIST_BRANCHES_ROUTE, GET_BRANCH_ROUTE, GET_TRACE_ROUTE, - GET_TRACE_PAGE_ROUTE, GET_TRACE_LIGHT_ROUTE, GET_SPAN_ROUTE, GET_TRACE_TRAJECTORY_ROUTE, @@ -62,7 +61,6 @@ export const OBSERVABILITY_ROUTES = [ LIST_BRANCHES_ROUTE, GET_BRANCH_ROUTE, GET_TRACE_ROUTE, - GET_TRACE_PAGE_ROUTE, GET_TRACE_LIGHT_ROUTE, GET_SPAN_ROUTE, GET_TRACE_TRAJECTORY_ROUTE, From d0482ccf0646bbbe39cc58c9bb6d161df0cbb02c Mon Sep 17 00:00:00 2001 From: Eric Pinzur Date: Mon, 16 Nov 2026 16:40:28 +0100 Subject: [PATCH 10/23] fix(observability): align trace page capabilities and contract checks --- .changeset/eager-meteors-wear.md | 2 +- client-sdks/client-js/src/client.ts | 5 +- .../client-js/src/resources/observability.ts | 5 +- .../src/resources/trace-page-types.ts | 1 - client-sdks/client-js/src/types.ts | 4 +- .../client-js/type-tests/trace-page.test-d.ts | 2 +- .../type-tests/tsconfig.thread-query.json | 1 + .../type-tests/tsconfig.trace-page.json | 4 - client-sdks/client-js/vitest.config.js | 1 + .../domains/observability/trace-page.test.ts | 1 + .../domains/observability/trace-page.ts | 240 +++++++++--------- 11 files changed, 136 insertions(+), 130 deletions(-) delete mode 100644 client-sdks/client-js/type-tests/tsconfig.trace-page.json diff --git a/.changeset/eager-meteors-wear.md b/.changeset/eager-meteors-wear.md index faf1d41e711..22a8680647d 100644 --- a/.changeset/eager-meteors-wear.md +++ b/.changeset/eager-meteors-wear.md @@ -2,4 +2,4 @@ '@mastra/client-js': minor --- -Add getTracePage() for supporting backends. +Add getTracePage() for Platform. diff --git a/client-sdks/client-js/src/client.ts b/client-sdks/client-js/src/client.ts index c94c773581f..d7498e0563d 100644 --- a/client-sdks/client-js/src/client.ts +++ b/client-sdks/client-js/src/client.ts @@ -1285,7 +1285,10 @@ export class MastraClient extends BaseResource { return this.observability.getTrace(traceId); } - /** Retrieves a bounded trace page from a supporting backend. */ + /** + * Retrieves a bounded trace page from Platform. OSS servers do not serve this endpoint. + * Check `capabilities.tracePage` before calling. + */ getTracePage(request: GetTracePageArgs): Promise { return this.observability.getTracePage(request); } diff --git a/client-sdks/client-js/src/resources/observability.ts b/client-sdks/client-js/src/resources/observability.ts index df384eba714..aac381152ed 100644 --- a/client-sdks/client-js/src/resources/observability.ts +++ b/client-sdks/client-js/src/resources/observability.ts @@ -235,7 +235,10 @@ export class Observability extends BaseResource { return this.request(`/observability/traces/${encodeURIComponent(traceId)}`); } - /** Retrieves a bounded trace page from a supporting backend. */ + /** + * Retrieves a bounded trace page from Platform. OSS servers do not serve this endpoint. + * Check `capabilities.tracePage` before calling. + */ getTracePage(request: GetTracePageArgs): Promise { const { traceId, ...body } = request; return this.request(`/observability/traces/${encodeURIComponent(traceId)}/page`, { method: 'POST', body }); diff --git a/client-sdks/client-js/src/resources/trace-page-types.ts b/client-sdks/client-js/src/resources/trace-page-types.ts index d375cc35e34..2cdf2acffd6 100644 --- a/client-sdks/client-js/src/resources/trace-page-types.ts +++ b/client-sdks/client-js/src/resources/trace-page-types.ts @@ -38,7 +38,6 @@ export type TracePageNode = Pick< details?: Serialized; }; export interface TracePageResponse { - version: 2; traceId: string; nodes: TracePageNode[]; missingSpanIds: string[]; diff --git a/client-sdks/client-js/src/types.ts b/client-sdks/client-js/src/types.ts index 13a56a36a96..db0f6c9073c 100644 --- a/client-sdks/client-js/src/types.ts +++ b/client-sdks/client-js/src/types.ts @@ -1496,8 +1496,8 @@ type BaseObservabilityCapabilitiesResponse = GeneratedResponse<'GET /observabili export type GetObservabilityCapabilitiesResponse = Omit & { capabilities: BaseObservabilityCapabilitiesResponse['capabilities'] & { /** Optional capabilities advertised by Platform's trace page endpoint. */ - traceNavigation?: boolean; - traceNavigationSearch?: boolean; + tracePage?: boolean; + tracePageSearch?: boolean; }; }; diff --git a/client-sdks/client-js/type-tests/trace-page.test-d.ts b/client-sdks/client-js/type-tests/trace-page.test-d.ts index 9b5f5b89b7c..496f3efcfd9 100644 --- a/client-sdks/client-js/type-tests/trace-page.test-d.ts +++ b/client-sdks/client-js/type-tests/trace-page.test-d.ts @@ -18,7 +18,7 @@ test('keeps Platform trace page types aligned without exposing an OSS server rou expectTypeOf().toMatchTypeOf(); expectTypeOf>().toEqualTypeOf>(); expectTypeOf>().toEqualTypeOf(); - expectTypeOf().toEqualTypeOf< + expectTypeOf().toEqualTypeOf< boolean | undefined >(); }); diff --git a/client-sdks/client-js/type-tests/tsconfig.thread-query.json b/client-sdks/client-js/type-tests/tsconfig.thread-query.json index a342b992e44..0381852df4f 100644 --- a/client-sdks/client-js/type-tests/tsconfig.thread-query.json +++ b/client-sdks/client-js/type-tests/tsconfig.thread-query.json @@ -4,6 +4,7 @@ "include": [ "../src/**/*.ts", "./thread-query.test-d.ts", + "./trace-page.test-d.ts", "./deletion.test-d.ts", "./request-context.test-d.ts", "./generate-legacy.test-d.ts" diff --git a/client-sdks/client-js/type-tests/tsconfig.trace-page.json b/client-sdks/client-js/type-tests/tsconfig.trace-page.json deleted file mode 100644 index 83b4bfeeb6d..00000000000 --- a/client-sdks/client-js/type-tests/tsconfig.trace-page.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "extends": "./tsconfig.json", - "include": ["../src/**/*.ts", "./trace-page.test-d.ts"] -} diff --git a/client-sdks/client-js/vitest.config.js b/client-sdks/client-js/vitest.config.js index 02c2c0c12d6..055a8c337ea 100644 --- a/client-sdks/client-js/vitest.config.js +++ b/client-sdks/client-js/vitest.config.js @@ -9,6 +9,7 @@ export default defineConfig({ enabled: true, include: [ 'type-tests/thread-query.test-d.ts', + 'type-tests/trace-page.test-d.ts', 'type-tests/deletion.test-d.ts', 'type-tests/request-context.test-d.ts', 'type-tests/generate-legacy.test-d.ts', diff --git a/packages/core/src/storage/domains/observability/trace-page.test.ts b/packages/core/src/storage/domains/observability/trace-page.test.ts index 8810e8f3f5c..7214f245174 100644 --- a/packages/core/src/storage/domains/observability/trace-page.test.ts +++ b/packages/core/src/storage/domains/observability/trace-page.test.ts @@ -38,6 +38,7 @@ describe('bounded trace exploration', () => { expect(pages.flatMap(page => page.nodes.map(node => node.spanId))).toEqual(['root', 'a', 'z']); expect(pages[0]!.nodes[1]).toMatchObject({ childCount: 1 }); expect(pages.at(-1)!.nextCursor).toBeUndefined(); + expect(pages[0]).not.toHaveProperty('version'); }); it('finds an unloaded deep match and continues its ancestor path across strict total node limits', async () => { diff --git a/packages/core/src/storage/domains/observability/trace-page.ts b/packages/core/src/storage/domains/observability/trace-page.ts index c40b8e42dbf..0e0e0c9e05d 100644 --- a/packages/core/src/storage/domains/observability/trace-page.ts +++ b/packages/core/src/storage/domains/observability/trace-page.ts @@ -64,7 +64,6 @@ export const tracePageNodeSchema = spanRecordSchema details: spanRecordSchema.optional(), }); export const tracePageResponseSchema = z.object({ - version: z.literal(2), traceId: z.string(), nodes: z.array(tracePageNodeSchema), missingSpanIds: z.array(z.string()), @@ -111,7 +110,11 @@ export interface TracePageSource { scopeKey: string; /** Return at most limit topology rows; the extra row detects overflow. */ structure(traceId: string, limit: number): Promise; - /** Read only the selected IDs, excluding payload fields for structure projection. */ + /** + * Read only the selected IDs, excluding payload fields for structure projection. + * Include a non-null `error` indicator for failed spans so status is computed correctly. + * Full projection must return complete SpanRecords, including the actual error payload. + */ spans(traceId: string, spanIds: string[], include: 'structure' | 'full'): Promise; /** Complete matching identities within this trace; fail rather than truncate. */ search?(traceId: string, text: string): Promise; @@ -150,6 +153,122 @@ export async function readTracePage( if (selection.search && !source.search) throw new TracePageError('UNSUPPORTED', 'Indexed in-trace search is unavailable'); const offset = continuation?.offset ?? 0; + const renderPage = async ( + ids: string[], + fingerprint: string, + topology?: { + nodes: Map; + children: Map; + matches: Set; + missingSpanIds: string[]; + unavailableAncestors: boolean; + }, + ): Promise => { + // Explicit-ID cursors depend on the requested positions, never on fetched membership. + if (continuation && continuation.fingerprint !== fingerprint) + throw new TracePageError('TRACE_CHANGED', 'Selection or request scope changed; restart this selection'); + if (continuation && offset >= ids.length) + throw new TracePageError('INVALID_CURSOR', 'Continuation is outside the selection'); + const pageIds = ids.slice(offset, offset + selection.limit); + const records = pageIds.length ? await source.spans(traceId, pageIds, selection.include) : []; + const recordsById = new Map(); + for (const record of records) { + if ( + record.traceId !== traceId || + !pageIds.includes(record.spanId) || + recordsById.has(record.spanId) || + !Number.isFinite(record.startedAt.getTime()) + ) + throw new TracePageError('INVALID_TOPOLOGY', 'Page read returned conflicting identities or timestamps'); + const node = topology?.nodes.get(record.spanId); + if ( + node && + ((node.parentSpanId ?? null) !== (record.parentSpanId ?? null) || + node.startedAt.getTime() !== record.startedAt.getTime()) + ) + throw new TracePageError('TRACE_CHANGED', 'Selected structure changed during the read'); + recordsById.set(record.spanId, record); + } + const result: TracePageResponse = { + traceId, + nodes: [], + missingSpanIds: [...(topology?.missingSpanIds ?? [])], + ancestorPaths: 'not-requested', + }; + let processed = 0; + const updateContinuation = () => { + const nextOffset = offset + processed; + if (nextOffset < ids.length) + result.nextCursor = Buffer.from( + JSON.stringify({ + version: 2, + selection, + fingerprint, + offset: nextOffset, + }), + ).toString('base64url'); + else delete result.nextCursor; + if (selection.search && selection.ancestors !== false) + result.ancestorPaths = result.nextCursor + ? 'continued' + : topology?.unavailableAncestors + ? 'unavailable' + : 'complete'; + }; + for (const spanId of pageIds) { + const record = recordsById.get(spanId); + if (!record) { + if (topology) throw new TracePageError('TRACE_CHANGED', 'Selected spans disappeared; restart this selection'); + result.missingSpanIds.push(spanId); + } else { + const { traceId, parentSpanId, name, spanType, isEvent, startedAt, endedAt, entityType, entityId, entityName } = + record; + result.nodes.push({ + traceId, + spanId, + parentSpanId, + name, + spanType, + isEvent, + startedAt, + endedAt, + entityType, + entityId, + entityName, + status: computeTraceStatus(record), + childCount: topology ? (topology.children.get(spanId)?.length ?? 0) : null, + match: topology?.matches.has(spanId) ?? false, + parentState: !parentSpanId + ? 'root' + : !topology + ? 'not-requested' + : topology.nodes.has(parentSpanId) + ? 'available' + : 'unavailable', + ...(selection.include === 'full' ? { details: record as SpanRecord } : {}), + }); + } + processed++; + updateContinuation(); + if (Buffer.byteLength(JSON.stringify(result)) > selection.maxBytes) { + if (record) result.nodes.pop(); + else result.missingSpanIds.pop(); + processed--; + updateContinuation(); + if (!processed) + throw new TracePageError( + 'PAYLOAD_TOO_LARGE', + `Span '${spanId}' exceeds maxBytes; request structure or use the single-span endpoint`, + ); + break; + } + } + updateContinuation(); + if (Buffer.byteLength(JSON.stringify(result)) > selection.maxBytes) + throw new TracePageError('LIMIT_EXCEEDED', 'Trace response metadata exceeds maxBytes'); + return result; + }; + if (selection.spanIds) { const ids = [...new Set(selection.spanIds)]; const fingerprint = hash([source.scopeKey, traceId, selection]); @@ -246,121 +365,4 @@ export async function readTracePage( unavailableAncestors: selected.some(node => node.parentSpanId && !nodes.has(node.parentSpanId)), }, ); - - async function renderPage( - ids: string[], - fingerprint: string, - topology?: { - nodes: Map; - children: Map; - matches: Set; - missingSpanIds: string[]; - unavailableAncestors: boolean; - }, - ): Promise { - // Explicit-ID cursors depend on the requested positions, never on fetched membership. - if (continuation && continuation.fingerprint !== fingerprint) - throw new TracePageError('TRACE_CHANGED', 'Selection or request scope changed; restart this selection'); - if (continuation && offset >= ids.length) - throw new TracePageError('INVALID_CURSOR', 'Continuation is outside the selection'); - const pageIds = ids.slice(offset, offset + selection!.limit); - const records = pageIds.length ? await source.spans(traceId, pageIds, selection!.include) : []; - const recordsById = new Map(); - for (const record of records) { - if ( - record.traceId !== traceId || - !pageIds.includes(record.spanId) || - recordsById.has(record.spanId) || - !Number.isFinite(record.startedAt.getTime()) - ) - throw new TracePageError('INVALID_TOPOLOGY', 'Page read returned conflicting identities or timestamps'); - const node = topology?.nodes.get(record.spanId); - if ( - node && - ((node.parentSpanId ?? null) !== (record.parentSpanId ?? null) || - node.startedAt.getTime() !== record.startedAt.getTime()) - ) - throw new TracePageError('TRACE_CHANGED', 'Selected structure changed during the read'); - recordsById.set(record.spanId, record); - } - const result: TracePageResponse = { - version: 2, - traceId, - nodes: [], - missingSpanIds: [...(topology?.missingSpanIds ?? [])], - ancestorPaths: 'not-requested', - }; - let processed = 0; - const updateContinuation = () => { - const nextOffset = offset + processed; - if (nextOffset < ids.length) - result.nextCursor = Buffer.from( - JSON.stringify({ - version: 2, - selection, - fingerprint, - offset: nextOffset, - }), - ).toString('base64url'); - else delete result.nextCursor; - if (selection!.search && selection!.ancestors !== false) - result.ancestorPaths = result.nextCursor - ? 'continued' - : topology?.unavailableAncestors - ? 'unavailable' - : 'complete'; - }; - for (const spanId of pageIds) { - const record = recordsById.get(spanId); - if (!record) { - if (topology) throw new TracePageError('TRACE_CHANGED', 'Selected spans disappeared; restart this selection'); - result.missingSpanIds.push(spanId); - } else { - const { traceId, parentSpanId, name, spanType, isEvent, startedAt, endedAt, entityType, entityId, entityName } = - record; - result.nodes.push({ - traceId, - spanId, - parentSpanId, - name, - spanType, - isEvent, - startedAt, - endedAt, - entityType, - entityId, - entityName, - status: computeTraceStatus(record), - childCount: topology ? (topology.children.get(spanId)?.length ?? 0) : null, - match: topology?.matches.has(spanId) ?? false, - parentState: !parentSpanId - ? 'root' - : !topology - ? 'not-requested' - : topology.nodes.has(parentSpanId) - ? 'available' - : 'unavailable', - ...(selection!.include === 'full' ? { details: record as SpanRecord } : {}), - }); - } - processed++; - updateContinuation(); - if (Buffer.byteLength(JSON.stringify(result)) > selection!.maxBytes) { - if (record) result.nodes.pop(); - else result.missingSpanIds.pop(); - processed--; - updateContinuation(); - if (!processed) - throw new TracePageError( - 'PAYLOAD_TOO_LARGE', - `Span '${spanId}' exceeds maxBytes; request structure or use the single-span endpoint`, - ); - break; - } - } - updateContinuation(); - if (Buffer.byteLength(JSON.stringify(result)) > selection!.maxBytes) - throw new TracePageError('LIMIT_EXCEEDED', 'Trace response metadata exceeds maxBytes'); - return result; - } } From 34a00ffd3baf7e02508ca071da4ffb2af981b421 Mon Sep 17 00:00:00 2001 From: Eric Pinzur Date: Tue, 17 Nov 2026 04:21:14 +0100 Subject: [PATCH 11/23] fix(client-js): retry experimental trace page reads Address finding 3: preserve the read retry policy for initial pages and continuations, with real HTTP regressions for transient and terminal failures. Review: https://github.com/mastra-ai/mastra/pull/30512#pullrequestreview-5925420901 --- .../observability.trace-page.test.ts | 60 ++++++++++++++++++- .../client-js/src/resources/observability.ts | 6 +- 2 files changed, 62 insertions(+), 4 deletions(-) diff --git a/client-sdks/client-js/src/resources/observability.trace-page.test.ts b/client-sdks/client-js/src/resources/observability.trace-page.test.ts index e23f634140a..c13819e2617 100644 --- a/client-sdks/client-js/src/resources/observability.trace-page.test.ts +++ b/client-sdks/client-js/src/resources/observability.trace-page.test.ts @@ -2,13 +2,21 @@ import { createServer } from 'node:http'; import type { AddressInfo } from 'node:net'; import { SpanType } from '@mastra/core/observability'; import { InMemoryStore, readTracePage } from '@mastra/core/storage'; -import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; import { MastraClient } from '../client'; // Real HTTP transport and real storage: proves the paged API does not serialize IDs or cursors as GET parameters. describe('legacy and paged trace reads over HTTP', () => { const storage = new InMemoryStore(); let client: MastraClient; + let failures: (number | 'disconnect')[] = []; + let requests: unknown[] = []; + const makeClient = (retries?: number) => + new MastraClient({ + baseUrl: `http://127.0.0.1:${(server.address() as AddressInfo).port}`, + retries, + backoffMs: 1, + }); const server = createServer(async (req, res) => { try { expect(req.url).toBe( @@ -17,6 +25,17 @@ describe('legacy and paged trace reads over HTTP', () => { const store = (await storage.getStore('observability'))!; const chunks: Buffer[] = []; for await (const chunk of req) chunks.push(Buffer.from(chunk)); + const body = chunks.length ? JSON.parse(Buffer.concat(chunks).toString()) : undefined; + requests.push(body); + const failure = failures.shift(); + if (failure === 'disconnect') { + req.socket.destroy(); + return; + } + if (failure) { + res.writeHead(failure).end(JSON.stringify({ error: 'Request failed' })); + return; + } const result = req.method === 'GET' ? await store.getTrace({ traceId: 'trace' }) @@ -27,7 +46,7 @@ describe('legacy and paged trace reads over HTTP', () => { spans: async (_, spanIds) => (await store.getSpans({ traceId: 'trace', spanIds })).spans, }, 'trace', - JSON.parse(Buffer.concat(chunks).toString()), + body, ); res.setHeader('content-type', 'application/json'); res.end(JSON.stringify(result)); @@ -49,7 +68,11 @@ describe('legacy and paged trace reads over HTTP', () => { })), }); await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); - client = new MastraClient({ baseUrl: `http://127.0.0.1:${(server.address() as AddressInfo).port}`, retries: 0 }); + client = makeClient(0); + }); + beforeEach(() => { + failures = []; + requests = []; }); afterAll(async () => { server.closeAllConnections(); @@ -68,4 +91,35 @@ describe('legacy and paged trace reads over HTTP', () => { expect(page.nodes).toHaveLength(1); expect(page.nodes[0]!.details?.input).toEqual({ value: 'b' }); }); + it.each([503, 'disconnect'] as const)( + 'retries a trace page after %s without changing its selection', + async failure => { + failures = [failure]; + const page = await makeClient().getTracePage({ traceId: 'trace', spanIds: ['b'], include: 'full' }); + expect(page.nodes.map(node => node.spanId)).toEqual(['b']); + expect(requests).toEqual([ + { spanIds: ['b'], include: 'full' }, + { spanIds: ['b'], include: 'full' }, + ]); + }, + ); + it('retries a continuation without losing or repeating a span', async () => { + const first = await client.getTracePage({ traceId: 'trace', limit: 1 }); + requests = []; + failures = [503]; + const next = await makeClient().getTracePage({ traceId: 'trace', cursor: first.nextCursor! }); + expect([...first.nodes, ...next.nodes].map(node => node.spanId)).toEqual(['a', 'b']); + expect(next.nextCursor).toBeUndefined(); + expect(requests).toEqual([{ cursor: first.nextCursor }, { cursor: first.nextCursor }]); + }); + it.each([400, 409, 413, 422, 501])('does not retry terminal HTTP %s', async status => { + failures = [status]; + await expect(makeClient().getTracePage({ traceId: 'trace' })).rejects.toMatchObject({ status }); + expect(requests).toEqual([{}]); + }); + it('honors explicitly disabled retries', async () => { + failures = [503]; + await expect(client.getTracePage({ traceId: 'trace' })).rejects.toMatchObject({ status: 503 }); + expect(requests).toEqual([{}]); + }); }); diff --git a/client-sdks/client-js/src/resources/observability.ts b/client-sdks/client-js/src/resources/observability.ts index aac381152ed..2e54642bcf4 100644 --- a/client-sdks/client-js/src/resources/observability.ts +++ b/client-sdks/client-js/src/resources/observability.ts @@ -241,7 +241,11 @@ export class Observability extends BaseResource { */ getTracePage(request: GetTracePageArgs): Promise { const { traceId, ...body } = request; - return this.request(`/observability/traces/${encodeURIComponent(traceId)}/page`, { method: 'POST', body }); + return this.request(`/observability/traces/${encodeURIComponent(traceId)}/page`, { + method: 'POST', + body, + retries: this.readRetries(), + }); } /** From 377df5594628af88a016348e4df0e6d8ca1c25af Mon Sep 17 00:00:00 2001 From: Eric Pinzur Date: Tue, 17 Nov 2026 04:22:16 +0100 Subject: [PATCH 12/23] test(core): cover trace page guards and byte boundaries Address finding 5 with topology limits, forged cursor validation, selection disappearance, and exact UTF-8 response budgets. Nine targeted guard mutations fail these regressions. Review: https://github.com/mastra-ai/mastra/pull/30512#pullrequestreview-5925420901 --- .../domains/observability/trace-page.test.ts | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/packages/core/src/storage/domains/observability/trace-page.test.ts b/packages/core/src/storage/domains/observability/trace-page.test.ts index 7214f245174..0bde64aa2d2 100644 --- a/packages/core/src/storage/domains/observability/trace-page.test.ts +++ b/packages/core/src/storage/domains/observability/trace-page.test.ts @@ -1,3 +1,4 @@ +import { createHash } from 'node:crypto'; import { describe, expect, it } from 'vitest'; import { SpanType } from '../../../observability/types'; import { readTracePage, tracePageRequestSchema } from './trace-page'; @@ -136,6 +137,104 @@ describe('bounded trace exploration', () => { }); }); +describe('trace page request boundaries', () => { + it('accepts exactly 50000 topology rows and rejects the overflow sentinel', async () => { + const rows = Array.from({ length: 50000 }, (_, i) => span(String(i).padStart(5, '0'))); + const page = (await readTracePage(source(rows), 'trace', { limit: 1 }))!; + expect(page.nodes.map(node => node.spanId)).toEqual(['00000']); + expect(page.nextCursor).toBeTruthy(); + rows.push(span('50000')); + await expect(readTracePage(source(rows), 'trace', { limit: 1 })).rejects.toMatchObject({ + code: 'LIMIT_EXCEEDED', + }); + }); + + it.each([{ limit: 201 }, { maxBytes: 4194305 }])('rejects request budgets above the cap: %j', request => { + expect(tracePageRequestSchema.safeParse(request).success).toBe(false); + }); + it('accepts the maximum request budgets', () => { + expect(tracePageRequestSchema.safeParse({ limit: 200, maxBytes: 4194304 }).success).toBe(true); + }); + + it.each([{ limit: 1000 }, { maxBytes: 4194305 }, { spanIds: ['a', 'b'], search: 'error' }])( + 'revalidates selection in a forged cursor: %j', + async changes => { + const src = source([span('a'), span('b')]); + const page = (await readTracePage(src, 'trace', { spanIds: ['a', 'b'], limit: 1 }))!; + const cursor = JSON.parse(Buffer.from(page.nextCursor!, 'base64url').toString()); + Object.assign(cursor.selection, changes); + // Even a recomputed fingerprint cannot authorize invalid selection or budgets. + cursor.fingerprint = createHash('sha256') + .update(JSON.stringify([src.scopeKey, 'trace', cursor.selection])) + .digest('hex'); + await expect( + readTracePage(src, 'trace', { cursor: Buffer.from(JSON.stringify(cursor)).toString('base64url') }), + ).rejects.toMatchObject({ code: 'INVALID_CURSOR' }); + }, + ); + + it.each([0, -1, 1.5, 2, 3, 50001])('rejects a cursor offset outside the selection: %s', async offset => { + const src = source([span('a'), span('b')]); + const page = (await readTracePage(src, 'trace', { spanIds: ['a', 'b'], limit: 1 }))!; + const cursor = JSON.parse(Buffer.from(page.nextCursor!, 'base64url').toString()); + cursor.offset = offset; + await expect( + readTracePage(src, 'trace', { cursor: Buffer.from(JSON.stringify(cursor)).toString('base64url') }), + ).rejects.toMatchObject({ code: 'INVALID_CURSOR' }); + }); + + it('rejects non-base64url characters even when the decoder would ignore them', async () => { + const src = source([span('a'), span('b')]); + const page = (await readTracePage(src, 'trace', { limit: 1 }))!; + await expect(readTracePage(src, 'trace', { cursor: `${page.nextCursor!}\n` })).rejects.toMatchObject({ + code: 'INVALID_CURSOR', + }); + }); + + it('limits cursor text before decoding an otherwise valid JSON continuation', async () => { + const src = source([span('a'), span('b')]); + const page = (await readTracePage(src, 'trace', { limit: 1 }))!; + const json = Buffer.from(page.nextCursor!, 'base64url').toString(); + const cursor = Buffer.from(json + ' '.repeat(45001)).toString('base64url'); + expect(cursor.length).toBeGreaterThan(60000); + await expect(readTracePage(src, 'trace', { cursor })).rejects.toThrow(); + expect(tracePageRequestSchema.safeParse({ cursor: 'a'.repeat(60000) }).success).toBe(true); + expect(tracePageRequestSchema.safeParse({ cursor: 'a'.repeat(60001) }).success).toBe(false); + }); + + it('requires a restart if a selected span disappears during hydration', async () => { + const src = source([span('a'), span('b')]); + const readSpans = src.spans; + src.spans = async (...args) => (await readSpans(...args)).filter(row => row.spanId !== 'b'); + await expect(readTracePage(src, 'trace', {})).rejects.toMatchObject({ code: 'TRACE_CHANGED' }); + }); +}); + +it.each(['browse', 'hydrate', 'search'] as const)( + 'accounts for exact UTF-8 and JSON byte boundaries in %s pages', + async mode => { + const rows = [span('a', null, '界"\n'.repeat(4000)), span('b', 'a', '界"\n'.repeat(4000))]; + const src = source(rows, ['b']); + const selection = + mode === 'hydrate' ? { spanIds: ['missing"界', 'a', 'b'] } : mode === 'search' ? { search: 'error' } : {}; + const request = { ...selection, include: 'full' as const, maxBytes: 65536 }; + const sample = (await readTracePage(src, 'trace', request))!; + rows[0]!.input += 'x'.repeat(65536 - Buffer.byteLength(JSON.stringify(sample))); + const exact = (await readTracePage(src, 'trace', request))!; + expect(Buffer.byteLength(JSON.stringify(exact))).toBe(65536); + expect(exact.nodes.map(node => node.spanId)).toEqual(['a', 'b']); + expect(exact.nextCursor).toBeUndefined(); + + rows[0]!.input += 'x'; + const pages = await drain(src, (await readTracePage(src, 'trace', request))!); + expect(pages).toHaveLength(2); + expect(pages.flatMap(page => page.nodes.map(node => node.spanId))).toEqual(['a', 'b']); + expect(pages.flatMap(page => page.missingSpanIds)).toEqual(mode === 'hydrate' ? ['missing"界'] : []); + for (const page of pages) expect(Buffer.byteLength(JSON.stringify(page))).toBeLessThanOrEqual(65536); + if (mode === 'search') expect(pages.map(page => page.ancestorPaths)).toEqual(['continued', 'complete']); + }, +); + it('reads each explicit ID only on its own page and survives unrelated arrivals', async () => { const rows = Array.from({ length: 100 }, (_, i) => span(String(i), null, 'large payload')); let reads = 0; From ac9aa37902138b1714d3f26d0836034cda68a3f7 Mon Sep 17 00:00:00 2001 From: Eric Pinzur Date: Tue, 17 Nov 2026 04:23:14 +0100 Subject: [PATCH 13/23] test(client-js): detect optional trace page field drift Address finding 6 with response and node key equality alongside assignability. Removing details or adding an optional response field fails the configured type suite. Review: https://github.com/mastra-ai/mastra/pull/30512#pullrequestreview-5925420901 --- client-sdks/client-js/type-tests/trace-page.test-d.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/client-sdks/client-js/type-tests/trace-page.test-d.ts b/client-sdks/client-js/type-tests/trace-page.test-d.ts index 496f3efcfd9..8c3d1cbe18b 100644 --- a/client-sdks/client-js/type-tests/trace-page.test-d.ts +++ b/client-sdks/client-js/type-tests/trace-page.test-d.ts @@ -16,6 +16,8 @@ test('keeps Platform trace page types aligned without exposing an OSS server rou expectTypeOf().toEqualTypeOf(); expectTypeOf().toMatchTypeOf(); expectTypeOf().toMatchTypeOf(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); expectTypeOf>().toEqualTypeOf>(); expectTypeOf>().toEqualTypeOf(); expectTypeOf().toEqualTypeOf< From bab98ea87562540dbc70dc50d0d22ec435bb686e Mon Sep 17 00:00:00 2001 From: Eric Pinzur Date: Tue, 17 Nov 2026 04:23:31 +0100 Subject: [PATCH 14/23] perf(core): count trace page payload bytes incrementally Address finding 4 by serializing each candidate node once and counting array separators separately from the changing response envelope. Preserve exact byte limits and continuation rewind. Review: https://github.com/mastra-ai/mastra/pull/30512#pullrequestreview-5925420901 --- .../domains/observability/trace-page.ts | 36 +++++++++++++++---- 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/packages/core/src/storage/domains/observability/trace-page.ts b/packages/core/src/storage/domains/observability/trace-page.ts index 0e0e0c9e05d..7c78fe24b36 100644 --- a/packages/core/src/storage/domains/observability/trace-page.ts +++ b/packages/core/src/storage/domains/observability/trace-page.ts @@ -195,6 +195,19 @@ export async function readTracePage( missingSpanIds: [...(topology?.missingSpanIds ?? [])], ancestorPaths: 'not-requested', }; + let nodeBytes = 0; + let missingIdBytes = result.missingSpanIds.reduce( + (bytes, spanId) => bytes + Buffer.byteLength(JSON.stringify(spanId)), + 0, + ); + // Serialize each selected payload once. The small envelope changes as the cursor + // advances; empty arrays already account for brackets, with commas added below. + const responseBytes = () => + Buffer.byteLength(JSON.stringify({ ...result, nodes: [], missingSpanIds: [] })) + + nodeBytes + + Math.max(0, result.nodes.length - 1) + + missingIdBytes + + Math.max(0, result.missingSpanIds.length - 1); let processed = 0; const updateContinuation = () => { const nextOffset = offset + processed; @@ -217,13 +230,16 @@ export async function readTracePage( }; for (const spanId of pageIds) { const record = recordsById.get(spanId); + let addedBytes: number; if (!record) { if (topology) throw new TracePageError('TRACE_CHANGED', 'Selected spans disappeared; restart this selection'); result.missingSpanIds.push(spanId); + addedBytes = Buffer.byteLength(JSON.stringify(spanId)); + missingIdBytes += addedBytes; } else { const { traceId, parentSpanId, name, spanType, isEvent, startedAt, endedAt, entityType, entityId, entityName } = record; - result.nodes.push({ + const node: TracePageNode = { traceId, spanId, parentSpanId, @@ -246,13 +262,21 @@ export async function readTracePage( ? 'available' : 'unavailable', ...(selection.include === 'full' ? { details: record as SpanRecord } : {}), - }); + }; + result.nodes.push(node); + addedBytes = Buffer.byteLength(JSON.stringify(node)); + nodeBytes += addedBytes; } processed++; updateContinuation(); - if (Buffer.byteLength(JSON.stringify(result)) > selection.maxBytes) { - if (record) result.nodes.pop(); - else result.missingSpanIds.pop(); + if (responseBytes() > selection.maxBytes) { + if (record) { + result.nodes.pop(); + nodeBytes -= addedBytes; + } else { + result.missingSpanIds.pop(); + missingIdBytes -= addedBytes; + } processed--; updateContinuation(); if (!processed) @@ -264,7 +288,7 @@ export async function readTracePage( } } updateContinuation(); - if (Buffer.byteLength(JSON.stringify(result)) > selection.maxBytes) + if (responseBytes() > selection.maxBytes) throw new TracePageError('LIMIT_EXCEEDED', 'Trace response metadata exceeds maxBytes'); return result; }; From 3ca8d25adf1c1f4ba89fced5803b9a1cbb829d93 Mon Sep 17 00:00:00 2001 From: Eric Pinzur Date: Tue, 17 Nov 2026 04:23:32 +0100 Subject: [PATCH 15/23] refactor(core): clarify internal trace page integration Address finding 7 by marking the experimental reader and source internal while preserving Platform runtime exports; remove the unused Core SDK argument alias. Review: https://github.com/mastra-ai/mastra/pull/30512#pullrequestreview-5925420901 --- .../src/storage/domains/observability/trace-page.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/core/src/storage/domains/observability/trace-page.ts b/packages/core/src/storage/domains/observability/trace-page.ts index 7c78fe24b36..bb5e51ccf79 100644 --- a/packages/core/src/storage/domains/observability/trace-page.ts +++ b/packages/core/src/storage/domains/observability/trace-page.ts @@ -39,7 +39,6 @@ export const tracePageRequestSchema = z.union([ z.object({ cursor: z.string().min(1).max(60000) }).strict(), ]); export type TracePageRequest = z.input; -export type GetTracePageArgs = TracePageRequest & { traceId: string }; /** Deliberately excludes unbounded payload, preview, metadata, and error fields. */ export const tracePageNodeSchema = spanRecordSchema @@ -105,7 +104,10 @@ export class TracePageError extends Error { } } -/** All reads must independently enforce trusted scope. A cursor is never authorization. */ +/** + * @internal Experimental Platform integration; not a supported OSS storage-adapter contract. + * All reads must independently enforce trusted scope. A cursor is never authorization. + */ export interface TracePageSource { scopeKey: string; /** Return at most limit topology rows; the extra row detects overflow. */ @@ -130,7 +132,10 @@ const cursorSchema = z }) .strict(); -/** Bounded wire pages over a capped topology read; this is not O(page size) database work. */ +/** + * @internal Experimental Platform reader; not a supported OSS storage-adapter contract. + * Bounded wire pages over a capped topology read; this is not O(page size) database work. + */ export async function readTracePage( source: TracePageSource, traceId: string, From 65ff2506c2bc580207eb8f2bac2147e927453b36 Mon Sep 17 00:00:00 2001 From: Eric Pinzur Date: Tue, 17 Nov 2026 04:23:42 +0100 Subject: [PATCH 16/23] fix(client-js): preserve legacy trace URL behavior Address finding 8 by removing the unrelated getTrace encoding change. The new getTracePage method continues to encode its trace ID. Review: https://github.com/mastra-ai/mastra/pull/30512#pullrequestreview-5925420901 --- client-sdks/client-js/src/resources/observability.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client-sdks/client-js/src/resources/observability.ts b/client-sdks/client-js/src/resources/observability.ts index 2e54642bcf4..d8b1c1868f4 100644 --- a/client-sdks/client-js/src/resources/observability.ts +++ b/client-sdks/client-js/src/resources/observability.ts @@ -232,7 +232,7 @@ export class Observability extends BaseResource { * @returns Promise containing the trace with all its spans */ getTrace(traceId: string): Promise { - return this.request(`/observability/traces/${encodeURIComponent(traceId)}`); + return this.request(`/observability/traces/${traceId}`); } /** From 5327ff3fbf5c9d68c9d6f9f505e55fdf82f8c0cc Mon Sep 17 00:00:00 2001 From: Eric Pinzur Date: Tue, 17 Nov 2026 04:24:45 +0100 Subject: [PATCH 17/23] fix(core): require search for ancestor selection Address finding 9: reject ignored ancestors options in browse and continuation requests while retaining both search modes. Review: https://github.com/mastra-ai/mastra/pull/30512#pullrequestreview-5925420901 --- .../domains/observability/trace-page.test.ts | 14 ++++++++++++++ .../storage/domains/observability/trace-page.ts | 3 +++ 2 files changed, 17 insertions(+) diff --git a/packages/core/src/storage/domains/observability/trace-page.test.ts b/packages/core/src/storage/domains/observability/trace-page.test.ts index 0bde64aa2d2..ab24a858aed 100644 --- a/packages/core/src/storage/domains/observability/trace-page.test.ts +++ b/packages/core/src/storage/domains/observability/trace-page.test.ts @@ -156,6 +156,20 @@ describe('trace page request boundaries', () => { expect(tracePageRequestSchema.safeParse({ limit: 200, maxBytes: 4194304 }).success).toBe(true); }); + it.each([true, false])('requires search when ancestors is %s, including in cursors', async ancestors => { + const src = source([span('a'), span('b', 'a')], ['b']); + expect(tracePageRequestSchema.safeParse({ ancestors }).success).toBe(false); + const page = (await readTracePage(src, 'trace', { limit: 1 }))!; + const cursor = JSON.parse(Buffer.from(page.nextCursor!, 'base64url').toString()); + cursor.selection.ancestors = ancestors; + await expect( + readTracePage(src, 'trace', { cursor: Buffer.from(JSON.stringify(cursor)).toString('base64url') }), + ).rejects.toMatchObject({ code: 'INVALID_CURSOR' }); + const search = (await readTracePage(src, 'trace', { search: 'error', ancestors }))!; + expect(search.nodes.map(node => node.spanId)).toEqual(ancestors ? ['a', 'b'] : ['b']); + expect(search.ancestorPaths).toBe(ancestors ? 'complete' : 'not-requested'); + }); + it.each([{ limit: 1000 }, { maxBytes: 4194305 }, { spanIds: ['a', 'b'], search: 'error' }])( 'revalidates selection in a forged cursor: %j', async changes => { diff --git a/packages/core/src/storage/domains/observability/trace-page.ts b/packages/core/src/storage/domains/observability/trace-page.ts index bb5e51ccf79..aeec60b855c 100644 --- a/packages/core/src/storage/domains/observability/trace-page.ts +++ b/packages/core/src/storage/domains/observability/trace-page.ts @@ -35,6 +35,9 @@ export const tracePageRequestSchema = z.union([ if (value.search && (value.anchorSpanId || value.depth !== undefined)) { ctx.addIssue({ code: 'custom', message: 'Search covers the entire trace; anchorSpanId is browse-only' }); } + if (value.ancestors !== undefined && !value.search) { + ctx.addIssue({ code: 'custom', message: 'ancestors requires search' }); + } }), z.object({ cursor: z.string().min(1).max(60000) }).strict(), ]); From 44ff0668cb1fa5af4322084cba33ee0b922bf697 Mon Sep 17 00:00:00 2001 From: Eric Pinzur Date: Tue, 17 Nov 2026 04:24:46 +0100 Subject: [PATCH 18/23] docs(core): describe experimental browse depth default Address finding 10 in schema metadata without defaulting a field that must stay absent for search and explicit-ID validation. Review: https://github.com/mastra-ai/mastra/pull/30512#pullrequestreview-5925420901 --- packages/core/src/storage/domains/observability/trace-page.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/src/storage/domains/observability/trace-page.ts b/packages/core/src/storage/domains/observability/trace-page.ts index aeec60b855c..a4e60be304d 100644 --- a/packages/core/src/storage/domains/observability/trace-page.ts +++ b/packages/core/src/storage/domains/observability/trace-page.ts @@ -13,7 +13,7 @@ const id = tracePageIdSchema; const selectionSchema = z .object({ anchorSpanId: id.optional(), - depth: z.number().int().min(0).max(32).optional(), + depth: z.number().int().min(0).max(32).optional().describe('Browse depth from the anchor or roots; defaults to 2.'), spanIds: z.array(id).min(1).max(100).optional(), search: z.string().trim().min(1).max(256).optional(), ancestors: z.boolean().optional(), From d8de2a663091da020afa8924d6838b23d106b24d Mon Sep 17 00:00:00 2001 From: Eric Pinzur Date: Tue, 17 Nov 2026 04:24:46 +0100 Subject: [PATCH 19/23] test(core): assert trace cursor character budget Address finding 11: check the 60000-character cursor cap separately from the request body byte budget for a maximum-size multibyte ID batch. Review: https://github.com/mastra-ai/mastra/pull/30512#pullrequestreview-5925420901 --- .../core/src/storage/domains/observability/trace-page.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/core/src/storage/domains/observability/trace-page.test.ts b/packages/core/src/storage/domains/observability/trace-page.test.ts index ab24a858aed..1a5d6f9cbf9 100644 --- a/packages/core/src/storage/domains/observability/trace-page.test.ts +++ b/packages/core/src/storage/domains/observability/trace-page.test.ts @@ -131,6 +131,7 @@ describe('bounded trace exploration', () => { const ids = Array.from({ length: 100 }, (_, i) => '界'.repeat(125) + String(i)); const src = source(ids.map(id => span(id))); const first = (await readTracePage(src, 'trace', { spanIds: ids, limit: 1 }))!; + expect(first.nextCursor!.length).toBeLessThanOrEqual(60000); expect(Buffer.byteLength(JSON.stringify({ cursor: first.nextCursor }))).toBeLessThan(65536); const second = await readTracePage(src, 'trace', { cursor: first.nextCursor! }); expect(second!.nodes[0]!.spanId).toBe(ids[1]); From 8872af9e8ff0f91cee86467264f03c9ff5927752 Mon Sep 17 00:00:00 2001 From: Eric Pinzur Date: Tue, 17 Nov 2026 04:24:46 +0100 Subject: [PATCH 20/23] docs(core): clarify unsigned trace continuation trust Address finding 12: fingerprints detect changes, not forgery. Keep independent scope enforcement and decoded-request validation explicit. Review: https://github.com/mastra-ai/mastra/pull/30512#pullrequestreview-5925420901 --- packages/core/src/storage/domains/observability/trace-page.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/core/src/storage/domains/observability/trace-page.ts b/packages/core/src/storage/domains/observability/trace-page.ts index a4e60be304d..97d3db6d7be 100644 --- a/packages/core/src/storage/domains/observability/trace-page.ts +++ b/packages/core/src/storage/domains/observability/trace-page.ts @@ -110,6 +110,8 @@ export class TracePageError extends Error { /** * @internal Experimental Platform integration; not a supported OSS storage-adapter contract. * All reads must independently enforce trusted scope. A cursor is never authorization. + * Cursors are unsigned base64url JSON, not integrity-protected tokens. Their fingerprint + * detects selection changes, not forgery; decoded requests must still be validated. */ export interface TracePageSource { scopeKey: string; From 54dc2e471f673b71772edbf0e10ce710f4eaf767 Mon Sep 17 00:00:00 2001 From: Eric Pinzur Date: Tue, 17 Nov 2026 04:26:38 +0100 Subject: [PATCH 21/23] test(client-js): include SDK suite in root discovery Rename the config to the extension scanned by root Vitest discovery so paging runtime and parity regressions run in CI. Preserve all five configured type-test files. Review: https://github.com/mastra-ai/mastra/pull/30512#pullrequestreview-5925420901 --- client-sdks/client-js/{vitest.config.js => vitest.config.ts} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename client-sdks/client-js/{vitest.config.js => vitest.config.ts} (100%) diff --git a/client-sdks/client-js/vitest.config.js b/client-sdks/client-js/vitest.config.ts similarity index 100% rename from client-sdks/client-js/vitest.config.js rename to client-sdks/client-js/vitest.config.ts From 84ac77505ebed790137dffec28d1c6272e0afb8e Mon Sep 17 00:00:00 2001 From: Eric Pinzur Date: Tue, 17 Nov 2026 07:33:02 +0100 Subject: [PATCH 22/23] fix(observability): preserve trace page execution diagnostics --- .../src/resources/trace-page-types.ts | 13 +++++ .../domains/observability/trace-page.test.ts | 49 ++++++++++++++++++- .../domains/observability/trace-page.ts | 41 +++++++++++++++- 3 files changed, 101 insertions(+), 2 deletions(-) diff --git a/client-sdks/client-js/src/resources/trace-page-types.ts b/client-sdks/client-js/src/resources/trace-page-types.ts index 2cdf2acffd6..d3ef63dab56 100644 --- a/client-sdks/client-js/src/resources/trace-page-types.ts +++ b/client-sdks/client-js/src/resources/trace-page-types.ts @@ -30,6 +30,19 @@ export type TracePageNode = Pick< | 'entityType' | 'entityId' | 'entityName' + | 'model' + | 'provider' + | 'outcome' + | 'stopReason' + | 'stopReasonContext' + | 'errorCount' + | 'externalParentSpanId' + | 'resumedFromSpanId' + | 'timeToFirstChunkMs' + | 'inputPreview' + | 'outputPreview' + | 'errorPreview' + | 'timeToFirstTokenMs' > & { status: 'success' | 'error' | 'running'; childCount: number | null; diff --git a/packages/core/src/storage/domains/observability/trace-page.test.ts b/packages/core/src/storage/domains/observability/trace-page.test.ts index 1a5d6f9cbf9..95c39516b6d 100644 --- a/packages/core/src/storage/domains/observability/trace-page.test.ts +++ b/packages/core/src/storage/domains/observability/trace-page.test.ts @@ -1,7 +1,7 @@ import { createHash } from 'node:crypto'; import { describe, expect, it } from 'vitest'; import { SpanType } from '../../../observability/types'; -import { readTracePage, tracePageRequestSchema } from './trace-page'; +import { readTracePage, tracePageRequestSchema, tracePageResponseSchema } from './trace-page'; import type { TracePageSource, TracePageResponse } from './trace-page'; import type { SpanRecord } from './tracing'; @@ -33,6 +33,53 @@ async function drain(src: TracePageSource, first: TracePageResponse): Promise { + it.each(['structure', 'full'] as const)( + 'preserves execution diagnostics in %s pages without requiring payload hydration', + async include => { + const diagnostics = { + model: 'gpt-6.1-sol', + provider: 'openai', + outcome: 'failed', + stopReason: 'max_steps', + stopReasonContext: { maxSteps: 10 }, + errorCount: 0, + externalParentSpanId: 'external', + resumedFromSpanId: 'previous', + timeToFirstChunkMs: 0, + timeToFirstTokenMs: 12, + inputPreview: 'question', + outputPreview: '', + errorPreview: 'limit reached', + }; + const record = { + ...span('root', null, { large: 'payload' }), + createdAt: new Date('2026-11-01'), + updatedAt: null, + ...diagnostics, + }; + const page = await readTracePage(source([record]), 'trace', { include }); + expect(tracePageResponseSchema.parse(page).nodes[0]).toMatchObject(diagnostics); + if (include === 'full') expect(page!.nodes[0]!.details).toEqual(record); + else expect(page!.nodes[0]).not.toHaveProperty('details'); + expect(page!.nodes[0]).not.toHaveProperty('input'); + }, + ); + + it.each(['inputPreview', 'stopReasonContext'] as const)( + 'counts %s toward the structure page byte budget', + async field => { + const value = 'x'.repeat(40000); + const diagnostics = { [field]: field === 'stopReasonContext' ? { reason: value } : value }; + const src = source([ + { ...span('a'), ...diagnostics }, + { ...span('b'), ...diagnostics }, + ]); + const pages = await drain(src, (await readTracePage(src, 'trace', { maxBytes: 65536 }))!); + expect(pages.map(page => page.nodes.map(node => node.spanId))).toEqual([['a'], ['b']]); + for (const page of pages) expect(Buffer.byteLength(JSON.stringify(page))).toBeLessThanOrEqual(65536); + }, + ); + it('loads a shallow breadth-first tree with deterministic sibling ordering and discoverable children', async () => { const src = source([span('z', 'root'), span('root'), span('deep', 'a'), span('a', 'root')]); const pages = await drain(src, (await readTracePage(src, 'trace', { depth: 1, limit: 2 }))!); diff --git a/packages/core/src/storage/domains/observability/trace-page.ts b/packages/core/src/storage/domains/observability/trace-page.ts index 97d3db6d7be..ee356566800 100644 --- a/packages/core/src/storage/domains/observability/trace-page.ts +++ b/packages/core/src/storage/domains/observability/trace-page.ts @@ -43,7 +43,7 @@ export const tracePageRequestSchema = z.union([ ]); export type TracePageRequest = z.input; -/** Deliberately excludes unbounded payload, preview, metadata, and error fields. */ +/** Execution diagnostics and previews; full payloads and metadata remain in details. */ export const tracePageNodeSchema = spanRecordSchema .pick({ traceId: true, @@ -57,6 +57,19 @@ export const tracePageNodeSchema = spanRecordSchema entityType: true, entityId: true, entityName: true, + model: true, + provider: true, + outcome: true, + stopReason: true, + stopReasonContext: true, + errorCount: true, + externalParentSpanId: true, + resumedFromSpanId: true, + timeToFirstChunkMs: true, + inputPreview: true, + outputPreview: true, + errorPreview: true, + timeToFirstTokenMs: true, }) .extend({ status: z.nativeEnum(TraceStatus), @@ -88,6 +101,19 @@ export type TracePageSummary = Pick< | 'entityType' | 'entityId' | 'entityName' + | 'model' + | 'provider' + | 'outcome' + | 'stopReason' + | 'stopReasonContext' + | 'errorCount' + | 'externalParentSpanId' + | 'resumedFromSpanId' + | 'timeToFirstChunkMs' + | 'inputPreview' + | 'outputPreview' + | 'errorPreview' + | 'timeToFirstTokenMs' > & { error?: unknown }; export class TracePageError extends Error { @@ -261,6 +287,19 @@ export async function readTracePage( entityType, entityId, entityName, + model: record.model, + provider: record.provider, + outcome: record.outcome, + stopReason: record.stopReason, + stopReasonContext: record.stopReasonContext, + errorCount: record.errorCount, + externalParentSpanId: record.externalParentSpanId, + resumedFromSpanId: record.resumedFromSpanId, + timeToFirstChunkMs: record.timeToFirstChunkMs, + inputPreview: record.inputPreview, + outputPreview: record.outputPreview, + errorPreview: record.errorPreview, + timeToFirstTokenMs: record.timeToFirstTokenMs, status: computeTraceStatus(record), childCount: topology ? (topology.children.get(spanId)?.length ?? 0) : null, match: topology?.matches.has(spanId) ?? false, From e728d57993a194cf6adf0d7ae144683b4730c460 Mon Sep 17 00:00:00 2001 From: Eric Pinzur Date: Tue, 17 Nov 2026 07:52:28 +0100 Subject: [PATCH 23/23] fix(observability): expose root context and omit unused trace details --- .../src/resources/trace-page-types.ts | 15 ++++- .../client-js/type-tests/trace-page.test-d.ts | 3 + .../domains/observability/trace-page.test.ts | 57 +++++++++++++++++++ .../domains/observability/trace-page.ts | 34 +++++++++-- 4 files changed, 104 insertions(+), 5 deletions(-) diff --git a/client-sdks/client-js/src/resources/trace-page-types.ts b/client-sdks/client-js/src/resources/trace-page-types.ts index d3ef63dab56..f5aec0927d5 100644 --- a/client-sdks/client-js/src/resources/trace-page-types.ts +++ b/client-sdks/client-js/src/resources/trace-page-types.ts @@ -43,12 +43,25 @@ export type TracePageNode = Pick< | 'outputPreview' | 'errorPreview' | 'timeToFirstTokenMs' + | 'experimentId' + | 'userId' + | 'organizationId' + | 'resourceId' + | 'runId' + | 'sessionId' + | 'threadId' + | 'requestId' + | 'environment' + | 'source' + | 'serviceName' + | 'tags' > & { + projectId?: string | null; status: 'success' | 'error' | 'running'; childCount: number | null; match: boolean; parentState: 'root' | 'available' | 'unavailable' | 'not-requested'; - details?: Serialized; + details?: Omit, 'scope' | 'links'>; }; export interface TracePageResponse { traceId: string; diff --git a/client-sdks/client-js/type-tests/trace-page.test-d.ts b/client-sdks/client-js/type-tests/trace-page.test-d.ts index 8c3d1cbe18b..1ab14d40086 100644 --- a/client-sdks/client-js/type-tests/trace-page.test-d.ts +++ b/client-sdks/client-js/type-tests/trace-page.test-d.ts @@ -18,6 +18,9 @@ test('keeps Platform trace page types aligned without exposing an OSS server rou expectTypeOf().toMatchTypeOf(); expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); + expectTypeOf>().toEqualTypeOf< + keyof NonNullable + >(); expectTypeOf>().toEqualTypeOf>(); expectTypeOf>().toEqualTypeOf(); expectTypeOf().toEqualTypeOf< diff --git a/packages/core/src/storage/domains/observability/trace-page.test.ts b/packages/core/src/storage/domains/observability/trace-page.test.ts index 95c39516b6d..5fa3a592aff 100644 --- a/packages/core/src/storage/domains/observability/trace-page.test.ts +++ b/packages/core/src/storage/domains/observability/trace-page.test.ts @@ -33,6 +33,63 @@ async function drain(src: TracePageSource, first: TracePageResponse): Promise { + it('keeps context on each actual root across pages, without treating orphan or anchored children as roots', async () => { + const context = { + experimentId: 'experiment', + userId: 'user', + organizationId: 'org', + projectId: 'project', + resourceId: 'resource', + runId: 'run', + sessionId: 'session', + threadId: 'thread', + requestId: 'request', + environment: 'test', + source: 'service', + serviceName: 'worker', + tags: ['test'], + }; + const src = source([ + { ...span('root-a'), ...context }, + { ...span('root-b'), ...context, userId: 'other' }, + { ...span('child', 'root-a'), ...context }, + { ...span('orphan', 'absent'), ...context }, + ]); + const pages = await drain(src, (await readTracePage(src, 'trace', { limit: 1 }))!); + const nodes = pages.flatMap(page => tracePageResponseSchema.parse(page).nodes); + expect(nodes.find(node => node.spanId === 'root-a')).toMatchObject(context); + expect(nodes.find(node => node.spanId === 'root-b')).toMatchObject({ ...context, userId: 'other' }); + for (const node of nodes.filter(node => node.parentSpanId)) { + for (const key of Object.keys(context)) expect(node).not.toHaveProperty(key); + } + for (const request of [{ anchorSpanId: 'child' }, { spanIds: ['child'] }]) { + const page = (await readTracePage(src, 'trace', request))!; + expect(page.nodes.map(node => node.spanId)).toEqual(['child']); + for (const key of Object.keys(context)) expect(page.nodes[0]).not.toHaveProperty(key); + } + const rootPage = await readTracePage(src, 'trace', { spanIds: ['root-b'] }); + expect(rootPage!.nodes[0]).toMatchObject({ ...context, userId: 'other' }); + }); + + it('counts root context toward the page byte budget', async () => { + const src = source([ + { ...span('a'), tags: ['x'.repeat(40000)] }, + { ...span('b'), tags: ['y'.repeat(40000)] }, + ]); + const pages = await drain(src, (await readTracePage(src, 'trace', { maxBytes: 65536 }))!); + expect(pages.map(page => page.nodes.map(node => node.spanId))).toEqual([['a'], ['b']]); + for (const page of pages) expect(Buffer.byteLength(JSON.stringify(page))).toBeLessThanOrEqual(65536); + }); + + it('omits scope and links from full details without charging their bytes to the page', async () => { + const record = { ...span('root'), scope: { hidden: 'x'.repeat(70000) }, links: [{ hidden: 'y'.repeat(70000) }] }; + const page = await readTracePage(source([record]), 'trace', { include: 'full', maxBytes: 65536 }); + expect(page!.nodes[0]!.details).not.toHaveProperty('scope'); + expect(page!.nodes[0]!.details).not.toHaveProperty('links'); + expect(record).toHaveProperty('scope'); + expect(record).toHaveProperty('links'); + }); + it.each(['structure', 'full'] as const)( 'preserves execution diagnostics in %s pages without requiring payload hydration', async include => { diff --git a/packages/core/src/storage/domains/observability/trace-page.ts b/packages/core/src/storage/domains/observability/trace-page.ts index ee356566800..ae79d2dcb31 100644 --- a/packages/core/src/storage/domains/observability/trace-page.ts +++ b/packages/core/src/storage/domains/observability/trace-page.ts @@ -43,6 +43,24 @@ export const tracePageRequestSchema = z.union([ ]); export type TracePageRequest = z.input; +// Trace context belongs to actual roots, not the anchor of an arbitrary subtree. +const tracePageRootContextSchema = spanRecordSchema + .pick({ + experimentId: true, + userId: true, + organizationId: true, + resourceId: true, + runId: true, + sessionId: true, + threadId: true, + requestId: true, + environment: true, + source: true, + serviceName: true, + tags: true, + }) + .extend({ projectId: z.string().nullish() }); + /** Execution diagnostics and previews; full payloads and metadata remain in details. */ export const tracePageNodeSchema = spanRecordSchema .pick({ @@ -72,11 +90,12 @@ export const tracePageNodeSchema = spanRecordSchema timeToFirstTokenMs: true, }) .extend({ + ...tracePageRootContextSchema.shape, status: z.nativeEnum(TraceStatus), childCount: z.number().int().nonnegative().nullable(), match: z.boolean(), parentState: z.enum(['root', 'available', 'unavailable', 'not-requested']), - details: spanRecordSchema.optional(), + details: spanRecordSchema.omit({ scope: true, links: true }).optional(), }); export const tracePageResponseSchema = z.object({ traceId: z.string(), @@ -114,7 +133,8 @@ export type TracePageSummary = Pick< | 'outputPreview' | 'errorPreview' | 'timeToFirstTokenMs' -> & { error?: unknown }; +> & + z.infer & { error?: unknown }; export class TracePageError extends Error { constructor( @@ -146,7 +166,7 @@ export interface TracePageSource { /** * Read only the selected IDs, excluding payload fields for structure projection. * Include a non-null `error` indicator for failed spans so status is computed correctly. - * Full projection must return complete SpanRecords, including the actual error payload. + * Full projection returns span details including the actual error payload; scope and links are omitted. */ spans(traceId: string, spanIds: string[], include: 'structure' | 'full'): Promise; /** Complete matching identities within this trace; fail rather than truncate. */ @@ -275,6 +295,11 @@ export async function readTracePage( } else { const { traceId, parentSpanId, name, spanType, isEvent, startedAt, endedAt, entityType, entityId, entityName } = record; + const details = selection.include === 'full' ? { ...(record as SpanRecord) } : undefined; + if (details) { + delete details.scope; + delete details.links; + } const node: TracePageNode = { traceId, spanId, @@ -300,6 +325,7 @@ export async function readTracePage( outputPreview: record.outputPreview, errorPreview: record.errorPreview, timeToFirstTokenMs: record.timeToFirstTokenMs, + ...(!parentSpanId ? tracePageRootContextSchema.parse(record) : {}), status: computeTraceStatus(record), childCount: topology ? (topology.children.get(spanId)?.length ?? 0) : null, match: topology?.matches.has(spanId) ?? false, @@ -310,7 +336,7 @@ export async function readTracePage( : topology.nodes.has(parentSpanId) ? 'available' : 'unavailable', - ...(selection.include === 'full' ? { details: record as SpanRecord } : {}), + ...(details ? { details } : {}), }; result.nodes.push(node); addedBytes = Buffer.byteLength(JSON.stringify(node));