From 75ed0cbce9be158e00f20b39ede0428dc3bcddbd Mon Sep 17 00:00:00 2001 From: Rhys Balevicius Date: Tue, 17 Nov 2026 03:57:57 +0000 Subject: [PATCH 1/5] fix(core): correct gateway catalogs, version routing and stream usage Authenticate and cache v2 model catalogs with bounded fallback and retry delays to reduce waits during outages. Keep organization catalogs out of the shared provider cache so they stay isolated across projects. Honor v1 project keys and registered gateway settings so Studio and model calls agree on the gateway version. Request streaming token usage so final usage chunks are reported. --- .changeset/quiet-gateways-count.md | 10 + .changeset/steady-gateway-versions.md | 5 + .../src/llm/model/gateways/mastra.test.ts | 396 ++++++++++++++++-- .../core/src/llm/model/gateways/mastra.ts | 207 +++++++-- .../src/llm/model/provider-registry.test.ts | 89 ++++ .../core/src/llm/model/provider-registry.ts | 9 +- .../server/handlers/gateway-memory-client.ts | 35 +- .../handlers/gateway-memory-client.v2.test.ts | 92 ++++ 8 files changed, 778 insertions(+), 65 deletions(-) create mode 100644 .changeset/quiet-gateways-count.md create mode 100644 .changeset/steady-gateway-versions.md diff --git a/.changeset/quiet-gateways-count.md b/.changeset/quiet-gateways-count.md new file mode 100644 index 00000000000..d641567a602 --- /dev/null +++ b/.changeset/quiet-gateways-count.md @@ -0,0 +1,10 @@ +--- +'@mastra/core': patch +--- + +Improved how `@mastra/core` works with the Mastra Gateway: + +- Streamed calls through the Gateway's chat endpoint now ask for token usage, so `agent.stream()` reports usage when the upstream provider sends it in a separate final chunk. Before, usage could be empty. +- The Gateway model list for organization keys is cached for a minute and shared between concurrent callers. If a refresh times out or the Gateway has an outage, the last list is kept for up to ten minutes. With no earlier list, the failure is reported again for 15 seconds without a new request, so Studio does not wait for the request timeout on every load during an outage. If the Gateway rejects the key (401 or 403), the cached list is dropped. The model list request now always sends the key in `Authorization` and no longer goes through a `customFetch`, which the v2 Gateway would not have authenticated. +- A `msk_` project key now always uses the v1 Gateway, even when `MASTRA_GATEWAY_VERSION=v2` is set. +- `mastra dev` no longer writes one organization's Gateway model list to the provider cache shared by other projects on the machine, whether the key comes from the environment or from a registered `MastraGateway`. A failing Gateway model list no longer stops the provider registry refresh. diff --git a/.changeset/steady-gateway-versions.md b/.changeset/steady-gateway-versions.md new file mode 100644 index 00000000000..3a31dbbd23d --- /dev/null +++ b/.changeset/steady-gateway-versions.md @@ -0,0 +1,5 @@ +--- +'@mastra/server': patch +--- + +A `msk_` project key now always uses the v1 Gateway memory API, even when `MASTRA_GATEWAY_VERSION=v2` is set. Studio follows a registered `MastraGateway` and its own key when deciding between the v1 and v2 Gateway, so it agrees with the model calls. diff --git a/packages/core/src/llm/model/gateways/mastra.test.ts b/packages/core/src/llm/model/gateways/mastra.test.ts index 470f07ecc9d..a67302fe677 100644 --- a/packages/core/src/llm/model/gateways/mastra.test.ts +++ b/packages/core/src/llm/model/gateways/mastra.test.ts @@ -1,6 +1,6 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { MastraGateway } from './mastra.js'; +import { _resetMastraGatewayCatalogCache, MastraGateway } from './mastra.js'; vi.mock('../provider-registry.js', () => ({ PROVIDER_REGISTRY: { openrouter: { models: ['openai/test-model'] } }, @@ -10,14 +10,22 @@ describe('MastraGateway', () => { beforeEach(() => { delete process.env.MASTRA_GATEWAY_API_KEY; delete process.env.MASTRA_GATEWAY_VERSION; + _resetMastraGatewayCatalogCache(); }); afterEach(() => { delete process.env.MASTRA_GATEWAY_API_KEY; delete process.env.MASTRA_GATEWAY_VERSION; vi.restoreAllMocks(); + vi.unstubAllGlobals(); }); + /** A v2 gateway whose catalogue request is answered by `gatewayFetch` in place of the global fetch. */ + function gatewayWith(apiKey: string, gatewayFetch: (...args: any[]) => Promise) { + vi.stubGlobal('fetch', gatewayFetch); + return new MastraGateway({ apiKey }); + } + it('reports disabled when MASTRA_GATEWAY_API_KEY is not set', () => { const gateway = new MastraGateway(); @@ -62,40 +70,38 @@ describe('MastraGateway', () => { { status: 200, headers: { 'content-type': 'application/json' } }, ), ); - const gateway = new MastraGateway({ - apiKey: 'deployment-key', - baseUrl: 'https://gateway.example/v1', - customFetch: gatewayFetch, - }); + vi.stubGlobal('fetch', gatewayFetch); + const gateway = new MastraGateway({ apiKey: 'deployment-key', baseUrl: 'https://gateway.example/v1' }); const providers = await gateway.fetchProviders(); expect(gatewayFetch).toHaveBeenCalledWith( 'https://gateway.example/v1/models', expect.objectContaining({ - headers: expect.objectContaining({ 'X-Memory-Gateway-Authorization': 'Bearer deployment-key' }), + headers: { Authorization: 'Bearer deployment-key' }, signal: expect.any(AbortSignal), }), ); expect(providers.mastra?.models).toEqual(['openai/chat-model']); }); - it('preserves the Gateway key when customFetch replaces Authorization with OAuth', async () => { + it('sends the v2 catalog request with the key in Authorization, not through a custom fetch', async () => { process.env.MASTRA_GATEWAY_VERSION = 'v2'; - const customFetch = vi.fn(async (_input: RequestInfo | URL, init?: RequestInit) => { - const headers = new Headers(init?.headers); - headers.set('Authorization', 'Bearer oauth-token'); - expect(headers.get('Authorization')).toBe('Bearer oauth-token'); - expect(headers.get('X-Memory-Gateway-Authorization')).toBe('Bearer deployment-key'); - return new Response(JSON.stringify({ data: [{ id: 'openai/chat-model', endpoints: ['chat'] }] }), { - status: 200, - }); - }); + const customFetch = vi.fn(async () => new Response('{}', { status: 200 })); + const globalFetch = vi.fn( + async (_input: RequestInfo | URL, _init?: RequestInit) => + new Response(JSON.stringify({ data: [{ id: 'openai/chat-model', endpoints: ['chat'] }] }), { status: 200 }), + ); + vi.stubGlobal('fetch', globalFetch); const gateway = new MastraGateway({ apiKey: 'deployment-key', customFetch }); const providers = await gateway.fetchProviders(); - expect(customFetch).toHaveBeenCalledOnce(); + expect(customFetch).not.toHaveBeenCalled(); + expect(globalFetch).toHaveBeenCalledOnce(); + const headers = new Headers(globalFetch.mock.calls[0]![1]?.headers); + expect(headers.get('Authorization')).toBe('Bearer deployment-key'); + expect(headers.get('X-Memory-Gateway-Authorization')).toBeNull(); expect(providers.mastra?.models).toEqual(['openai/chat-model']); }); @@ -103,16 +109,16 @@ describe('MastraGateway', () => { process.env.MASTRA_GATEWAY_VERSION = 'v2'; const controller = new AbortController(); const timeout = vi.spyOn(AbortSignal, 'timeout').mockReturnValue(controller.signal); - const customFetch = vi.fn( + const gatewayFetch = vi.fn( (_input: RequestInfo | URL, init?: RequestInit) => new Promise((_resolve, reject) => { init?.signal?.addEventListener('abort', () => reject(init.signal?.reason), { once: true }); }), ); - const gateway = new MastraGateway({ apiKey: 'deployment-key', customFetch }); + const gateway = gatewayWith('deployment-key', gatewayFetch); const pending = gateway.fetchProviders(); - await vi.waitFor(() => expect(customFetch).toHaveBeenCalledOnce()); + await vi.waitFor(() => expect(gatewayFetch).toHaveBeenCalledOnce()); controller.abort(new DOMException('catalog deadline', 'TimeoutError')); await expect(pending).rejects.toThrow('catalog deadline'); @@ -124,7 +130,7 @@ describe('MastraGateway', () => { async () => new Response(JSON.stringify({ data: [{ id: 'openai/chat-model', endpoints: ['chat'] }] }), { status: 200 }), ); - const gateway = new MastraGateway({ apiKey: 'mskv2_org-key', customFetch: gatewayFetch }); + const gateway = gatewayWith('mskv2_org-key', gatewayFetch); const providers = await gateway.fetchProviders(); @@ -133,15 +139,353 @@ describe('MastraGateway', () => { expect(providers.mastra?.modelReleaseDates).toBeUndefined(); }); + describe('isV2', () => { + it.each([ + { version: undefined, key: 'msk_0123456789abcdef0123456789abcdef', expected: false }, + { version: 'v2', key: 'msk_0123456789abcdef0123456789abcdef', expected: false }, + { version: 'v2', key: 'deployment-key', expected: true }, + { version: 'v2', key: undefined, expected: true }, + { version: undefined, key: 'mskv2_org-key', expected: true }, + { version: undefined, key: 'msk-gateway-org-key', expected: true }, + { version: undefined, key: 'other-key', expected: false }, + ])('with MASTRA_GATEWAY_VERSION=$version and key $key is $expected', ({ version, key, expected }) => { + if (version) process.env.MASTRA_GATEWAY_VERSION = version; + expect(new MastraGateway({ apiKey: key }).isV2()).toBe(expected); + }); + + it('lets a v1 config key win over a v2 env key and MASTRA_GATEWAY_VERSION', () => { + process.env.MASTRA_GATEWAY_VERSION = 'v2'; + process.env.MASTRA_GATEWAY_API_KEY = 'mskv2_org-key'; + expect(new MastraGateway({ apiKey: 'msk_0123456789abcdef0123456789abcdef' }).isV2()).toBe(false); + }); + }); + it('does not show a static v1 model list if v2 catalog loading fails', async () => { process.env.MASTRA_GATEWAY_VERSION = 'v2'; - const gateway = new MastraGateway({ - apiKey: 'deployment-key', - customFetch: vi.fn(async () => new Response('unavailable', { status: 503 })), - }); + const gateway = gatewayWith( + 'deployment-key', + vi.fn(async () => new Response('unavailable', { status: 503 })), + ); await expect(gateway.fetchProviders()).rejects.toThrow('Gateway model catalog unavailable (503)'); }); + describe('v2 catalog cache', () => { + const catalog = (ids: string[]) => + new Response(JSON.stringify({ data: ids.map(id => ({ id, endpoints: ['chat'] })) }), { status: 200 }); + + beforeEach(() => { + process.env.MASTRA_GATEWAY_VERSION = 'v2'; + }); + + it('reuses the catalog for a minute, then fetches it again', async () => { + const now = vi.spyOn(Date, 'now').mockReturnValue(1_000_000); + const gatewayFetch = vi.fn(async () => catalog(['openai/chat-model'])); + const gateway = gatewayWith('deployment-key', gatewayFetch); + + await gateway.fetchProviders(); + now.mockReturnValue(1_000_000 + 59_000); + await gateway.fetchProviders(); + expect(gatewayFetch).toHaveBeenCalledOnce(); + + now.mockReturnValue(1_000_000 + 61_000); + await gateway.fetchProviders(); + expect(gatewayFetch).toHaveBeenCalledTimes(2); + }); + + it('shares one request between concurrent callers', async () => { + let respond!: (response: Response) => void; + const gatewayFetch = vi.fn(() => new Promise(resolve => (respond = resolve))); + const gateway = gatewayWith('deployment-key', gatewayFetch); + + const first = gateway.fetchProviders(); + const second = gatewayWith('deployment-key', gatewayFetch).fetchProviders(); + await vi.waitFor(() => expect(gatewayFetch).toHaveBeenCalledOnce()); + respond(catalog(['openai/chat-model'])); + + expect((await first).mastra?.models).toEqual(['openai/chat-model']); + expect((await second).mastra?.models).toEqual(['openai/chat-model']); + expect(gatewayFetch).toHaveBeenCalledOnce(); + }); + + it('keeps catalogs for different keys apart', async () => { + const gatewayFetch = vi.fn(async (_input: RequestInfo | URL, init?: RequestInit) => { + const key = new Headers(init?.headers).get('Authorization'); + return catalog([key === 'Bearer key-a' ? 'openai/model-a' : 'openai/model-b']); + }); + + const a = await gatewayWith('key-a', gatewayFetch).fetchProviders(); + const b = await gatewayWith('key-b', gatewayFetch).fetchProviders(); + + expect(a.mastra?.models).toEqual(['openai/model-a']); + expect(b.mastra?.models).toEqual(['openai/model-b']); + }); + + it('serves the last good catalog for up to ten minutes when a refresh fails', async () => { + const now = vi.spyOn(Date, 'now').mockReturnValue(1_000_000); + const gatewayFetch = vi.fn(async () => catalog(['openai/chat-model'])); + const gateway = gatewayWith('deployment-key', gatewayFetch); + await gateway.fetchProviders(); + + gatewayFetch.mockImplementation(async () => new Response('unavailable', { status: 503 })); + now.mockReturnValue(1_000_000 + 9 * 60_000); + expect((await gateway.fetchProviders()).mastra?.models).toEqual(['openai/chat-model']); + + now.mockReturnValue(1_000_000 + 11 * 60_000); + await expect(gateway.fetchProviders()).rejects.toThrow('Gateway model catalog unavailable (503)'); + }); + + it.each([401, 403])('drops the cached catalog when the key is rejected with %i', async status => { + const now = vi.spyOn(Date, 'now').mockReturnValue(1_000_000); + const gatewayFetch = vi.fn(async () => catalog(['openai/chat-model'])); + const gateway = gatewayWith('deployment-key', gatewayFetch); + await gateway.fetchProviders(); + + gatewayFetch.mockImplementation(async () => new Response('rejected', { status })); + now.mockReturnValue(1_000_000 + 61_000); + await expect(gateway.fetchProviders()).rejects.toThrow(`Gateway model catalog unavailable (${status})`); + + gatewayFetch.mockImplementation(async () => catalog(['openai/other-model'])); + expect((await gateway.fetchProviders()).mastra?.models).toEqual(['openai/other-model']); + }); + + it('serves the last good catalog after a network error', async () => { + const now = vi.spyOn(Date, 'now').mockReturnValue(1_000_000); + const gatewayFetch = vi.fn(async () => catalog(['openai/chat-model'])); + const gateway = gatewayWith('deployment-key', gatewayFetch); + await gateway.fetchProviders(); + + gatewayFetch.mockRejectedValue(new TypeError('fetch failed')); + now.mockReturnValue(1_000_000 + 61_000); + expect((await gateway.fetchProviders()).mastra?.models).toEqual(['openai/chat-model']); + }); + + it('waits before retrying after a failed refresh', async () => { + const now = vi.spyOn(Date, 'now').mockReturnValue(1_000_000); + const gatewayFetch = vi.fn(async () => catalog(['openai/chat-model'])); + const gateway = gatewayWith('deployment-key', gatewayFetch); + await gateway.fetchProviders(); + + gatewayFetch.mockImplementation(async () => new Response('unavailable', { status: 503 })); + now.mockReturnValue(1_000_000 + 61_000); + await gateway.fetchProviders(); + expect(gatewayFetch).toHaveBeenCalledTimes(2); + + now.mockReturnValue(1_000_000 + 61_000 + 14_000); + expect((await gateway.fetchProviders()).mastra?.models).toEqual(['openai/chat-model']); + expect(gatewayFetch).toHaveBeenCalledTimes(2); + + now.mockReturnValue(1_000_000 + 61_000 + 16_000); + expect((await gateway.fetchProviders()).mastra?.models).toEqual(['openai/chat-model']); + expect(gatewayFetch).toHaveBeenCalledTimes(3); + }); + + it('keeps catalogs fetched through different global fetch implementations apart', async () => { + const firstFetch = vi.fn(async () => catalog(['openai/model-a'])); + const secondFetch = vi.fn(async () => catalog(['openai/model-b'])); + + const a = await gatewayWith('same-key', firstFetch).fetchProviders(); + const b = await gatewayWith('same-key', secondFetch).fetchProviders(); + + expect(a.mastra?.models).toEqual(['openai/model-a']); + expect(b.mastra?.models).toEqual(['openai/model-b']); + expect(secondFetch).toHaveBeenCalledOnce(); + }); + + it('keeps at most 100 catalogs, dropping the oldest', async () => { + const gatewayFetch = vi.fn(async () => catalog(['openai/chat-model'])); + for (let i = 0; i <= 100; i++) { + await gatewayWith(`key-${i}`, gatewayFetch).fetchProviders(); + } + expect(gatewayFetch).toHaveBeenCalledTimes(101); + + await gatewayWith('key-100', gatewayFetch).fetchProviders(); + expect(gatewayFetch).toHaveBeenCalledTimes(101); + await gatewayWith('key-0', gatewayFetch).fetchProviders(); + expect(gatewayFetch).toHaveBeenCalledTimes(102); + }); + + it('remembers a failure with no earlier catalog for 15 seconds', async () => { + const now = vi.spyOn(Date, 'now').mockReturnValue(1_000_000); + const gatewayFetch = vi.fn(async () => new Response('unavailable', { status: 503 })); + const gateway = gatewayWith('deployment-key', gatewayFetch); + + await expect(gateway.fetchProviders()).rejects.toThrow('Gateway model catalog unavailable (503)'); + expect(gatewayFetch).toHaveBeenCalledOnce(); + + now.mockReturnValue(1_000_000 + 14_000); + await expect(gateway.fetchProviders()).rejects.toThrow('Gateway model catalog unavailable (503)'); + expect(gatewayFetch).toHaveBeenCalledOnce(); + + now.mockReturnValue(1_000_000 + 16_000); + await expect(gateway.fetchProviders()).rejects.toThrow('Gateway model catalog unavailable (503)'); + expect(gatewayFetch).toHaveBeenCalledTimes(2); + }); + + it('remembers a network error with no earlier catalog', async () => { + const now = vi.spyOn(Date, 'now').mockReturnValue(1_000_000); + const gatewayFetch = vi.fn(async () => { + throw new TypeError('fetch failed'); + }); + const gateway = gatewayWith('deployment-key', gatewayFetch); + + await expect(gateway.fetchProviders()).rejects.toThrow('fetch failed'); + now.mockReturnValue(1_000_000 + 5_000); + await expect(gateway.fetchProviders()).rejects.toThrow('fetch failed'); + expect(gatewayFetch).toHaveBeenCalledOnce(); + }); + + it('clears a remembered failure on success, and the next failure starts a new window', async () => { + const now = vi.spyOn(Date, 'now').mockReturnValue(1_000_000); + const gatewayFetch = vi.fn(async () => new Response('unavailable', { status: 503 })); + const gateway = gatewayWith('deployment-key', gatewayFetch); + await expect(gateway.fetchProviders()).rejects.toThrow('(503)'); + + gatewayFetch.mockImplementation(async () => catalog(['openai/chat-model'])); + now.mockReturnValue(1_000_000 + 16_000); + expect((await gateway.fetchProviders()).mastra?.models).toEqual(['openai/chat-model']); + expect(gatewayFetch).toHaveBeenCalledTimes(2); + + // Past the stale window there is no list to fall back on, so a failure is remembered again. + gatewayFetch.mockImplementation(async () => new Response('unavailable', { status: 503 })); + now.mockReturnValue(1_000_000 + 16_000 + 11 * 60_000); + await expect(gateway.fetchProviders()).rejects.toThrow('(503)'); + expect(gatewayFetch).toHaveBeenCalledTimes(3); + now.mockReturnValue(1_000_000 + 16_000 + 11 * 60_000 + 14_000); + await expect(gateway.fetchProviders()).rejects.toThrow('(503)'); + expect(gatewayFetch).toHaveBeenCalledTimes(3); + now.mockReturnValue(1_000_000 + 16_000 + 11 * 60_000 + 16_000); + await expect(gateway.fetchProviders()).rejects.toThrow('(503)'); + expect(gatewayFetch).toHaveBeenCalledTimes(4); + }); + + it.each([401, 403])('asks the gateway again on every call after a %i', async status => { + vi.spyOn(Date, 'now').mockReturnValue(1_000_000); + const gatewayFetch = vi.fn(async () => new Response('rejected', { status })); + const gateway = gatewayWith('deployment-key', gatewayFetch); + + await expect(gateway.fetchProviders()).rejects.toThrow(`(${status})`); + await expect(gateway.fetchProviders()).rejects.toThrow(`(${status})`); + expect(gatewayFetch).toHaveBeenCalledTimes(2); + + gatewayFetch.mockImplementation(async () => catalog(['openai/chat-model'])); + expect((await gateway.fetchProviders()).mastra?.models).toEqual(['openai/chat-model']); + }); + + it('shares one failing request between concurrent callers', async () => { + let respond!: (response: Response) => void; + const gatewayFetch = vi.fn(() => new Promise(resolve => (respond = resolve))); + const gateway = gatewayWith('deployment-key', gatewayFetch); + + const first = gateway.fetchProviders(); + const second = gateway.fetchProviders(); + await vi.waitFor(() => expect(gatewayFetch).toHaveBeenCalledOnce()); + respond(new Response('unavailable', { status: 503 })); + + await expect(first).rejects.toThrow('(503)'); + await expect(second).rejects.toThrow('(503)'); + expect(gatewayFetch).toHaveBeenCalledOnce(); + }); + + it('counts remembered failures toward the 100 catalog limit', async () => { + const gatewayFetch = vi.fn(async () => new Response('unavailable', { status: 503 })); + for (let i = 0; i <= 100; i++) { + await expect(gatewayWith(`key-${i}`, gatewayFetch).fetchProviders()).rejects.toThrow('(503)'); + } + expect(gatewayFetch).toHaveBeenCalledTimes(101); + + await expect(gatewayWith('key-100', gatewayFetch).fetchProviders()).rejects.toThrow('(503)'); + expect(gatewayFetch).toHaveBeenCalledTimes(101); + await expect(gatewayWith('key-0', gatewayFetch).fetchProviders()).rejects.toThrow('(503)'); + expect(gatewayFetch).toHaveBeenCalledTimes(102); + }); + + it('forgets remembered failures when the cache is reset', async () => { + const gatewayFetch = vi.fn(async () => new Response('unavailable', { status: 503 })); + const gateway = gatewayWith('deployment-key', gatewayFetch); + await expect(gateway.fetchProviders()).rejects.toThrow('(503)'); + + _resetMastraGatewayCatalogCache(); + await expect(gateway.fetchProviders()).rejects.toThrow('(503)'); + expect(gatewayFetch).toHaveBeenCalledTimes(2); + }); + }); + + describe('resolveLanguageModel streaming usage', () => { + const prompt = [{ role: 'user', content: [{ type: 'text', text: 'hi' }] }]; + + function captureBodies() { + const bodies: Record[] = []; + const fetchImpl = vi.fn(async (_input: any, init: any) => { + bodies.push(JSON.parse(String(init?.body))); + return new Response('{}', { status: 200, headers: { 'content-type': 'application/json' } }); + }); + return { bodies, fetchImpl }; + } + + async function send(model: any) { + await model.doStream({ prompt }).catch(() => {}); + await model.doGenerate({ prompt }).catch(() => {}); + } + + it('asks for usage on streaming requests only with an API key', async () => { + const { bodies, fetchImpl } = captureBodies(); + vi.stubGlobal('fetch', fetchImpl); + try { + const model = new MastraGateway({ apiKey: 'gateway-key' }).resolveLanguageModel({ + modelId: 'gpt-4o', + providerId: 'openai', + apiKey: 'gateway-key', + }); + await send(model); + } finally { + vi.unstubAllGlobals(); + } + + expect(bodies).toHaveLength(2); + expect(bodies[0]).toMatchObject({ stream: true, stream_options: { include_usage: true } }); + expect(bodies[1]).not.toHaveProperty('stream_options'); + expect(bodies[1]).not.toHaveProperty('usage'); + }); + + it('reports usage sent in a separate final chunk', async () => { + const frames = [ + { id: 'c1', choices: [{ index: 0, delta: { role: 'assistant', content: 'ok' } }] }, + { id: 'c1', choices: [{ index: 0, delta: {}, finish_reason: 'stop' }] }, + { id: 'c1', choices: [], usage: { prompt_tokens: 7, completion_tokens: 3, total_tokens: 10 } }, + ]; + const body = frames.map(frame => `data: ${JSON.stringify(frame)}\n\n`).join('') + 'data: [DONE]\n\n'; + const customFetch = vi.fn( + async () => new Response(body, { status: 200, headers: { 'content-type': 'text/event-stream' } }), + ); + const model: any = new MastraGateway({ + apiKey: 'gateway-key', + customFetch: customFetch as any, + }).resolveLanguageModel({ modelId: 'gpt-4o', providerId: 'openai', apiKey: 'gateway-key' }); + + const { stream } = await model.doStream({ prompt }); + const parts: any[] = []; + for await (const part of stream) parts.push(part); + + const finish = parts.find(part => part.type === 'finish'); + expect(finish?.usage).toMatchObject({ inputTokens: { total: 7 }, outputTokens: { total: 3 } }); + }); + + it('asks for usage on streaming requests only with a custom fetch', async () => { + const { bodies, fetchImpl } = captureBodies(); + const model = new MastraGateway({ apiKey: 'gateway-key', customFetch: fetchImpl as any }).resolveLanguageModel({ + modelId: 'gpt-4o', + providerId: 'openai', + apiKey: 'gateway-key', + }); + await send(model); + + expect(bodies).toHaveLength(2); + expect(bodies[0]).toMatchObject({ stream: true, stream_options: { include_usage: true } }); + expect(bodies[1]).not.toHaveProperty('stream_options'); + expect(bodies[1]).not.toHaveProperty('usage'); + }); + }); + describe('resolveLanguageModel gateway authorization header', () => { const GATEWAY_AUTH_HEADER = 'X-Memory-Gateway-Authorization'; diff --git a/packages/core/src/llm/model/gateways/mastra.ts b/packages/core/src/llm/model/gateways/mastra.ts index cf6f9b451b4..5a38839a898 100644 --- a/packages/core/src/llm/model/gateways/mastra.ts +++ b/packages/core/src/llm/model/gateways/mastra.ts @@ -1,3 +1,4 @@ +import { createHash } from 'node:crypto'; import { createAnthropic } from '@ai-sdk/anthropic-v6'; import { createOpenRouter } from '@openrouter/ai-sdk-provider-v6'; import { MastraError } from '../../../error/index.js'; @@ -64,15 +65,19 @@ export class MastraGateway extends MastraModelGateway { /** * Gateway v2 deployments are signaled by MASTRA_GATEWAY_VERSION=v2 (set by the platform) - * or by a v2 (organization) API key prefix, which predates the env var signal. + * or by a v2 (organization) API key prefix, which predates the env var signal. A v1 project + * key (`msk_`) is always served by v1, so it wins over the env var. * Keep in sync with isGatewayV2 in @mastra/server's gateway-memory-client. * Public so the server can ask a registered gateway instance for its - * effective v2 state (the instance may carry a v2 key via config.apiKey + * effective v2 state (the instance may carry a key via config.apiKey * that env-only checks cannot see). */ isV2(): boolean { - if (process.env['MASTRA_GATEWAY_VERSION'] === 'v2') return true; const key = this.config?.apiKey ?? process.env['MASTRA_GATEWAY_API_KEY']; + // A v1 project key (`msk_`) is always served by v1, so it wins over the env var. + // The v2 prefixes (`msk-gateway-`, `mskv2_`) do not start with `msk_`. + if (key?.startsWith('msk_')) return false; + if (process.env['MASTRA_GATEWAY_VERSION'] === 'v2') return true; return !!key && (key.startsWith('msk-gateway-') || key.startsWith('mskv2_')); } @@ -82,27 +87,12 @@ export class MastraGateway extends MastraModelGateway { let models = openrouterConfig?.models ?? []; if (isV2) { const apiKey = await this.getApiKey(); - const response = await (this.config?.customFetch ?? fetch)(`${this.getBaseUrl()}/v1/models`, { - headers: this.config?.customFetch ? withGatewayAuthHeader(apiKey) : { Authorization: `Bearer ${apiKey}` }, - signal: AbortSignal.timeout(10_000), - }); - if (!response.ok) throw new Error(`Gateway model catalog unavailable (${response.status})`); - const body: unknown = await response.json(); - if (!body || typeof body !== 'object' || !('data' in body) || !Array.isArray(body.data)) { - throw new Error('Gateway model catalog has an invalid response'); - } - // The Mastra gateway model uses the OpenAI-compatible chat transport. - // Models offered only through another endpoint cannot be selected here. - models = body.data - .filter( - (item): item is { id: string; endpoints: string[] } => - !!item && - typeof item === 'object' && - typeof item.id === 'string' && - Array.isArray(item.endpoints) && - item.endpoints.includes('chat'), - ) - .map(item => item.id); + // Gateway v2 reads the key only from Authorization or x-api-key. A custom fetch is a v1 + // pass-through that owns Authorization, so the catalogue request uses the plain global fetch. + const fetchImpl = globalThis.fetch; + models = await loadCachedV2Catalog(fetchImpl, `${this.getBaseUrl()}\n${hashApiKey(apiKey)}`, () => + this.fetchV2Catalog(fetchImpl, apiKey), + ); } const providers = { @@ -121,6 +111,32 @@ export class MastraGateway extends MastraModelGateway { return providers; } + private async fetchV2Catalog(fetchImpl: typeof fetch, apiKey: string): Promise { + const response = await fetchImpl(`${this.getBaseUrl()}/v1/models`, { + headers: { Authorization: `Bearer ${apiKey}` }, + signal: AbortSignal.timeout(10_000), + }); + if (!response.ok) { + throw new GatewayCatalogError(`Gateway model catalog unavailable (${response.status})`, response.status); + } + const body: unknown = await response.json(); + if (!body || typeof body !== 'object' || !('data' in body) || !Array.isArray(body.data)) { + throw new Error('Gateway model catalog has an invalid response'); + } + // The Mastra gateway model uses the OpenAI-compatible chat transport. + // Models offered only through another endpoint cannot be selected here. + return body.data + .filter( + (item): item is { id: string; endpoints: string[] } => + !!item && + typeof item === 'object' && + typeof item.id === 'string' && + Array.isArray(item.endpoints) && + item.endpoints.includes('chat'), + ) + .map(item => item.id); + } + async buildUrl(_modelId: string): Promise { return `${this.getBaseUrl()}/v1`; } @@ -169,6 +185,9 @@ export class MastraGateway extends MastraModelGateway { baseURL, headers: withGatewayAuthHeader(apiKey, headers), fetch: this.config.customFetch, + // Strict mode adds stream_options.include_usage to streaming requests only, + // so routes that report usage in a separate final chunk still return it. + compatibility: 'strict', }).chat(fullModelId) as unknown as GatewayLanguageModel; } @@ -180,6 +199,146 @@ export class MastraGateway extends MastraModelGateway { 'User-Agent': MASTRA_USER_AGENT, ...headers, }, + // Strict mode adds stream_options.include_usage to streaming requests only. + compatibility: 'strict', }).chat(fullModelId) as unknown as GatewayLanguageModel; } } + +/** How long a v2 model catalogue is reused before it is fetched again. */ +const V2_CATALOG_TTL_MS = 60_000; +/** How long the last good v2 catalogue is still served while refreshes fail. */ +const V2_CATALOG_STALE_IF_ERROR_MS = 10 * 60_000; +/** + * How long to wait after a failed request before trying again: the last good catalogue is served + * meanwhile, or, when there is none, the failure is thrown again. + */ +const V2_CATALOG_RETRY_AFTER_ERROR_MS = 15_000; +/** Most catalogues kept per fetch implementation. */ +const V2_CATALOG_MAX_ENTRIES = 100; + +/** A catalogue request that got an HTTP error status. */ +class GatewayCatalogError extends Error { + constructor( + message: string, + readonly status: number, + ) { + super(message); + this.name = 'GatewayCatalogError'; + } +} + +interface V2CatalogEntry { + models?: string[]; + fetchedAt?: number; + retryAt?: number; + pending?: Promise; + /** A failure with no list to fall back on, rethrown until `failedUntil` without a new request. */ + failure?: unknown; + failedUntil?: number; +} + +/** + * v2 model catalogues, first by the global fetch in effect when the request was made, then by + * base URL and API key hash. A replaced global fetch (instrumentation, test doubles) gets its own + * entries, so a list from one is never served through another. The raw key is never stored. + */ +let v2CatalogCaches = new WeakMap>(); + +/** Only an in-memory cache key, so the raw key is not held. Not a password hash. */ +function hashApiKey(apiKey: string): string { + return createHash('sha256').update(apiKey).digest('hex'); +} + +/** A 401 or 403 means the key was rejected, so a list cached for it must not be served. */ +function isRejectedKey(error: unknown): boolean { + return error instanceof GatewayCatalogError && (error.status === 401 || error.status === 403); +} + +/** + * Drops settled entries past the stale window (keeping a failure that is still remembered), then + * the oldest settled ones over the size cap. + */ +function pruneV2Catalogs(cache: Map, now: number): void { + for (const [key, entry] of cache) { + const failureRemembered = entry.failedUntil !== undefined && now < entry.failedUntil; + const listExpired = entry.fetchedAt === undefined || now - entry.fetchedAt >= V2_CATALOG_STALE_IF_ERROR_MS; + if (!entry.pending && !failureRemembered && listExpired) cache.delete(key); + } + for (const [key, entry] of cache) { + if (cache.size <= V2_CATALOG_MAX_ENTRIES) break; + if (!entry.pending) cache.delete(key); + } +} + +/** + * Returns the cached v2 catalogue for `cacheKey`, fetching it when it is missing or older than + * the TTL. Concurrent callers share one request. When a refresh fails with a timeout, a network + * error or a server error, the last good catalogue is returned for a bounded time, and the next + * attempt waits a short while. Without a catalogue to fall back on, the error is thrown, and the + * same error is thrown again without a new request for that short while, so callers during an + * outage do not each wait for the request timeout. A rejected key (401 or 403) drops the cached + * catalogue and is never remembered, so every call asks the gateway again. + */ +function loadCachedV2Catalog( + fetchImpl: typeof fetch, + cacheKey: string, + load: () => Promise, +): Promise { + let cache = v2CatalogCaches.get(fetchImpl); + if (!cache) { + cache = new Map(); + v2CatalogCaches.set(fetchImpl, cache); + } + const entries = cache; + const entry = entries.get(cacheKey); + const now = Date.now(); + if (entry?.models && entry.fetchedAt !== undefined) { + const age = now - entry.fetchedAt; + const fresh = age < V2_CATALOG_TTL_MS; + const backingOff = entry.retryAt !== undefined && now < entry.retryAt && age < V2_CATALOG_STALE_IF_ERROR_MS; + if (fresh || backingOff) return Promise.resolve(entry.models); + } + if (entry?.failedUntil !== undefined && now < entry.failedUntil) return Promise.reject(entry.failure); + if (entry?.pending) return entry.pending; + + const pending = load().then( + models => { + const settledAt = Date.now(); + entries.delete(cacheKey); + entries.set(cacheKey, { models, fetchedAt: settledAt }); + pruneV2Catalogs(entries, settledAt); + return models; + }, + error => { + const previous = entries.get(cacheKey); + const settledAt = Date.now(); + if ( + !isRejectedKey(error) && + previous?.models && + previous.fetchedAt !== undefined && + settledAt - previous.fetchedAt < V2_CATALOG_STALE_IF_ERROR_MS + ) { + entries.set(cacheKey, { + models: previous.models, + fetchedAt: previous.fetchedAt, + retryAt: settledAt + V2_CATALOG_RETRY_AFTER_ERROR_MS, + }); + return previous.models; + } + entries.delete(cacheKey); + if (!isRejectedKey(error)) { + entries.set(cacheKey, { failure: error, failedUntil: settledAt + V2_CATALOG_RETRY_AFTER_ERROR_MS }); + pruneV2Catalogs(entries, settledAt); + } + throw error; + }, + ); + entries.set(cacheKey, { ...entry, pending }); + return pending; +} + +/** @internal Clear the cached v2 model catalogues. For testing only. */ +export function _resetMastraGatewayCatalogCache(): void { + v2CatalogCaches = new WeakMap(); +} diff --git a/packages/core/src/llm/model/provider-registry.test.ts b/packages/core/src/llm/model/provider-registry.test.ts index 3af1f3b27aa..1674bf3271c 100644 --- a/packages/core/src/llm/model/provider-registry.test.ts +++ b/packages/core/src/llm/model/provider-registry.test.ts @@ -933,6 +933,95 @@ describe('GatewayRegistry Auto-Refresh', () => { fs.rmSync(tmpDir, { recursive: true, force: true }); }); + describe('Mastra gateway in the shared cache', () => { + const envKeys = ['MASTRA_GATEWAY_VERSION', 'MASTRA_GATEWAY_API_KEY'] as const; + let savedEnv: Record = {}; + + beforeEach(() => { + savedEnv = Object.fromEntries(envKeys.map(key => [key, process.env[key]])); + for (const key of envKeys) delete process.env[key]; + }); + + afterEach(() => { + for (const key of envKeys) { + if (savedEnv[key] === undefined) delete process.env[key]; + else process.env[key] = savedEnv[key]; + } + GatewayRegistry.getInstance({ useDynamicLoading: true }).registerCustomGateways([]); + }); + + async function sync(mastraCatalog: () => Promise>) { + const registry = GatewayRegistry.getInstance({ useDynamicLoading: true }); + const writtenFiles: string[] = []; + vi.spyOn(fs.promises, 'writeFile').mockImplementation(async (filePath: any, data: any) => { + writtenFiles.push(`${filePath.toString()}\n${String(data)}`); + }); + vi.spyOn(fs.promises, 'rename').mockResolvedValue(); + vi.spyOn(fs.promises, 'copyFile').mockResolvedValue(); + vi.spyOn(fs, 'mkdirSync').mockReturnValue(undefined); + vi.spyOn(ModelsDevGateway.prototype, 'fetchProviders').mockResolvedValue({ + 'test-provider': { + name: 'Test Provider', + models: ['model-a'], + apiKeyEnvVar: 'TEST_API_KEY', + gateway: 'models-dev', + } as ProviderConfig, + }); + vi.spyOn(NetlifyGateway.prototype, 'fetchProviders').mockResolvedValue({}); + const mastraFetchProvidersSpy = vi + .spyOn(MastraGateway.prototype, 'fetchProviders') + .mockImplementation(mastraCatalog); + + await registry.syncGateways(true); + + const registryWrites = writtenFiles.filter(entry => entry.includes('provider-registry.json')); + expect(registryWrites.length).toBeGreaterThan(0); + for (const entry of registryWrites) expect(entry).toContain('test-provider'); + return { mastraFetchProvidersSpy, registryWrites }; + } + + const failingCatalog = async () => { + throw new Error('Gateway model catalog unavailable (401)'); + }; + + it('does not sync the default Mastra gateway in v2 mode', async () => { + process.env.MASTRA_GATEWAY_VERSION = 'v2'; + process.env.MASTRA_GATEWAY_API_KEY = 'deployment-key'; + + const { mastraFetchProvidersSpy, registryWrites } = await sync(failingCatalog); + + expect(mastraFetchProvidersSpy).not.toHaveBeenCalled(); + for (const entry of registryWrites) expect(entry).not.toContain('"mastra"'); + }); + + it('does not sync a registered Mastra gateway that holds an organization key', async () => { + GatewayRegistry.getInstance({ useDynamicLoading: true }).registerCustomGateways([ + new MastraGateway({ apiKey: 'mskv2_org-key' }), + ]); + + const { mastraFetchProvidersSpy, registryWrites } = await sync(failingCatalog); + + expect(mastraFetchProvidersSpy).not.toHaveBeenCalled(); + for (const entry of registryWrites) expect(entry).not.toContain('"mastra"'); + }); + + it('still syncs the Mastra gateway with a v1 project key', async () => { + process.env.MASTRA_GATEWAY_API_KEY = 'msk_0123456789abcdef0123456789abcdef'; + + const { mastraFetchProvidersSpy, registryWrites } = await sync(async () => ({ + mastra: { + name: 'Gateway', + models: ['openai/chat-model'], + apiKeyEnvVar: 'MASTRA_GATEWAY_API_KEY', + gateway: 'mastra', + } as ProviderConfig, + })); + + expect(mastraFetchProvidersSpy).toHaveBeenCalled(); + for (const entry of registryWrites) expect(entry).toContain('"mastra"'); + }); + }); + it('should skip syncGateways when MASTRA_OFFLINE is set to true', async () => { process.env.MASTRA_OFFLINE = 'true'; diff --git a/packages/core/src/llm/model/provider-registry.ts b/packages/core/src/llm/model/provider-registry.ts index fd8b562ff60..1ed63815f1a 100644 --- a/packages/core/src/llm/model/provider-registry.ts +++ b/packages/core/src/llm/model/provider-registry.ts @@ -866,8 +866,13 @@ export class GatewayRegistry { ...(writeToSrc ? [] : [new MastraGateway()]), ]; - // Combine default and custom gateways - const gateways = [...defaultGateways, ...this.customGateways]; + // Combine default and custom gateways. A Mastra gateway in v2 mode, default or registered, + // lists models for one organization's key, so it is not written to the machine-wide cache, + // and its failures cannot stop the rest of the refresh. + const gateways = [...defaultGateways, ...this.customGateways].filter(gateway => { + const candidate = gateway as { id?: string; isV2?: () => boolean }; + return !(candidate.id === 'mastra' && typeof candidate.isV2 === 'function' && candidate.isV2()); + }); // Fetch provider data const { diff --git a/packages/server/src/server/handlers/gateway-memory-client.ts b/packages/server/src/server/handlers/gateway-memory-client.ts index 271a6ec90c9..011958b3142 100644 --- a/packages/server/src/server/handlers/gateway-memory-client.ts +++ b/packages/server/src/server/handlers/gateway-memory-client.ts @@ -293,24 +293,29 @@ function isV2GatewayKey(key: string): boolean { } /** - * Whether this deployment targets Gateway v2. Signaled either by the - * MASTRA_GATEWAY_VERSION=v2 runtime env var (set by the platform), by a v2 - * (organization) API key prefix (which predates the env var signal), or by a - * registered Mastra gateway instance that carries a v2 key via config.apiKey - * (pass `mastra` so instance-level config is consulted too). + * Whether this deployment targets Gateway v2. Signaled by the MASTRA_GATEWAY_VERSION=v2 runtime + * env var (set by the platform) or by a v2 (organization) API key prefix, which predates the env + * var signal. A v1 project key (`msk_`) is always served by v1, so it wins over the env var. + * + * Pass `mastra` so the registered Mastra gateway answers: it holds the key model calls use, which + * may be set in its config rather than the environment, and its `isV2()` applies the same rules. + * Keep the environment rules below in sync with `MastraGateway.isV2` in `@mastra/core`. */ export function isGatewayV2(mastra?: { listGateways?: () => Record | undefined }): boolean { - if (process.env.MASTRA_GATEWAY_VERSION === 'v2') return true; - const key = process.env.MASTRA_GATEWAY_API_KEY; - if (!!key && isV2GatewayKey(key)) return true; - // A user-registered MastraGateway can hold a v2 key in its config rather - // than the environment; ask the instance for its effective v2 state. const gateways = mastra?.listGateways?.() ?? {}; for (const gateway of Object.values(gateways)) { const candidate = gateway as { id?: string; isV2?: () => boolean }; - if (candidate?.id === 'mastra' && typeof candidate.isV2 === 'function' && candidate.isV2()) return true; + if (candidate?.id === 'mastra' && typeof candidate.isV2 === 'function') return candidate.isV2(); } - return false; + return isGatewayV2ByEnvironment(); +} + +/** Whether the environment alone (MASTRA_GATEWAY_API_KEY and MASTRA_GATEWAY_VERSION) says v2. */ +function isGatewayV2ByEnvironment(): boolean { + const key = process.env.MASTRA_GATEWAY_API_KEY; + if (key?.startsWith('msk_')) return false; + if (process.env.MASTRA_GATEWAY_VERSION === 'v2') return true; + return !!key && isV2GatewayKey(key); } // Singleton-ish lazy instance @@ -320,10 +325,14 @@ let _gatewayClient: GatewayMemoryClient | null | undefined; * Get a GatewayMemoryClient instance, or null if gateway is not configured. * Pass `mastra` so a registered v2 gateway configuration takes precedence * over a v1 environment key. + * + * The client is built from MASTRA_GATEWAY_API_KEY and MASTRA_GATEWAY_URL, so it is built only when + * the environment itself says v1. A registered gateway that is v1 only because of a key in its own + * config, which the server cannot use here, does not make the environment key a v1 key. */ export function getGatewayClient(mastra?: Parameters[0]): GatewayMemoryClient | null { // Gateway v2 has no memory REST API; Studio must use the agent's own memory configuration. - if (isGatewayV2(mastra)) return null; + if (isGatewayV2(mastra) || isGatewayV2ByEnvironment()) return null; if (_gatewayClient !== undefined) return _gatewayClient; const url = process.env.MASTRA_GATEWAY_URL || 'https://gateway-api.mastra.ai'; const key = process.env.MASTRA_GATEWAY_API_KEY; diff --git a/packages/server/src/server/handlers/gateway-memory-client.v2.test.ts b/packages/server/src/server/handlers/gateway-memory-client.v2.test.ts index 9cb5398dd88..87b4c059ad9 100644 --- a/packages/server/src/server/handlers/gateway-memory-client.v2.test.ts +++ b/packages/server/src/server/handlers/gateway-memory-client.v2.test.ts @@ -1,3 +1,4 @@ +import { MastraGateway } from '@mastra/core/llm'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { getGatewayClient, isGatewayV2 } from './gateway-memory-client.js'; @@ -21,6 +22,18 @@ describe('Gateway memory client in v2 deployments', () => { expect(isGatewayV2()).toBe(false); }); + it('is not v2 for a v1 key even when MASTRA_GATEWAY_VERSION is v2', () => { + vi.stubEnv('MASTRA_GATEWAY_VERSION', 'v2'); + vi.stubEnv('MASTRA_GATEWAY_API_KEY', 'msk_v1-key'); + expect(isGatewayV2()).toBe(false); + }); + + it('is v2 when MASTRA_GATEWAY_VERSION is v2 and the key has no v1 prefix', () => { + vi.stubEnv('MASTRA_GATEWAY_VERSION', 'v2'); + vi.stubEnv('MASTRA_GATEWAY_API_KEY', 'deployment-key'); + expect(isGatewayV2()).toBe(true); + }); + it('detects v2 from a registered Mastra gateway instance carrying a v2 config key', () => { vi.stubEnv('MASTRA_GATEWAY_VERSION', ''); vi.stubEnv('MASTRA_GATEWAY_API_KEY', ''); @@ -35,6 +48,24 @@ describe('Gateway memory client in v2 deployments', () => { expect(getGatewayClient(mastra)).toBeNull(); }); + it('agrees with a registered gateway whose v1 config key wins over MASTRA_GATEWAY_VERSION', () => { + vi.stubEnv('MASTRA_GATEWAY_VERSION', 'v2'); + vi.stubEnv('MASTRA_GATEWAY_API_KEY', ''); + const gateway = new MastraGateway({ apiKey: 'msk_0123456789abcdef0123456789abcdef' }); + const mastra = { listGateways: () => ({ mastra: gateway }) }; + expect(gateway.isV2()).toBe(false); + expect(isGatewayV2(mastra)).toBe(false); + }); + + it('does not build a v1 memory client from an organization env key', () => { + vi.stubEnv('MASTRA_GATEWAY_VERSION', ''); + vi.stubEnv('MASTRA_GATEWAY_API_KEY', 'mskv2_org-key'); + const gateway = new MastraGateway({ apiKey: 'msk_0123456789abcdef0123456789abcdef' }); + const mastra = { listGateways: () => ({ mastra: gateway }) }; + expect(isGatewayV2(mastra)).toBe(false); + expect(getGatewayClient(mastra)).toBeNull(); + }); + it('is not v2 when the registered Mastra gateway instance reports v1', () => { vi.stubEnv('MASTRA_GATEWAY_VERSION', ''); vi.stubEnv('MASTRA_GATEWAY_API_KEY', ''); @@ -42,3 +73,64 @@ describe('Gateway memory client in v2 deployments', () => { expect(isGatewayV2(mastra)).toBe(false); }); }); + +describe('v2 detection and the v1 memory client across configurations', () => { + afterEach(() => vi.unstubAllEnvs()); + + const envKeys = { + none: '', + project: 'msk_0123456789abcdef0123456789abcdef', + organization: 'mskv2_org-key', + unprefixed: 'deployment-key', + } as const; + const registeredKeys = { none: undefined, v1: 'msk_fedcba9876543210fedcba9876543210', v2: 'mskv2_registered' } as const; + + // [env key, MASTRA_GATEWAY_VERSION, registered gateway key, isGatewayV2, v1 memory client built] + const table: Array<[keyof typeof envKeys, 'unset' | 'v2', keyof typeof registeredKeys, boolean, boolean]> = [ + ['none', 'unset', 'none', false, false], + ['none', 'v2', 'none', true, false], + ['none', 'unset', 'v1', false, false], + ['none', 'v2', 'v1', false, false], + ['none', 'unset', 'v2', true, false], + ['none', 'v2', 'v2', true, false], + ['project', 'unset', 'none', false, true], + ['project', 'v2', 'none', false, true], + ['project', 'unset', 'v1', false, true], + ['project', 'v2', 'v1', false, true], + ['project', 'unset', 'v2', true, false], + ['project', 'v2', 'v2', true, false], + ['organization', 'unset', 'none', true, false], + ['organization', 'v2', 'none', true, false], + ['organization', 'unset', 'v1', false, false], + ['organization', 'v2', 'v1', false, false], + ['organization', 'unset', 'v2', true, false], + ['organization', 'v2', 'v2', true, false], + ['unprefixed', 'unset', 'none', false, true], + ['unprefixed', 'v2', 'none', true, false], + ['unprefixed', 'unset', 'v1', false, true], + ['unprefixed', 'v2', 'v1', false, false], + ['unprefixed', 'unset', 'v2', true, false], + ['unprefixed', 'v2', 'v2', true, false], + ]; + + it.each(table)( + 'env key %s, MASTRA_GATEWAY_VERSION %s, registered key %s: v2 is %s, v1 memory client built is %s', + async (envKey, version, registeredKey, expectedV2, expectedClient) => { + vi.stubEnv('MASTRA_GATEWAY_API_KEY', envKeys[envKey]); + vi.stubEnv('MASTRA_GATEWAY_VERSION', version === 'v2' ? 'v2' : ''); + const registered = registeredKeys[registeredKey]; + // Mastra always lists a Mastra gateway: the registered one, or a default one without a config key. + const gateway = registered ? new MastraGateway({ apiKey: registered }) : new MastraGateway(); + const mastra = { listGateways: () => ({ mastra: gateway }) }; + + // A fresh module, so the cached client from one case cannot answer for another. + vi.resetModules(); + const client = await import('./gateway-memory-client.js'); + + expect(client.isGatewayV2(mastra)).toBe(expectedV2); + expect(client.getGatewayClient(mastra) !== null).toBe(expectedClient); + if (!registered) expect(client.isGatewayV2()).toBe(expectedV2); + }, + ); +}); + From 39730d03408c9bb1757c4c25700d4ee1608ec73d Mon Sep 17 00:00:00 2001 From: Rhys Balevicius Date: Tue, 17 Nov 2026 03:58:31 +0000 Subject: [PATCH 2/5] fix(core): correct gateway diagnostics and studio model selection Persist request origins so saved errors distinguish gateway failures from provider failures without retaining URL credentials or paths. Restore missing-memory warnings when the gateway does not manage memory, without delaying runs. Keep the gateway visible when its catalog fails to load. Use gateway credentials when selecting models for instruction enhancement. --- .changeset/saved-error-origins.md | 8 ++ .changeset/steady-studio-providers.md | 8 ++ .../mastra-gateway-duck-typing.test.ts | 105 +++++++++++++++++- packages/core/src/agent/agent.ts | 33 ++++-- .../src/agent/message-list/state/types.ts | 8 +- packages/core/src/error/index.ts | 1 + .../core/src/error/provider-error.test.ts | 1 + packages/core/src/error/provider-error.ts | 22 +++- .../shared/provider-error-details.test.ts | 20 +++- .../src/loop/shared/provider-error-details.ts | 17 +++ .../record-terminal-error-message.test.ts | 45 ++++++++ .../shared/record-terminal-error-message.ts | 5 +- .../persistence-failed-turns.scenario.test.ts | 5 +- .../server/src/server/handlers/agent.test.ts | 101 +++++++++++++++++ .../server/src/server/handlers/agents.test.ts | 6 +- packages/server/src/server/handlers/agents.ts | 36 +++++- 16 files changed, 403 insertions(+), 18 deletions(-) create mode 100644 .changeset/saved-error-origins.md create mode 100644 .changeset/steady-studio-providers.md diff --git a/.changeset/saved-error-origins.md b/.changeset/saved-error-origins.md new file mode 100644 index 00000000000..42d4d00325d --- /dev/null +++ b/.changeset/saved-error-origins.md @@ -0,0 +1,8 @@ +--- +'@mastra/core': patch +--- + +Improved how `@mastra/core` reports on Mastra Gateway models: + +- The warning "No memory is configured but resourceId and threadId were passed in args" is shown again for Gateway models when the Gateway does not keep memory for your key, such as an organization key. It stays hidden when the Gateway manages memory. The check runs in the background and does not delay the run. +- Error parts saved to thread history for a failed model call now include `requestOrigin`, the scheme, host and port the request was sent to (for example `https://gateway-api.mastra.ai`). A UI that rebuilds the error from history can tell a Gateway error from a provider error with the same status. The path, query string and any credentials in the URL are not saved. diff --git a/.changeset/steady-studio-providers.md b/.changeset/steady-studio-providers.md new file mode 100644 index 00000000000..e88159d6aa3 --- /dev/null +++ b/.changeset/steady-studio-providers.md @@ -0,0 +1,8 @@ +--- +'@mastra/server': patch +--- + +Fixed Studio for projects that use the Mastra Gateway: + +- The Gateway provider stays in the provider list, with no models, when its model list cannot be loaded (for example a revoked key or an outage). Before, it disappeared. +- "Enhance instructions" now works for agents on gateway models such as `mastra/...` when only the gateway's key, for example `MASTRA_GATEWAY_API_KEY`, is set. Before, it returned "No model with a configured API key found". A gateway model whose credential is missing is skipped in favour of the next connected model. diff --git a/packages/core/src/agent/__tests__/mastra-gateway-duck-typing.test.ts b/packages/core/src/agent/__tests__/mastra-gateway-duck-typing.test.ts index caf26e4fc3d..4802fafdf11 100644 --- a/packages/core/src/agent/__tests__/mastra-gateway-duck-typing.test.ts +++ b/packages/core/src/agent/__tests__/mastra-gateway-duck-typing.test.ts @@ -1,4 +1,5 @@ -import { describe, expect, it, vi } from 'vitest'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { ModelRouterLanguageModel } from '../../llm/model/router'; import { Agent } from '../agent'; describe('mastra gateway duck typing', () => { @@ -71,4 +72,106 @@ describe('mastra gateway duck typing', () => { expect.anything(), ); }); + + describe('with a Mastra gateway router model', () => { + const WARNING = 'No memory is configured but resourceId and threadId were passed in args'; + + afterEach(() => { + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + }); + + function routerModel(apiKey: string) { + vi.stubEnv('MASTRA_GATEWAY_API_KEY', apiKey); + const model = new ModelRouterLanguageModel('mastra/openai/gpt-4o'); + expect(model.gatewayId).toBe('mastra'); + const respond = async () => + ({ + stream: new ReadableStream({ + start(controller) { + controller.enqueue({ type: 'text-start', id: 'text-1' }); + controller.enqueue({ type: 'text-delta', id: 'text-1', delta: 'ok' }); + controller.enqueue({ type: 'text-end', id: 'text-1' }); + controller.enqueue({ + type: 'finish', + finishReason: 'stop', + usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 }, + }); + controller.close(); + }, + }), + rawCall: { rawPrompt: null, rawSettings: {} }, + warnings: [], + }) as any; + vi.spyOn(model, 'doStream').mockImplementation(respond); + vi.spyOn(model, 'doGenerate').mockImplementation(respond); + return model; + } + + async function run(model: ModelRouterLanguageModel) { + const warn = vi.fn(); + const debug = vi.fn(); + const agent = new Agent({ + id: 'test-agent', + name: 'test-agent', + instructions: 'You are a helpful assistant.', + model, + }); + agent.__setLogger({ debug, info: vi.fn(), warn, error: vi.fn(), trace: vi.fn() } as any); + const result = await agent.generate('hello', { + memory: { thread: { id: 'thread-1' }, resource: 'resource-1' }, + }); + return { result, warn, debug }; + } + + const settle = () => new Promise(resolve => setTimeout(resolve, 0)); + + it('warns for an organization key, whose memory stays local', async () => { + const { result, warn } = await run(routerModel('mskv2_org-key')); + + expect(result.text).toBe('ok'); + await vi.waitFor(() => expect(warn).toHaveBeenCalledWith(WARNING, { agent: 'test-agent' })); + }); + + it('does not warn for a project key, whose memory the gateway keeps', async () => { + const model = routerModel('msk_0123456789abcdef0123456789abcdef'); + const check = vi.spyOn(model, 'isGatewayMemoryManaged'); + const { warn } = await run(model); + + await vi.waitFor(() => expect(check).toHaveBeenCalled()); + await expect(check.mock.results[0]!.value).resolves.toBe(true); + await settle(); + expect(warn).not.toHaveBeenCalledWith(WARNING, expect.anything()); + }); + + it('finishes the run and does not warn when the check fails', async () => { + const model = routerModel('mskv2_org-key'); + const check = vi.spyOn(model, 'isGatewayMemoryManaged').mockRejectedValue(new Error('credential unavailable')); + const { result, warn, debug } = await run(model); + + expect(result.text).toBe('ok'); + await vi.waitFor(() => expect(check).toHaveBeenCalled()); + await settle(); + expect(warn).not.toHaveBeenCalledWith(WARNING, expect.anything()); + expect(debug).toHaveBeenCalledWith('Could not tell whether the gateway manages memory for this model', { + agent: 'test-agent', + error: expect.objectContaining({ message: 'credential unavailable' }), + }); + }); + + it('does not wait for a slow check before running the model', async () => { + const model = routerModel('mskv2_org-key'); + let release: ((value: boolean) => void) | undefined; + vi.spyOn(model, 'isGatewayMemoryManaged').mockImplementation( + () => new Promise(resolve => (release = resolve)), + ); + const { result, warn } = await run(model); + + expect(result.text).toBe('ok'); + expect(warn).not.toHaveBeenCalledWith(WARNING, expect.anything()); + expect(release).toBeDefined(); + release!(false); + await vi.waitFor(() => expect(warn).toHaveBeenCalledWith(WARNING, { agent: 'test-agent' })); + }); + }); }); diff --git a/packages/core/src/agent/agent.ts b/packages/core/src/agent/agent.ts index df77b6d89be..de2431c2328 100644 --- a/packages/core/src/agent/agent.ts +++ b/packages/core/src/agent/agent.ts @@ -9281,14 +9281,31 @@ export class Agent< // Only warn for ids the caller passed in `memory` args. Reserved request-context ids are set // per request (auth middleware, AgentController) and inherited by every agent in that request, // including internal memory-less agents such as the OM Observer and Reflector. - if ( - options.memory?.resource && - options.memory?.thread && - !this.hasOwnMemory() && - !this.#knowledge && - !isGatewayModel - ) { - this.logger.warn('No memory is configured but resourceId and threadId were passed in args', { agent: this.name }); + if (options.memory?.resource && options.memory?.thread && !this.hasOwnMemory() && !this.#knowledge) { + const warnMissingMemory = () => + this.logger.warn('No memory is configured but resourceId and threadId were passed in args', { + agent: this.name, + }); + const gatewayModel = resolvedModel as unknown as { isGatewayMemoryManaged?: () => Promise }; + if (!isGatewayModel) { + warnMissingMemory(); + } else if (typeof gatewayModel.isGatewayMemoryManaged === 'function') { + // The gateway keeps memory only for some credentials, so ask the model. The check may + // resolve a credential, so it runs in the background and never delays or fails the run. + // A check that fails leaves the warning out, as it was before the model could answer. + void Promise.resolve() + .then(() => gatewayModel.isGatewayMemoryManaged!()) + .then( + managed => { + if (!managed) warnMissingMemory(); + }, + error => + this.logger.debug('Could not tell whether the gateway manages memory for this model', { + agent: this.name, + error, + }), + ); + } } // Apply null→undefined transform for OpenAI structured output validation. diff --git a/packages/core/src/agent/message-list/state/types.ts b/packages/core/src/agent/message-list/state/types.ts index dfda172a0e8..3ca9a47cf9e 100644 --- a/packages/core/src/agent/message-list/state/types.ts +++ b/packages/core/src/agent/message-list/state/types.ts @@ -40,7 +40,7 @@ export type MastraStepStartPart = { // surfaces (stream chunk, onError, result.error) and are never written to // thread history. Provider HTTP failures (AI SDK `APICallError`) also keep the // status code, a bounded copy of the response body, the provider request id, -// the model the request named and the provider facts a reload classifies by +// the model the request named, the origin it was sent to and the provider facts a reload classifies by // (every code, the provider's own reason sentence), because their `message` is // often just the HTTP status text. export type MastraErrorPart = { @@ -51,6 +51,12 @@ export type MastraErrorPart = { responseBody?: string; requestId?: string; modelId?: string; + /** + * Origin (scheme, host and port, e.g. `https://gateway-api.mastra.ai`) the failed model + * request was sent to, so a reader can tell which service answered. Never the path, query + * string or credentials. Absent on messages saved before it was recorded. + */ + requestOrigin?: string; } & ProviderErrorFacts; } & MastraPartExtensions; diff --git a/packages/core/src/error/index.ts b/packages/core/src/error/index.ts index 0ecc2116bfd..36c14665a39 100644 --- a/packages/core/src/error/index.ts +++ b/packages/core/src/error/index.ts @@ -15,5 +15,6 @@ export { providerErrorReason, providerModelId, providerRequestId, + providerRequestOrigin, } from './provider-error'; export type { ProviderFailureKind } from './provider-error'; diff --git a/packages/core/src/error/provider-error.test.ts b/packages/core/src/error/provider-error.test.ts index 9a3fc9ade42..3faefc74332 100644 --- a/packages/core/src/error/provider-error.test.ts +++ b/packages/core/src/error/provider-error.test.ts @@ -474,6 +474,7 @@ describe('toPersistedErrorIdentity', () => { message: 'Model call failed', statusCode: 400, reason: 'prompt is too long: 1 > 0 maximum', + requestOrigin: 'https://api.example.com', }); }); diff --git a/packages/core/src/error/provider-error.ts b/packages/core/src/error/provider-error.ts index 78f651dbf60..4387aff060b 100644 --- a/packages/core/src/error/provider-error.ts +++ b/packages/core/src/error/provider-error.ts @@ -1,4 +1,9 @@ -import { capPersistedText, readRequestedModel, readRequestId } from '../loop/shared/provider-error-details'; +import { + capPersistedText, + readRequestedModel, + readRequestId, + readRequestOrigin, +} from '../loop/shared/provider-error-details'; export type ProviderFailureKind = | 'auth' @@ -235,6 +240,15 @@ export function providerModelId(error: unknown): string | undefined { return firstAlongCauses(error, modelIdOf, 0); } +/** + * The origin (scheme, host and port) a failed model request was sent to, so a reader can tell + * which service answered. Read only from an AI SDK API call error, whose `url` is the request it + * made; a `url` on any other error may name some other resource. + */ +export function providerRequestOrigin(error: unknown): string | undefined { + return firstAlongCauses(error, requestOriginOf, 0); +} + function firstAlongCauses( error: unknown, read: (error: object) => string | undefined, @@ -248,6 +262,12 @@ function requestIdOf(error: object): string | undefined { return nonBlank(readField(error, 'requestId')) ?? readRequestId(readField(error, 'responseHeaders')); } +function requestOriginOf(error: object): string | undefined { + const isApiCallError = + readField(error, 'name') === 'AI_APICallError' || readField(error, 'requestBodyValues') !== undefined; + return isApiCallError ? readRequestOrigin(readField(error, 'url')) : undefined; +} + function modelIdOf(error: object): string | undefined { return nonBlank(readField(error, 'modelId')) ?? readRequestedModel(readField(error, 'requestBodyValues')); } diff --git a/packages/core/src/loop/shared/provider-error-details.test.ts b/packages/core/src/loop/shared/provider-error-details.test.ts index c67a533319d..acbd5cf9345 100644 --- a/packages/core/src/loop/shared/provider-error-details.test.ts +++ b/packages/core/src/loop/shared/provider-error-details.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; -import { MAX_PERSISTED_RESPONSE_BODY_LENGTH, describeProviderError } from './provider-error-details'; +import { MAX_PERSISTED_RESPONSE_BODY_LENGTH, describeProviderError, readRequestOrigin } from './provider-error-details'; function providerError(responseBody: string, requestId = 'req_123') { return Object.assign(new Error('Bad Request'), { @@ -44,3 +44,21 @@ describe('describeProviderError', () => { expect(describeProviderError('boom', new Error('boom'), 1000)).toBeUndefined(); }); }); + +describe('readRequestOrigin', () => { + it.each([ + ['https://gateway-api.mastra.ai/v1/chat/completions', 'https://gateway-api.mastra.ai'], + ['http://127.0.0.1:4180/v1/chat/completions?x=1', 'http://127.0.0.1:4180'], + ['https://user:pass@api.example.com:443/path', 'https://api.example.com'], + [' https://api.example.com/v1 ', 'https://api.example.com'], + ])('keeps only the origin of %s', (url, origin) => { + expect(readRequestOrigin(url)).toBe(origin); + }); + + it.each([undefined, null, 42, '', ' ', 'not a url', '/relative', 'file:///etc/passwd', 'data:text/plain,hi'])( + 'gives undefined for %s', + url => { + expect(readRequestOrigin(url)).toBeUndefined(); + }, + ); +}); diff --git a/packages/core/src/loop/shared/provider-error-details.ts b/packages/core/src/loop/shared/provider-error-details.ts index 83b505f9799..ec570f0f317 100644 --- a/packages/core/src/loop/shared/provider-error-details.ts +++ b/packages/core/src/loop/shared/provider-error-details.ts @@ -42,6 +42,23 @@ export function readRequestId(headers: unknown): string | undefined { return undefined; } +/** + * The origin (scheme, host and port) of a request URL. The path, query string and any + * credentials in the URL are never kept. Anything that is not an absolute http or https URL + * gives undefined. + */ +export function readRequestOrigin(url: unknown): string | undefined { + if (typeof url !== 'string' || !url.trim()) return undefined; + let parsed: URL; + try { + parsed = new URL(url.trim()); + } catch { + return undefined; + } + if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return undefined; + return parsed.origin; +} + export function readRequestedModel(requestBody: unknown): string | undefined { if (!requestBody || typeof requestBody !== 'object') return undefined; const model = readField(requestBody, 'model'); diff --git a/packages/core/src/loop/shared/record-terminal-error-message.test.ts b/packages/core/src/loop/shared/record-terminal-error-message.test.ts index 28e5f9411e2..2cbde067586 100644 --- a/packages/core/src/loop/shared/record-terminal-error-message.test.ts +++ b/packages/core/src/loop/shared/record-terminal-error-message.test.ts @@ -46,6 +46,7 @@ describe('toPersistedErrorIdentity', () => { responseBody: '{"error":{"code":"insufficient_quota"}}', modelId: 'claude-opus-4-5', codes: ['insufficient_quota'], + requestOrigin: 'https://api.example.com', }); expect(Object.keys(toPersistedErrorIdentity(new Error('boom')))).toEqual(['name', 'message']); expect( @@ -58,6 +59,50 @@ describe('toPersistedErrorIdentity', () => { ).toMatchObject({ statusCode: 400, reason: 'prompt is too long' }); }); + it('keeps only the origin of the URL a failed model request was sent to', () => { + const error = Object.assign(new Error('Too Many Requests'), { + name: 'AI_APICallError', + statusCode: 429, + url: 'http://user:secret@127.0.0.1:4180/v1/chat/completions?api_key=secret#frag', + requestBodyValues: { model: 'stub/ok' }, + }); + + const identity = toPersistedErrorIdentity(error); + + expect(identity.requestOrigin).toBe('http://127.0.0.1:4180'); + expect(JSON.stringify(identity)).not.toContain('secret'); + expect(JSON.stringify(identity)).not.toContain('/v1/chat/completions'); + }); + + it('reads the request origin from a model request error wrapped as a cause', () => { + const apiCallError = Object.assign(new Error('Service Unavailable'), { + name: 'AI_APICallError', + statusCode: 503, + url: 'https://gateway-api.mastra.ai/v1/chat/completions', + requestBodyValues: {}, + }); + const wrapped = new Error('Model call failed', { cause: apiCallError }); + + expect(toPersistedErrorIdentity(wrapped).requestOrigin).toBe('https://gateway-api.mastra.ai'); + }); + + it.each([ + ['a malformed URL', 'not a url'], + ['a relative URL', '/v1/chat/completions'], + ['a non-http scheme', 'file:///etc/passwd'], + ['a non-string URL', 42], + ])('leaves the request origin out for %s', (_label, url) => { + const error = Object.assign(new Error('Bad Request'), { statusCode: 400, url, requestBodyValues: {} }); + + expect(toPersistedErrorIdentity(error)).not.toHaveProperty('requestOrigin'); + }); + + it('ignores a url on an error that is not a model request error', () => { + const error = Object.assign(new Error('Not Found'), { url: 'https://internal.example/resource' }); + + expect(toPersistedErrorIdentity(error)).not.toHaveProperty('requestOrigin'); + }); + it('keeps a bounded response body and the provider request id', () => { const error = Object.assign(new Error('Bad Request'), { statusCode: 400, diff --git a/packages/core/src/loop/shared/record-terminal-error-message.ts b/packages/core/src/loop/shared/record-terminal-error-message.ts index 2e33daeada8..edc0b14a82d 100644 --- a/packages/core/src/loop/shared/record-terminal-error-message.ts +++ b/packages/core/src/loop/shared/record-terminal-error-message.ts @@ -16,6 +16,7 @@ * * Only `{ name, message }` is persisted, plus `statusCode`, a bounded * `responseBody`, the provider `requestId`, the `modelId` the request named, + * the `requestOrigin` the request was sent to (scheme, host and port only), * every provider code and the innermost `reason` for provider failures — * without those, an `APICallError` whose message is just "Bad Request" cannot * be diagnosed or classified after the fact. The rest of the request body, @@ -26,7 +27,7 @@ */ import type { MastraDBMessage, MastraErrorPart, MessageList } from '../../agent/message-list'; -import { providerErrorFacts } from '../../error/provider-error'; +import { providerErrorFacts, providerRequestOrigin } from '../../error/provider-error'; import { readProviderErrorDetails } from './provider-error-details'; const FALLBACK_ERROR_NAME = 'Error'; @@ -58,11 +59,13 @@ export function toPersistedErrorIdentity(error: unknown): MastraErrorPart['error return { name: FALLBACK_ERROR_NAME, message: FALLBACK_ERROR_MESSAGE }; } + const requestOrigin = providerRequestOrigin(error); return { name: readErrorField(error, 'name') ?? FALLBACK_ERROR_NAME, message: readErrorField(error, 'message') ?? FALLBACK_ERROR_MESSAGE, ...readProviderErrorDetails(error), ...providerErrorFacts(error), + ...(requestOrigin ? { requestOrigin } : {}), }; } diff --git a/packages/core/src/loop/test-utils/aimock/scenarios/persistence-failed-turns.scenario.test.ts b/packages/core/src/loop/test-utils/aimock/scenarios/persistence-failed-turns.scenario.test.ts index fb579d76728..2e7d2140598 100644 --- a/packages/core/src/loop/test-utils/aimock/scenarios/persistence-failed-turns.scenario.test.ts +++ b/packages/core/src/loop/test-utils/aimock/scenarios/persistence-failed-turns.scenario.test.ts @@ -95,7 +95,10 @@ describeForAllEngines('AIMock loop scenario: persistence of failed turns', engin .poll(async () => (await memory.recall({ threadId, resourceId })).messages.map(m => m.role)) .toEqual(['user', 'assistant']); const [errorPart] = errorParts((await memory.recall({ threadId, resourceId })).messages); - expect(errorPart).toMatchObject({ type: 'error', error: { statusCode: 400 } }); + expect(errorPart).toMatchObject({ + type: 'error', + error: { statusCode: 400, requestOrigin: new URL(getMock().url).origin }, + }); expect(JSON.parse((errorPart as { error: { responseBody: string } }).error.responseBody)).toMatchObject({ error: { message: 'PROVIDER_400_EXPLANATION', type: 'invalid_request_error' }, }); diff --git a/packages/server/src/server/handlers/agent.test.ts b/packages/server/src/server/handlers/agent.test.ts index 384c37fb369..d48853ad971 100644 --- a/packages/server/src/server/handlers/agent.test.ts +++ b/packages/server/src/server/handlers/agent.test.ts @@ -1329,5 +1329,106 @@ describe('Agent Handlers', () => { } } }); + + it('uses a Mastra gateway model when only MASTRA_GATEWAY_API_KEY is set', async () => { + vi.stubEnv('OPENAI_API_KEY', ''); + vi.stubEnv('MASTRA_GATEWAY_API_KEY', 'test-gateway-key'); + onTestFinished(() => vi.unstubAllEnvs()); + const mastra = makeMastraMock({ + agents: { 'gateway-agent': makeMockAgent({ name: 'gateway-agent', model: 'mastra/openai/gpt-5-mini' }) }, + }); + const generateSpy = vi + .spyOn(Agent.prototype, 'generate') + .mockResolvedValue({ object: { explanation: 'Clearer.', new_prompt: 'Be clear.' } } as any); + onTestFinished(() => generateSpy.mockRestore()); + + const result = await ENHANCE_INSTRUCTIONS_ROUTE.handler({ + ...createTestServerContext({ mastra }), + agentId: 'gateway-agent', + instructions: 'Be helpful.', + }); + + expect(result).toEqual({ explanation: 'Clearer.', new_prompt: 'Be clear.' }); + expect(generateSpy).toHaveBeenCalledOnce(); + }); + + it('skips a gateway model without credentials and uses a later connected model', async () => { + vi.stubEnv('OPENAI_API_KEY', ''); + vi.stubEnv('NETLIFY_TOKEN', ''); + vi.stubEnv('NETLIFY_SITE_ID', ''); + vi.stubEnv('GOOGLE_GENERATIVE_AI_API_KEY', 'test-google-key'); + onTestFinished(() => vi.unstubAllEnvs()); + const mastra = makeMastraMock({ + agents: { + 'fallback-agent': makeMockAgent({ + name: 'fallback-agent', + model: [{ model: 'netlify/openai/gpt-4o' }, { model: 'google/gemini-2.5-flash' }], + }), + }, + }); + const generateSpy = vi + .spyOn(Agent.prototype, 'generate') + .mockResolvedValue({ object: { explanation: 'Clearer.', new_prompt: 'Be clear.' } } as any); + onTestFinished(() => generateSpy.mockRestore()); + + await ENHANCE_INSTRUCTIONS_ROUTE.handler({ + ...createTestServerContext({ mastra }), + agentId: 'fallback-agent', + instructions: 'Be helpful.', + }); + + const enhancer = generateSpy.mock.contexts[0] as Agent; + const chosen = (await enhancer.getModel()) as { provider: string; modelId: string }; + expect(chosen.provider).toBe('google'); + expect(chosen.modelId).toBe('gemini-2.5-flash'); + }); + + it('does not treat a lab provider key as the credential for a gateway model', async () => { + vi.stubEnv('OPENAI_API_KEY', 'test-openai-key'); + vi.stubEnv('NETLIFY_TOKEN', ''); + vi.stubEnv('NETLIFY_SITE_ID', ''); + vi.stubEnv('GOOGLE_GENERATIVE_AI_API_KEY', 'test-google-key'); + onTestFinished(() => vi.unstubAllEnvs()); + const mastra = makeMastraMock({ + agents: { + 'fallback-agent': makeMockAgent({ + name: 'fallback-agent', + model: [{ model: 'netlify/openai/gpt-4o' }, { model: 'google/gemini-2.5-flash' }], + }), + }, + }); + const generateSpy = vi + .spyOn(Agent.prototype, 'generate') + .mockResolvedValue({ object: { explanation: 'Clearer.', new_prompt: 'Be clear.' } } as any); + onTestFinished(() => generateSpy.mockRestore()); + + await ENHANCE_INSTRUCTIONS_ROUTE.handler({ + ...createTestServerContext({ mastra }), + agentId: 'fallback-agent', + instructions: 'Be helpful.', + }); + + const enhancer = generateSpy.mock.contexts[0] as Agent; + const chosen = (await enhancer.getModel()) as { provider: string; modelId: string }; + expect(chosen.provider).toBe('google'); + expect(chosen.modelId).toBe('gemini-2.5-flash'); + }); + + it('returns 400 for a Mastra gateway model when no gateway key is set', async () => { + vi.stubEnv('OPENAI_API_KEY', ''); + vi.stubEnv('MASTRA_GATEWAY_API_KEY', ''); + onTestFinished(() => vi.unstubAllEnvs()); + const mastra = makeMastraMock({ + agents: { 'gateway-agent': makeMockAgent({ name: 'gateway-agent', model: 'mastra/openai/gpt-5-mini' }) }, + }); + + await expect( + ENHANCE_INSTRUCTIONS_ROUTE.handler({ + ...createTestServerContext({ mastra }), + agentId: 'gateway-agent', + instructions: 'Be helpful.', + }), + ).rejects.toMatchObject({ status: 400 }); + }); }); }); diff --git a/packages/server/src/server/handlers/agents.test.ts b/packages/server/src/server/handlers/agents.test.ts index de07cd56444..f6612d4467e 100644 --- a/packages/server/src/server/handlers/agents.test.ts +++ b/packages/server/src/server/handlers/agents.test.ts @@ -526,7 +526,7 @@ describe('getProvidersHandler', () => { expect(result.providers.find(provider => provider.id === 'mastra')?.models).toEqual(['live-chat-model']); }); - it('does not fall back to cached Mastra models when the v2 catalog fails', async () => { + it('keeps the Mastra gateway listed with no models when the v2 catalog fails', async () => { process.env.MASTRA_GATEWAY_VERSION = 'v2'; process.env.MASTRA_GATEWAY_API_KEY = 'test-key'; Object.assign(globalThis, { @@ -552,7 +552,9 @@ describe('getProvidersHandler', () => { requestContext: new RequestContext(), }); - expect(result.providers.find(provider => provider.id === 'mastra')).toBeUndefined(); + const gateway = result.providers.find(provider => provider.id === 'mastra'); + expect(gateway).toMatchObject({ name: 'Gateway', envVar: 'MASTRA_GATEWAY_API_KEY', connected: true }); + expect(gateway?.models).toEqual([]); expect(result.providers.find(provider => provider.id === 'openai')).toBeDefined(); }); }); diff --git a/packages/server/src/server/handlers/agents.ts b/packages/server/src/server/handlers/agents.ts index 8594cf14b16..1f3cc90f28b 100644 --- a/packages/server/src/server/handlers/agents.ts +++ b/packages/server/src/server/handlers/agents.ts @@ -2093,6 +2093,18 @@ export async function buildProvidersList(mastra: Context['mastra']): Promise>): Promise { + const router = model as { gatewayId?: unknown; resolveApiProviderId?: () => Promise }; + if ( + typeof router.gatewayId === 'string' && + router.gatewayId !== 'models.dev' && + typeof router.resolveApiProviderId === 'function' + ) { + return (await router.resolveApiProviderId()) !== undefined; + } + return isProviderConnected(model.provider); +} + // Helper to find the first model with a connected provider async function findConnectedModel(agent: Agent): Promise> | null> { const modelList = await agent.getModelList(); @@ -4383,7 +4415,7 @@ async function findConnectedModel(agent: Agent): Promise Date: Tue, 17 Nov 2026 04:00:59 +0000 Subject: [PATCH 3/5] fix(factory): explain gateway refusals with appropriate recovery actions Distinguish gateway billing, routing, context and rate limit failures in the web UI and Slack so users get relevant recovery actions. Offer credits only when adding them can help. Recognize v1 billing replies by their structure and configured gateways by request origin, including saved errors that record it. --- .changeset/plain-gates-refuse.md | 5 + mastracode/factory-ui/src/api/types.ts | 6 +- .../factory-ui/src/hooks/use-providers.ts | 9 + .../chat/components/RunErrorNotice.tsx | 5 +- .../__tests__/TranscriptErrors.msw.test.tsx | 33 ++ .../integrations/slack/run-error-card.test.ts | 17 + .../src/integrations/slack/run-error-card.ts | 3 +- mastracode/factory/src/routes/config.test.ts | 24 + mastracode/factory/src/routes/config.ts | 30 +- .../factory/src/routes/mastra-gateway.test.ts | 50 ++ .../factory/src/routes/mastra-gateway.ts | 21 + mastracode/factory/src/run-error.test.ts | 459 ++++++++++++++++++ mastracode/factory/src/run-error.ts | 364 ++++++++++++-- 13 files changed, 980 insertions(+), 46 deletions(-) create mode 100644 .changeset/plain-gates-refuse.md create mode 100644 mastracode/factory/src/routes/mastra-gateway.test.ts diff --git a/.changeset/plain-gates-refuse.md b/.changeset/plain-gates-refuse.md new file mode 100644 index 00000000000..8d9fcd28104 --- /dev/null +++ b/.changeset/plain-gates-refuse.md @@ -0,0 +1,5 @@ +--- +'@mastra/factory': patch +--- + +Factory now explains Mastra Gateway v2 refusals correctly in the web UI and in Slack. An organization budget, a key or organization spend limit, an unsupported billing plan, a routing policy refusal and a context that is too large each get their own message, and "Add credits" is offered only where adding credits helps. v1 Gateway billing replies are recognized by their structure rather than by a link in their text, so they are still named once the Gateway links to the Mastra dashboard, and the v1 monthly spend limit no longer offers to add credits. A rate limit now says whether the gateway's own limit on the key or organization was reached or the model provider behind the gateway is limiting requests, and no longer suggests changing credentials. A gateway at the configured base URL, such as staging or a self-hosted one, is recognized as well as the production one. The web UI recognizes v2 refusals from any gateway by their response body. In the web UI, rate limits and errors without a gateway reply body are recognized only for runs saved by a version of `@mastra/core` that records the request origin; older saved errors keep the generic provider message. diff --git a/mastracode/factory-ui/src/api/types.ts b/mastracode/factory-ui/src/api/types.ts index eed60c3b7d8..50fefde34e8 100644 --- a/mastracode/factory-ui/src/api/types.ts +++ b/mastracode/factory-ui/src/api/types.ts @@ -133,6 +133,8 @@ export interface ProvidersResponse { providers: ProviderInfo[]; /** Tenant mode: whether the caller may write org-wide keys. Absent locally. */ orgKeyAdmin?: boolean; + /** Origin of the Mastra Gateway this server sends model calls to. Absent from servers that predate it. */ + mastraGatewayOrigin?: string; } export interface CustomProvidersResponse { @@ -244,9 +246,7 @@ export interface OAuthStartResponse { } export type OAuthPollResponse = - | { status: 'pending'; nextPollMs: number } - | { status: 'complete' } - | { status: 'failed'; error: string }; + { status: 'pending'; nextPollMs: number } | { status: 'complete' } | { status: 'failed'; error: string }; export interface UpdateOMResponse { ok: true; diff --git a/mastracode/factory-ui/src/hooks/use-providers.ts b/mastracode/factory-ui/src/hooks/use-providers.ts index 96e451eac7c..52e8a6cec65 100644 --- a/mastracode/factory-ui/src/hooks/use-providers.ts +++ b/mastracode/factory-ui/src/hooks/use-providers.ts @@ -41,6 +41,15 @@ export function useProvidersQuery() { }); } +/** Origin of the server's Mastra Gateway, so run errors from a staging or self-hosted gateway are recognized. */ +export function useMastraGatewayOrigin(): string | undefined { + const { client } = useApiConfig(); + return useQuery({ + ...providersQueryOptions(client), + select: (body: ProvidersResponse) => body.mastraGatewayOrigin, + }).data; +} + /** * Whether the caller may write org-wide keys (tenant mode). Shares the * providers cache entry — no extra request. Fails open when the server diff --git a/mastracode/factory-ui/src/ui/domains/chat/components/RunErrorNotice.tsx b/mastracode/factory-ui/src/ui/domains/chat/components/RunErrorNotice.tsx index 60d1c2533f2..ac633b59f2b 100644 --- a/mastracode/factory-ui/src/ui/domains/chat/components/RunErrorNotice.tsx +++ b/mastracode/factory-ui/src/ui/domains/chat/components/RunErrorNotice.tsx @@ -17,6 +17,7 @@ import type { RunErrorRecovery } from '@mastra/factory/run-error'; import { useContext } from 'react'; import type { ComponentType } from 'react'; +import { useMastraGatewayOrigin } from '../../../../hooks/use-providers'; import { ChatSessionContext } from '../context/ChatSessionContext'; import { ChatThreadMessagesContext } from '../context/ChatThreadMessagesContext'; import { AddGatewayCreditsAction } from './AddGatewayCreditsAction'; @@ -37,7 +38,9 @@ const RECOVERY_ACTIONS: Record { expect(addCredits).toHaveAttribute('target', '_blank'); }); + describe("a gateway's own rate limit from a self-hosted gateway", () => { + const gatewayLimit = (): MastraDBMessage => { + const message = 'Gateway key requests limit exceeded. Retry in 12s.'; + // Saved by a core version that records the request's origin; this core's type does not name it yet. + const error = { + name: 'AI_APICallError', + message, + statusCode: 429, + responseBody: JSON.stringify({ error: { type: 'rate_limit_error', code: 'rate_limit_exceeded', message } }), + modelId: 'anthropic/claude-opus-4-5', + requestOrigin: 'http://127.0.0.1:4180', + }; + return { ...failedReply(), content: { format: 2, parts: [{ type: 'error', error }] } }; + }; + + it('names the gateway once the server reports that origin as its gateway', async () => { + server.use( + http.get(`${TEST_BASE_URL}/web/config/providers`, () => + HttpResponse.json({ providers: [], mastraGatewayOrigin: 'http://127.0.0.1:4180' }), + ), + ); + renderWithProviders(); + + expect(await screen.findByText('Mastra Gateway rate limit reached')).toBeVisible(); + }); + + it('stays a provider rate limit while the server names no gateway origin', async () => { + renderWithProviders(); + + expect(await screen.findByText('Provider rate limit reached')).toBeVisible(); + }); + }); + it('continues a timed-out run from the card', async () => { const sentMessages: unknown[] = []; server.use( diff --git a/mastracode/factory/src/integrations/slack/run-error-card.test.ts b/mastracode/factory/src/integrations/slack/run-error-card.test.ts index ad986f88ae6..42ccd942680 100644 --- a/mastracode/factory/src/integrations/slack/run-error-card.test.ts +++ b/mastracode/factory/src/integrations/slack/run-error-card.test.ts @@ -1,3 +1,7 @@ +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + import { prepareCredentialTenant, setCredentialStoreProvider } from '@mastra/code-sdk/agents/credential-resolver'; import type { CredentialTenant } from '@mastra/code-sdk/agents/credential-resolver'; import { ProviderAuthRequiredError } from '@mastra/code-sdk/auth/provider-auth-error'; @@ -213,4 +217,17 @@ describe('slackRunErrorCard', () => { expect(card).not.toContain('https://'); } }); + + it('recognizes a refusal from the gateway this server is configured to use', () => { + // An empty settings directory, so only the environment names the gateway. + vi.stubEnv('MASTRA_APP_DATA_DIR', mkdtempSync(join(tmpdir(), 'slack-gateway-settings-'))); + vi.stubEnv('MASTRA_GATEWAY_URL', 'https://gateway.staging.example.com/v1'); + const outOfCredits = Object.assign(new Error('Payment Required'), { + name: 'AI_APICallError', + statusCode: 402, + url: 'https://gateway.staging.example.com/v1/chat/completions', + }); + + expect(rendered(slackRunErrorCard(outOfCredits, slackRun()))).toContain('Out of Mastra Gateway credits'); + }); }); diff --git a/mastracode/factory/src/integrations/slack/run-error-card.ts b/mastracode/factory/src/integrations/slack/run-error-card.ts index 80980ca4a73..7b179bda0d8 100644 --- a/mastracode/factory/src/integrations/slack/run-error-card.ts +++ b/mastracode/factory/src/integrations/slack/run-error-card.ts @@ -4,6 +4,7 @@ import { providerRequestId } from '@mastra/core/error'; import type { RequestContext } from '@mastra/core/request-context'; import { Actions, Card, CardText, LinkButton } from 'chat'; +import { mastraGatewayBaseUrl } from '../../routes/mastra-gateway.js'; import { readFactorySessionScope } from '../../rules/binding-context.js'; import type { RunErrorRecovery } from '../../run-error.js'; import { describeRunError } from '../../run-error.js'; @@ -12,7 +13,7 @@ import { modelSettingsLink } from './channel-reply-processor.js'; import { sessionLinkFromRequest } from './channel-session-link-processor.js'; export function slackRunErrorCard(error: Error, { requestContext }: ChannelErrorContext) { - const { title, message, recoveries, model } = describeRunError(error); + const { title, message, recoveries, model } = describeRunError(error, { gatewayBaseUrl: mastraGatewayBaseUrl() }); const requestId = providerRequestId(error); const facts = [model && `Model: ${model}`, requestId && `Request id: ${requestId}`].filter(Boolean).join(' · '); const credential = model && recoveries.includes('credentials') ? credentialLine(requestContext, model) : undefined; diff --git a/mastracode/factory/src/routes/config.test.ts b/mastracode/factory/src/routes/config.test.ts index 371f1ab78a6..218a7476847 100644 --- a/mastracode/factory/src/routes/config.test.ts +++ b/mastracode/factory/src/routes/config.test.ts @@ -1018,6 +1018,30 @@ describe('Mastra Gateway routes with a tenant', () => { expect(await gatewayRow(buildApp(fakePlatform()))).toMatchObject({ connectWithMastraAccount: true }); expect(await gatewayRow(buildApp())).not.toHaveProperty('connectWithMastraAccount'); }); + describe('the gateway origin for the web UI', () => { + beforeEach(async () => { + // An empty settings directory, so a gateway URL in the developer's own settings cannot apply. + vi.stubEnv('MASTRA_APP_DATA_DIR', await fs.mkdtemp(join(tmpdir(), 'factory-gateway-origin-'))); + }); + afterEach(() => { + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + }); + + it.each([ + ['https://gateway-api.mastra.ai', 'https://gateway-api.mastra.ai'], + ['http://127.0.0.1:4180/v1/', 'http://127.0.0.1:4180'], + ])('reports the origin of the configured gateway %s', async (configured, origin) => { + vi.stubEnv('MASTRA_GATEWAY_URL', configured); + vi.spyOn(console, 'warn').mockImplementation(() => {}); + + const body = (await (await buildApp(fakePlatform()).request('/web/config/providers')).json()) as { + mastraGatewayOrigin?: string; + }; + + expect(body.mastraGatewayOrigin).toBe(origin); + }); + }); }); describe('GET /web/config/models', () => { diff --git a/mastracode/factory/src/routes/config.ts b/mastracode/factory/src/routes/config.ts index 9058ebb0b71..994c18160d9 100644 --- a/mastracode/factory/src/routes/config.ts +++ b/mastracode/factory/src/routes/config.ts @@ -44,7 +44,13 @@ import type { WorkItemsStorage } from '../storage/domains/work-items/base.js'; import { THINKING_LEVELS } from './contracts.js'; import { customEndpointUrlProblem } from './custom-endpoint-url.js'; import { customProviderOwner } from './custom-provider-source.js'; -import { createGatewayKey, getGatewayCredits, MASTRA_SIGN_IN_REQUIRED, MastraGatewayError } from './mastra-gateway.js'; +import { + createGatewayKey, + getGatewayCredits, + MASTRA_SIGN_IN_REQUIRED, + MastraGatewayError, + mastraGatewayBaseUrl, +} from './mastra-gateway.js'; import { credentialLabel, getAuthProviderId, @@ -81,15 +87,7 @@ import { resolveTenantProviderDefaultModel } from './tenant-credentials.js'; * the server process's own credentials (e.g. Amazon Bedrock's AWS chain). */ export type ProviderCredentialSource = - | 'deployment' - | 'oauth' - | 'stored' - | 'env' - | 'none' - | 'oauth-user' - | 'oauth-org' - | 'stored-user' - | 'stored-org'; + 'deployment' | 'oauth' | 'stored' | 'env' | 'none' | 'oauth-user' | 'oauth-org' | 'stored-user' | 'stored-org'; /** A model provider with the current source of its credentials. */ export interface ProviderInfo { @@ -205,6 +203,15 @@ async function listCatalogModels( return [...models, ...gatewayModels]; } +function originOf(url: string): string | undefined { + try { + const origin = new URL(url).origin; + return origin === 'null' ? undefined : origin; + } catch { + return undefined; + } +} + /** * Build a deduplicated, sorted list of providers from the model catalog, * annotated with where each provider's credential currently comes from. @@ -1197,6 +1204,7 @@ export class ConfigRoutes extends Route { // keys, so the settings UI can gate the "Everyone in org" option. const tenant = auth.tenant(loose(c)); const orgKeyAdmin = tenant ? await auth.isOrganizationAdmin(loose(c), tenantOrgId(tenant)) : undefined; + const gatewayOrigin = originOf(mastraGatewayBaseUrl()); const customProviders = options.customProviders; const listCustomProviders = customProviders ? async () => { @@ -1215,6 +1223,8 @@ export class ConfigRoutes extends Route { connectWithMastraAccount, }), ...(orgKeyAdmin !== undefined ? { orgKeyAdmin } : {}), + // Lets the web UI recognize a saved run error from this server's gateway, wherever it is hosted. + ...(gatewayOrigin ? { mastraGatewayOrigin: gatewayOrigin } : {}), }); } catch (error) { return c.json({ error: error instanceof Error ? error.message : String(error) }, 500); diff --git a/mastracode/factory/src/routes/mastra-gateway.test.ts b/mastracode/factory/src/routes/mastra-gateway.test.ts new file mode 100644 index 00000000000..4842fee25d5 --- /dev/null +++ b/mastracode/factory/src/routes/mastra-gateway.test.ts @@ -0,0 +1,50 @@ +import { mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { mastraGatewayBaseUrl } from './mastra-gateway.js'; + +function withSettings(settings?: unknown) { + const dir = mkdtempSync(join(tmpdir(), 'mastra-gateway-url-')); + if (settings !== undefined) writeFileSync(join(dir, 'settings.json'), JSON.stringify(settings)); + vi.stubEnv('MASTRA_APP_DATA_DIR', dir); +} + +describe('mastraGatewayBaseUrl', () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it('prefers the settings over the environment, as model calls do', () => { + withSettings({ memoryGateway: { baseUrl: 'https://gateway.settings.example.com/v1/' } }); + vi.stubEnv('MASTRA_GATEWAY_URL', 'https://gateway.env.example.com'); + + expect(mastraGatewayBaseUrl()).toBe('https://gateway.settings.example.com'); + }); + + it('uses the environment when the settings name no gateway', () => { + withSettings(); + vi.stubEnv('MASTRA_GATEWAY_URL', 'http://127.0.0.1:4180/v1'); + + expect(mastraGatewayBaseUrl()).toBe('http://127.0.0.1:4180'); + }); + + it.each([ + ['a number', 42], + ['a blank string', ' '], + ])('ignores a setting that is %s', (_shape, baseUrl) => { + withSettings({ memoryGateway: { baseUrl } }); + vi.stubEnv('MASTRA_GATEWAY_URL', 'https://gateway.env.example.com'); + + expect(mastraGatewayBaseUrl()).toBe('https://gateway.env.example.com'); + }); + + it('falls back to the production gateway', () => { + withSettings(); + vi.stubEnv('MASTRA_GATEWAY_URL', ''); + + expect(mastraGatewayBaseUrl()).toBe('https://gateway-api.mastra.ai'); + }); +}); diff --git a/mastracode/factory/src/routes/mastra-gateway.ts b/mastracode/factory/src/routes/mastra-gateway.ts index 58b72ea8071..9e75b2d2e98 100644 --- a/mastracode/factory/src/routes/mastra-gateway.ts +++ b/mastracode/factory/src/routes/mastra-gateway.ts @@ -1,8 +1,29 @@ +import { loadSettings } from '@mastra/code-sdk/onboarding/settings'; import type { Context } from 'hono'; import { z } from 'zod'; import type { RouteAuth } from './route.js'; +/** + * The Mastra Gateway this server sends model calls to: the `memoryGateway.baseUrl` setting, then + * `MASTRA_GATEWAY_URL`, then production, without a trailing slash or `/v1`. This is the order the SDK's + * per-request model resolution reads them in, which is where run calls go. (The SDK's startup reads the + * environment first, but only for its boot-time gateway, which does not send model calls.) A setting + * that is not a non-empty string is ignored. + */ +export function mastraGatewayBaseUrl(): string { + let configured: unknown; + try { + configured = loadSettings().memoryGateway?.baseUrl; + } catch { + // An unreadable settings file leaves the environment and the default in charge. + } + const fromSettings = typeof configured === 'string' && configured.trim() ? configured.trim() : undefined; + const fromEnv = process.env.MASTRA_GATEWAY_URL?.trim() || undefined; + const raw = fromSettings ?? fromEnv ?? 'https://gateway-api.mastra.ai'; + return raw.replace(/\/+$/, '').replace(/\/v1$/, ''); +} + const MASTRA_PROJECTS_URL = process.env.MASTRA_PROJECTS_URL || 'https://projects.mastra.ai'; /** The Starter plan's gateway credit pool, used when the customer names none. */ diff --git a/mastracode/factory/src/run-error.test.ts b/mastracode/factory/src/run-error.test.ts index 15710b9254a..eca019b38b3 100644 --- a/mastracode/factory/src/run-error.test.ts +++ b/mastracode/factory/src/run-error.test.ts @@ -1,3 +1,4 @@ +import { toPersistedErrorIdentity } from '@mastra/core/error'; import { describe, expect, it } from 'vitest'; import { describeRunError } from './run-error'; @@ -75,6 +76,78 @@ describe('describeRunError', () => { }); }); + describe('Mastra Gateway v1 billing refusals', () => { + // The v1 reply as the gateway sends it: a billing `type`, a `help_url`, and no `code`. The live + // error is called without a gateway URL or `mastra/` model, so only the reply can identify it. + const v1Refusal = (error: Record) => ({ + name: 'AI_APICallError', + message: error.message!, + statusCode: 402, + responseBody: JSON.stringify({ error }), + requestBodyValues: { model: 'anthropic/claude-opus-4-5' }, + }); + const outOfCredits = { + message: "You don't have inference credits left. Add credits at https://projects.mastra.ai/", + type: 'billing_error', + help_url: 'https://projects.mastra.ai/', + }; + const outOfMemoryTokens = { + message: "You don't have memory tokens left. Add credits or upgrade your plan at https://projects.mastra.ai/", + type: 'billing_error', + help_url: 'https://projects.mastra.ai/', + }; + const spendLimit = { + message: 'Your organization reached its monthly Gateway spend limit. Raise it at https://projects.mastra.ai/', + type: 'spend_limit_reached', + help_url: 'https://projects.mastra.ai/', + }; + + it.each([ + ['inference credits', outOfCredits], + ['memory tokens', outOfMemoryTokens], + ])('offers credits when %s run out, live and rebuilt from history', (_what, error) => { + const live = v1Refusal(error); + for (const failure of [live, toPersistedErrorIdentity(live)]) { + expect(describeRunError(failure)).toMatchObject({ + title: 'Out of Mastra Gateway credits', + recoveries: ['credits', 'continue', 'model'], + }); + } + }); + + it('sends a monthly spend limit to an organization admin instead of offering credits', () => { + const live = v1Refusal(spendLimit); + for (const failure of [live, toPersistedErrorIdentity(live)]) { + expect(describeRunError(failure)).toMatchObject({ + title: 'Mastra Gateway spend limit reached', + recoveries: ['model'], + }); + } + }); + + it('still recognizes the older replies that linked to the gateway host', () => { + const olderSpendLimit = v1Refusal({ + message: 'Your organization reached its monthly Gateway spend limit. Raise it at https://gateway.mastra.ai/', + type: 'spend_limit_reached', + help_url: 'https://gateway.mastra.ai/', + }); + expect(describeRunError(toPersistedErrorIdentity(olderSpendLimit))).toMatchObject({ + title: 'Mastra Gateway spend limit reached', + }); + }); + + it.each([ + ['a billing type with a help link elsewhere', { ...outOfCredits, help_url: 'https://billing.example.com/' }], + ['a dashboard link in its text only', { message: outOfCredits.message, type: 'insufficient_quota' }], + ['a look-alike help host', { ...outOfCredits, help_url: 'https://projects.mastra.ai.example.com/' }], + ])("keeps another provider's 402 with %s a provider quota error", (_shape, error) => { + expect(describeRunError(toPersistedErrorIdentity(v1Refusal(error)))).toMatchObject({ + title: 'Provider quota exhausted', + recoveries: ['credentials', 'model'], + }); + }); + }); + it('recognizes the gateway by the host it called when its reply names no gateway', () => { const outOfCredits = { name: 'AI_APICallError', @@ -113,6 +186,392 @@ describe('describeRunError', () => { expect(describeRunError(outOfCredits)).toMatchObject({ title: 'Provider quota exhausted' }); }); + describe('Mastra Gateway v2 refusals', () => { + // What v2 sends for a reason code it has no special reply for, and what history keeps of it: + // no request URL, and the model the request named without the `mastra/` prefix. + const gatewayRefusal = (status: number, code: string, message = code) => { + const responseBody = JSON.stringify({ error: { type: 'gateway_error', code, message } }); + const live = { + name: 'AI_APICallError', + message, + statusCode: status, + responseBody, + requestBodyValues: { model: 'anthropic/claude-opus-4-5' }, + }; + return toPersistedErrorIdentity(live); + }; + + it('keeps no request URL or gateway prefix in the saved part these cases are rebuilt from', () => { + const saved = gatewayRefusal(402, 'insufficient_credit'); + expect(saved).not.toHaveProperty('url'); + expect(saved).toMatchObject({ + modelId: 'anthropic/claude-opus-4-5', + codes: expect.arrayContaining(['gateway_error', 'insufficient_credit']), + }); + }); + + it('offers to add credits when the organization is out of credits', () => { + expect(describeRunError(gatewayRefusal(402, 'insufficient_credit'))).toEqual({ + title: 'Out of Mastra Gateway credits', + message: + 'Your organization has no gateway credits left. Add credits, then continue the run, or choose another model.', + recoveries: ['credits', 'continue', 'model'], + model: 'anthropic/claude-opus-4-5', + }); + }); + + it('names the organization budget, and still offers credits, which lift a starting budget', () => { + const description = describeRunError(gatewayRefusal(402, 'organization_budget_exhausted')); + expect(description).toMatchObject({ + title: 'Mastra Gateway budget reached', + recoveries: ['credits', 'continue', 'model'], + }); + expect(description.message).toContain('used its Mastra Gateway budget'); + expect(description.message).toContain('adding credits lifts it'); + }); + + it.each(['organization_budget_insufficient', 'organization_budget_unbounded_request'])( + 'says a %s request needs more than the budget has left', + code => { + const description = describeRunError(gatewayRefusal(402, code)); + expect(description).toMatchObject({ + title: 'Mastra Gateway budget reached', + recoveries: ['credits', 'continue', 'model'], + }); + expect(description.message).toContain('This request needs more than your organization has left'); + }, + ); + + it.each(['key_budget_exhausted', 'spend_limit_reached'])( + 'sends %s to an organization admin, without offering credits that would not help', + code => { + const description = describeRunError(gatewayRefusal(402, code)); + expect(description).toMatchObject({ title: 'Mastra Gateway spend limit reached', recoveries: ['model'] }); + expect(description.message).toContain('Ask an organization admin to raise the limit'); + }, + ); + + it('refuses usage the billing plan does not cover, without offering credits', () => { + expect(describeRunError(gatewayRefusal(402, 'billing_overage_unsupported'))).toMatchObject({ + title: 'Usage not covered by the billing plan', + recoveries: ['model'], + }); + }); + + it('keeps offering credits for a 402 code it does not know yet', () => { + expect(describeRunError(gatewayRefusal(402, 'some_new_billing_code'))).toMatchObject({ + title: 'Out of Mastra Gateway credits', + recoveries: ['credits', 'continue', 'model'], + }); + }); + + it('quotes the sentence the gateway wrote instead of the code', () => { + const description = describeRunError( + gatewayRefusal(402, 'spend_limit_reached', 'This key reached its monthly spend limit of $50.'), + ); + expect(description.message).toBe( + 'This key reached its monthly spend limit of $50. Ask an organization admin to raise the limit, or choose another model.', + ); + }); + + it.each(['no_eligible_route', 'model_not_allowed'])( + 'offers another model, not credentials, when routing policy refuses with %s', + code => { + expect(describeRunError(gatewayRefusal(403, code))).toMatchObject({ + title: 'Model not available through the Mastra Gateway', + recoveries: ['model'], + }); + }, + ); + + it.each([ + [413, 'context_too_large'], + [422, 'context_unaccountable'], + ])('treats a %i %s as a conversation over the model limit', (status, code) => { + expect(describeRunError(gatewayRefusal(status, code))).toMatchObject({ + title: 'Conversation exceeds the model’s limit', + recoveries: ['model', 'new_session'], + }); + }); + + it('reads the code from a Messages route reply, which repeats it as the message', () => { + const messagesReply = { + name: 'AI_APICallError', + message: 'organization_budget_exhausted', + statusCode: 402, + responseBody: JSON.stringify({ + type: 'error', + error: { type: 'gateway_error', message: 'organization_budget_exhausted' }, + }), + }; + expect(describeRunError(toPersistedErrorIdentity(messagesReply))).toMatchObject({ + title: 'Mastra Gateway budget reached', + }); + }); + + it("describes the gateway's own rate limit as the key's or organization's, not the provider account's", () => { + const limited = { + name: 'AI_APICallError', + message: 'Gateway key requests limit exceeded. Retry in 12s.', + statusCode: 429, + url: 'https://gateway-api.mastra.ai/v1/chat/completions', + responseBody: JSON.stringify({ + error: { + type: 'rate_limit_error', + code: 'rate_limit_exceeded', + message: 'Gateway key requests limit exceeded. Retry in 12s.', + }, + }), + }; + expect(describeRunError(limited)).toEqual({ + title: 'Mastra Gateway rate limit reached', + message: 'Gateway key requests limit exceeded. Retry in 12s. Continue once it resets, or choose another model.', + recoveries: ['continue', 'model'], + model: undefined, + }); + }); + + it('recognizes a gateway at the configured base URL', () => { + const staging = { + name: 'AI_APICallError', + message: 'Payment required', + statusCode: 402, + url: 'http://localhost:8787/v1/chat/completions', + }; + expect(describeRunError(staging)).toMatchObject({ title: 'Provider quota exhausted' }); + expect(describeRunError(staging, { gatewayBaseUrl: 'http://localhost:8787' })).toMatchObject({ + title: 'Out of Mastra Gateway credits', + }); + expect(describeRunError(staging, { gatewayBaseUrl: 'http://localhost:9999' })).toMatchObject({ + title: 'Provider quota exhausted', + }); + expect(describeRunError(staging, { gatewayBaseUrl: 'https://localhost:8787' })).toMatchObject({ + title: 'Provider quota exhausted', + }); + }); + + describe('a code another provider can also send', () => { + // A direct provider's saved error: no gateway type, no request origin, no `mastra/` model. + const direct = (status: number, code: string) => + toPersistedErrorIdentity({ + name: 'AI_APICallError', + message: 'Refused.', + statusCode: status, + responseBody: JSON.stringify({ error: { type: 'invalid_request_error', code, message: 'Refused.' } }), + requestBodyValues: { model: 'acme/model-1' }, + }); + + it.each([ + [403, 'model_not_allowed', 'Provider access failed'], + [402, 'insufficient_credit', 'Provider quota exhausted'], + [402, 'spend_limit_reached', 'Provider quota exhausted'], + ])('keeps a direct provider’s %i %s its own refusal', (status, code, title) => { + expect(describeRunError(direct(status, code))).toMatchObject({ title, recoveries: ['credentials', 'model'] }); + }); + + it('names the gateway when the saved request origin is the gateway’s', () => { + const saved = { ...direct(403, 'model_not_allowed'), requestOrigin: 'https://gateway-api.mastra.ai' }; + expect(describeRunError(saved)).toMatchObject({ title: 'Model not available through the Mastra Gateway' }); + }); + + it('names the gateway from saved codes alone only for a code no one else sends', () => { + const savedCodes = (codes: string[]) => ({ + name: 'AI_APICallError', + message: 'Refused.', + statusCode: 402, + codes, + }); + expect(describeRunError(savedCodes(['insufficient_credit']))).toMatchObject({ + title: 'Provider quota exhausted', + }); + expect(describeRunError(savedCodes(['key_budget_exhausted']))).toMatchObject({ + title: 'Mastra Gateway spend limit reached', + }); + }); + }); + + it('bounds a long gateway sentence before showing it', () => { + const long = 'The budget is spent. '.repeat(60).trim(); + const { message } = describeRunError(gatewayRefusal(402, 'spend_limit_reached', long)); + expect(message.length).toBeLessThan(600); + expect(message).toContain('…'); + }); + + it.each([ + [401, 'some_new_key_reason'], + [403, 'identity_policy_denied'], + [403, 'byok_connection_inactive'], + ])('keeps the credential recovery for a %i %s refusal of the key or connection', (status, code) => { + expect(describeRunError(gatewayRefusal(status, code))).toMatchObject({ + title: 'Provider access failed', + recoveries: ['credentials', 'model'], + }); + }); + + it('offers another model when no route in the key’s region serves it', () => { + expect(describeRunError(gatewayRefusal(403, 'no_route_in_region'))).toMatchObject({ + title: 'Model not available through the Mastra Gateway', + recoveries: ['model'], + }); + }); + + it('names the provider behind the gateway when it is the one limiting requests', () => { + const limited = toPersistedErrorIdentity({ + name: 'AI_APICallError', + message: 'Upstream provider rate limit reached. Try again shortly.', + statusCode: 429, + responseBody: JSON.stringify({ + error: { + type: 'rate_limit_error', + code: 'provider_rate_limited', + message: 'Upstream provider rate limit reached. Try again shortly.', + }, + }), + }); + expect(describeRunError(limited)).toMatchObject({ + title: 'Model provider rate limit reached', + recoveries: ['continue', 'model'], + }); + }); + + it.each(['key_budget_capacity_held', 'organization_budget_capacity_held'])( + 'asks to continue in a few seconds when %s', + code => { + expect(describeRunError(gatewayRefusal(429, code))).toMatchObject({ + title: 'Mastra Gateway budget busy', + recoveries: ['continue', 'model'], + }); + }, + ); + + it('recognizes the gateway from the saved codes alone when the body is gone', () => { + const withoutBody = { + name: 'AI_APICallError', + message: 'key_budget_exhausted', + statusCode: 402, + codes: ['gateway_error', 'key_budget_exhausted'], + }; + expect(describeRunError(withoutBody)).toMatchObject({ title: 'Mastra Gateway spend limit reached' }); + }); + + it.each([ + [ + 'an OpenAI-style', + { type: 'requests', code: 'rate_limit_exceeded', message: 'Rate limit reached for requests' }, + ], + ['an uncoded', { message: 'rate limited', type: 'rate_limit_error', code: null }], + ])( + "names the provider behind the gateway, not the session's credential, for %s 429 it relayed", + (_shape, error) => { + const relayed = { + name: 'AI_APICallError', + message: error.message, + statusCode: 429, + url: 'https://gateway-api.mastra.ai/v1/chat/completions', + responseBody: JSON.stringify({ error }), + }; + expect(describeRunError(relayed)).toEqual({ + title: 'Model provider rate limit reached', + message: + 'The model provider behind the Mastra Gateway is limiting requests. Continue shortly, or choose another model.', + recoveries: ['continue', 'model'], + model: undefined, + }); + }, + ); + + describe('a saved 429, rebuilt from history', () => { + // History keeps no request URL. Core saves the request's origin as `requestOrigin`; a part + // saved by an older core has none, which is what leaving the argument out stands for. + const saved429 = (url: string, error: Record, requestOrigin?: string) => { + const { requestOrigin: _recorded, ...part } = toPersistedErrorIdentity({ + name: 'AI_APICallError', + message: String(error.message), + statusCode: 429, + url, + responseBody: JSON.stringify({ error }), + }) as Record; + return { ...part, ...(requestOrigin ? { requestOrigin } : {}) }; + }; + const gatewayLimit = { + type: 'rate_limit_error', + code: 'rate_limit_exceeded', + message: 'Gateway key requests limit exceeded. Retry in 12s.', + }; + const relayedLimit = { message: 'rate limited', type: 'rate_limit_error', code: null }; + + it("names the gateway's own limit from the origin core saves with the error", () => { + const saved = toPersistedErrorIdentity({ + name: 'AI_APICallError', + message: gatewayLimit.message, + statusCode: 429, + url: 'https://gateway-api.mastra.ai/v1/chat/completions', + responseBody: JSON.stringify({ error: gatewayLimit }), + }); + expect(saved).toMatchObject({ requestOrigin: 'https://gateway-api.mastra.ai' }); + expect(saved).not.toHaveProperty('url'); + expect(describeRunError(saved)).toMatchObject({ title: 'Mastra Gateway rate limit reached' }); + }); + + it('stays a provider rate limit without a request origin, since its type and code are shared', () => { + const saved = saved429('https://gateway-api.mastra.ai/v1/chat/completions', gatewayLimit); + expect(saved).not.toHaveProperty('url'); + expect(describeRunError(saved)).toMatchObject({ title: 'Provider rate limit reached' }); + }); + + it("names the gateway's own limit when the request went to the production gateway", () => { + const saved = saved429( + 'https://gateway-api.mastra.ai/v1/chat/completions', + gatewayLimit, + 'https://gateway-api.mastra.ai', + ); + expect(describeRunError(saved)).toMatchObject({ + title: 'Mastra Gateway rate limit reached', + recoveries: ['continue', 'model'], + }); + }); + + it('names the provider behind the gateway for a limit the gateway relayed', () => { + const saved = saved429( + 'https://gateway-api.mastra.ai/v1/chat/completions', + relayedLimit, + 'https://gateway-api.mastra.ai', + ); + expect(describeRunError(saved)).toMatchObject({ + title: 'Model provider rate limit reached', + recoveries: ['continue', 'model'], + }); + }); + + it('recognizes a non-production gateway only when the configured origin is passed', () => { + const saved = saved429('http://127.0.0.1:4180/v1/chat/completions', gatewayLimit, 'http://127.0.0.1:4180'); + expect(describeRunError(saved)).toMatchObject({ title: 'Provider rate limit reached' }); + expect(describeRunError(saved, { gatewayBaseUrl: 'http://127.0.0.1:4180' })).toMatchObject({ + title: 'Mastra Gateway rate limit reached', + }); + }); + }); + + it('keeps an Anthropic 403 a credential problem', () => { + const anthropicDenied = { + name: 'AI_APICallError', + message: 'Your API key does not have permission to use the specified resource.', + statusCode: 403, + responseBody: JSON.stringify({ + type: 'error', + error: { + type: 'permission_error', + message: 'Your API key does not have permission to use the specified resource.', + }, + }), + }; + expect(describeRunError(anthropicDenied)).toMatchObject({ + title: 'Provider access failed', + recoveries: ['credentials', 'model'], + }); + }); + }); + it('quotes a plain-text body the provider sent instead of JSON', () => { const textBody = { name: 'AI_APICallError', diff --git a/mastracode/factory/src/run-error.ts b/mastracode/factory/src/run-error.ts index f0625e653a5..802061e163b 100644 --- a/mastracode/factory/src/run-error.ts +++ b/mastracode/factory/src/run-error.ts @@ -25,24 +25,39 @@ const UNEXPECTED_ERROR = 'An unexpected error stopped the run. Copy the error de const NO_REASON_AVAILABLE = 'The provider rejected the request. A specific reason isn’t available here. Copy the error details and check the server logs.'; -export function describeRunError(error: RunErrorIdentity): RunErrorDescription { - return { ...explainRunError(error), model: providerModelId(error) }; +export interface DescribeRunErrorOptions { + /** + * The Mastra Gateway base URL this server sends requests to. Requests to its host are recognized as + * gateway replies, so a staging, local or self-hosted gateway is handled like the production one. + */ + gatewayBaseUrl?: string; } -function explainRunError(error: RunErrorIdentity): Omit { +export function describeRunError(error: RunErrorIdentity, options: DescribeRunErrorOptions = {}): RunErrorDescription { + return { ...explainRunError(error, options), model: providerModelId(error) }; +} + +const CONTEXT_OVERFLOW: Omit = { + title: 'Conversation exceeds the model’s limit', + message: 'Choose a model with a larger context window or start a new session with a shorter prompt.', + recoveries: ['model', 'new_session'], +}; + +function explainRunError( + error: RunErrorIdentity, + options: DescribeRunErrorOptions, +): Omit { if (error.name === PROVIDER_AUTH_REQUIRED_ERROR) { return { title: 'Provider sign-in required', message: error.message, recoveries: ['credentials', 'model'] }; } - switch (classifyProviderError(error)) { + const kind = classifyProviderError(error); + const gateway = gatewayReply(error); + if (isMastraGatewayError(error, gateway, options)) { + const refusal = explainGatewayRefusal(kind, gateway); + if (refusal) return refusal; + } + switch (kind) { case 'quota': - if (isMastraGatewayError(error)) { - return { - title: 'Out of Mastra Gateway credits', - message: - 'Your organization has no gateway credits left. Add credits, then continue the run, or choose another model.', - recoveries: ['credits', 'continue', 'model'], - }; - } return { title: 'Provider quota exhausted', message: afterProviderReason( @@ -58,11 +73,7 @@ function explainRunError(error: RunErrorIdentity): Omit, + reply: GatewayReply | undefined, +): Omit | undefined { + const code = reply?.code; + const said = (fallback: string, next: string) => `${(reply?.message ?? fallback).replace(/\.?$/, '.')} ${next}`; + if (code && GATEWAY_CONTEXT_OVERFLOW.has(code)) return CONTEXT_OVERFLOW; + if (code === ORGANIZATION_BUDGET_EXHAUSTED) { + return { + title: 'Mastra Gateway budget reached', + message: said( + `Your organization has used its Mastra Gateway budget. ${STARTING_BUDGET} If your organization has already added credits, an organization admin can raise its spend limit`, + ADD_CREDITS_THEN_CONTINUE, + ), + recoveries: ['credits', 'continue', 'model'], + }; + } + if (code && ORGANIZATION_BUDGET_SHORT.has(code)) { + return { + title: 'Mastra Gateway budget reached', + message: said( + `This request needs more than your organization has left in its Mastra Gateway budget. ${STARTING_BUDGET}`, + ADD_CREDITS_THEN_CONTINUE, + ), + recoveries: ['credits', 'continue', 'model'], + }; + } + if (code && SPEND_LIMIT_REACHED.has(code)) { + return { + title: 'Mastra Gateway spend limit reached', + message: said( + 'A spend limit set on this gateway key or your organization was reached, and adding credits does not raise it', + 'Ask an organization admin to raise the limit, or choose another model.', + ), + recoveries: ['model'], + }; + } + if (code === BILLING_OVERAGE_UNSUPPORTED) { + return { + title: 'Usage not covered by the billing plan', + message: said( + 'Your organization’s billing plan does not allow this Mastra Gateway usage', + 'Choose another model, or ask an organization admin to review the plan.', + ), + recoveries: ['model'], + }; + } + if (kind === 'quota') { + // A credit refusal from v1, or a v2 code this version does not know yet. + return { + title: 'Out of Mastra Gateway credits', + message: OUT_OF_CREDITS.has(code ?? '') + ? said('Your organization has no gateway credits left', ADD_CREDITS_THEN_CONTINUE) + : `Your organization has no gateway credits left. ${ADD_CREDITS_THEN_CONTINUE}`, + recoveries: ['credits', 'continue', 'model'], + }; + } + // Any other refusal of the key or connection keeps the credential recovery. + if (kind === 'auth' && code !== undefined && ROUTE_REFUSED.has(code)) { + return { + title: 'Model not available through the Mastra Gateway', + message: said( + 'The Mastra Gateway has no route your organization may use for this model or feature', + 'Choose another model for this session.', + ), + recoveries: ['model'], + }; + } + if (kind !== 'rate_limited') return undefined; + if (code === UPSTREAM_RATE_LIMITED) { + return { + title: 'Model provider rate limit reached', + message: said( + 'The model provider behind the Mastra Gateway is limiting requests', + 'Continue shortly, or choose another model.', + ), + recoveries: ['continue', 'model'], + }; + } + if (code && BUDGET_CAPACITY_HELD.has(code)) { + return { + title: 'Mastra Gateway budget busy', + message: said( + 'Requests still in flight hold the rest of the gateway budget for this key or organization', + 'Continue in a few seconds, or choose another model.', + ), + recoveries: ['continue', 'model'], + }; + } + // The gateway's limit reply pairs these two; a provider reply relayed through it names its own type. + const gatewayLimit = code === GATEWAY_RATE_LIMITED && reply?.type === 'rate_limit_error'; + if (gatewayLimit || reply?.type === 'gateway_error') { + // The gateway's own limit on the key or organization, not the provider account's. + return { + title: 'Mastra Gateway rate limit reached', + message: said( + 'The gateway’s rate limit for this key or organization was reached', + 'Continue once it resets, or choose another model.', + ), + recoveries: ['continue', 'model'], + }; + } + // A provider's own limit relayed through the gateway: the provider account is the gateway's route, + // not one of this session's credentials, and the relayed text names no one. + return { + title: 'Model provider rate limit reached', + message: + 'The model provider behind the Mastra Gateway is limiting requests. Continue shortly, or choose another model.', + recoveries: ['continue', 'model'], + }; +} + +/** + * The Mastra Gateway answered: a `mastra/…` model, a request sent to its host (production, or the + * configured base URL), a v1 billing reply (a billing `type` with a `help_url` on a mastra.ai host, or + * from older gateways a message naming `gateway.mastra.ai`), or a v2 reply naming itself a gateway + * error or carrying a reason code only the gateway sends. The request reaches the + * gateway without the `mastra/` prefix, and history keeps no request URL, so the web UI rebuilds a saved + * error from its request origin, when one was recorded, and otherwise from the reply body. + */ +function isMastraGatewayError( + error: RunErrorIdentity, + reply: GatewayReply | undefined, + options: DescribeRunErrorOptions, +): boolean { return ( providerModelId(error)?.startsWith(MASTRA_GATEWAY_PREFIX) === true || - calledMastraGateway(error) || - /gateway\.mastra\.ai/.test(providerErrorReason(error) ?? error.message) + calledMastraGateway(error, configuredGatewayOrigin(options.gatewayBaseUrl)) || + /gateway\.mastra\.ai/.test(providerErrorReason(error) ?? error.message) || + reply?.v1Billing === true || + reply?.type === 'gateway_error' || + (reply?.code !== undefined && onlyTheGatewaySends(reply.code)) ); } -/** Whether the request, or a provider error it wraps, went to the gateway's host. */ -function calledMastraGateway(error: unknown, depth = 0): boolean { - if (depth > 4 || typeof error !== 'object' || error === null) return false; - const url = 'url' in error ? error.url : undefined; - const cause = 'cause' in error ? error.cause : undefined; - return (typeof url === 'string' && MASTRA_GATEWAY_HOST.test(hostOf(url))) || calledMastraGateway(cause, depth + 1); +/** + * The configured gateway's origin: scheme, host and port. A gateway served under a path prefix shares + * its origin with anything else on that host, so a provider at the same origin would also match. + */ +function configuredGatewayOrigin(baseUrl: string | undefined): string | undefined { + if (!baseUrl) return undefined; + try { + const { origin } = new URL(baseUrl); + return origin === 'null' ? undefined : origin; + } catch { + return undefined; + } } -function hostOf(url: string): string { +/** + * Whether the request, or a provider error it wraps, went to the gateway's host. A saved error part + * keeps no URL, only the request's origin as `requestOrigin` on versions of `@mastra/core` that record it. + */ +function calledMastraGateway(error: unknown, configuredOrigin: string | undefined, depth = 0): boolean { + if (depth > 4 || typeof error !== 'object' || error === null) return false; + const url = 'url' in error ? error.url : 'requestOrigin' in error ? error.requestOrigin : undefined; + const cause = 'cause' in error ? error.cause : undefined; + return ( + (typeof url === 'string' && isGatewayUrl(url, configuredOrigin)) || + calledMastraGateway(cause, configuredOrigin, depth + 1) + ); +} + +function isGatewayUrl(url: string, configuredOrigin: string | undefined): boolean { try { - return new URL(url).hostname; + const parsed = new URL(url); + return ( + MASTRA_GATEWAY_HOST.test(parsed.hostname) || + (configuredOrigin !== undefined && parsed.origin === configuredOrigin) + ); } catch { - return ''; + return false; } } +/** + * The gateway's error object from the response body, on the error or a cause it wraps, with the codes + * a saved error part keeps as a fallback. The Messages route sends no `code` and repeats the code as + * the message, so a code-shaped message is read as the code. + */ +function gatewayReply(error: unknown, depth = 0): GatewayReply | undefined { + if (depth > 4 || typeof error !== 'object' || error === null) return undefined; + const fromBody = 'responseBody' in error ? replyFromBody(error.responseBody) : undefined; + if (fromBody) return fromBody; + const fromCause = gatewayReply('cause' in error ? error.cause : undefined, depth + 1); + if (fromCause) return fromCause; + const codes = 'codes' in error && Array.isArray(error.codes) ? error.codes.filter(isString) : []; + const code = codes.find(candidate => GATEWAY_CODES.has(candidate)); + if (code || codes.includes('gateway_error')) { + return { ...(codes.includes('gateway_error') ? { type: 'gateway_error' } : {}), ...(code ? { code } : {}) }; + } + return undefined; +} + +const CODE_SHAPED = /^[a-z][a-z0-9_]*$/; + +function replyFromBody(body: unknown): GatewayReply | undefined { + if (typeof body !== 'string') return undefined; + let parsed: unknown; + try { + parsed = JSON.parse(body); + } catch { + return undefined; + } + const inner = typeof parsed === 'object' && parsed !== null && 'error' in parsed ? parsed.error : undefined; + if (typeof inner !== 'object' || inner === null) return undefined; + const type = 'type' in inner && isString(inner.type) ? inner.type : undefined; + const rawMessage = 'message' in inner && isString(inner.message) ? inner.message.trim() : undefined; + const code = + 'code' in inner && isString(inner.code) + ? inner.code + : type === 'gateway_error' && rawMessage && CODE_SHAPED.test(rawMessage) + ? rawMessage + : undefined; + const sentence = rawMessage && rawMessage !== code && !CODE_SHAPED.test(rawMessage) ? rawMessage : undefined; + const message = + sentence && sentence.length > MAX_GATEWAY_MESSAGE_LENGTH + ? `${sentence.slice(0, MAX_GATEWAY_MESSAGE_LENGTH - 1)}…` + : sentence; + if (!type && !code) return undefined; + const helpUrl = 'help_url' in inner && isString(inner.help_url) ? inner.help_url : undefined; + const v1Billing = type !== undefined && V1_BILLING_TYPES.has(type) && helpUrl !== undefined && onMastraHost(helpUrl); + // The v1 spend limit names itself only by its type; read it as the v2 code of the same meaning. + const reason = code ?? (v1Billing && type === 'spend_limit_reached' ? type : undefined); + return { + ...(type ? { type } : {}), + ...(reason ? { code: reason } : {}), + ...(message ? { message } : {}), + ...(v1Billing ? { v1Billing: true as const } : {}), + }; +} + +function onMastraHost(url: string): boolean { + try { + const parsed = new URL(url); + return parsed.protocol === 'https:' && MASTRA_HOST.test(parsed.hostname); + } catch { + return false; + } +} + +function isString(value: unknown): value is string { + return typeof value === 'string' && value.trim().length > 0; +} + function afterProviderReason(error: RunErrorIdentity, next: string, fallback?: string): string { const reason = providerErrorReason(error) ?? fallback; return reason ? `${reason.replace(/\.?$/, '.')} ${next}` : next; From 5a3854805375d6fcbba8acdb73d132c8ee0a70a0 Mon Sep 17 00:00:00 2001 From: Rhys Balevicius Date: Tue, 17 Nov 2026 04:02:52 +0000 Subject: [PATCH 4/5] fix(factory): handle gateway catalog failures in settings Fetch gateway models with the tenant's key so settings show the models its organization can use. Bound catalog waits and keep connection controls available during failures. Reuse cached catalogs through transient outages without substituting the server key's models. --- .changeset/steady-gateway-catalogues.md | 5 + mastracode/factory/src/routes/config.test.ts | 190 ++++++++++++++++++ mastracode/factory/src/routes/config.ts | 38 +++- .../src/routes/gateway-catalog.test.ts | 135 +++++++++++++ .../factory/src/routes/gateway-catalog.ts | 70 +++++++ 5 files changed, 431 insertions(+), 7 deletions(-) create mode 100644 .changeset/steady-gateway-catalogues.md create mode 100644 mastracode/factory/src/routes/gateway-catalog.test.ts create mode 100644 mastracode/factory/src/routes/gateway-catalog.ts diff --git a/.changeset/steady-gateway-catalogues.md b/.changeset/steady-gateway-catalogues.md new file mode 100644 index 00000000000..6430816036c --- /dev/null +++ b/.changeset/steady-gateway-catalogues.md @@ -0,0 +1,5 @@ +--- +'@mastra/factory': patch +--- + +The provider and model settings no longer fail when the Mastra Gateway model catalogue cannot be fetched. The Gateway row stays in the provider list so it can still be connected or disconnected. An organization that connected its own v2 Gateway key now sees the models that key can use instead of the bundled list. diff --git a/mastracode/factory/src/routes/config.test.ts b/mastracode/factory/src/routes/config.test.ts index 218a7476847..657bbfaf8e7 100644 --- a/mastracode/factory/src/routes/config.test.ts +++ b/mastracode/factory/src/routes/config.test.ts @@ -22,6 +22,7 @@ import { createFactoryStorageForTests } from '../storage/test-utils.js'; import type { FactoryStorageTestSeed } from '../storage/test-utils.js'; import { buildProviderAccess, ConfigRoutes, listProviders, resolveDeploymentModelProviders } from './config.js'; import type { ModelCatalogEntry } from './config.js'; +import { resetMastraGatewayCatalogCacheForTests } from './gateway-catalog.js'; import type { RouteAuth } from './route.js'; import { registerTenantCredentialResolver, @@ -1018,6 +1019,7 @@ describe('Mastra Gateway routes with a tenant', () => { expect(await gatewayRow(buildApp(fakePlatform()))).toMatchObject({ connectWithMastraAccount: true }); expect(await gatewayRow(buildApp())).not.toHaveProperty('connectWithMastraAccount'); }); + describe('the gateway origin for the web UI', () => { beforeEach(async () => { // An empty settings directory, so a gateway URL in the developer's own settings cannot apply. @@ -1042,6 +1044,194 @@ describe('Mastra Gateway routes with a tenant', () => { expect(body.mastraGatewayOrigin).toBe(origin); }); }); + + describe('with a v2 gateway catalogue', () => { + const MODELS_URL = 'https://gateway-api.mastra.ai/v1/models'; + const gateway = setupServer(); + const catalogRequests: Array = []; + // The gateway client caches a list per key for the life of the process, so each test uses its own key. + let testNumber = 0; + let tenantKey: string; + + beforeAll(() => gateway.listen({ onUnhandledRequest: 'error' })); + afterAll(() => gateway.close()); + beforeEach(async () => { + resetMastraGatewayCatalogCacheForTests(); + catalogRequests.length = 0; + tenantKey = `msk-gateway-tenant${++testNumber}`; + // An empty settings directory, so a gateway URL in the developer's own settings cannot apply. + vi.stubEnv('MASTRA_APP_DATA_DIR', await fs.mkdtemp(join(tmpdir(), 'factory-gateway-catalog-'))); + vi.stubEnv('MASTRA_GATEWAY_URL', 'https://gateway-api.mastra.ai'); + }); + afterEach(() => { + gateway.resetHandlers(); + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + }); + + const serveCatalog = (respond: () => Response) => + gateway.use( + http.get(MODELS_URL, ({ request }) => { + catalogRequests.push(request.headers.get('authorization')); + return respond(); + }), + ); + + const readSettings = async (app: Hono) => { + const providersRes = await app.request('/web/config/providers'); + const modelsRes = await app.request('/web/config/models'); + return { + providersStatus: providersRes.status, + modelsStatus: modelsRes.status, + gatewayRow: ((await providersRes.json()) as { providers: Array<{ provider: string }> }).providers.find( + p => p.provider === 'mastra', + ), + modelIds: ((await modelsRes.json()) as { models: Array<{ id: string }> }).models.map(m => m.id), + }; + }; + + it("lists what the tenant's own v2 key may use, fetched once for both settings requests", async () => { + await seed.credentials.setCredential({ orgId: 'org1' }, 'mastra-gateway', { + type: 'api_key', + key: tenantKey, + }); + serveCatalog(() => + Response.json({ + data: [ + { id: 'anthropic/claude-opus-4-5', endpoints: ['chat'] }, + { id: 'openai/gpt-image-1', endpoints: ['images'] }, + ], + }), + ); + + const settings = await readSettings(buildApp(fakePlatform())); + + expect(settings.modelIds.filter(id => id.startsWith('mastra/'))).toEqual(['mastra/anthropic/claude-opus-4-5']); + expect(settings.gatewayRow).toMatchObject({ source: 'stored-org' }); + expect(catalogRequests).toEqual([`Bearer ${tenantKey}`]); + }); + + it('keeps settings working and the gateway row reachable when the catalogue fails', async () => { + await seed.credentials.setCredential({ orgId: 'org1' }, 'mastra-gateway', { + type: 'api_key', + key: tenantKey, + }); + serveCatalog(() => new Response(null, { status: 503 })); + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + + const settings = await readSettings(buildApp(fakePlatform())); + + expect(settings).toMatchObject({ providersStatus: 200, modelsStatus: 200 }); + expect(settings.gatewayRow).toMatchObject({ provider: 'mastra', source: 'stored-org', orgCredential: 'api_key' }); + expect(settings.modelIds.some(id => id.startsWith('mastra/'))).toBe(false); + expect(warn).toHaveBeenCalledTimes(1); + expect(JSON.stringify(warn.mock.calls)).not.toContain(tenantKey); + }); + + it('serves the last catalogue it fetched while the gateway is failing', async () => { + await seed.credentials.setCredential({ orgId: 'org1' }, 'mastra-gateway', { + type: 'api_key', + key: tenantKey, + }); + let up = true; + serveCatalog(() => + up + ? Response.json({ data: [{ id: 'anthropic/claude-opus-4-5', endpoints: ['chat'] }] }) + : new Response(null, { status: 503 }), + ); + vi.spyOn(console, 'warn').mockImplementation(() => {}); + const app = buildApp(fakePlatform()); + await readSettings(app); + up = false; + vi.spyOn(Date, 'now').mockReturnValue(Date.now() + 5 * 60_000); + + const settings = await readSettings(app); + + expect(catalogRequests).toHaveLength(2); + expect(settings.modelIds).toContain('mastra/anthropic/claude-opus-4-5'); + }); + + it("never offers the server key's gateway models in place of a tenant key's failing catalogue", async () => { + await seed.credentials.setCredential({ orgId: 'org1' }, 'mastra-gateway', { + type: 'api_key', + key: tenantKey, + }); + serveCatalog(() => new Response(null, { status: 503 })); + vi.spyOn(console, 'warn').mockImplementation(() => {}); + serverGatewayKey.mockReturnValue('msk-gateway-deploy'); + const app = new Hono(); + app.use('*', async (c, next) => { + c.set('factoryAuthUser' as never, user as never); + await next(); + }); + mountApiRoutes( + app as any, + new ConfigRoutes({ + auth: fakeRouteAuth(), + // The boot-time catalogue lists the server key's gateway models. + controller: { + listAvailableModels: async () => [ + { modelName: 'anthropic-model', provider: 'anthropic', hasApiKey: false }, + { id: 'mastra/openai/gpt-5', modelName: 'openai/gpt-5', provider: 'mastra', hasApiKey: true }, + ], + }, + modelCredentials: seed.credentials, + memorySettings: seed.memorySettings, + }).routes(), + ); + + const settings = await readSettings(app); + + expect(settings.modelIds.some(id => id.startsWith('mastra/'))).toBe(false); + expect(settings.gatewayRow).toMatchObject({ provider: 'mastra', source: 'stored-org' }); + }); + + it("asks the gateway named in the settings for the server key's catalogue, as model calls do", async () => { + const appData = await fs.mkdtemp(join(tmpdir(), 'factory-gateway-settings-')); + await fs.writeFile( + join(appData, 'settings.json'), + JSON.stringify({ memoryGateway: { baseUrl: 'https://gateway.staging.test/v1' } }), + ); + vi.stubEnv('MASTRA_APP_DATA_DIR', appData); + vi.stubEnv('MASTRA_GATEWAY_API_KEY', 'msk-gateway-server'); + serverGatewayKey.mockReturnValue('msk-gateway-server'); + const staging: Array = []; + gateway.use( + http.get('https://gateway.staging.test/v1/models', ({ request }) => { + staging.push(request.headers.get('authorization')); + return Response.json({ data: [{ id: 'anthropic/claude-opus-4-5', endpoints: ['chat'] }] }); + }), + ); + + const settings = await readSettings(buildApp(fakePlatform())); + + expect(staging).toEqual(['Bearer msk-gateway-server']); + expect(settings.modelIds).toContain('mastra/anthropic/claude-opus-4-5'); + }); + + it('keeps the gateway row for connecting a key by hand when nothing else can list it', async () => { + vi.stubEnv('MASTRA_GATEWAY_VERSION', 'v2'); + vi.stubEnv('MASTRA_GATEWAY_API_KEY', ''); + vi.spyOn(console, 'warn').mockImplementation(() => {}); + + const settings = await readSettings(buildApp()); + + expect(settings.providersStatus).toBe(200); + expect(settings.gatewayRow).toMatchObject({ provider: 'mastra', source: 'none' }); + expect(settings.gatewayRow).not.toHaveProperty('connectWithMastraAccount'); + }); + + it('keeps the one-click connect when the server is in v2 mode without a key', async () => { + vi.stubEnv('MASTRA_GATEWAY_VERSION', 'v2'); + vi.stubEnv('MASTRA_GATEWAY_API_KEY', ''); + vi.spyOn(console, 'warn').mockImplementation(() => {}); + + const settings = await readSettings(buildApp(fakePlatform())); + + expect(settings).toMatchObject({ providersStatus: 200, modelsStatus: 200 }); + expect(settings.gatewayRow).toMatchObject({ provider: 'mastra', source: 'none', connectWithMastraAccount: true }); + }); + }); }); describe('GET /web/config/models', () => { diff --git a/mastracode/factory/src/routes/config.ts b/mastracode/factory/src/routes/config.ts index 994c18160d9..3a27dcb28b6 100644 --- a/mastracode/factory/src/routes/config.ts +++ b/mastracode/factory/src/routes/config.ts @@ -10,7 +10,7 @@ import { AMAZON_BEDROCK_GATEWAY_ID } from '@mastra/code-sdk/providers/amazon-bed import { GOOGLE_VERTEX_GATEWAY_ID } from '@mastra/code-sdk/providers/google-vertex-gateway'; import { MASTRA_GATEWAY_PREFIX, normalizeLegacyProviderId } from '@mastra/code-sdk/providers/model-ids'; import type { ThinkingLevelSetting } from '@mastra/code-sdk/thinking'; -import { getModelReasoningOptions, MastraGateway, PROVIDER_REGISTRY } from '@mastra/core/llm'; +import { getModelReasoningOptions, PROVIDER_REGISTRY } from '@mastra/core/llm'; import type { ModelReasoningOption } from '@mastra/core/llm'; import { RequestContext } from '@mastra/core/request-context'; import type { ApiRoute } from '@mastra/core/server'; @@ -44,6 +44,7 @@ import type { WorkItemsStorage } from '../storage/domains/work-items/base.js'; import { THINKING_LEVELS } from './contracts.js'; import { customEndpointUrlProblem } from './custom-endpoint-url.js'; import { customProviderOwner } from './custom-provider-source.js'; +import { fetchMastraGatewayCatalog } from './gateway-catalog.js'; import { createGatewayKey, getGatewayCredits, @@ -185,22 +186,32 @@ async function isReservedProviderId(controller: ModelCatalog, providerId: string * The controller's model catalog, plus the Mastra Gateway's models for tenant * callers. The boot-time catalog lists `mastra/...` models only when the server * itself has a gateway key; tenants bring their own (connected in one click). + * A tenant's own gateway key wins over the server's at run time, so its + * catalogue is fetched with that key: a v2 key sees what its organization may + * use. A catalogue that cannot be fetched leaves the gateway's models out + * rather than failing the caller's settings, and the server key's list is + * never shown in place of a tenant key's. */ async function listCatalogModels( controller: ModelCatalog, tenantCredentials: CredentialRecord[] | undefined, ): Promise { const models = await controller.listAvailableModels(); - if (!tenantCredentials || models.some(m => m.provider === MASTRA_GATEWAY_CATALOG_PROVIDER)) return models; - const gateway = (await new MastraGateway().fetchProviders())[MASTRA_GATEWAY_CATALOG_PROVIDER]; - const gatewayModels = (gateway?.models ?? []).map(modelName => ({ + if (!tenantCredentials) return models; + const tenantKey = tenantGatewayKey(tenantCredentials); + if (!tenantKey && models.some(m => m.provider === MASTRA_GATEWAY_CATALOG_PROVIDER)) return models; + const gateway = await fetchMastraGatewayCatalog({ apiKey: tenantKey, baseUrl: mastraGatewayBaseUrl() }); + // The server key's models are not what a tenant's own key may use, so they never stand in for its list. + const withoutGateway = tenantKey ? models.filter(m => m.provider !== MASTRA_GATEWAY_CATALOG_PROVIDER) : models; + if (!gateway) return withoutGateway; + const gatewayModels = gateway.models.map(modelName => ({ id: `${MASTRA_GATEWAY_CATALOG_PROVIDER}/${modelName}`, provider: MASTRA_GATEWAY_CATALOG_PROVIDER, modelName, hasApiKey: false, - releaseDate: gateway?.modelReleaseDates?.[modelName], + releaseDate: gateway.modelReleaseDates?.[modelName], })); - return [...models, ...gatewayModels]; + return [...models.filter(m => m.provider !== MASTRA_GATEWAY_CATALOG_PROVIDER), ...gatewayModels]; } function originOf(url: string): string | undefined { @@ -212,6 +223,12 @@ function originOf(url: string): string | undefined { } } +/** The tenant's stored gateway key, personal before organization, as model resolution picks it. */ +function tenantGatewayKey(tenantCredentials: CredentialRecord[]): string | undefined { + const credential = grantingTenantRecord(tenantCredentials, MASTRA_GATEWAY_PROVIDER)?.credential; + return credential?.type === 'api_key' ? credential.key.trim() || undefined : undefined; +} + /** * Build a deduplicated, sorted list of providers from the model catalog, * annotated with where each provider's credential currently comes from. @@ -239,8 +256,15 @@ export async function listProviders({ }): Promise { const models = await listCatalogModels(controller, tenantCredentials); const seen = new Map(); + // The gateway's connect and disconnect controls stay reachable for tenant callers when its model + // list is empty or unavailable, as they are when the bundled list is shown. + const keepGatewayRow = + tenantCredentials !== undefined && !models.some(m => m.provider === MASTRA_GATEWAY_CATALOG_PROVIDER); + const rows: Array> = keepGatewayRow + ? [...models, { provider: MASTRA_GATEWAY_CATALOG_PROVIDER, hasApiKey: false }] + : models; - for (const model of models) { + for (const model of rows) { if (seen.has(model.provider)) continue; if (tenantCredentials && isDeploymentModelProvider(model.provider)) { diff --git a/mastracode/factory/src/routes/gateway-catalog.test.ts b/mastracode/factory/src/routes/gateway-catalog.test.ts new file mode 100644 index 00000000000..4c18b115aef --- /dev/null +++ b/mastracode/factory/src/routes/gateway-catalog.test.ts @@ -0,0 +1,135 @@ +import { http, HttpResponse } from 'msw'; +import { setupServer } from 'msw/node'; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { fetchMastraGatewayCatalog, resetMastraGatewayCatalogCacheForTests } from './gateway-catalog.js'; + +const BASE_URL = 'https://gateway.test'; +const MODELS_URL = `${BASE_URL}/v1/models`; + +const gateway = setupServer(); +let reply: (key: string | null) => Response | Promise; +const requests: Array = []; + +const catalog = (...ids: string[]) => Response.json({ data: ids.map(id => ({ id, endpoints: ['chat'] })) }); +const fetchFor = (apiKey: string) => fetchMastraGatewayCatalog({ apiKey, baseUrl: BASE_URL }); +const minutesLater = (minutes: number) => vi.spyOn(Date, 'now').mockReturnValue(start + minutes * 60_000); + +let start: number; +// The gateway client caches a list per key for the life of the process, so each test uses its own key. +let testNumber = 0; +let key: string; + +beforeAll(() => gateway.listen({ onUnhandledRequest: 'error' })); +afterAll(() => gateway.close()); +beforeEach(() => { + resetMastraGatewayCatalogCacheForTests(); + requests.length = 0; + start = Date.now(); + key = `mskv2_test${++testNumber}`; + vi.spyOn(console, 'warn').mockImplementation(() => {}); + gateway.use( + http.get(MODELS_URL, ({ request }) => { + const authorization = request.headers.get('authorization'); + requests.push(authorization); + return reply(authorization); + }), + ); +}); +afterEach(() => { + gateway.resetHandlers(); + vi.restoreAllMocks(); + vi.useRealTimers(); +}); + +describe('fetchMastraGatewayCatalog', () => { + it('returns the list the key may use, fetched once for repeated calls', async () => { + reply = () => catalog('anthropic/claude-opus-4-5'); + + expect((await fetchFor(key))?.models).toEqual(['anthropic/claude-opus-4-5']); + expect((await fetchFor(key))?.models).toEqual(['anthropic/claude-opus-4-5']); + expect(requests).toEqual([`Bearer ${key}`]); + }); + + it('keeps the last list through a gateway outage', async () => { + reply = () => catalog('anthropic/claude-opus-4-5'); + await fetchFor(key); + reply = () => new Response(null, { status: 503 }); + + minutesLater(5); + expect((await fetchFor(key))?.models).toEqual(['anthropic/claude-opus-4-5']); + expect(requests).toHaveLength(2); + }); + + it('keeps the last list when the gateway cannot be reached', async () => { + reply = () => catalog('anthropic/claude-opus-4-5'); + await fetchFor(key); + reply = () => HttpResponse.error(); + + minutesLater(5); + expect((await fetchFor(key))?.models).toEqual(['anthropic/claude-opus-4-5']); + }); + + it.each([401, 403])('drops the list once the gateway refuses the key with %i', async status => { + reply = () => catalog('anthropic/claude-opus-4-5'); + await fetchFor(key); + reply = () => new Response(null, { status }); + + minutesLater(5); + expect(await fetchFor(key)).toBeUndefined(); + minutesLater(5.1); + expect(await fetchFor(key)).toBeUndefined(); + }); + + it("never answers one key with another key's list", async () => { + const other = `${key}_other`; + reply = authorization => + authorization === `Bearer ${key}` ? catalog('anthropic/claude-opus-4-5') : catalog('openai/gpt-5'); + await fetchFor(key); + + expect((await fetchFor(other))?.models).toEqual(['openai/gpt-5']); + expect(requests).toEqual([`Bearer ${key}`, `Bearer ${other}`]); + }); + + it('answers with nothing when the gateway fails, without throwing, and logs it once', async () => { + reply = () => new Response(null, { status: 503 }); + + expect(await Promise.all([fetchFor(key), fetchFor(key)])).toEqual([undefined, undefined]); + minutesLater(1); + expect(await fetchFor(key)).toBeUndefined(); + + expect(console.warn).toHaveBeenCalledTimes(1); + expect(JSON.stringify(vi.mocked(console.warn).mock.calls)).not.toContain(key); + }); + + it('logs again when a gateway that recovered fails once more', async () => { + reply = () => new Response(null, { status: 503 }); + await fetchFor(key); + reply = () => catalog('anthropic/claude-opus-4-5'); + minutesLater(1); + await fetchFor(key); + reply = () => new Response(null, { status: 401 }); + minutesLater(5); + await fetchFor(key); + + expect(console.warn).toHaveBeenCalledTimes(2); + }); + + it('stops waiting for a gateway that hangs, and has the list once the fetch lands', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + let release!: () => void; + const released = new Promise(resolve => (release = resolve)); + reply = async () => { + await released; + return catalog('anthropic/claude-opus-4-5'); + }; + + const waiting = fetchFor(key); + await vi.advanceTimersByTimeAsync(3_000); + expect(await waiting).toBeUndefined(); + + release(); + await vi.waitFor(async () => expect((await fetchFor(key))?.models).toEqual(['anthropic/claude-opus-4-5'])); + expect(requests).toHaveLength(1); + }); +}); diff --git a/mastracode/factory/src/routes/gateway-catalog.ts b/mastracode/factory/src/routes/gateway-catalog.ts new file mode 100644 index 00000000000..d767c9eea3b --- /dev/null +++ b/mastracode/factory/src/routes/gateway-catalog.ts @@ -0,0 +1,70 @@ +import { createHash } from 'node:crypto'; + +import { MastraGateway } from '@mastra/core/llm'; +import type { ProviderConfig } from '@mastra/core/llm'; + +import { MASTRA_GATEWAY_CATALOG_PROVIDER } from './provider-credentials.js'; + +/** + * The longest a settings request waits for the gateway. The gateway client allows a fetch 10 s; a + * caller that stops waiting gets no gateway models this time, and the fetch still fills the + * gateway client's cache for the next request. + */ +const WAIT_MS = 3_000; +/** Keys are per tenant, so the set of keys already warned about is bounded. */ +const MAX_WARNED = 256; + +type GatewayCatalog = ProviderConfig | undefined; + +/** Hashed keys whose catalogue is failing and has been logged, until a fetch succeeds again. */ +const warned = new Set(); + +/** + * The Mastra Gateway's model catalogue as `apiKey` sees it, or the server's own when no key is + * given. A v2 key gets its organization's live list and a v1 key the bundled one, as + * `MastraGateway.fetchProviders` decides. + * + * The gateway client caches a v2 list per key and base URL, shares one fetch between concurrent + * callers, keeps serving the last list through a timeout, network error or 5xx, and drops it when + * the gateway refuses the key. This function adds what a settings page needs on top: it never + * throws, it waits a bounded time, and it logs a failing gateway once per key. + */ +export async function fetchMastraGatewayCatalog({ + apiKey, + baseUrl, +}: { + apiKey?: string; + baseUrl?: string; +}): Promise { + // Hashed so the key itself is never held or logged. + const logKey = createHash('sha256') + .update(`${baseUrl ?? ''}\n${apiKey ?? ''}`) + .digest('hex'); + const fetching = new MastraGateway({ apiKey, baseUrl }) + .fetchProviders() + .then(providers => { + warned.delete(logKey); + return providers[MASTRA_GATEWAY_CATALOG_PROVIDER]; + }) + .catch((error: unknown) => { + if (!warned.has(logKey)) { + if (warned.size >= MAX_WARNED) warned.delete(warned.values().next().value!); + warned.add(logKey); + console.warn('[Factory Config] Mastra Gateway model catalogue unavailable', { + error: error instanceof Error ? error.message : String(error), + }); + } + return undefined; + }); + + let timer: ReturnType | undefined; + const waited = new Promise(resolve => { + timer = setTimeout(() => resolve(undefined), WAIT_MS); + timer.unref?.(); + }); + return Promise.race([fetching, waited]).finally(() => clearTimeout(timer)); +} + +export function resetMastraGatewayCatalogCacheForTests() { + warned.clear(); +} From bb6c76b52eb49387585da8870b6a13a2d9839cac Mon Sep 17 00:00:00 2001 From: Rhys Balevicius Date: Tue, 17 Nov 2026 05:17:40 +0000 Subject: [PATCH 5/5] style(factory): restore repository formatting --- mastracode/factory-ui/src/api/types.ts | 4 +++- mastracode/factory/src/routes/config.ts | 10 +++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/mastracode/factory-ui/src/api/types.ts b/mastracode/factory-ui/src/api/types.ts index 9202acb642e..97f1240f132 100644 --- a/mastracode/factory-ui/src/api/types.ts +++ b/mastracode/factory-ui/src/api/types.ts @@ -249,7 +249,9 @@ export interface OAuthStartResponse { } export type OAuthPollResponse = - { status: 'pending'; nextPollMs: number } | { status: 'complete' } | { status: 'failed'; error: string }; + | { status: 'pending'; nextPollMs: number } + | { status: 'complete' } + | { status: 'failed'; error: string }; export interface UpdateOMResponse { ok: true; diff --git a/mastracode/factory/src/routes/config.ts b/mastracode/factory/src/routes/config.ts index 994c18160d9..bdf4f16cb1d 100644 --- a/mastracode/factory/src/routes/config.ts +++ b/mastracode/factory/src/routes/config.ts @@ -87,7 +87,15 @@ import { resolveTenantProviderDefaultModel } from './tenant-credentials.js'; * the server process's own credentials (e.g. Amazon Bedrock's AWS chain). */ export type ProviderCredentialSource = - 'deployment' | 'oauth' | 'stored' | 'env' | 'none' | 'oauth-user' | 'oauth-org' | 'stored-user' | 'stored-org'; + | 'deployment' + | 'oauth' + | 'stored' + | 'env' + | 'none' + | 'oauth-user' + | 'oauth-org' + | 'stored-user' + | 'stored-org'; /** A model provider with the current source of its credentials. */ export interface ProviderInfo {