From 75ed0cbce9be158e00f20b39ede0428dc3bcddbd Mon Sep 17 00:00:00 2001 From: Rhys Balevicius Date: Tue, 17 Nov 2026 03:57:57 +0000 Subject: [PATCH 1/2] fix(core): correct gateway catalogs, version routing and stream usage Authenticate and cache v2 model catalogs with bounded fallback and retry delays to reduce waits during outages. Keep organization catalogs out of the shared provider cache so they stay isolated across projects. Honor v1 project keys and registered gateway settings so Studio and model calls agree on the gateway version. Request streaming token usage so final usage chunks are reported. --- .changeset/quiet-gateways-count.md | 10 + .changeset/steady-gateway-versions.md | 5 + .../src/llm/model/gateways/mastra.test.ts | 396 ++++++++++++++++-- .../core/src/llm/model/gateways/mastra.ts | 207 +++++++-- .../src/llm/model/provider-registry.test.ts | 89 ++++ .../core/src/llm/model/provider-registry.ts | 9 +- .../server/handlers/gateway-memory-client.ts | 35 +- .../handlers/gateway-memory-client.v2.test.ts | 92 ++++ 8 files changed, 778 insertions(+), 65 deletions(-) create mode 100644 .changeset/quiet-gateways-count.md create mode 100644 .changeset/steady-gateway-versions.md diff --git a/.changeset/quiet-gateways-count.md b/.changeset/quiet-gateways-count.md new file mode 100644 index 00000000000..d641567a602 --- /dev/null +++ b/.changeset/quiet-gateways-count.md @@ -0,0 +1,10 @@ +--- +'@mastra/core': patch +--- + +Improved how `@mastra/core` works with the Mastra Gateway: + +- Streamed calls through the Gateway's chat endpoint now ask for token usage, so `agent.stream()` reports usage when the upstream provider sends it in a separate final chunk. Before, usage could be empty. +- The Gateway model list for organization keys is cached for a minute and shared between concurrent callers. If a refresh times out or the Gateway has an outage, the last list is kept for up to ten minutes. With no earlier list, the failure is reported again for 15 seconds without a new request, so Studio does not wait for the request timeout on every load during an outage. If the Gateway rejects the key (401 or 403), the cached list is dropped. The model list request now always sends the key in `Authorization` and no longer goes through a `customFetch`, which the v2 Gateway would not have authenticated. +- A `msk_` project key now always uses the v1 Gateway, even when `MASTRA_GATEWAY_VERSION=v2` is set. +- `mastra dev` no longer writes one organization's Gateway model list to the provider cache shared by other projects on the machine, whether the key comes from the environment or from a registered `MastraGateway`. A failing Gateway model list no longer stops the provider registry refresh. diff --git a/.changeset/steady-gateway-versions.md b/.changeset/steady-gateway-versions.md new file mode 100644 index 00000000000..3a31dbbd23d --- /dev/null +++ b/.changeset/steady-gateway-versions.md @@ -0,0 +1,5 @@ +--- +'@mastra/server': patch +--- + +A `msk_` project key now always uses the v1 Gateway memory API, even when `MASTRA_GATEWAY_VERSION=v2` is set. Studio follows a registered `MastraGateway` and its own key when deciding between the v1 and v2 Gateway, so it agrees with the model calls. diff --git a/packages/core/src/llm/model/gateways/mastra.test.ts b/packages/core/src/llm/model/gateways/mastra.test.ts index 470f07ecc9d..a67302fe677 100644 --- a/packages/core/src/llm/model/gateways/mastra.test.ts +++ b/packages/core/src/llm/model/gateways/mastra.test.ts @@ -1,6 +1,6 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { MastraGateway } from './mastra.js'; +import { _resetMastraGatewayCatalogCache, MastraGateway } from './mastra.js'; vi.mock('../provider-registry.js', () => ({ PROVIDER_REGISTRY: { openrouter: { models: ['openai/test-model'] } }, @@ -10,14 +10,22 @@ describe('MastraGateway', () => { beforeEach(() => { delete process.env.MASTRA_GATEWAY_API_KEY; delete process.env.MASTRA_GATEWAY_VERSION; + _resetMastraGatewayCatalogCache(); }); afterEach(() => { delete process.env.MASTRA_GATEWAY_API_KEY; delete process.env.MASTRA_GATEWAY_VERSION; vi.restoreAllMocks(); + vi.unstubAllGlobals(); }); + /** A v2 gateway whose catalogue request is answered by `gatewayFetch` in place of the global fetch. */ + function gatewayWith(apiKey: string, gatewayFetch: (...args: any[]) => Promise) { + vi.stubGlobal('fetch', gatewayFetch); + return new MastraGateway({ apiKey }); + } + it('reports disabled when MASTRA_GATEWAY_API_KEY is not set', () => { const gateway = new MastraGateway(); @@ -62,40 +70,38 @@ describe('MastraGateway', () => { { status: 200, headers: { 'content-type': 'application/json' } }, ), ); - const gateway = new MastraGateway({ - apiKey: 'deployment-key', - baseUrl: 'https://gateway.example/v1', - customFetch: gatewayFetch, - }); + vi.stubGlobal('fetch', gatewayFetch); + const gateway = new MastraGateway({ apiKey: 'deployment-key', baseUrl: 'https://gateway.example/v1' }); const providers = await gateway.fetchProviders(); expect(gatewayFetch).toHaveBeenCalledWith( 'https://gateway.example/v1/models', expect.objectContaining({ - headers: expect.objectContaining({ 'X-Memory-Gateway-Authorization': 'Bearer deployment-key' }), + headers: { Authorization: 'Bearer deployment-key' }, signal: expect.any(AbortSignal), }), ); expect(providers.mastra?.models).toEqual(['openai/chat-model']); }); - it('preserves the Gateway key when customFetch replaces Authorization with OAuth', async () => { + it('sends the v2 catalog request with the key in Authorization, not through a custom fetch', async () => { process.env.MASTRA_GATEWAY_VERSION = 'v2'; - const customFetch = vi.fn(async (_input: RequestInfo | URL, init?: RequestInit) => { - const headers = new Headers(init?.headers); - headers.set('Authorization', 'Bearer oauth-token'); - expect(headers.get('Authorization')).toBe('Bearer oauth-token'); - expect(headers.get('X-Memory-Gateway-Authorization')).toBe('Bearer deployment-key'); - return new Response(JSON.stringify({ data: [{ id: 'openai/chat-model', endpoints: ['chat'] }] }), { - status: 200, - }); - }); + const customFetch = vi.fn(async () => new Response('{}', { status: 200 })); + const globalFetch = vi.fn( + async (_input: RequestInfo | URL, _init?: RequestInit) => + new Response(JSON.stringify({ data: [{ id: 'openai/chat-model', endpoints: ['chat'] }] }), { status: 200 }), + ); + vi.stubGlobal('fetch', globalFetch); const gateway = new MastraGateway({ apiKey: 'deployment-key', customFetch }); const providers = await gateway.fetchProviders(); - expect(customFetch).toHaveBeenCalledOnce(); + expect(customFetch).not.toHaveBeenCalled(); + expect(globalFetch).toHaveBeenCalledOnce(); + const headers = new Headers(globalFetch.mock.calls[0]![1]?.headers); + expect(headers.get('Authorization')).toBe('Bearer deployment-key'); + expect(headers.get('X-Memory-Gateway-Authorization')).toBeNull(); expect(providers.mastra?.models).toEqual(['openai/chat-model']); }); @@ -103,16 +109,16 @@ describe('MastraGateway', () => { process.env.MASTRA_GATEWAY_VERSION = 'v2'; const controller = new AbortController(); const timeout = vi.spyOn(AbortSignal, 'timeout').mockReturnValue(controller.signal); - const customFetch = vi.fn( + const gatewayFetch = vi.fn( (_input: RequestInfo | URL, init?: RequestInit) => new Promise((_resolve, reject) => { init?.signal?.addEventListener('abort', () => reject(init.signal?.reason), { once: true }); }), ); - const gateway = new MastraGateway({ apiKey: 'deployment-key', customFetch }); + const gateway = gatewayWith('deployment-key', gatewayFetch); const pending = gateway.fetchProviders(); - await vi.waitFor(() => expect(customFetch).toHaveBeenCalledOnce()); + await vi.waitFor(() => expect(gatewayFetch).toHaveBeenCalledOnce()); controller.abort(new DOMException('catalog deadline', 'TimeoutError')); await expect(pending).rejects.toThrow('catalog deadline'); @@ -124,7 +130,7 @@ describe('MastraGateway', () => { async () => new Response(JSON.stringify({ data: [{ id: 'openai/chat-model', endpoints: ['chat'] }] }), { status: 200 }), ); - const gateway = new MastraGateway({ apiKey: 'mskv2_org-key', customFetch: gatewayFetch }); + const gateway = gatewayWith('mskv2_org-key', gatewayFetch); const providers = await gateway.fetchProviders(); @@ -133,15 +139,353 @@ describe('MastraGateway', () => { expect(providers.mastra?.modelReleaseDates).toBeUndefined(); }); + describe('isV2', () => { + it.each([ + { version: undefined, key: 'msk_0123456789abcdef0123456789abcdef', expected: false }, + { version: 'v2', key: 'msk_0123456789abcdef0123456789abcdef', expected: false }, + { version: 'v2', key: 'deployment-key', expected: true }, + { version: 'v2', key: undefined, expected: true }, + { version: undefined, key: 'mskv2_org-key', expected: true }, + { version: undefined, key: 'msk-gateway-org-key', expected: true }, + { version: undefined, key: 'other-key', expected: false }, + ])('with MASTRA_GATEWAY_VERSION=$version and key $key is $expected', ({ version, key, expected }) => { + if (version) process.env.MASTRA_GATEWAY_VERSION = version; + expect(new MastraGateway({ apiKey: key }).isV2()).toBe(expected); + }); + + it('lets a v1 config key win over a v2 env key and MASTRA_GATEWAY_VERSION', () => { + process.env.MASTRA_GATEWAY_VERSION = 'v2'; + process.env.MASTRA_GATEWAY_API_KEY = 'mskv2_org-key'; + expect(new MastraGateway({ apiKey: 'msk_0123456789abcdef0123456789abcdef' }).isV2()).toBe(false); + }); + }); + it('does not show a static v1 model list if v2 catalog loading fails', async () => { process.env.MASTRA_GATEWAY_VERSION = 'v2'; - const gateway = new MastraGateway({ - apiKey: 'deployment-key', - customFetch: vi.fn(async () => new Response('unavailable', { status: 503 })), - }); + const gateway = gatewayWith( + 'deployment-key', + vi.fn(async () => new Response('unavailable', { status: 503 })), + ); await expect(gateway.fetchProviders()).rejects.toThrow('Gateway model catalog unavailable (503)'); }); + describe('v2 catalog cache', () => { + const catalog = (ids: string[]) => + new Response(JSON.stringify({ data: ids.map(id => ({ id, endpoints: ['chat'] })) }), { status: 200 }); + + beforeEach(() => { + process.env.MASTRA_GATEWAY_VERSION = 'v2'; + }); + + it('reuses the catalog for a minute, then fetches it again', async () => { + const now = vi.spyOn(Date, 'now').mockReturnValue(1_000_000); + const gatewayFetch = vi.fn(async () => catalog(['openai/chat-model'])); + const gateway = gatewayWith('deployment-key', gatewayFetch); + + await gateway.fetchProviders(); + now.mockReturnValue(1_000_000 + 59_000); + await gateway.fetchProviders(); + expect(gatewayFetch).toHaveBeenCalledOnce(); + + now.mockReturnValue(1_000_000 + 61_000); + await gateway.fetchProviders(); + expect(gatewayFetch).toHaveBeenCalledTimes(2); + }); + + it('shares one request between concurrent callers', async () => { + let respond!: (response: Response) => void; + const gatewayFetch = vi.fn(() => new Promise(resolve => (respond = resolve))); + const gateway = gatewayWith('deployment-key', gatewayFetch); + + const first = gateway.fetchProviders(); + const second = gatewayWith('deployment-key', gatewayFetch).fetchProviders(); + await vi.waitFor(() => expect(gatewayFetch).toHaveBeenCalledOnce()); + respond(catalog(['openai/chat-model'])); + + expect((await first).mastra?.models).toEqual(['openai/chat-model']); + expect((await second).mastra?.models).toEqual(['openai/chat-model']); + expect(gatewayFetch).toHaveBeenCalledOnce(); + }); + + it('keeps catalogs for different keys apart', async () => { + const gatewayFetch = vi.fn(async (_input: RequestInfo | URL, init?: RequestInit) => { + const key = new Headers(init?.headers).get('Authorization'); + return catalog([key === 'Bearer key-a' ? 'openai/model-a' : 'openai/model-b']); + }); + + const a = await gatewayWith('key-a', gatewayFetch).fetchProviders(); + const b = await gatewayWith('key-b', gatewayFetch).fetchProviders(); + + expect(a.mastra?.models).toEqual(['openai/model-a']); + expect(b.mastra?.models).toEqual(['openai/model-b']); + }); + + it('serves the last good catalog for up to ten minutes when a refresh fails', async () => { + const now = vi.spyOn(Date, 'now').mockReturnValue(1_000_000); + const gatewayFetch = vi.fn(async () => catalog(['openai/chat-model'])); + const gateway = gatewayWith('deployment-key', gatewayFetch); + await gateway.fetchProviders(); + + gatewayFetch.mockImplementation(async () => new Response('unavailable', { status: 503 })); + now.mockReturnValue(1_000_000 + 9 * 60_000); + expect((await gateway.fetchProviders()).mastra?.models).toEqual(['openai/chat-model']); + + now.mockReturnValue(1_000_000 + 11 * 60_000); + await expect(gateway.fetchProviders()).rejects.toThrow('Gateway model catalog unavailable (503)'); + }); + + it.each([401, 403])('drops the cached catalog when the key is rejected with %i', async status => { + const now = vi.spyOn(Date, 'now').mockReturnValue(1_000_000); + const gatewayFetch = vi.fn(async () => catalog(['openai/chat-model'])); + const gateway = gatewayWith('deployment-key', gatewayFetch); + await gateway.fetchProviders(); + + gatewayFetch.mockImplementation(async () => new Response('rejected', { status })); + now.mockReturnValue(1_000_000 + 61_000); + await expect(gateway.fetchProviders()).rejects.toThrow(`Gateway model catalog unavailable (${status})`); + + gatewayFetch.mockImplementation(async () => catalog(['openai/other-model'])); + expect((await gateway.fetchProviders()).mastra?.models).toEqual(['openai/other-model']); + }); + + it('serves the last good catalog after a network error', async () => { + const now = vi.spyOn(Date, 'now').mockReturnValue(1_000_000); + const gatewayFetch = vi.fn(async () => catalog(['openai/chat-model'])); + const gateway = gatewayWith('deployment-key', gatewayFetch); + await gateway.fetchProviders(); + + gatewayFetch.mockRejectedValue(new TypeError('fetch failed')); + now.mockReturnValue(1_000_000 + 61_000); + expect((await gateway.fetchProviders()).mastra?.models).toEqual(['openai/chat-model']); + }); + + it('waits before retrying after a failed refresh', async () => { + const now = vi.spyOn(Date, 'now').mockReturnValue(1_000_000); + const gatewayFetch = vi.fn(async () => catalog(['openai/chat-model'])); + const gateway = gatewayWith('deployment-key', gatewayFetch); + await gateway.fetchProviders(); + + gatewayFetch.mockImplementation(async () => new Response('unavailable', { status: 503 })); + now.mockReturnValue(1_000_000 + 61_000); + await gateway.fetchProviders(); + expect(gatewayFetch).toHaveBeenCalledTimes(2); + + now.mockReturnValue(1_000_000 + 61_000 + 14_000); + expect((await gateway.fetchProviders()).mastra?.models).toEqual(['openai/chat-model']); + expect(gatewayFetch).toHaveBeenCalledTimes(2); + + now.mockReturnValue(1_000_000 + 61_000 + 16_000); + expect((await gateway.fetchProviders()).mastra?.models).toEqual(['openai/chat-model']); + expect(gatewayFetch).toHaveBeenCalledTimes(3); + }); + + it('keeps catalogs fetched through different global fetch implementations apart', async () => { + const firstFetch = vi.fn(async () => catalog(['openai/model-a'])); + const secondFetch = vi.fn(async () => catalog(['openai/model-b'])); + + const a = await gatewayWith('same-key', firstFetch).fetchProviders(); + const b = await gatewayWith('same-key', secondFetch).fetchProviders(); + + expect(a.mastra?.models).toEqual(['openai/model-a']); + expect(b.mastra?.models).toEqual(['openai/model-b']); + expect(secondFetch).toHaveBeenCalledOnce(); + }); + + it('keeps at most 100 catalogs, dropping the oldest', async () => { + const gatewayFetch = vi.fn(async () => catalog(['openai/chat-model'])); + for (let i = 0; i <= 100; i++) { + await gatewayWith(`key-${i}`, gatewayFetch).fetchProviders(); + } + expect(gatewayFetch).toHaveBeenCalledTimes(101); + + await gatewayWith('key-100', gatewayFetch).fetchProviders(); + expect(gatewayFetch).toHaveBeenCalledTimes(101); + await gatewayWith('key-0', gatewayFetch).fetchProviders(); + expect(gatewayFetch).toHaveBeenCalledTimes(102); + }); + + it('remembers a failure with no earlier catalog for 15 seconds', async () => { + const now = vi.spyOn(Date, 'now').mockReturnValue(1_000_000); + const gatewayFetch = vi.fn(async () => new Response('unavailable', { status: 503 })); + const gateway = gatewayWith('deployment-key', gatewayFetch); + + await expect(gateway.fetchProviders()).rejects.toThrow('Gateway model catalog unavailable (503)'); + expect(gatewayFetch).toHaveBeenCalledOnce(); + + now.mockReturnValue(1_000_000 + 14_000); + await expect(gateway.fetchProviders()).rejects.toThrow('Gateway model catalog unavailable (503)'); + expect(gatewayFetch).toHaveBeenCalledOnce(); + + now.mockReturnValue(1_000_000 + 16_000); + await expect(gateway.fetchProviders()).rejects.toThrow('Gateway model catalog unavailable (503)'); + expect(gatewayFetch).toHaveBeenCalledTimes(2); + }); + + it('remembers a network error with no earlier catalog', async () => { + const now = vi.spyOn(Date, 'now').mockReturnValue(1_000_000); + const gatewayFetch = vi.fn(async () => { + throw new TypeError('fetch failed'); + }); + const gateway = gatewayWith('deployment-key', gatewayFetch); + + await expect(gateway.fetchProviders()).rejects.toThrow('fetch failed'); + now.mockReturnValue(1_000_000 + 5_000); + await expect(gateway.fetchProviders()).rejects.toThrow('fetch failed'); + expect(gatewayFetch).toHaveBeenCalledOnce(); + }); + + it('clears a remembered failure on success, and the next failure starts a new window', async () => { + const now = vi.spyOn(Date, 'now').mockReturnValue(1_000_000); + const gatewayFetch = vi.fn(async () => new Response('unavailable', { status: 503 })); + const gateway = gatewayWith('deployment-key', gatewayFetch); + await expect(gateway.fetchProviders()).rejects.toThrow('(503)'); + + gatewayFetch.mockImplementation(async () => catalog(['openai/chat-model'])); + now.mockReturnValue(1_000_000 + 16_000); + expect((await gateway.fetchProviders()).mastra?.models).toEqual(['openai/chat-model']); + expect(gatewayFetch).toHaveBeenCalledTimes(2); + + // Past the stale window there is no list to fall back on, so a failure is remembered again. + gatewayFetch.mockImplementation(async () => new Response('unavailable', { status: 503 })); + now.mockReturnValue(1_000_000 + 16_000 + 11 * 60_000); + await expect(gateway.fetchProviders()).rejects.toThrow('(503)'); + expect(gatewayFetch).toHaveBeenCalledTimes(3); + now.mockReturnValue(1_000_000 + 16_000 + 11 * 60_000 + 14_000); + await expect(gateway.fetchProviders()).rejects.toThrow('(503)'); + expect(gatewayFetch).toHaveBeenCalledTimes(3); + now.mockReturnValue(1_000_000 + 16_000 + 11 * 60_000 + 16_000); + await expect(gateway.fetchProviders()).rejects.toThrow('(503)'); + expect(gatewayFetch).toHaveBeenCalledTimes(4); + }); + + it.each([401, 403])('asks the gateway again on every call after a %i', async status => { + vi.spyOn(Date, 'now').mockReturnValue(1_000_000); + const gatewayFetch = vi.fn(async () => new Response('rejected', { status })); + const gateway = gatewayWith('deployment-key', gatewayFetch); + + await expect(gateway.fetchProviders()).rejects.toThrow(`(${status})`); + await expect(gateway.fetchProviders()).rejects.toThrow(`(${status})`); + expect(gatewayFetch).toHaveBeenCalledTimes(2); + + gatewayFetch.mockImplementation(async () => catalog(['openai/chat-model'])); + expect((await gateway.fetchProviders()).mastra?.models).toEqual(['openai/chat-model']); + }); + + it('shares one failing request between concurrent callers', async () => { + let respond!: (response: Response) => void; + const gatewayFetch = vi.fn(() => new Promise(resolve => (respond = resolve))); + const gateway = gatewayWith('deployment-key', gatewayFetch); + + const first = gateway.fetchProviders(); + const second = gateway.fetchProviders(); + await vi.waitFor(() => expect(gatewayFetch).toHaveBeenCalledOnce()); + respond(new Response('unavailable', { status: 503 })); + + await expect(first).rejects.toThrow('(503)'); + await expect(second).rejects.toThrow('(503)'); + expect(gatewayFetch).toHaveBeenCalledOnce(); + }); + + it('counts remembered failures toward the 100 catalog limit', async () => { + const gatewayFetch = vi.fn(async () => new Response('unavailable', { status: 503 })); + for (let i = 0; i <= 100; i++) { + await expect(gatewayWith(`key-${i}`, gatewayFetch).fetchProviders()).rejects.toThrow('(503)'); + } + expect(gatewayFetch).toHaveBeenCalledTimes(101); + + await expect(gatewayWith('key-100', gatewayFetch).fetchProviders()).rejects.toThrow('(503)'); + expect(gatewayFetch).toHaveBeenCalledTimes(101); + await expect(gatewayWith('key-0', gatewayFetch).fetchProviders()).rejects.toThrow('(503)'); + expect(gatewayFetch).toHaveBeenCalledTimes(102); + }); + + it('forgets remembered failures when the cache is reset', async () => { + const gatewayFetch = vi.fn(async () => new Response('unavailable', { status: 503 })); + const gateway = gatewayWith('deployment-key', gatewayFetch); + await expect(gateway.fetchProviders()).rejects.toThrow('(503)'); + + _resetMastraGatewayCatalogCache(); + await expect(gateway.fetchProviders()).rejects.toThrow('(503)'); + expect(gatewayFetch).toHaveBeenCalledTimes(2); + }); + }); + + describe('resolveLanguageModel streaming usage', () => { + const prompt = [{ role: 'user', content: [{ type: 'text', text: 'hi' }] }]; + + function captureBodies() { + const bodies: Record[] = []; + const fetchImpl = vi.fn(async (_input: any, init: any) => { + bodies.push(JSON.parse(String(init?.body))); + return new Response('{}', { status: 200, headers: { 'content-type': 'application/json' } }); + }); + return { bodies, fetchImpl }; + } + + async function send(model: any) { + await model.doStream({ prompt }).catch(() => {}); + await model.doGenerate({ prompt }).catch(() => {}); + } + + it('asks for usage on streaming requests only with an API key', async () => { + const { bodies, fetchImpl } = captureBodies(); + vi.stubGlobal('fetch', fetchImpl); + try { + const model = new MastraGateway({ apiKey: 'gateway-key' }).resolveLanguageModel({ + modelId: 'gpt-4o', + providerId: 'openai', + apiKey: 'gateway-key', + }); + await send(model); + } finally { + vi.unstubAllGlobals(); + } + + expect(bodies).toHaveLength(2); + expect(bodies[0]).toMatchObject({ stream: true, stream_options: { include_usage: true } }); + expect(bodies[1]).not.toHaveProperty('stream_options'); + expect(bodies[1]).not.toHaveProperty('usage'); + }); + + it('reports usage sent in a separate final chunk', async () => { + const frames = [ + { id: 'c1', choices: [{ index: 0, delta: { role: 'assistant', content: 'ok' } }] }, + { id: 'c1', choices: [{ index: 0, delta: {}, finish_reason: 'stop' }] }, + { id: 'c1', choices: [], usage: { prompt_tokens: 7, completion_tokens: 3, total_tokens: 10 } }, + ]; + const body = frames.map(frame => `data: ${JSON.stringify(frame)}\n\n`).join('') + 'data: [DONE]\n\n'; + const customFetch = vi.fn( + async () => new Response(body, { status: 200, headers: { 'content-type': 'text/event-stream' } }), + ); + const model: any = new MastraGateway({ + apiKey: 'gateway-key', + customFetch: customFetch as any, + }).resolveLanguageModel({ modelId: 'gpt-4o', providerId: 'openai', apiKey: 'gateway-key' }); + + const { stream } = await model.doStream({ prompt }); + const parts: any[] = []; + for await (const part of stream) parts.push(part); + + const finish = parts.find(part => part.type === 'finish'); + expect(finish?.usage).toMatchObject({ inputTokens: { total: 7 }, outputTokens: { total: 3 } }); + }); + + it('asks for usage on streaming requests only with a custom fetch', async () => { + const { bodies, fetchImpl } = captureBodies(); + const model = new MastraGateway({ apiKey: 'gateway-key', customFetch: fetchImpl as any }).resolveLanguageModel({ + modelId: 'gpt-4o', + providerId: 'openai', + apiKey: 'gateway-key', + }); + await send(model); + + expect(bodies).toHaveLength(2); + expect(bodies[0]).toMatchObject({ stream: true, stream_options: { include_usage: true } }); + expect(bodies[1]).not.toHaveProperty('stream_options'); + expect(bodies[1]).not.toHaveProperty('usage'); + }); + }); + describe('resolveLanguageModel gateway authorization header', () => { const GATEWAY_AUTH_HEADER = 'X-Memory-Gateway-Authorization'; diff --git a/packages/core/src/llm/model/gateways/mastra.ts b/packages/core/src/llm/model/gateways/mastra.ts index cf6f9b451b4..5a38839a898 100644 --- a/packages/core/src/llm/model/gateways/mastra.ts +++ b/packages/core/src/llm/model/gateways/mastra.ts @@ -1,3 +1,4 @@ +import { createHash } from 'node:crypto'; import { createAnthropic } from '@ai-sdk/anthropic-v6'; import { createOpenRouter } from '@openrouter/ai-sdk-provider-v6'; import { MastraError } from '../../../error/index.js'; @@ -64,15 +65,19 @@ export class MastraGateway extends MastraModelGateway { /** * Gateway v2 deployments are signaled by MASTRA_GATEWAY_VERSION=v2 (set by the platform) - * or by a v2 (organization) API key prefix, which predates the env var signal. + * or by a v2 (organization) API key prefix, which predates the env var signal. A v1 project + * key (`msk_`) is always served by v1, so it wins over the env var. * Keep in sync with isGatewayV2 in @mastra/server's gateway-memory-client. * Public so the server can ask a registered gateway instance for its - * effective v2 state (the instance may carry a v2 key via config.apiKey + * effective v2 state (the instance may carry a key via config.apiKey * that env-only checks cannot see). */ isV2(): boolean { - if (process.env['MASTRA_GATEWAY_VERSION'] === 'v2') return true; const key = this.config?.apiKey ?? process.env['MASTRA_GATEWAY_API_KEY']; + // A v1 project key (`msk_`) is always served by v1, so it wins over the env var. + // The v2 prefixes (`msk-gateway-`, `mskv2_`) do not start with `msk_`. + if (key?.startsWith('msk_')) return false; + if (process.env['MASTRA_GATEWAY_VERSION'] === 'v2') return true; return !!key && (key.startsWith('msk-gateway-') || key.startsWith('mskv2_')); } @@ -82,27 +87,12 @@ export class MastraGateway extends MastraModelGateway { let models = openrouterConfig?.models ?? []; if (isV2) { const apiKey = await this.getApiKey(); - const response = await (this.config?.customFetch ?? fetch)(`${this.getBaseUrl()}/v1/models`, { - headers: this.config?.customFetch ? withGatewayAuthHeader(apiKey) : { Authorization: `Bearer ${apiKey}` }, - signal: AbortSignal.timeout(10_000), - }); - if (!response.ok) throw new Error(`Gateway model catalog unavailable (${response.status})`); - const body: unknown = await response.json(); - if (!body || typeof body !== 'object' || !('data' in body) || !Array.isArray(body.data)) { - throw new Error('Gateway model catalog has an invalid response'); - } - // The Mastra gateway model uses the OpenAI-compatible chat transport. - // Models offered only through another endpoint cannot be selected here. - models = body.data - .filter( - (item): item is { id: string; endpoints: string[] } => - !!item && - typeof item === 'object' && - typeof item.id === 'string' && - Array.isArray(item.endpoints) && - item.endpoints.includes('chat'), - ) - .map(item => item.id); + // Gateway v2 reads the key only from Authorization or x-api-key. A custom fetch is a v1 + // pass-through that owns Authorization, so the catalogue request uses the plain global fetch. + const fetchImpl = globalThis.fetch; + models = await loadCachedV2Catalog(fetchImpl, `${this.getBaseUrl()}\n${hashApiKey(apiKey)}`, () => + this.fetchV2Catalog(fetchImpl, apiKey), + ); } const providers = { @@ -121,6 +111,32 @@ export class MastraGateway extends MastraModelGateway { return providers; } + private async fetchV2Catalog(fetchImpl: typeof fetch, apiKey: string): Promise { + const response = await fetchImpl(`${this.getBaseUrl()}/v1/models`, { + headers: { Authorization: `Bearer ${apiKey}` }, + signal: AbortSignal.timeout(10_000), + }); + if (!response.ok) { + throw new GatewayCatalogError(`Gateway model catalog unavailable (${response.status})`, response.status); + } + const body: unknown = await response.json(); + if (!body || typeof body !== 'object' || !('data' in body) || !Array.isArray(body.data)) { + throw new Error('Gateway model catalog has an invalid response'); + } + // The Mastra gateway model uses the OpenAI-compatible chat transport. + // Models offered only through another endpoint cannot be selected here. + return body.data + .filter( + (item): item is { id: string; endpoints: string[] } => + !!item && + typeof item === 'object' && + typeof item.id === 'string' && + Array.isArray(item.endpoints) && + item.endpoints.includes('chat'), + ) + .map(item => item.id); + } + async buildUrl(_modelId: string): Promise { return `${this.getBaseUrl()}/v1`; } @@ -169,6 +185,9 @@ export class MastraGateway extends MastraModelGateway { baseURL, headers: withGatewayAuthHeader(apiKey, headers), fetch: this.config.customFetch, + // Strict mode adds stream_options.include_usage to streaming requests only, + // so routes that report usage in a separate final chunk still return it. + compatibility: 'strict', }).chat(fullModelId) as unknown as GatewayLanguageModel; } @@ -180,6 +199,146 @@ export class MastraGateway extends MastraModelGateway { 'User-Agent': MASTRA_USER_AGENT, ...headers, }, + // Strict mode adds stream_options.include_usage to streaming requests only. + compatibility: 'strict', }).chat(fullModelId) as unknown as GatewayLanguageModel; } } + +/** How long a v2 model catalogue is reused before it is fetched again. */ +const V2_CATALOG_TTL_MS = 60_000; +/** How long the last good v2 catalogue is still served while refreshes fail. */ +const V2_CATALOG_STALE_IF_ERROR_MS = 10 * 60_000; +/** + * How long to wait after a failed request before trying again: the last good catalogue is served + * meanwhile, or, when there is none, the failure is thrown again. + */ +const V2_CATALOG_RETRY_AFTER_ERROR_MS = 15_000; +/** Most catalogues kept per fetch implementation. */ +const V2_CATALOG_MAX_ENTRIES = 100; + +/** A catalogue request that got an HTTP error status. */ +class GatewayCatalogError extends Error { + constructor( + message: string, + readonly status: number, + ) { + super(message); + this.name = 'GatewayCatalogError'; + } +} + +interface V2CatalogEntry { + models?: string[]; + fetchedAt?: number; + retryAt?: number; + pending?: Promise; + /** A failure with no list to fall back on, rethrown until `failedUntil` without a new request. */ + failure?: unknown; + failedUntil?: number; +} + +/** + * v2 model catalogues, first by the global fetch in effect when the request was made, then by + * base URL and API key hash. A replaced global fetch (instrumentation, test doubles) gets its own + * entries, so a list from one is never served through another. The raw key is never stored. + */ +let v2CatalogCaches = new WeakMap>(); + +/** Only an in-memory cache key, so the raw key is not held. Not a password hash. */ +function hashApiKey(apiKey: string): string { + return createHash('sha256').update(apiKey).digest('hex'); +} + +/** A 401 or 403 means the key was rejected, so a list cached for it must not be served. */ +function isRejectedKey(error: unknown): boolean { + return error instanceof GatewayCatalogError && (error.status === 401 || error.status === 403); +} + +/** + * Drops settled entries past the stale window (keeping a failure that is still remembered), then + * the oldest settled ones over the size cap. + */ +function pruneV2Catalogs(cache: Map, now: number): void { + for (const [key, entry] of cache) { + const failureRemembered = entry.failedUntil !== undefined && now < entry.failedUntil; + const listExpired = entry.fetchedAt === undefined || now - entry.fetchedAt >= V2_CATALOG_STALE_IF_ERROR_MS; + if (!entry.pending && !failureRemembered && listExpired) cache.delete(key); + } + for (const [key, entry] of cache) { + if (cache.size <= V2_CATALOG_MAX_ENTRIES) break; + if (!entry.pending) cache.delete(key); + } +} + +/** + * Returns the cached v2 catalogue for `cacheKey`, fetching it when it is missing or older than + * the TTL. Concurrent callers share one request. When a refresh fails with a timeout, a network + * error or a server error, the last good catalogue is returned for a bounded time, and the next + * attempt waits a short while. Without a catalogue to fall back on, the error is thrown, and the + * same error is thrown again without a new request for that short while, so callers during an + * outage do not each wait for the request timeout. A rejected key (401 or 403) drops the cached + * catalogue and is never remembered, so every call asks the gateway again. + */ +function loadCachedV2Catalog( + fetchImpl: typeof fetch, + cacheKey: string, + load: () => Promise, +): Promise { + let cache = v2CatalogCaches.get(fetchImpl); + if (!cache) { + cache = new Map(); + v2CatalogCaches.set(fetchImpl, cache); + } + const entries = cache; + const entry = entries.get(cacheKey); + const now = Date.now(); + if (entry?.models && entry.fetchedAt !== undefined) { + const age = now - entry.fetchedAt; + const fresh = age < V2_CATALOG_TTL_MS; + const backingOff = entry.retryAt !== undefined && now < entry.retryAt && age < V2_CATALOG_STALE_IF_ERROR_MS; + if (fresh || backingOff) return Promise.resolve(entry.models); + } + if (entry?.failedUntil !== undefined && now < entry.failedUntil) return Promise.reject(entry.failure); + if (entry?.pending) return entry.pending; + + const pending = load().then( + models => { + const settledAt = Date.now(); + entries.delete(cacheKey); + entries.set(cacheKey, { models, fetchedAt: settledAt }); + pruneV2Catalogs(entries, settledAt); + return models; + }, + error => { + const previous = entries.get(cacheKey); + const settledAt = Date.now(); + if ( + !isRejectedKey(error) && + previous?.models && + previous.fetchedAt !== undefined && + settledAt - previous.fetchedAt < V2_CATALOG_STALE_IF_ERROR_MS + ) { + entries.set(cacheKey, { + models: previous.models, + fetchedAt: previous.fetchedAt, + retryAt: settledAt + V2_CATALOG_RETRY_AFTER_ERROR_MS, + }); + return previous.models; + } + entries.delete(cacheKey); + if (!isRejectedKey(error)) { + entries.set(cacheKey, { failure: error, failedUntil: settledAt + V2_CATALOG_RETRY_AFTER_ERROR_MS }); + pruneV2Catalogs(entries, settledAt); + } + throw error; + }, + ); + entries.set(cacheKey, { ...entry, pending }); + return pending; +} + +/** @internal Clear the cached v2 model catalogues. For testing only. */ +export function _resetMastraGatewayCatalogCache(): void { + v2CatalogCaches = new WeakMap(); +} diff --git a/packages/core/src/llm/model/provider-registry.test.ts b/packages/core/src/llm/model/provider-registry.test.ts index 3af1f3b27aa..1674bf3271c 100644 --- a/packages/core/src/llm/model/provider-registry.test.ts +++ b/packages/core/src/llm/model/provider-registry.test.ts @@ -933,6 +933,95 @@ describe('GatewayRegistry Auto-Refresh', () => { fs.rmSync(tmpDir, { recursive: true, force: true }); }); + describe('Mastra gateway in the shared cache', () => { + const envKeys = ['MASTRA_GATEWAY_VERSION', 'MASTRA_GATEWAY_API_KEY'] as const; + let savedEnv: Record = {}; + + beforeEach(() => { + savedEnv = Object.fromEntries(envKeys.map(key => [key, process.env[key]])); + for (const key of envKeys) delete process.env[key]; + }); + + afterEach(() => { + for (const key of envKeys) { + if (savedEnv[key] === undefined) delete process.env[key]; + else process.env[key] = savedEnv[key]; + } + GatewayRegistry.getInstance({ useDynamicLoading: true }).registerCustomGateways([]); + }); + + async function sync(mastraCatalog: () => Promise>) { + const registry = GatewayRegistry.getInstance({ useDynamicLoading: true }); + const writtenFiles: string[] = []; + vi.spyOn(fs.promises, 'writeFile').mockImplementation(async (filePath: any, data: any) => { + writtenFiles.push(`${filePath.toString()}\n${String(data)}`); + }); + vi.spyOn(fs.promises, 'rename').mockResolvedValue(); + vi.spyOn(fs.promises, 'copyFile').mockResolvedValue(); + vi.spyOn(fs, 'mkdirSync').mockReturnValue(undefined); + vi.spyOn(ModelsDevGateway.prototype, 'fetchProviders').mockResolvedValue({ + 'test-provider': { + name: 'Test Provider', + models: ['model-a'], + apiKeyEnvVar: 'TEST_API_KEY', + gateway: 'models-dev', + } as ProviderConfig, + }); + vi.spyOn(NetlifyGateway.prototype, 'fetchProviders').mockResolvedValue({}); + const mastraFetchProvidersSpy = vi + .spyOn(MastraGateway.prototype, 'fetchProviders') + .mockImplementation(mastraCatalog); + + await registry.syncGateways(true); + + const registryWrites = writtenFiles.filter(entry => entry.includes('provider-registry.json')); + expect(registryWrites.length).toBeGreaterThan(0); + for (const entry of registryWrites) expect(entry).toContain('test-provider'); + return { mastraFetchProvidersSpy, registryWrites }; + } + + const failingCatalog = async () => { + throw new Error('Gateway model catalog unavailable (401)'); + }; + + it('does not sync the default Mastra gateway in v2 mode', async () => { + process.env.MASTRA_GATEWAY_VERSION = 'v2'; + process.env.MASTRA_GATEWAY_API_KEY = 'deployment-key'; + + const { mastraFetchProvidersSpy, registryWrites } = await sync(failingCatalog); + + expect(mastraFetchProvidersSpy).not.toHaveBeenCalled(); + for (const entry of registryWrites) expect(entry).not.toContain('"mastra"'); + }); + + it('does not sync a registered Mastra gateway that holds an organization key', async () => { + GatewayRegistry.getInstance({ useDynamicLoading: true }).registerCustomGateways([ + new MastraGateway({ apiKey: 'mskv2_org-key' }), + ]); + + const { mastraFetchProvidersSpy, registryWrites } = await sync(failingCatalog); + + expect(mastraFetchProvidersSpy).not.toHaveBeenCalled(); + for (const entry of registryWrites) expect(entry).not.toContain('"mastra"'); + }); + + it('still syncs the Mastra gateway with a v1 project key', async () => { + process.env.MASTRA_GATEWAY_API_KEY = 'msk_0123456789abcdef0123456789abcdef'; + + const { mastraFetchProvidersSpy, registryWrites } = await sync(async () => ({ + mastra: { + name: 'Gateway', + models: ['openai/chat-model'], + apiKeyEnvVar: 'MASTRA_GATEWAY_API_KEY', + gateway: 'mastra', + } as ProviderConfig, + })); + + expect(mastraFetchProvidersSpy).toHaveBeenCalled(); + for (const entry of registryWrites) expect(entry).toContain('"mastra"'); + }); + }); + it('should skip syncGateways when MASTRA_OFFLINE is set to true', async () => { process.env.MASTRA_OFFLINE = 'true'; diff --git a/packages/core/src/llm/model/provider-registry.ts b/packages/core/src/llm/model/provider-registry.ts index fd8b562ff60..1ed63815f1a 100644 --- a/packages/core/src/llm/model/provider-registry.ts +++ b/packages/core/src/llm/model/provider-registry.ts @@ -866,8 +866,13 @@ export class GatewayRegistry { ...(writeToSrc ? [] : [new MastraGateway()]), ]; - // Combine default and custom gateways - const gateways = [...defaultGateways, ...this.customGateways]; + // Combine default and custom gateways. A Mastra gateway in v2 mode, default or registered, + // lists models for one organization's key, so it is not written to the machine-wide cache, + // and its failures cannot stop the rest of the refresh. + const gateways = [...defaultGateways, ...this.customGateways].filter(gateway => { + const candidate = gateway as { id?: string; isV2?: () => boolean }; + return !(candidate.id === 'mastra' && typeof candidate.isV2 === 'function' && candidate.isV2()); + }); // Fetch provider data const { diff --git a/packages/server/src/server/handlers/gateway-memory-client.ts b/packages/server/src/server/handlers/gateway-memory-client.ts index 271a6ec90c9..011958b3142 100644 --- a/packages/server/src/server/handlers/gateway-memory-client.ts +++ b/packages/server/src/server/handlers/gateway-memory-client.ts @@ -293,24 +293,29 @@ function isV2GatewayKey(key: string): boolean { } /** - * Whether this deployment targets Gateway v2. Signaled either by the - * MASTRA_GATEWAY_VERSION=v2 runtime env var (set by the platform), by a v2 - * (organization) API key prefix (which predates the env var signal), or by a - * registered Mastra gateway instance that carries a v2 key via config.apiKey - * (pass `mastra` so instance-level config is consulted too). + * Whether this deployment targets Gateway v2. Signaled by the MASTRA_GATEWAY_VERSION=v2 runtime + * env var (set by the platform) or by a v2 (organization) API key prefix, which predates the env + * var signal. A v1 project key (`msk_`) is always served by v1, so it wins over the env var. + * + * Pass `mastra` so the registered Mastra gateway answers: it holds the key model calls use, which + * may be set in its config rather than the environment, and its `isV2()` applies the same rules. + * Keep the environment rules below in sync with `MastraGateway.isV2` in `@mastra/core`. */ export function isGatewayV2(mastra?: { listGateways?: () => Record | undefined }): boolean { - if (process.env.MASTRA_GATEWAY_VERSION === 'v2') return true; - const key = process.env.MASTRA_GATEWAY_API_KEY; - if (!!key && isV2GatewayKey(key)) return true; - // A user-registered MastraGateway can hold a v2 key in its config rather - // than the environment; ask the instance for its effective v2 state. const gateways = mastra?.listGateways?.() ?? {}; for (const gateway of Object.values(gateways)) { const candidate = gateway as { id?: string; isV2?: () => boolean }; - if (candidate?.id === 'mastra' && typeof candidate.isV2 === 'function' && candidate.isV2()) return true; + if (candidate?.id === 'mastra' && typeof candidate.isV2 === 'function') return candidate.isV2(); } - return false; + return isGatewayV2ByEnvironment(); +} + +/** Whether the environment alone (MASTRA_GATEWAY_API_KEY and MASTRA_GATEWAY_VERSION) says v2. */ +function isGatewayV2ByEnvironment(): boolean { + const key = process.env.MASTRA_GATEWAY_API_KEY; + if (key?.startsWith('msk_')) return false; + if (process.env.MASTRA_GATEWAY_VERSION === 'v2') return true; + return !!key && isV2GatewayKey(key); } // Singleton-ish lazy instance @@ -320,10 +325,14 @@ let _gatewayClient: GatewayMemoryClient | null | undefined; * Get a GatewayMemoryClient instance, or null if gateway is not configured. * Pass `mastra` so a registered v2 gateway configuration takes precedence * over a v1 environment key. + * + * The client is built from MASTRA_GATEWAY_API_KEY and MASTRA_GATEWAY_URL, so it is built only when + * the environment itself says v1. A registered gateway that is v1 only because of a key in its own + * config, which the server cannot use here, does not make the environment key a v1 key. */ export function getGatewayClient(mastra?: Parameters[0]): GatewayMemoryClient | null { // Gateway v2 has no memory REST API; Studio must use the agent's own memory configuration. - if (isGatewayV2(mastra)) return null; + if (isGatewayV2(mastra) || isGatewayV2ByEnvironment()) return null; if (_gatewayClient !== undefined) return _gatewayClient; const url = process.env.MASTRA_GATEWAY_URL || 'https://gateway-api.mastra.ai'; const key = process.env.MASTRA_GATEWAY_API_KEY; diff --git a/packages/server/src/server/handlers/gateway-memory-client.v2.test.ts b/packages/server/src/server/handlers/gateway-memory-client.v2.test.ts index 9cb5398dd88..87b4c059ad9 100644 --- a/packages/server/src/server/handlers/gateway-memory-client.v2.test.ts +++ b/packages/server/src/server/handlers/gateway-memory-client.v2.test.ts @@ -1,3 +1,4 @@ +import { MastraGateway } from '@mastra/core/llm'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { getGatewayClient, isGatewayV2 } from './gateway-memory-client.js'; @@ -21,6 +22,18 @@ describe('Gateway memory client in v2 deployments', () => { expect(isGatewayV2()).toBe(false); }); + it('is not v2 for a v1 key even when MASTRA_GATEWAY_VERSION is v2', () => { + vi.stubEnv('MASTRA_GATEWAY_VERSION', 'v2'); + vi.stubEnv('MASTRA_GATEWAY_API_KEY', 'msk_v1-key'); + expect(isGatewayV2()).toBe(false); + }); + + it('is v2 when MASTRA_GATEWAY_VERSION is v2 and the key has no v1 prefix', () => { + vi.stubEnv('MASTRA_GATEWAY_VERSION', 'v2'); + vi.stubEnv('MASTRA_GATEWAY_API_KEY', 'deployment-key'); + expect(isGatewayV2()).toBe(true); + }); + it('detects v2 from a registered Mastra gateway instance carrying a v2 config key', () => { vi.stubEnv('MASTRA_GATEWAY_VERSION', ''); vi.stubEnv('MASTRA_GATEWAY_API_KEY', ''); @@ -35,6 +48,24 @@ describe('Gateway memory client in v2 deployments', () => { expect(getGatewayClient(mastra)).toBeNull(); }); + it('agrees with a registered gateway whose v1 config key wins over MASTRA_GATEWAY_VERSION', () => { + vi.stubEnv('MASTRA_GATEWAY_VERSION', 'v2'); + vi.stubEnv('MASTRA_GATEWAY_API_KEY', ''); + const gateway = new MastraGateway({ apiKey: 'msk_0123456789abcdef0123456789abcdef' }); + const mastra = { listGateways: () => ({ mastra: gateway }) }; + expect(gateway.isV2()).toBe(false); + expect(isGatewayV2(mastra)).toBe(false); + }); + + it('does not build a v1 memory client from an organization env key', () => { + vi.stubEnv('MASTRA_GATEWAY_VERSION', ''); + vi.stubEnv('MASTRA_GATEWAY_API_KEY', 'mskv2_org-key'); + const gateway = new MastraGateway({ apiKey: 'msk_0123456789abcdef0123456789abcdef' }); + const mastra = { listGateways: () => ({ mastra: gateway }) }; + expect(isGatewayV2(mastra)).toBe(false); + expect(getGatewayClient(mastra)).toBeNull(); + }); + it('is not v2 when the registered Mastra gateway instance reports v1', () => { vi.stubEnv('MASTRA_GATEWAY_VERSION', ''); vi.stubEnv('MASTRA_GATEWAY_API_KEY', ''); @@ -42,3 +73,64 @@ describe('Gateway memory client in v2 deployments', () => { expect(isGatewayV2(mastra)).toBe(false); }); }); + +describe('v2 detection and the v1 memory client across configurations', () => { + afterEach(() => vi.unstubAllEnvs()); + + const envKeys = { + none: '', + project: 'msk_0123456789abcdef0123456789abcdef', + organization: 'mskv2_org-key', + unprefixed: 'deployment-key', + } as const; + const registeredKeys = { none: undefined, v1: 'msk_fedcba9876543210fedcba9876543210', v2: 'mskv2_registered' } as const; + + // [env key, MASTRA_GATEWAY_VERSION, registered gateway key, isGatewayV2, v1 memory client built] + const table: Array<[keyof typeof envKeys, 'unset' | 'v2', keyof typeof registeredKeys, boolean, boolean]> = [ + ['none', 'unset', 'none', false, false], + ['none', 'v2', 'none', true, false], + ['none', 'unset', 'v1', false, false], + ['none', 'v2', 'v1', false, false], + ['none', 'unset', 'v2', true, false], + ['none', 'v2', 'v2', true, false], + ['project', 'unset', 'none', false, true], + ['project', 'v2', 'none', false, true], + ['project', 'unset', 'v1', false, true], + ['project', 'v2', 'v1', false, true], + ['project', 'unset', 'v2', true, false], + ['project', 'v2', 'v2', true, false], + ['organization', 'unset', 'none', true, false], + ['organization', 'v2', 'none', true, false], + ['organization', 'unset', 'v1', false, false], + ['organization', 'v2', 'v1', false, false], + ['organization', 'unset', 'v2', true, false], + ['organization', 'v2', 'v2', true, false], + ['unprefixed', 'unset', 'none', false, true], + ['unprefixed', 'v2', 'none', true, false], + ['unprefixed', 'unset', 'v1', false, true], + ['unprefixed', 'v2', 'v1', false, false], + ['unprefixed', 'unset', 'v2', true, false], + ['unprefixed', 'v2', 'v2', true, false], + ]; + + it.each(table)( + 'env key %s, MASTRA_GATEWAY_VERSION %s, registered key %s: v2 is %s, v1 memory client built is %s', + async (envKey, version, registeredKey, expectedV2, expectedClient) => { + vi.stubEnv('MASTRA_GATEWAY_API_KEY', envKeys[envKey]); + vi.stubEnv('MASTRA_GATEWAY_VERSION', version === 'v2' ? 'v2' : ''); + const registered = registeredKeys[registeredKey]; + // Mastra always lists a Mastra gateway: the registered one, or a default one without a config key. + const gateway = registered ? new MastraGateway({ apiKey: registered }) : new MastraGateway(); + const mastra = { listGateways: () => ({ mastra: gateway }) }; + + // A fresh module, so the cached client from one case cannot answer for another. + vi.resetModules(); + const client = await import('./gateway-memory-client.js'); + + expect(client.isGatewayV2(mastra)).toBe(expectedV2); + expect(client.getGatewayClient(mastra) !== null).toBe(expectedClient); + if (!registered) expect(client.isGatewayV2()).toBe(expectedV2); + }, + ); +}); + From 39730d03408c9bb1757c4c25700d4ee1608ec73d Mon Sep 17 00:00:00 2001 From: Rhys Balevicius Date: Tue, 17 Nov 2026 03:58:31 +0000 Subject: [PATCH 2/2] fix(core): correct gateway diagnostics and studio model selection Persist request origins so saved errors distinguish gateway failures from provider failures without retaining URL credentials or paths. Restore missing-memory warnings when the gateway does not manage memory, without delaying runs. Keep the gateway visible when its catalog fails to load. Use gateway credentials when selecting models for instruction enhancement. --- .changeset/saved-error-origins.md | 8 ++ .changeset/steady-studio-providers.md | 8 ++ .../mastra-gateway-duck-typing.test.ts | 105 +++++++++++++++++- packages/core/src/agent/agent.ts | 33 ++++-- .../src/agent/message-list/state/types.ts | 8 +- packages/core/src/error/index.ts | 1 + .../core/src/error/provider-error.test.ts | 1 + packages/core/src/error/provider-error.ts | 22 +++- .../shared/provider-error-details.test.ts | 20 +++- .../src/loop/shared/provider-error-details.ts | 17 +++ .../record-terminal-error-message.test.ts | 45 ++++++++ .../shared/record-terminal-error-message.ts | 5 +- .../persistence-failed-turns.scenario.test.ts | 5 +- .../server/src/server/handlers/agent.test.ts | 101 +++++++++++++++++ .../server/src/server/handlers/agents.test.ts | 6 +- packages/server/src/server/handlers/agents.ts | 36 +++++- 16 files changed, 403 insertions(+), 18 deletions(-) create mode 100644 .changeset/saved-error-origins.md create mode 100644 .changeset/steady-studio-providers.md diff --git a/.changeset/saved-error-origins.md b/.changeset/saved-error-origins.md new file mode 100644 index 00000000000..42d4d00325d --- /dev/null +++ b/.changeset/saved-error-origins.md @@ -0,0 +1,8 @@ +--- +'@mastra/core': patch +--- + +Improved how `@mastra/core` reports on Mastra Gateway models: + +- The warning "No memory is configured but resourceId and threadId were passed in args" is shown again for Gateway models when the Gateway does not keep memory for your key, such as an organization key. It stays hidden when the Gateway manages memory. The check runs in the background and does not delay the run. +- Error parts saved to thread history for a failed model call now include `requestOrigin`, the scheme, host and port the request was sent to (for example `https://gateway-api.mastra.ai`). A UI that rebuilds the error from history can tell a Gateway error from a provider error with the same status. The path, query string and any credentials in the URL are not saved. diff --git a/.changeset/steady-studio-providers.md b/.changeset/steady-studio-providers.md new file mode 100644 index 00000000000..e88159d6aa3 --- /dev/null +++ b/.changeset/steady-studio-providers.md @@ -0,0 +1,8 @@ +--- +'@mastra/server': patch +--- + +Fixed Studio for projects that use the Mastra Gateway: + +- The Gateway provider stays in the provider list, with no models, when its model list cannot be loaded (for example a revoked key or an outage). Before, it disappeared. +- "Enhance instructions" now works for agents on gateway models such as `mastra/...` when only the gateway's key, for example `MASTRA_GATEWAY_API_KEY`, is set. Before, it returned "No model with a configured API key found". A gateway model whose credential is missing is skipped in favour of the next connected model. diff --git a/packages/core/src/agent/__tests__/mastra-gateway-duck-typing.test.ts b/packages/core/src/agent/__tests__/mastra-gateway-duck-typing.test.ts index caf26e4fc3d..4802fafdf11 100644 --- a/packages/core/src/agent/__tests__/mastra-gateway-duck-typing.test.ts +++ b/packages/core/src/agent/__tests__/mastra-gateway-duck-typing.test.ts @@ -1,4 +1,5 @@ -import { describe, expect, it, vi } from 'vitest'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { ModelRouterLanguageModel } from '../../llm/model/router'; import { Agent } from '../agent'; describe('mastra gateway duck typing', () => { @@ -71,4 +72,106 @@ describe('mastra gateway duck typing', () => { expect.anything(), ); }); + + describe('with a Mastra gateway router model', () => { + const WARNING = 'No memory is configured but resourceId and threadId were passed in args'; + + afterEach(() => { + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + }); + + function routerModel(apiKey: string) { + vi.stubEnv('MASTRA_GATEWAY_API_KEY', apiKey); + const model = new ModelRouterLanguageModel('mastra/openai/gpt-4o'); + expect(model.gatewayId).toBe('mastra'); + const respond = async () => + ({ + stream: new ReadableStream({ + start(controller) { + controller.enqueue({ type: 'text-start', id: 'text-1' }); + controller.enqueue({ type: 'text-delta', id: 'text-1', delta: 'ok' }); + controller.enqueue({ type: 'text-end', id: 'text-1' }); + controller.enqueue({ + type: 'finish', + finishReason: 'stop', + usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 }, + }); + controller.close(); + }, + }), + rawCall: { rawPrompt: null, rawSettings: {} }, + warnings: [], + }) as any; + vi.spyOn(model, 'doStream').mockImplementation(respond); + vi.spyOn(model, 'doGenerate').mockImplementation(respond); + return model; + } + + async function run(model: ModelRouterLanguageModel) { + const warn = vi.fn(); + const debug = vi.fn(); + const agent = new Agent({ + id: 'test-agent', + name: 'test-agent', + instructions: 'You are a helpful assistant.', + model, + }); + agent.__setLogger({ debug, info: vi.fn(), warn, error: vi.fn(), trace: vi.fn() } as any); + const result = await agent.generate('hello', { + memory: { thread: { id: 'thread-1' }, resource: 'resource-1' }, + }); + return { result, warn, debug }; + } + + const settle = () => new Promise(resolve => setTimeout(resolve, 0)); + + it('warns for an organization key, whose memory stays local', async () => { + const { result, warn } = await run(routerModel('mskv2_org-key')); + + expect(result.text).toBe('ok'); + await vi.waitFor(() => expect(warn).toHaveBeenCalledWith(WARNING, { agent: 'test-agent' })); + }); + + it('does not warn for a project key, whose memory the gateway keeps', async () => { + const model = routerModel('msk_0123456789abcdef0123456789abcdef'); + const check = vi.spyOn(model, 'isGatewayMemoryManaged'); + const { warn } = await run(model); + + await vi.waitFor(() => expect(check).toHaveBeenCalled()); + await expect(check.mock.results[0]!.value).resolves.toBe(true); + await settle(); + expect(warn).not.toHaveBeenCalledWith(WARNING, expect.anything()); + }); + + it('finishes the run and does not warn when the check fails', async () => { + const model = routerModel('mskv2_org-key'); + const check = vi.spyOn(model, 'isGatewayMemoryManaged').mockRejectedValue(new Error('credential unavailable')); + const { result, warn, debug } = await run(model); + + expect(result.text).toBe('ok'); + await vi.waitFor(() => expect(check).toHaveBeenCalled()); + await settle(); + expect(warn).not.toHaveBeenCalledWith(WARNING, expect.anything()); + expect(debug).toHaveBeenCalledWith('Could not tell whether the gateway manages memory for this model', { + agent: 'test-agent', + error: expect.objectContaining({ message: 'credential unavailable' }), + }); + }); + + it('does not wait for a slow check before running the model', async () => { + const model = routerModel('mskv2_org-key'); + let release: ((value: boolean) => void) | undefined; + vi.spyOn(model, 'isGatewayMemoryManaged').mockImplementation( + () => new Promise(resolve => (release = resolve)), + ); + const { result, warn } = await run(model); + + expect(result.text).toBe('ok'); + expect(warn).not.toHaveBeenCalledWith(WARNING, expect.anything()); + expect(release).toBeDefined(); + release!(false); + await vi.waitFor(() => expect(warn).toHaveBeenCalledWith(WARNING, { agent: 'test-agent' })); + }); + }); }); diff --git a/packages/core/src/agent/agent.ts b/packages/core/src/agent/agent.ts index df77b6d89be..de2431c2328 100644 --- a/packages/core/src/agent/agent.ts +++ b/packages/core/src/agent/agent.ts @@ -9281,14 +9281,31 @@ export class Agent< // Only warn for ids the caller passed in `memory` args. Reserved request-context ids are set // per request (auth middleware, AgentController) and inherited by every agent in that request, // including internal memory-less agents such as the OM Observer and Reflector. - if ( - options.memory?.resource && - options.memory?.thread && - !this.hasOwnMemory() && - !this.#knowledge && - !isGatewayModel - ) { - this.logger.warn('No memory is configured but resourceId and threadId were passed in args', { agent: this.name }); + if (options.memory?.resource && options.memory?.thread && !this.hasOwnMemory() && !this.#knowledge) { + const warnMissingMemory = () => + this.logger.warn('No memory is configured but resourceId and threadId were passed in args', { + agent: this.name, + }); + const gatewayModel = resolvedModel as unknown as { isGatewayMemoryManaged?: () => Promise }; + if (!isGatewayModel) { + warnMissingMemory(); + } else if (typeof gatewayModel.isGatewayMemoryManaged === 'function') { + // The gateway keeps memory only for some credentials, so ask the model. The check may + // resolve a credential, so it runs in the background and never delays or fails the run. + // A check that fails leaves the warning out, as it was before the model could answer. + void Promise.resolve() + .then(() => gatewayModel.isGatewayMemoryManaged!()) + .then( + managed => { + if (!managed) warnMissingMemory(); + }, + error => + this.logger.debug('Could not tell whether the gateway manages memory for this model', { + agent: this.name, + error, + }), + ); + } } // Apply null→undefined transform for OpenAI structured output validation. diff --git a/packages/core/src/agent/message-list/state/types.ts b/packages/core/src/agent/message-list/state/types.ts index dfda172a0e8..3ca9a47cf9e 100644 --- a/packages/core/src/agent/message-list/state/types.ts +++ b/packages/core/src/agent/message-list/state/types.ts @@ -40,7 +40,7 @@ export type MastraStepStartPart = { // surfaces (stream chunk, onError, result.error) and are never written to // thread history. Provider HTTP failures (AI SDK `APICallError`) also keep the // status code, a bounded copy of the response body, the provider request id, -// the model the request named and the provider facts a reload classifies by +// the model the request named, the origin it was sent to and the provider facts a reload classifies by // (every code, the provider's own reason sentence), because their `message` is // often just the HTTP status text. export type MastraErrorPart = { @@ -51,6 +51,12 @@ export type MastraErrorPart = { responseBody?: string; requestId?: string; modelId?: string; + /** + * Origin (scheme, host and port, e.g. `https://gateway-api.mastra.ai`) the failed model + * request was sent to, so a reader can tell which service answered. Never the path, query + * string or credentials. Absent on messages saved before it was recorded. + */ + requestOrigin?: string; } & ProviderErrorFacts; } & MastraPartExtensions; diff --git a/packages/core/src/error/index.ts b/packages/core/src/error/index.ts index 0ecc2116bfd..36c14665a39 100644 --- a/packages/core/src/error/index.ts +++ b/packages/core/src/error/index.ts @@ -15,5 +15,6 @@ export { providerErrorReason, providerModelId, providerRequestId, + providerRequestOrigin, } from './provider-error'; export type { ProviderFailureKind } from './provider-error'; diff --git a/packages/core/src/error/provider-error.test.ts b/packages/core/src/error/provider-error.test.ts index 9a3fc9ade42..3faefc74332 100644 --- a/packages/core/src/error/provider-error.test.ts +++ b/packages/core/src/error/provider-error.test.ts @@ -474,6 +474,7 @@ describe('toPersistedErrorIdentity', () => { message: 'Model call failed', statusCode: 400, reason: 'prompt is too long: 1 > 0 maximum', + requestOrigin: 'https://api.example.com', }); }); diff --git a/packages/core/src/error/provider-error.ts b/packages/core/src/error/provider-error.ts index 78f651dbf60..4387aff060b 100644 --- a/packages/core/src/error/provider-error.ts +++ b/packages/core/src/error/provider-error.ts @@ -1,4 +1,9 @@ -import { capPersistedText, readRequestedModel, readRequestId } from '../loop/shared/provider-error-details'; +import { + capPersistedText, + readRequestedModel, + readRequestId, + readRequestOrigin, +} from '../loop/shared/provider-error-details'; export type ProviderFailureKind = | 'auth' @@ -235,6 +240,15 @@ export function providerModelId(error: unknown): string | undefined { return firstAlongCauses(error, modelIdOf, 0); } +/** + * The origin (scheme, host and port) a failed model request was sent to, so a reader can tell + * which service answered. Read only from an AI SDK API call error, whose `url` is the request it + * made; a `url` on any other error may name some other resource. + */ +export function providerRequestOrigin(error: unknown): string | undefined { + return firstAlongCauses(error, requestOriginOf, 0); +} + function firstAlongCauses( error: unknown, read: (error: object) => string | undefined, @@ -248,6 +262,12 @@ function requestIdOf(error: object): string | undefined { return nonBlank(readField(error, 'requestId')) ?? readRequestId(readField(error, 'responseHeaders')); } +function requestOriginOf(error: object): string | undefined { + const isApiCallError = + readField(error, 'name') === 'AI_APICallError' || readField(error, 'requestBodyValues') !== undefined; + return isApiCallError ? readRequestOrigin(readField(error, 'url')) : undefined; +} + function modelIdOf(error: object): string | undefined { return nonBlank(readField(error, 'modelId')) ?? readRequestedModel(readField(error, 'requestBodyValues')); } diff --git a/packages/core/src/loop/shared/provider-error-details.test.ts b/packages/core/src/loop/shared/provider-error-details.test.ts index c67a533319d..acbd5cf9345 100644 --- a/packages/core/src/loop/shared/provider-error-details.test.ts +++ b/packages/core/src/loop/shared/provider-error-details.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; -import { MAX_PERSISTED_RESPONSE_BODY_LENGTH, describeProviderError } from './provider-error-details'; +import { MAX_PERSISTED_RESPONSE_BODY_LENGTH, describeProviderError, readRequestOrigin } from './provider-error-details'; function providerError(responseBody: string, requestId = 'req_123') { return Object.assign(new Error('Bad Request'), { @@ -44,3 +44,21 @@ describe('describeProviderError', () => { expect(describeProviderError('boom', new Error('boom'), 1000)).toBeUndefined(); }); }); + +describe('readRequestOrigin', () => { + it.each([ + ['https://gateway-api.mastra.ai/v1/chat/completions', 'https://gateway-api.mastra.ai'], + ['http://127.0.0.1:4180/v1/chat/completions?x=1', 'http://127.0.0.1:4180'], + ['https://user:pass@api.example.com:443/path', 'https://api.example.com'], + [' https://api.example.com/v1 ', 'https://api.example.com'], + ])('keeps only the origin of %s', (url, origin) => { + expect(readRequestOrigin(url)).toBe(origin); + }); + + it.each([undefined, null, 42, '', ' ', 'not a url', '/relative', 'file:///etc/passwd', 'data:text/plain,hi'])( + 'gives undefined for %s', + url => { + expect(readRequestOrigin(url)).toBeUndefined(); + }, + ); +}); diff --git a/packages/core/src/loop/shared/provider-error-details.ts b/packages/core/src/loop/shared/provider-error-details.ts index 83b505f9799..ec570f0f317 100644 --- a/packages/core/src/loop/shared/provider-error-details.ts +++ b/packages/core/src/loop/shared/provider-error-details.ts @@ -42,6 +42,23 @@ export function readRequestId(headers: unknown): string | undefined { return undefined; } +/** + * The origin (scheme, host and port) of a request URL. The path, query string and any + * credentials in the URL are never kept. Anything that is not an absolute http or https URL + * gives undefined. + */ +export function readRequestOrigin(url: unknown): string | undefined { + if (typeof url !== 'string' || !url.trim()) return undefined; + let parsed: URL; + try { + parsed = new URL(url.trim()); + } catch { + return undefined; + } + if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return undefined; + return parsed.origin; +} + export function readRequestedModel(requestBody: unknown): string | undefined { if (!requestBody || typeof requestBody !== 'object') return undefined; const model = readField(requestBody, 'model'); diff --git a/packages/core/src/loop/shared/record-terminal-error-message.test.ts b/packages/core/src/loop/shared/record-terminal-error-message.test.ts index 28e5f9411e2..2cbde067586 100644 --- a/packages/core/src/loop/shared/record-terminal-error-message.test.ts +++ b/packages/core/src/loop/shared/record-terminal-error-message.test.ts @@ -46,6 +46,7 @@ describe('toPersistedErrorIdentity', () => { responseBody: '{"error":{"code":"insufficient_quota"}}', modelId: 'claude-opus-4-5', codes: ['insufficient_quota'], + requestOrigin: 'https://api.example.com', }); expect(Object.keys(toPersistedErrorIdentity(new Error('boom')))).toEqual(['name', 'message']); expect( @@ -58,6 +59,50 @@ describe('toPersistedErrorIdentity', () => { ).toMatchObject({ statusCode: 400, reason: 'prompt is too long' }); }); + it('keeps only the origin of the URL a failed model request was sent to', () => { + const error = Object.assign(new Error('Too Many Requests'), { + name: 'AI_APICallError', + statusCode: 429, + url: 'http://user:secret@127.0.0.1:4180/v1/chat/completions?api_key=secret#frag', + requestBodyValues: { model: 'stub/ok' }, + }); + + const identity = toPersistedErrorIdentity(error); + + expect(identity.requestOrigin).toBe('http://127.0.0.1:4180'); + expect(JSON.stringify(identity)).not.toContain('secret'); + expect(JSON.stringify(identity)).not.toContain('/v1/chat/completions'); + }); + + it('reads the request origin from a model request error wrapped as a cause', () => { + const apiCallError = Object.assign(new Error('Service Unavailable'), { + name: 'AI_APICallError', + statusCode: 503, + url: 'https://gateway-api.mastra.ai/v1/chat/completions', + requestBodyValues: {}, + }); + const wrapped = new Error('Model call failed', { cause: apiCallError }); + + expect(toPersistedErrorIdentity(wrapped).requestOrigin).toBe('https://gateway-api.mastra.ai'); + }); + + it.each([ + ['a malformed URL', 'not a url'], + ['a relative URL', '/v1/chat/completions'], + ['a non-http scheme', 'file:///etc/passwd'], + ['a non-string URL', 42], + ])('leaves the request origin out for %s', (_label, url) => { + const error = Object.assign(new Error('Bad Request'), { statusCode: 400, url, requestBodyValues: {} }); + + expect(toPersistedErrorIdentity(error)).not.toHaveProperty('requestOrigin'); + }); + + it('ignores a url on an error that is not a model request error', () => { + const error = Object.assign(new Error('Not Found'), { url: 'https://internal.example/resource' }); + + expect(toPersistedErrorIdentity(error)).not.toHaveProperty('requestOrigin'); + }); + it('keeps a bounded response body and the provider request id', () => { const error = Object.assign(new Error('Bad Request'), { statusCode: 400, diff --git a/packages/core/src/loop/shared/record-terminal-error-message.ts b/packages/core/src/loop/shared/record-terminal-error-message.ts index 2e33daeada8..edc0b14a82d 100644 --- a/packages/core/src/loop/shared/record-terminal-error-message.ts +++ b/packages/core/src/loop/shared/record-terminal-error-message.ts @@ -16,6 +16,7 @@ * * Only `{ name, message }` is persisted, plus `statusCode`, a bounded * `responseBody`, the provider `requestId`, the `modelId` the request named, + * the `requestOrigin` the request was sent to (scheme, host and port only), * every provider code and the innermost `reason` for provider failures — * without those, an `APICallError` whose message is just "Bad Request" cannot * be diagnosed or classified after the fact. The rest of the request body, @@ -26,7 +27,7 @@ */ import type { MastraDBMessage, MastraErrorPart, MessageList } from '../../agent/message-list'; -import { providerErrorFacts } from '../../error/provider-error'; +import { providerErrorFacts, providerRequestOrigin } from '../../error/provider-error'; import { readProviderErrorDetails } from './provider-error-details'; const FALLBACK_ERROR_NAME = 'Error'; @@ -58,11 +59,13 @@ export function toPersistedErrorIdentity(error: unknown): MastraErrorPart['error return { name: FALLBACK_ERROR_NAME, message: FALLBACK_ERROR_MESSAGE }; } + const requestOrigin = providerRequestOrigin(error); return { name: readErrorField(error, 'name') ?? FALLBACK_ERROR_NAME, message: readErrorField(error, 'message') ?? FALLBACK_ERROR_MESSAGE, ...readProviderErrorDetails(error), ...providerErrorFacts(error), + ...(requestOrigin ? { requestOrigin } : {}), }; } diff --git a/packages/core/src/loop/test-utils/aimock/scenarios/persistence-failed-turns.scenario.test.ts b/packages/core/src/loop/test-utils/aimock/scenarios/persistence-failed-turns.scenario.test.ts index fb579d76728..2e7d2140598 100644 --- a/packages/core/src/loop/test-utils/aimock/scenarios/persistence-failed-turns.scenario.test.ts +++ b/packages/core/src/loop/test-utils/aimock/scenarios/persistence-failed-turns.scenario.test.ts @@ -95,7 +95,10 @@ describeForAllEngines('AIMock loop scenario: persistence of failed turns', engin .poll(async () => (await memory.recall({ threadId, resourceId })).messages.map(m => m.role)) .toEqual(['user', 'assistant']); const [errorPart] = errorParts((await memory.recall({ threadId, resourceId })).messages); - expect(errorPart).toMatchObject({ type: 'error', error: { statusCode: 400 } }); + expect(errorPart).toMatchObject({ + type: 'error', + error: { statusCode: 400, requestOrigin: new URL(getMock().url).origin }, + }); expect(JSON.parse((errorPart as { error: { responseBody: string } }).error.responseBody)).toMatchObject({ error: { message: 'PROVIDER_400_EXPLANATION', type: 'invalid_request_error' }, }); diff --git a/packages/server/src/server/handlers/agent.test.ts b/packages/server/src/server/handlers/agent.test.ts index 384c37fb369..d48853ad971 100644 --- a/packages/server/src/server/handlers/agent.test.ts +++ b/packages/server/src/server/handlers/agent.test.ts @@ -1329,5 +1329,106 @@ describe('Agent Handlers', () => { } } }); + + it('uses a Mastra gateway model when only MASTRA_GATEWAY_API_KEY is set', async () => { + vi.stubEnv('OPENAI_API_KEY', ''); + vi.stubEnv('MASTRA_GATEWAY_API_KEY', 'test-gateway-key'); + onTestFinished(() => vi.unstubAllEnvs()); + const mastra = makeMastraMock({ + agents: { 'gateway-agent': makeMockAgent({ name: 'gateway-agent', model: 'mastra/openai/gpt-5-mini' }) }, + }); + const generateSpy = vi + .spyOn(Agent.prototype, 'generate') + .mockResolvedValue({ object: { explanation: 'Clearer.', new_prompt: 'Be clear.' } } as any); + onTestFinished(() => generateSpy.mockRestore()); + + const result = await ENHANCE_INSTRUCTIONS_ROUTE.handler({ + ...createTestServerContext({ mastra }), + agentId: 'gateway-agent', + instructions: 'Be helpful.', + }); + + expect(result).toEqual({ explanation: 'Clearer.', new_prompt: 'Be clear.' }); + expect(generateSpy).toHaveBeenCalledOnce(); + }); + + it('skips a gateway model without credentials and uses a later connected model', async () => { + vi.stubEnv('OPENAI_API_KEY', ''); + vi.stubEnv('NETLIFY_TOKEN', ''); + vi.stubEnv('NETLIFY_SITE_ID', ''); + vi.stubEnv('GOOGLE_GENERATIVE_AI_API_KEY', 'test-google-key'); + onTestFinished(() => vi.unstubAllEnvs()); + const mastra = makeMastraMock({ + agents: { + 'fallback-agent': makeMockAgent({ + name: 'fallback-agent', + model: [{ model: 'netlify/openai/gpt-4o' }, { model: 'google/gemini-2.5-flash' }], + }), + }, + }); + const generateSpy = vi + .spyOn(Agent.prototype, 'generate') + .mockResolvedValue({ object: { explanation: 'Clearer.', new_prompt: 'Be clear.' } } as any); + onTestFinished(() => generateSpy.mockRestore()); + + await ENHANCE_INSTRUCTIONS_ROUTE.handler({ + ...createTestServerContext({ mastra }), + agentId: 'fallback-agent', + instructions: 'Be helpful.', + }); + + const enhancer = generateSpy.mock.contexts[0] as Agent; + const chosen = (await enhancer.getModel()) as { provider: string; modelId: string }; + expect(chosen.provider).toBe('google'); + expect(chosen.modelId).toBe('gemini-2.5-flash'); + }); + + it('does not treat a lab provider key as the credential for a gateway model', async () => { + vi.stubEnv('OPENAI_API_KEY', 'test-openai-key'); + vi.stubEnv('NETLIFY_TOKEN', ''); + vi.stubEnv('NETLIFY_SITE_ID', ''); + vi.stubEnv('GOOGLE_GENERATIVE_AI_API_KEY', 'test-google-key'); + onTestFinished(() => vi.unstubAllEnvs()); + const mastra = makeMastraMock({ + agents: { + 'fallback-agent': makeMockAgent({ + name: 'fallback-agent', + model: [{ model: 'netlify/openai/gpt-4o' }, { model: 'google/gemini-2.5-flash' }], + }), + }, + }); + const generateSpy = vi + .spyOn(Agent.prototype, 'generate') + .mockResolvedValue({ object: { explanation: 'Clearer.', new_prompt: 'Be clear.' } } as any); + onTestFinished(() => generateSpy.mockRestore()); + + await ENHANCE_INSTRUCTIONS_ROUTE.handler({ + ...createTestServerContext({ mastra }), + agentId: 'fallback-agent', + instructions: 'Be helpful.', + }); + + const enhancer = generateSpy.mock.contexts[0] as Agent; + const chosen = (await enhancer.getModel()) as { provider: string; modelId: string }; + expect(chosen.provider).toBe('google'); + expect(chosen.modelId).toBe('gemini-2.5-flash'); + }); + + it('returns 400 for a Mastra gateway model when no gateway key is set', async () => { + vi.stubEnv('OPENAI_API_KEY', ''); + vi.stubEnv('MASTRA_GATEWAY_API_KEY', ''); + onTestFinished(() => vi.unstubAllEnvs()); + const mastra = makeMastraMock({ + agents: { 'gateway-agent': makeMockAgent({ name: 'gateway-agent', model: 'mastra/openai/gpt-5-mini' }) }, + }); + + await expect( + ENHANCE_INSTRUCTIONS_ROUTE.handler({ + ...createTestServerContext({ mastra }), + agentId: 'gateway-agent', + instructions: 'Be helpful.', + }), + ).rejects.toMatchObject({ status: 400 }); + }); }); }); diff --git a/packages/server/src/server/handlers/agents.test.ts b/packages/server/src/server/handlers/agents.test.ts index de07cd56444..f6612d4467e 100644 --- a/packages/server/src/server/handlers/agents.test.ts +++ b/packages/server/src/server/handlers/agents.test.ts @@ -526,7 +526,7 @@ describe('getProvidersHandler', () => { expect(result.providers.find(provider => provider.id === 'mastra')?.models).toEqual(['live-chat-model']); }); - it('does not fall back to cached Mastra models when the v2 catalog fails', async () => { + it('keeps the Mastra gateway listed with no models when the v2 catalog fails', async () => { process.env.MASTRA_GATEWAY_VERSION = 'v2'; process.env.MASTRA_GATEWAY_API_KEY = 'test-key'; Object.assign(globalThis, { @@ -552,7 +552,9 @@ describe('getProvidersHandler', () => { requestContext: new RequestContext(), }); - expect(result.providers.find(provider => provider.id === 'mastra')).toBeUndefined(); + const gateway = result.providers.find(provider => provider.id === 'mastra'); + expect(gateway).toMatchObject({ name: 'Gateway', envVar: 'MASTRA_GATEWAY_API_KEY', connected: true }); + expect(gateway?.models).toEqual([]); expect(result.providers.find(provider => provider.id === 'openai')).toBeDefined(); }); }); diff --git a/packages/server/src/server/handlers/agents.ts b/packages/server/src/server/handlers/agents.ts index 8594cf14b16..1f3cc90f28b 100644 --- a/packages/server/src/server/handlers/agents.ts +++ b/packages/server/src/server/handlers/agents.ts @@ -2093,6 +2093,18 @@ export async function buildProvidersList(mastra: Context['mastra']): Promise>): Promise { + const router = model as { gatewayId?: unknown; resolveApiProviderId?: () => Promise }; + if ( + typeof router.gatewayId === 'string' && + router.gatewayId !== 'models.dev' && + typeof router.resolveApiProviderId === 'function' + ) { + return (await router.resolveApiProviderId()) !== undefined; + } + return isProviderConnected(model.provider); +} + // Helper to find the first model with a connected provider async function findConnectedModel(agent: Agent): Promise> | null> { const modelList = await agent.getModelList(); @@ -4383,7 +4415,7 @@ async function findConnectedModel(agent: Agent): Promise