diff --git a/.changeset/code-sdk-gateway-v2-oauth.md b/.changeset/code-sdk-gateway-v2-oauth.md new file mode 100644 index 00000000000..0b3f0b1bedd --- /dev/null +++ b/.changeset/code-sdk-gateway-v2-oauth.md @@ -0,0 +1,7 @@ +--- +'@mastra/code-sdk': patch +--- + +Fixed `mastra/anthropic/*` and `mastra/openai/*` models failing on every turn with "Invalid API key format" when a Mastra Gateway organization key (`msk-gateway-...` or `mskv2_...`) is set and you are signed in with Claude or Codex OAuth. The request now goes through the Mastra Gateway with the gateway key and is billed there, and web search follows the same route. These requests are handled like other `mastra/` models without a sign-in, so the Claude and Codex specific request settings, such as the thinking level, are not sent. Project keys (`msk_...`) keep using your Claude or Codex sign-in through the gateway as before, also when `MASTRA_GATEWAY_VERSION=v2` is set. + +Errors from the Mastra Gateway itself are now worded for the gateway. A 402 says the gateway refused the request because of credits or a budget and shows the gateway's code, instead of "Usage quota exhausted for this provider account". A 403 with a gateway code such as `no_eligible_route` or `model_not_allowed` says the gateway does not allow this model or feature for your key. diff --git a/.changeset/gateway-v2-custom-fetch-key.md b/.changeset/gateway-v2-custom-fetch-key.md new file mode 100644 index 00000000000..10eb3a33e87 --- /dev/null +++ b/.changeset/gateway-v2-custom-fetch-key.md @@ -0,0 +1,5 @@ +--- +'@mastra/core': patch +--- + +Fixed `MastraGateway` with a `customFetch` failing with "Invalid API key format" for Mastra Gateway organization keys (`msk-gateway-...` and `mskv2_...`). The gateway that serves these keys reads the key only from `Authorization` and cannot forward a provider sign-in, so model calls whose key is an organization key, including a key set on the model itself, now send it in `Authorization` to the chat endpoint. Project keys (`msk_...`) still pass the provider sign-in through `customFetch` as before. diff --git a/.changeset/mastracode-gateway-error-hints.md b/.changeset/mastracode-gateway-error-hints.md new file mode 100644 index 00000000000..e0d1714506d --- /dev/null +++ b/.changeset/mastracode-gateway-error-hints.md @@ -0,0 +1,5 @@ +--- +'mastracode': patch +--- + +Gateway refusals now get gateway hints. A credit or budget refusal from the Mastra Gateway points to Mastra billing, and a policy or routing refusal suggests choosing another model with `/models` instead of `/connect`. `/mastra-gateway` no longer says that memory mode changes take effect on restart. diff --git a/.changeset/quiet-gateways-count.md b/.changeset/quiet-gateways-count.md new file mode 100644 index 00000000000..d641567a602 --- /dev/null +++ b/.changeset/quiet-gateways-count.md @@ -0,0 +1,10 @@ +--- +'@mastra/core': patch +--- + +Improved how `@mastra/core` works with the Mastra Gateway: + +- Streamed calls through the Gateway's chat endpoint now ask for token usage, so `agent.stream()` reports usage when the upstream provider sends it in a separate final chunk. Before, usage could be empty. +- The Gateway model list for organization keys is cached for a minute and shared between concurrent callers. If a refresh times out or the Gateway has an outage, the last list is kept for up to ten minutes. With no earlier list, the failure is reported again for 15 seconds without a new request, so Studio does not wait for the request timeout on every load during an outage. If the Gateway rejects the key (401 or 403), the cached list is dropped. The model list request now always sends the key in `Authorization` and no longer goes through a `customFetch`, which the v2 Gateway would not have authenticated. +- A `msk_` project key now always uses the v1 Gateway, even when `MASTRA_GATEWAY_VERSION=v2` is set. +- `mastra dev` no longer writes one organization's Gateway model list to the provider cache shared by other projects on the machine, whether the key comes from the environment or from a registered `MastraGateway`. A failing Gateway model list no longer stops the provider registry refresh. diff --git a/.changeset/steady-gateway-versions.md b/.changeset/steady-gateway-versions.md new file mode 100644 index 00000000000..3a31dbbd23d --- /dev/null +++ b/.changeset/steady-gateway-versions.md @@ -0,0 +1,5 @@ +--- +'@mastra/server': patch +--- + +A `msk_` project key now always uses the v1 Gateway memory API, even when `MASTRA_GATEWAY_VERSION=v2` is set. Studio follows a registered `MastraGateway` and its own key when deciding between the v1 and v2 Gateway, so it agrees with the model calls. diff --git a/mastracode/sdk/src/agents/__tests__/model.test.ts b/mastracode/sdk/src/agents/__tests__/model.test.ts index b598ab315cb..c7894f1f5d1 100644 --- a/mastracode/sdk/src/agents/__tests__/model.test.ts +++ b/mastracode/sdk/src/agents/__tests__/model.test.ts @@ -183,6 +183,14 @@ vi.mock('@mastra/core/llm', () => ({ this.apiKey = config?.apiKey; this.baseUrl = config?.baseUrl; this.customFetch = config?.customFetch; + // Mirrors the real gateway: a project key selects v1, then the version variable or an + // organization key selects Gateway v2. + this.isV2 = vi.fn(() => { + const key = config?.apiKey ?? process.env.MASTRA_GATEWAY_API_KEY; + if (key?.startsWith('msk_')) return false; + if (process.env.MASTRA_GATEWAY_VERSION === 'v2') return true; + return !!key && (key.startsWith('msk-gateway-') || key.startsWith('mskv2_')); + }); this.resolveLanguageModel = vi.fn((args: Record) => ({ __provider: 'mastra-gateway-delegate', args, @@ -1198,6 +1206,91 @@ describe('resolveModel', () => { expect(openaiCodexProvider).not.toHaveBeenCalled(); }); + describe('with a Gateway v2 organization key', () => { + function useGatewayKey(key: string) { + mockAuthStorageInstance.getStoredApiKey.mockImplementation((providerId: string) => + providerId === 'mastra-gateway' ? key : undefined, + ); + } + + function signInWithOAuth() { + mockAuthStorageInstance.get.mockReturnValue({ + type: 'oauth', + access: 'oauth-access-token', + refresh: 'oauth-refresh-token', + expires: Date.now() + 60_000, + }); + } + + it.each(['msk-gateway-us_org_key', 'msk-gateway-eu_org_key', 'mskv2_org_key'])( + 'sends a Claude OAuth model to the gateway with the %s key instead of the stored sign-in', + key => { + useGatewayKey(key); + signInWithOAuth(); + + const result = resolveModel('mastra/anthropic/claude-opus-4.6') as Record; + + expect(result.__provider).toBe('mastra-gateway-delegate'); + expect(result.args).toEqual({ + providerId: 'anthropic', + modelId: 'claude-opus-4-6', + apiKey: key, + headers: undefined, + }); + expect(createAnthropic).not.toHaveBeenCalled(); + expect(buildAnthropicOAuthFetch).not.toHaveBeenCalled(); + expect(opencodeClaudeMaxProvider).not.toHaveBeenCalled(); + }, + ); + + it('sends a Codex OAuth model to the gateway with the gateway key instead of the stored sign-in', () => { + useGatewayKey('msk-gateway-eu_org_key'); + signInWithOAuth(); + + const result = resolveModel('mastra/openai/gpt-4o') as Record; + + expect(result.__provider).toBe('mastra-gateway-delegate'); + expect(result.args).toEqual({ + providerId: 'openai', + modelId: 'gpt-4o', + apiKey: 'msk-gateway-eu_org_key', + headers: undefined, + }); + expect(createOpenAI).not.toHaveBeenCalled(); + expect(buildOpenAICodexOAuthFetch).not.toHaveBeenCalled(); + expect(openaiCodexProvider).not.toHaveBeenCalled(); + }); + + it('keeps the Codex OAuth pass-through for a v1 project key when MASTRA_GATEWAY_VERSION is v2', () => { + process.env.MASTRA_GATEWAY_VERSION = 'v2'; + useGatewayKey('msk_project_key'); + signInWithOAuth(); + + const result = resolveModel('mastra/openai/gpt-4o') as Record; + + expect(result.__provider).toBe('openai-direct'); + expect(buildOpenAICodexOAuthFetch).toHaveBeenCalled(); + const opts = vi.mocked(createOpenAI).mock.calls[0]?.[0] as Record | undefined; + expect((opts?.headers as Record)?.['X-Memory-Gateway-Authorization']).toBe( + 'Bearer msk_project_key', + ); + }); + + it('keeps the Claude OAuth pass-through for a v1 project key', () => { + useGatewayKey('msk_project_key'); + signInWithOAuth(); + + const result = resolveModel('mastra/anthropic/claude-sonnet-4') as Record; + + expect(result.__provider).toBe('anthropic-direct'); + expect(buildAnthropicOAuthFetch).toHaveBeenCalled(); + const opts = vi.mocked(createAnthropic).mock.calls[0]?.[0] as Record | undefined; + expect((opts?.headers as Record)?.['X-Memory-Gateway-Authorization']).toBe( + 'Bearer msk_project_key', + ); + }); + }); + it('routes explicit mastra-prefixed anthropic API key model through gateway without customFetch', () => { mockAuthStorageInstance.get.mockReturnValue({ type: 'api_key', key: 'sk-stored-key' }); diff --git a/mastracode/sdk/src/agents/__tests__/web-search-route.test.ts b/mastracode/sdk/src/agents/__tests__/web-search-route.test.ts index 92313de8dd9..bcdcb473430 100644 --- a/mastracode/sdk/src/agents/__tests__/web-search-route.test.ts +++ b/mastracode/sdk/src/agents/__tests__/web-search-route.test.ts @@ -49,15 +49,17 @@ function requestContextFor(modelId: string): RequestContext { } /** Runs one agent turn with the model and tools mastracode builds, returning the first request sent. */ -async function firstRequest(modelId: string): Promise<{ url: string; body: any }> { - const requests: { url: string; body: any }[] = []; +async function firstRequest(modelId: string): Promise<{ url: string; body: any; headers: Headers }> { + const requests: { url: string; body: any; headers: Headers }[] = []; vi.stubGlobal( 'fetch', vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { const url = input instanceof Request ? input.url : String(input); // OAuth fetch wrappers forward a Request with a streamed body. const text = typeof init?.body === 'string' ? init.body : input instanceof Request ? await input.text() : ''; - if (text) requests.push({ url, body: JSON.parse(text) }); + const headers = new Headers(input instanceof Request ? input.headers : undefined); + new Headers(init?.headers).forEach((value, name) => headers.set(name, value)); + if (text) requests.push({ url, body: JSON.parse(text), headers }); // Only the request matters; a client error ends the turn without retries. return new Response(JSON.stringify({ error: { message: 'stubbed' } }), { status: 400, @@ -151,6 +153,41 @@ describe('provider web search follows the model transport', () => { }); }); + it('keeps the Claude sign-in pass-through for a project key when MASTRA_GATEWAY_VERSION is v2', async () => { + vi.stubEnv('MASTRA_GATEWAY_API_KEY', 'msk_project-key'); + vi.stubEnv('MASTRA_GATEWAY_VERSION', 'v2'); + signInWithOAuth('anthropic'); + const modelId = 'mastra/anthropic/claude-sonnet-4-5'; + expect(resolveProviderWebSearchRoute(modelId)).toBe('anthropic'); + + const { url, headers } = await firstRequest(modelId); + expect(url).toMatch(/\/v1\/messages/); + expect(headers.get('x-memory-gateway-authorization')).toBe('Bearer msk_project-key'); + }); + + describe('Mastra gateway alias with a Gateway v2 organization key', () => { + it.each([ + ['anthropic', 'mastra/anthropic/claude-sonnet-4-5', 'msk-gateway-us_org-key'], + ['openai-codex', 'mastra/openai/gpt-5.5', 'msk-gateway-eu_org-key'], + ['anthropic', 'mastra/anthropic/claude-sonnet-4-5', 'mskv2_org-key'], + ] as const)( + 'sends a %s OAuth sign-in turn for %s to the gateway chat endpoint with the %s key', + async (credentialId, modelId, gatewayKey) => { + vi.stubEnv('MASTRA_GATEWAY_API_KEY', gatewayKey); + signInWithOAuth(credentialId); + expect(resolveProviderWebSearchRoute(modelId)).toBe('mastra-gateway'); + + const { url, body, headers } = await firstRequest(modelId); + expect(url).toMatch(/\/v1\/chat\/completions$/); + expect(body.model).toBe(modelId.slice('mastra/'.length)); + expect(headers.get('authorization')).toBe(`Bearer ${gatewayKey}`); + expect(headers.get('x-memory-gateway-authorization')).toBeNull(); + expect(body.tools).toContainEqual({ type: 'openrouter:web_search' }); + expect(JSON.stringify(body.tools)).not.toMatch(/openai:web_search|anthropic:web_search|web_search_2025/); + }, + ); + }); + it.each([ ['anthropic/claude-sonnet-4-5', 'anthropic', /api\.anthropic\.com\/v1\/messages/, 'web_search_20250305'], ['openai/gpt-5.5', 'openai', /api\.openai\.com\/v1\/responses/, 'web_search'], diff --git a/mastracode/sdk/src/agents/mastracode-gateway.ts b/mastracode/sdk/src/agents/mastracode-gateway.ts index a81b232650e..3981ff2d742 100644 --- a/mastracode/sdk/src/agents/mastracode-gateway.ts +++ b/mastracode/sdk/src/agents/mastracode-gateway.ts @@ -305,6 +305,18 @@ export function hasProviderEnvironmentKey(providerId: string): boolean { /** Servers call custom endpoints too: a redirect could aim that call at an address the saved URL never named. */ const fetchWithoutRedirects: typeof fetch = (input, init) => fetch(input, { ...init, redirect: 'error' }); +/** + * Whether the Mastra gateway can carry a provider sign-in for this gateway key, with the provider + * token in `Authorization` and the gateway key in `GATEWAY_AUTH_HEADER`. Only the v1 gateway + * (project keys) supports that. Gateway v2 (organization keys) reads its key from `Authorization` + * and cannot forward a provider credential, so a v2 key always takes the gateway's own route. + * The gateway chooses v1 or v2 from the key itself, so the key is classified on its own: a project + * key (`msk_…`) keeps the pass-through even where `MASTRA_GATEWAY_VERSION=v2` is set. + */ +export function mastraGatewayPassesProviderSignIn(mastraGatewayApiKey: string): boolean { + return !new MastraGateway({ apiKey: mastraGatewayApiKey }).isV2(); +} + export class MastraCodeGateway extends MastraModelGateway { readonly id = MASTRACODE_GATEWAY_ID; readonly name = 'MastraCode Gateway'; @@ -604,7 +616,7 @@ export class MastraCodeGateway extends MastraModelGateway { const thinkingMiddleware = createAnthropicThinkingMiddleware(bareModelId, this.#thinkingLevel); if (this.#routeThroughMastraGateway) { - if (storedCred?.type === 'oauth') { + if (storedCred?.type === 'oauth' && mastraGatewayPassesProviderSignIn(args.apiKey)) { const anthropic = createAnthropic({ apiKey: 'oauth-gateway-placeholder', baseURL: `${this.#mastraGatewayBaseUrl}/v1`, @@ -678,7 +690,7 @@ export class MastraCodeGateway extends MastraModelGateway { const storedCred = this.#credentials.get('openai-codex'); if (this.#routeThroughMastraGateway) { - if (storedCred?.type === 'oauth') { + if (storedCred?.type === 'oauth' && mastraGatewayPassesProviderSignIn(args.apiKey)) { const resolvedModelId = remapOpenAIModelForCodexOAuth(`openai/${args.modelId}`); const resolvedBareModelId = resolvedModelId.substring(OPENAI_PREFIX.length); const effectiveLevel = getEffectiveThinkingLevel(args.modelId, this.#thinkingLevel ?? 'medium'); diff --git a/mastracode/sdk/src/agents/model.ts b/mastracode/sdk/src/agents/model.ts index e3b444583af..ed62d977df9 100644 --- a/mastracode/sdk/src/agents/model.ts +++ b/mastracode/sdk/src/agents/model.ts @@ -35,6 +35,7 @@ import { MastraCodeGateway, getGlobalAuthStorage, hasProviderEnvironmentKey, + mastraGatewayPassesProviderSignIn, reloadAuthStorage, stripMastraGatewayPrefix, } from './mastracode-gateway.js'; @@ -358,9 +359,10 @@ function mastraGatewayApiKey(tenantCredentialStore: CredentialStore | undefined) /** * Which provider-executed web search tool fits the transport a model resolves to: * - `anthropic` / `openai`: the native provider API (direct keys, and Claude or - * Codex OAuth, which keep the native API even through the Mastra gateway). - * - `mastra-gateway`: a `mastra/` alias with a gateway key and no OAuth, which - * `resolveModelForRequest` sends to the gateway's OpenRouter chat endpoint. + * Codex OAuth, which keep the native API through a v1 Mastra gateway key). + * - `mastra-gateway`: a `mastra/` alias with a gateway key and no OAuth, or with a + * v2 gateway key, which `resolveModelForRequest` sends to the gateway's + * OpenRouter chat endpoint. * Other providers have no built-in search. Mirrors the routing in * `resolveModelForRequest` and `MastraCodeGateway`, so the registered tool and * the prompt guidance follow the request the model will actually make. @@ -412,8 +414,10 @@ export function resolveProviderWebSearchRoute( tenantCredentialStore ?? getGlobalAuthStorage(), options?.requestContext, ); - if (credentialStore.get(NATIVE_OAUTH_CREDENTIAL_IDS[providerId])?.type === 'oauth') return providerId; - return mastraGatewayApiKey(tenantCredentialStore) ? 'mastra-gateway' : providerId; + const gatewayKey = mastraGatewayApiKey(tenantCredentialStore); + const hasOAuth = credentialStore.get(NATIVE_OAUTH_CREDENTIAL_IDS[providerId])?.type === 'oauth'; + if (hasOAuth && (!gatewayKey || mastraGatewayPassesProviderSignIn(gatewayKey))) return providerId; + return gatewayKey ? 'mastra-gateway' : providerId; } /** diff --git a/mastracode/sdk/src/utils/__tests__/errors.test.ts b/mastracode/sdk/src/utils/__tests__/errors.test.ts index 08c6313d027..419cd9cbcc3 100644 --- a/mastracode/sdk/src/utils/__tests__/errors.test.ts +++ b/mastracode/sdk/src/utils/__tests__/errors.test.ts @@ -119,4 +119,190 @@ describe('parseError', () => { expect(parsed.type).toBe('content_filter'); expect(parsed.detail).toBe(providerMessage); }); + + describe('Mastra Gateway refusals', () => { + const gatewayBaseUrl = 'https://gateway.example.test'; + + // The provider SDKs set `data` to the body as their own error schema parsed it, which keeps only + // the fields that schema declares, and `responseBody` to the raw text. + it('keeps the gateway code that a provider error schema dropped from the parsed data', () => { + const responseBody = JSON.stringify({ + type: 'error', + error: { + type: 'gateway_error', + code: 'organization_budget_insufficient', + message: 'The budget cannot cover this request.', + }, + }); + const error = Object.assign(new Error('The budget cannot cover this request.'), { + statusCode: 402, + url: `${gatewayBaseUrl}/v1/messages`, + data: { type: 'error', error: { type: 'gateway_error', message: 'The budget cannot cover this request.' } }, + responseBody, + }); + + const parsed = parseError(error, { gatewayBaseUrl }); + + expect(parsed.source).toBe('gateway'); + expect(parsed.detail).toBe('The budget cannot cover this request. (organization_budget_insufficient)'); + }); + + it('recognises a gateway refusal whose parsed data lost the gateway marker', () => { + const error = Object.assign(new Error('no_eligible_route'), { + statusCode: 403, + url: `${gatewayBaseUrl}/v1/chat/completions`, + data: { error: { message: 'no_eligible_route' } }, + responseBody: JSON.stringify({ + error: { type: 'gateway_error', code: 'no_eligible_route', message: 'no_eligible_route' }, + }), + }); + + const parsed = parseError(error, { gatewayBaseUrl }); + + expect(parsed.source).toBe('gateway'); + expect(parsed.detail).toBe('no_eligible_route'); + }); + + it('reads the gateway marker from parsed data when no raw body is kept', () => { + const error = Object.assign(new Error('no_eligible_route'), { + statusCode: 403, + url: `${gatewayBaseUrl}/v1/chat/completions`, + data: { error: { type: 'gateway_error', code: 'no_eligible_route', message: 'no_eligible_route' } }, + }); + + expect(parseError(error, { gatewayBaseUrl }).source).toBe('gateway'); + }); + + it('keeps provider wording when both the parsed data and the raw body carry a provider type', () => { + const body = { + error: { + type: 'request_forbidden', + code: 'unsupported_country_region_territory', + message: 'Country not supported', + }, + }; + const error = Object.assign(new Error('Country not supported'), { + statusCode: 403, + url: `${gatewayBaseUrl}/v1/responses`, + data: body, + responseBody: JSON.stringify(body), + }); + + expect(parseError(error, { gatewayBaseUrl }).source).toBeUndefined(); + }); + + function gatewayError(status: number, code: string, message = code, url = `${gatewayBaseUrl}/v1/chat/completions`) { + return Object.assign(new Error(message), { + statusCode: status, + url, + responseBody: JSON.stringify({ error: { type: 'gateway_error', code, message } }), + }); + } + + it('words a gateway 402 as a credit or budget refusal and keeps the gateway code', () => { + const parsed = parseError(gatewayError(402, 'organization_budget_unbounded_request'), { gatewayBaseUrl }); + + expect(parsed.type).toBe('quota'); + expect(parsed.source).toBe('gateway'); + expect(parsed.message).toBe('The Mastra Gateway refused this request because of your credits or a budget.'); + expect(parsed.message).not.toContain('provider account'); + expect(parsed.detail).toBe('organization_budget_unbounded_request'); + expect(parsed.retryable).toBe(false); + }); + + it('shows a readable gateway message alongside its code', () => { + const parsed = parseError( + gatewayError(402, 'organization_budget_insufficient', 'The organization budget cannot cover this request.'), + { gatewayBaseUrl: `${gatewayBaseUrl}/v1/` }, + ); + + expect(parsed.detail).toBe( + 'The organization budget cannot cover this request. (organization_budget_insufficient)', + ); + }); + + it.each(['no_eligible_route', 'model_not_allowed'])( + 'words a gateway 403 %s as a policy or routing refusal', + code => { + const parsed = parseError(gatewayError(403, code), { gatewayBaseUrl }); + + expect(parsed.type).toBe('auth'); + expect(parsed.source).toBe('gateway'); + expect(parsed.message).toBe( + "The Mastra Gateway's policy or routing does not allow this model or feature for your key.", + ); + expect(parsed.detail).toBe(code); + }, + ); + + it('reads the gateway status and body from a wrapped cause', () => { + const error = new Error('Agent stream failed', { cause: gatewayError(402, 'organization_credits_exhausted') }); + + const parsed = parseError(error, { gatewayBaseUrl }); + + expect(parsed.source).toBe('gateway'); + expect(parsed.detail).toBe('organization_credits_exhausted'); + }); + + it('keeps the provider wording for the same errors from another origin', () => { + const quota = parseError( + gatewayError(402, 'insufficient_quota', 'Payment Required', 'https://api.openai.com/v1/responses'), + { + gatewayBaseUrl, + }, + ); + const denied = parseError( + gatewayError(403, 'permission_denied', 'forbidden', 'https://api.anthropic.com/v1/messages'), + { + gatewayBaseUrl, + }, + ); + + expect(quota.source).toBeUndefined(); + expect(quota.message).toBe('Usage quota exhausted for this provider account.'); + expect(denied.source).toBeUndefined(); + expect(denied.message).toBe('Access denied. You may not have permission to use this model.'); + }); + + it.each([ + [403, 'request_forbidden', 'unsupported_country_region_territory'], + [402, 'invalid_request_error', 'insufficient_quota'], + ])('keeps the provider wording for a provider %i the gateway forwarded with type %s', (status, type, code) => { + const error = Object.assign(new Error(code), { + statusCode: status, + url: `${gatewayBaseUrl}/v1/responses`, + responseBody: JSON.stringify({ error: { type, code, message: code } }), + }); + + const parsed = parseError(error, { gatewayBaseUrl }); + + expect(parsed.source).toBeUndefined(); + expect(parsed.message).not.toContain('Mastra Gateway'); + }); + + it('words a gateway 403 from the messages endpoint, which sends the code as the message', () => { + const error = Object.assign(new Error('model_not_allowed'), { + statusCode: 403, + url: `${gatewayBaseUrl}/v1/messages`, + responseBody: JSON.stringify({ type: 'error', error: { type: 'gateway_error', message: 'model_not_allowed' } }), + }); + + const parsed = parseError(error, { gatewayBaseUrl }); + + expect(parsed.source).toBe('gateway'); + expect(parsed.detail).toBe('model_not_allowed'); + }); + + it('keeps the access-denied wording for a gateway 403 without a gateway code', () => { + const error = Object.assign(new Error('forbidden'), { + status: 403, + requestUrl: `${gatewayBaseUrl}/v1/responses`, + }); + + const parsed = parseError(error, { gatewayBaseUrl }); + + expect(parsed.source).toBeUndefined(); + expect(parsed.message).toBe('Access denied. You may not have permission to use this model.'); + }); + }); }); diff --git a/mastracode/sdk/src/utils/errors.ts b/mastracode/sdk/src/utils/errors.ts index d9b3fe9d85f..1a18befb451 100644 --- a/mastracode/sdk/src/utils/errors.ts +++ b/mastracode/sdk/src/utils/errors.ts @@ -22,6 +22,13 @@ export interface ParsedError { retryDelay?: number; /** Original error for debugging */ originalError: Error; + /** Set when the Mastra Gateway itself refused the request, rather than the model provider */ + source?: 'gateway'; +} + +export interface ParseErrorOptions { + /** Base URL of the configured Mastra Gateway. Failed requests to its origin are worded as gateway refusals. */ + gatewayBaseUrl?: string; } export type ErrorType = @@ -96,6 +103,93 @@ function extractRequestUrl(error: unknown): string | undefined { return undefined; } +function originOf(url: string | undefined): string | undefined { + if (!url) return undefined; + try { + return new URL(url).origin; + } catch { + return undefined; + } +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null; +} + +function nonEmptyString(value: unknown): string | undefined { + return typeof value === 'string' && value.trim().length > 0 ? value.trim() : undefined; +} + +function parseJsonBody(text: unknown): unknown { + if (typeof text !== 'string') return undefined; + try { + return JSON.parse(text); + } catch { + return undefined; + } +} + +/** + * The code and message from a Mastra Gateway error body, `{ "error": { "type": "gateway_error", "code", "message" } }`. + * The message may only repeat the code, so both are kept. Errors the gateway forwards from a provider + * carry the provider's own `type`, so only `gateway_error` bodies count as the gateway's refusal. + * The raw response body is read before the provider's parsed `data`, because a provider's error schema + * can drop fields it does not declare, such as the code on the Anthropic messages shape. + */ +function gatewayErrorOf(error: unknown): { code?: string; message?: string } | undefined { + for (const candidate of [error, isRecord(error) ? error['cause'] : undefined]) { + if (!isRecord(candidate)) continue; + for (const body of [parseJsonBody(candidate['responseBody']), candidate['data']]) { + const inner = isRecord(body) ? body['error'] : undefined; + if (isRecord(inner) && inner['type'] === 'gateway_error') { + return { code: nonEmptyString(inner['code']), message: nonEmptyString(inner['message']) }; + } + } + } + return undefined; +} + +function describeGatewayError({ code, message }: { code?: string; message?: string }): string | undefined { + if (message && code && message !== code) return `${message} (${code})`; + return message ?? code; +} + +/** A refusal by the Mastra Gateway itself, worded for the gateway instead of the provider account. */ +function parseGatewayRefusal( + error: unknown, + status: unknown, + requestUrl: string | undefined, + err: Error, + fallbackDetail: string | undefined, +): ParsedError | undefined { + const gatewayError = gatewayErrorOf(error); + if (!gatewayError) return undefined; + const detail = describeGatewayError(gatewayError) ?? fallbackDetail; + if (status === 402) { + return { + message: 'The Mastra Gateway refused this request because of your credits or a budget.', + detail, + requestUrl, + type: 'quota', + retryable: false, + originalError: err, + source: 'gateway', + }; + } + if (status === 403) { + return { + message: "The Mastra Gateway's policy or routing does not allow this model or feature for your key.", + detail, + requestUrl, + type: 'auth', + retryable: false, + originalError: err, + source: 'gateway', + }; + } + return undefined; +} + /** A flattened error crossing the wire: the server sends `{ name, message }`, not an `Error`. */ function isSerializedError(value: unknown): value is { name: string; message: string } { return ( @@ -170,7 +264,7 @@ function errorTypeOf(error: unknown, message: string, status: unknown): ErrorTyp return 'unknown'; } -export function parseError(error: unknown): ParsedError { +export function parseError(error: unknown, options: ParseErrorOptions = {}): ParsedError { const err = toError(error); const errorObj = (typeof error === 'object' && error !== null ? error : {}) as Record; const detail = summarizeErrorDetail(error); @@ -181,8 +275,16 @@ export function parseError(error: unknown): ParsedError { return { message: err.message, detail, requestUrl, type: 'auth', retryable: false, originalError: err }; } + const status = errorObj.statusCode ?? errorObj.status; + const gatewayOrigin = originOf(options.gatewayBaseUrl); + if (gatewayOrigin && originOf(requestUrl) === gatewayOrigin) { + const cause = isRecord(errorObj.cause) ? errorObj.cause : {}; + const refusal = parseGatewayRefusal(error, status ?? cause.statusCode ?? cause.status, requestUrl, err, detail); + if (refusal) return refusal; + } + const message = err.message.toLowerCase(); - const type = errorTypeOf(error, message, errorObj.statusCode ?? errorObj.status); + const type = errorTypeOf(error, message, status); switch (type) { case 'quota': @@ -205,7 +307,7 @@ export function parseError(error: unknown): ParsedError { case 'auth': return { message: - (errorObj.statusCode ?? errorObj.status) === 403 + status === 403 ? 'Access denied. You may not have permission to use this model.' : 'Authentication failed. Check the credential configured for this provider.', detail, diff --git a/mastracode/tui/src/tui/commands/__tests__/mastra-gateway.test.ts b/mastracode/tui/src/tui/commands/__tests__/mastra-gateway.test.ts index bf4c2ab0475..c1bda70c876 100644 --- a/mastracode/tui/src/tui/commands/__tests__/mastra-gateway.test.ts +++ b/mastracode/tui/src/tui/commands/__tests__/mastra-gateway.test.ts @@ -102,9 +102,7 @@ describe('handleMastraGatewayCommand', () => { expect(mockSaveSettings).toHaveBeenCalledWith({ memoryGateway: { baseUrl: 'http://localhost:4111' } }); expect(mockGatewayRegistryGetInstance).toHaveBeenCalledWith({ useDynamicLoading: true }); expect(mockGatewayRegistrySyncGateways).toHaveBeenCalledWith(true); - expect(ctx.showInfo).toHaveBeenLastCalledWith( - 'Gateway configured. Memory mode changes take effect on next restart.', - ); + expect(ctx.showInfo).toHaveBeenLastCalledWith('Gateway configured.'); }); it('prefills the existing API key and stores the localhost URL without prompting for custom input', async () => { @@ -164,6 +162,6 @@ describe('handleMastraGatewayCommand', () => { expect(process.env.MASTRA_GATEWAY_API_KEY).toBeUndefined(); expect(process.env.MASTRA_GATEWAY_URL).toBeUndefined(); expect(mockGatewayRegistrySyncGateways).toHaveBeenCalledWith(true); - expect(ctx.showInfo).toHaveBeenLastCalledWith('Gateway cleared. Memory mode changes take effect on next restart.'); + expect(ctx.showInfo).toHaveBeenLastCalledWith('Gateway cleared.'); }); }); diff --git a/mastracode/tui/src/tui/commands/mastra-gateway.ts b/mastracode/tui/src/tui/commands/mastra-gateway.ts index 085bb234fe6..cf3a8afdc10 100644 --- a/mastracode/tui/src/tui/commands/mastra-gateway.ts +++ b/mastracode/tui/src/tui/commands/mastra-gateway.ts @@ -74,7 +74,7 @@ export async function handleMastraGatewayCommand(ctx: SlashCommandContext): Prom settings.memoryGateway = {}; saveSettings(settings); await refreshGatewayModels(ctx); - ctx.showInfo('Gateway cleared. Memory mode changes take effect on next restart.'); + ctx.showInfo('Gateway cleared.'); return; } else if (keyAnswer.length > 0) { authStorage.setStoredApiKey(MASTRA_GATEWAY_PROVIDER, keyAnswer, 'MASTRA_GATEWAY_API_KEY'); @@ -109,5 +109,5 @@ export async function handleMastraGatewayCommand(ctx: SlashCommandContext): Prom saveSettings(settings); await refreshGatewayModels(ctx); - ctx.showInfo('Gateway configured. Memory mode changes take effect on next restart.'); + ctx.showInfo('Gateway configured.'); } diff --git a/mastracode/tui/src/tui/display.test.ts b/mastracode/tui/src/tui/display.test.ts index 1ad74eb492a..5e2f7d07df8 100644 --- a/mastracode/tui/src/tui/display.test.ts +++ b/mastracode/tui/src/tui/display.test.ts @@ -5,6 +5,11 @@ import { describe, expect, it, vi } from 'vitest'; import { showError, showFormattedError, showInfo, showLines, showProgress } from './display.js'; import type { TUIState } from './state.js'; +vi.mock('@mastra/code-sdk/onboarding/settings', async importOriginal => ({ + ...(await importOriginal()), + loadSettings: () => ({ memoryGateway: { baseUrl: 'https://gateway.example.test' } }), +})); + function createState(): TUIState { return { chatContainer: new Container(), @@ -95,6 +100,48 @@ describe('showFormattedError', () => { expect(text).toContain('rejected as a result of our safety system'); expect(text).toContain('Hint: Retry, rephrase, or use /model to switch models'); }); + + function apiError(status: number, url: string, code: string) { + return Object.assign(new Error(code), { + statusCode: status, + url, + responseBody: JSON.stringify({ error: { type: 'gateway_error', code, message: code } }), + }); + } + + it('points a gateway credit refusal at Mastra billing instead of the provider account', () => { + const state = createState(); + + showFormattedError( + state, + apiError(402, 'https://gateway.example.test/v1/chat/completions', 'organization_budget_insufficient'), + ); + + const text = renderedText(state); + expect(text).toContain('The Mastra Gateway refused this request because of your credits or a budget.'); + expect(text).toContain('organization_budget_insufficient'); + expect(text).toContain('Mastra billing'); + expect(text).not.toContain('provider account'); + }); + + it('suggests another model for a gateway policy refusal instead of /connect', () => { + const state = createState(); + + showFormattedError(state, apiError(403, 'https://gateway.example.test/v1/chat/completions', 'no_eligible_route')); + + const text = renderedText(state); + expect(text).toContain('does not allow this model or feature for your key'); + expect(text).toContain('Hint: Use /models to choose another model'); + expect(text).not.toContain('/connect'); + }); + + it('keeps the /connect hint for a provider 403', () => { + const state = createState(); + + showFormattedError(state, apiError(403, 'https://api.anthropic.com/v1/messages', 'permission_error')); + + expect(renderedText(state)).toContain('Hint: Use /connect to authenticate with a provider'); + }); }); /** Minimal chat child standing in for an inline prompt component (#21966). */ diff --git a/mastracode/tui/src/tui/display.ts b/mastracode/tui/src/tui/display.ts index 497d3580ddf..ba998d94512 100644 --- a/mastracode/tui/src/tui/display.ts +++ b/mastracode/tui/src/tui/display.ts @@ -4,7 +4,9 @@ import { Container, Text } from '@earendil-works/pi-tui'; +import { loadSettings, MASTRA_GATEWAY_DEFAULT_URL } from '@mastra/code-sdk/onboarding/settings'; import { parseError } from '@mastra/code-sdk/utils/errors'; +import type { ParsedError } from '@mastra/code-sdk/utils/errors'; import type { AgentControllerEvent } from '@mastra/core/agent-controller'; import { insertChatComponentWithBoundarySpacing, @@ -105,7 +107,7 @@ export function showFormattedError( | Error, ): void { const error = 'error' in event ? event.error : event; - const parsed = parseError(error); + const parsed = parseError(error, { gatewayBaseUrl: configuredGatewayBaseUrl() }); // Show the main error message let errorText = `Error: ${parsed.message}`; @@ -133,9 +135,11 @@ export function showFormattedError( const om = state.local?.session.om; const omRole = /reflect/i.test(error.message) ? om?.reflector : om?.observer; const hint = withOMGuidance( - state.factory && parsed.type === 'auth' - ? `Add the provider credential in Factory (${state.factory.url})` - : getErrorHint(parsed.type), + parsed.source === 'gateway' + ? getGatewayErrorHint(parsed) + : state.factory && parsed.type === 'auth' + ? `Add the provider credential in Factory (${state.factory.url})` + : getErrorHint(parsed.type), isObservationalMemoryError ? omRole?.modelId() : undefined, ); if (hint) { @@ -154,6 +158,28 @@ function withOMGuidance(typeHint: string | null, omModelId: string | undefined): .join('. '); } +/** The Mastra Gateway URL model requests use, so its refusals can be told apart from provider errors. */ +function configuredGatewayBaseUrl(): string { + let baseUrl: string | undefined; + try { + baseUrl = loadSettings().memoryGateway?.baseUrl; + } catch { + // Unreadable settings fall back to the environment and the hosted default, like model resolution. + } + return baseUrl ?? process.env['MASTRA_GATEWAY_URL'] ?? MASTRA_GATEWAY_DEFAULT_URL; +} + +function getGatewayErrorHint(parsed: ParsedError): string | null { + switch (parsed.type) { + case 'quota': + return 'Check your credits and budgets in Mastra billing for the organization that owns the Gateway key'; + case 'auth': + return 'Use /models to choose another model'; + default: + return getErrorHint(parsed.type); + } +} + function getErrorHint(errorType: string): string | null { switch (errorType) { case 'auth': diff --git a/packages/core/src/llm/model/gateways/mastra.test.ts b/packages/core/src/llm/model/gateways/mastra.test.ts index 3fc7745c284..734165c714f 100644 --- a/packages/core/src/llm/model/gateways/mastra.test.ts +++ b/packages/core/src/llm/model/gateways/mastra.test.ts @@ -1,6 +1,6 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { MastraGateway } from './mastra.js'; +import { _resetMastraGatewayCatalogCache, MastraGateway } from './mastra.js'; vi.mock('../provider-registry.js', () => ({ PROVIDER_REGISTRY: { openrouter: { models: ['openai/test-model'] } }, @@ -10,14 +10,22 @@ describe('MastraGateway', () => { beforeEach(() => { delete process.env.MASTRA_GATEWAY_API_KEY; delete process.env.MASTRA_GATEWAY_VERSION; + _resetMastraGatewayCatalogCache(); }); afterEach(() => { delete process.env.MASTRA_GATEWAY_API_KEY; delete process.env.MASTRA_GATEWAY_VERSION; vi.restoreAllMocks(); + vi.unstubAllGlobals(); }); + /** A v2 gateway whose catalogue request is answered by `gatewayFetch` in place of the global fetch. */ + function gatewayWith(apiKey: string, gatewayFetch: (...args: any[]) => Promise) { + vi.stubGlobal('fetch', gatewayFetch); + return new MastraGateway({ apiKey }); + } + it('reports disabled when MASTRA_GATEWAY_API_KEY is not set', () => { const gateway = new MastraGateway(); @@ -62,18 +70,15 @@ describe('MastraGateway', () => { { status: 200, headers: { 'content-type': 'application/json' } }, ), ); - const gateway = new MastraGateway({ - apiKey: 'deployment-key', - baseUrl: 'https://gateway.example/v1', - customFetch: gatewayFetch, - }); + vi.stubGlobal('fetch', gatewayFetch); + const gateway = new MastraGateway({ apiKey: 'deployment-key', baseUrl: 'https://gateway.example/v1' }); const providers = await gateway.fetchProviders(); expect(gatewayFetch).toHaveBeenCalledWith( 'https://gateway.example/v1/models', expect.objectContaining({ - headers: expect.objectContaining({ 'X-Memory-Gateway-Authorization': 'Bearer deployment-key' }), + headers: { Authorization: 'Bearer deployment-key' }, signal: expect.any(AbortSignal), }), ); @@ -87,38 +92,41 @@ describe('MastraGateway', () => { ['https://gateway.example/api/', 'https://gateway.example/api/v1/models'], ])('normalizes the base URL %s', async (baseUrl, expected) => { process.env.MASTRA_GATEWAY_VERSION = 'v2'; - const customFetch = vi.fn(async () => new Response(JSON.stringify({ data: [] }), { status: 200 })); - await new MastraGateway({ apiKey: 'deployment-key', baseUrl, customFetch }).fetchProviders(); - expect(customFetch).toHaveBeenCalledWith(expected, expect.anything()); + const gatewayFetch = vi.fn(async () => new Response(JSON.stringify({ data: [] }), { status: 200 })); + vi.stubGlobal('fetch', gatewayFetch); + await new MastraGateway({ apiKey: 'deployment-key', baseUrl }).fetchProviders(); + expect(gatewayFetch).toHaveBeenCalledWith(expected, expect.anything()); }); it('normalizes a base URL with a pathological run of slashes in linear time', async () => { process.env.MASTRA_GATEWAY_VERSION = 'v2'; - const customFetch = vi.fn(async () => new Response(JSON.stringify({ data: [] }), { status: 200 })); + const gatewayFetch = vi.fn(async () => new Response(JSON.stringify({ data: [] }), { status: 200 })); + vi.stubGlobal('fetch', gatewayFetch); const slashes = '/'.repeat(100_000); const baseUrl = `https://gateway.example${slashes}api${slashes}`; const started = performance.now(); - await new MastraGateway({ apiKey: 'deployment-key', baseUrl, customFetch }).fetchProviders(); + await new MastraGateway({ apiKey: 'deployment-key', baseUrl }).fetchProviders(); expect(performance.now() - started).toBeLessThan(500); - expect(customFetch).toHaveBeenCalledWith(`https://gateway.example${slashes}api/v1/models`, expect.anything()); + expect(gatewayFetch).toHaveBeenCalledWith(`https://gateway.example${slashes}api/v1/models`, expect.anything()); }); - it('preserves the Gateway key when customFetch replaces Authorization with OAuth', async () => { + it('sends the v2 catalog request with the key in Authorization, not through a custom fetch', async () => { process.env.MASTRA_GATEWAY_VERSION = 'v2'; - const customFetch = vi.fn(async (_input: RequestInfo | URL, init?: RequestInit) => { - const headers = new Headers(init?.headers); - headers.set('Authorization', 'Bearer oauth-token'); - expect(headers.get('Authorization')).toBe('Bearer oauth-token'); - expect(headers.get('X-Memory-Gateway-Authorization')).toBe('Bearer deployment-key'); - return new Response(JSON.stringify({ data: [{ id: 'openai/chat-model', endpoints: ['chat'] }] }), { - status: 200, - }); - }); + const customFetch = vi.fn(async () => new Response('{}', { status: 200 })); + const globalFetch = vi.fn( + async (_input: RequestInfo | URL, _init?: RequestInit) => + new Response(JSON.stringify({ data: [{ id: 'openai/chat-model', endpoints: ['chat'] }] }), { status: 200 }), + ); + vi.stubGlobal('fetch', globalFetch); const gateway = new MastraGateway({ apiKey: 'deployment-key', customFetch }); const providers = await gateway.fetchProviders(); - expect(customFetch).toHaveBeenCalledOnce(); + expect(customFetch).not.toHaveBeenCalled(); + expect(globalFetch).toHaveBeenCalledOnce(); + const headers = new Headers(globalFetch.mock.calls[0]![1]?.headers); + expect(headers.get('Authorization')).toBe('Bearer deployment-key'); + expect(headers.get('X-Memory-Gateway-Authorization')).toBeNull(); expect(providers.mastra?.models).toEqual(['openai/chat-model']); }); @@ -126,16 +134,16 @@ describe('MastraGateway', () => { process.env.MASTRA_GATEWAY_VERSION = 'v2'; const controller = new AbortController(); const timeout = vi.spyOn(AbortSignal, 'timeout').mockReturnValue(controller.signal); - const customFetch = vi.fn( + const gatewayFetch = vi.fn( (_input: RequestInfo | URL, init?: RequestInit) => new Promise((_resolve, reject) => { init?.signal?.addEventListener('abort', () => reject(init.signal?.reason), { once: true }); }), ); - const gateway = new MastraGateway({ apiKey: 'deployment-key', customFetch }); + const gateway = gatewayWith('deployment-key', gatewayFetch); const pending = gateway.fetchProviders(); - await vi.waitFor(() => expect(customFetch).toHaveBeenCalledOnce()); + await vi.waitFor(() => expect(gatewayFetch).toHaveBeenCalledOnce()); controller.abort(new DOMException('catalog deadline', 'TimeoutError')); await expect(pending).rejects.toThrow('catalog deadline'); @@ -147,7 +155,7 @@ describe('MastraGateway', () => { async () => new Response(JSON.stringify({ data: [{ id: 'openai/chat-model', endpoints: ['chat'] }] }), { status: 200 }), ); - const gateway = new MastraGateway({ apiKey: 'mskv2_org-key', customFetch: gatewayFetch }); + const gateway = gatewayWith('mskv2_org-key', gatewayFetch); const providers = await gateway.fetchProviders(); @@ -156,15 +164,353 @@ describe('MastraGateway', () => { expect(providers.mastra?.modelReleaseDates).toBeUndefined(); }); + describe('isV2', () => { + it.each([ + { version: undefined, key: 'msk_0123456789abcdef0123456789abcdef', expected: false }, + { version: 'v2', key: 'msk_0123456789abcdef0123456789abcdef', expected: false }, + { version: 'v2', key: 'deployment-key', expected: true }, + { version: 'v2', key: undefined, expected: true }, + { version: undefined, key: 'mskv2_org-key', expected: true }, + { version: undefined, key: 'msk-gateway-org-key', expected: true }, + { version: undefined, key: 'other-key', expected: false }, + ])('with MASTRA_GATEWAY_VERSION=$version and key $key is $expected', ({ version, key, expected }) => { + if (version) process.env.MASTRA_GATEWAY_VERSION = version; + expect(new MastraGateway({ apiKey: key }).isV2()).toBe(expected); + }); + + it('lets a v1 config key win over a v2 env key and MASTRA_GATEWAY_VERSION', () => { + process.env.MASTRA_GATEWAY_VERSION = 'v2'; + process.env.MASTRA_GATEWAY_API_KEY = 'mskv2_org-key'; + expect(new MastraGateway({ apiKey: 'msk_0123456789abcdef0123456789abcdef' }).isV2()).toBe(false); + }); + }); + it('does not show a static v1 model list if v2 catalog loading fails', async () => { process.env.MASTRA_GATEWAY_VERSION = 'v2'; - const gateway = new MastraGateway({ - apiKey: 'deployment-key', - customFetch: vi.fn(async () => new Response('unavailable', { status: 503 })), - }); + const gateway = gatewayWith( + 'deployment-key', + vi.fn(async () => new Response('unavailable', { status: 503 })), + ); await expect(gateway.fetchProviders()).rejects.toThrow('Gateway model catalog unavailable (503)'); }); + describe('v2 catalog cache', () => { + const catalog = (ids: string[]) => + new Response(JSON.stringify({ data: ids.map(id => ({ id, endpoints: ['chat'] })) }), { status: 200 }); + + beforeEach(() => { + process.env.MASTRA_GATEWAY_VERSION = 'v2'; + }); + + it('reuses the catalog for a minute, then fetches it again', async () => { + const now = vi.spyOn(Date, 'now').mockReturnValue(1_000_000); + const gatewayFetch = vi.fn(async () => catalog(['openai/chat-model'])); + const gateway = gatewayWith('deployment-key', gatewayFetch); + + await gateway.fetchProviders(); + now.mockReturnValue(1_000_000 + 59_000); + await gateway.fetchProviders(); + expect(gatewayFetch).toHaveBeenCalledOnce(); + + now.mockReturnValue(1_000_000 + 61_000); + await gateway.fetchProviders(); + expect(gatewayFetch).toHaveBeenCalledTimes(2); + }); + + it('shares one request between concurrent callers', async () => { + let respond!: (response: Response) => void; + const gatewayFetch = vi.fn(() => new Promise(resolve => (respond = resolve))); + const gateway = gatewayWith('deployment-key', gatewayFetch); + + const first = gateway.fetchProviders(); + const second = gatewayWith('deployment-key', gatewayFetch).fetchProviders(); + await vi.waitFor(() => expect(gatewayFetch).toHaveBeenCalledOnce()); + respond(catalog(['openai/chat-model'])); + + expect((await first).mastra?.models).toEqual(['openai/chat-model']); + expect((await second).mastra?.models).toEqual(['openai/chat-model']); + expect(gatewayFetch).toHaveBeenCalledOnce(); + }); + + it('keeps catalogs for different keys apart', async () => { + const gatewayFetch = vi.fn(async (_input: RequestInfo | URL, init?: RequestInit) => { + const key = new Headers(init?.headers).get('Authorization'); + return catalog([key === 'Bearer key-a' ? 'openai/model-a' : 'openai/model-b']); + }); + + const a = await gatewayWith('key-a', gatewayFetch).fetchProviders(); + const b = await gatewayWith('key-b', gatewayFetch).fetchProviders(); + + expect(a.mastra?.models).toEqual(['openai/model-a']); + expect(b.mastra?.models).toEqual(['openai/model-b']); + }); + + it('serves the last good catalog for up to ten minutes when a refresh fails', async () => { + const now = vi.spyOn(Date, 'now').mockReturnValue(1_000_000); + const gatewayFetch = vi.fn(async () => catalog(['openai/chat-model'])); + const gateway = gatewayWith('deployment-key', gatewayFetch); + await gateway.fetchProviders(); + + gatewayFetch.mockImplementation(async () => new Response('unavailable', { status: 503 })); + now.mockReturnValue(1_000_000 + 9 * 60_000); + expect((await gateway.fetchProviders()).mastra?.models).toEqual(['openai/chat-model']); + + now.mockReturnValue(1_000_000 + 11 * 60_000); + await expect(gateway.fetchProviders()).rejects.toThrow('Gateway model catalog unavailable (503)'); + }); + + it.each([401, 403])('drops the cached catalog when the key is rejected with %i', async status => { + const now = vi.spyOn(Date, 'now').mockReturnValue(1_000_000); + const gatewayFetch = vi.fn(async () => catalog(['openai/chat-model'])); + const gateway = gatewayWith('deployment-key', gatewayFetch); + await gateway.fetchProviders(); + + gatewayFetch.mockImplementation(async () => new Response('rejected', { status })); + now.mockReturnValue(1_000_000 + 61_000); + await expect(gateway.fetchProviders()).rejects.toThrow(`Gateway model catalog unavailable (${status})`); + + gatewayFetch.mockImplementation(async () => catalog(['openai/other-model'])); + expect((await gateway.fetchProviders()).mastra?.models).toEqual(['openai/other-model']); + }); + + it('serves the last good catalog after a network error', async () => { + const now = vi.spyOn(Date, 'now').mockReturnValue(1_000_000); + const gatewayFetch = vi.fn(async () => catalog(['openai/chat-model'])); + const gateway = gatewayWith('deployment-key', gatewayFetch); + await gateway.fetchProviders(); + + gatewayFetch.mockRejectedValue(new TypeError('fetch failed')); + now.mockReturnValue(1_000_000 + 61_000); + expect((await gateway.fetchProviders()).mastra?.models).toEqual(['openai/chat-model']); + }); + + it('waits before retrying after a failed refresh', async () => { + const now = vi.spyOn(Date, 'now').mockReturnValue(1_000_000); + const gatewayFetch = vi.fn(async () => catalog(['openai/chat-model'])); + const gateway = gatewayWith('deployment-key', gatewayFetch); + await gateway.fetchProviders(); + + gatewayFetch.mockImplementation(async () => new Response('unavailable', { status: 503 })); + now.mockReturnValue(1_000_000 + 61_000); + await gateway.fetchProviders(); + expect(gatewayFetch).toHaveBeenCalledTimes(2); + + now.mockReturnValue(1_000_000 + 61_000 + 14_000); + expect((await gateway.fetchProviders()).mastra?.models).toEqual(['openai/chat-model']); + expect(gatewayFetch).toHaveBeenCalledTimes(2); + + now.mockReturnValue(1_000_000 + 61_000 + 16_000); + expect((await gateway.fetchProviders()).mastra?.models).toEqual(['openai/chat-model']); + expect(gatewayFetch).toHaveBeenCalledTimes(3); + }); + + it('keeps catalogs fetched through different global fetch implementations apart', async () => { + const firstFetch = vi.fn(async () => catalog(['openai/model-a'])); + const secondFetch = vi.fn(async () => catalog(['openai/model-b'])); + + const a = await gatewayWith('same-key', firstFetch).fetchProviders(); + const b = await gatewayWith('same-key', secondFetch).fetchProviders(); + + expect(a.mastra?.models).toEqual(['openai/model-a']); + expect(b.mastra?.models).toEqual(['openai/model-b']); + expect(secondFetch).toHaveBeenCalledOnce(); + }); + + it('keeps at most 100 catalogs, dropping the oldest', async () => { + const gatewayFetch = vi.fn(async () => catalog(['openai/chat-model'])); + for (let i = 0; i <= 100; i++) { + await gatewayWith(`key-${i}`, gatewayFetch).fetchProviders(); + } + expect(gatewayFetch).toHaveBeenCalledTimes(101); + + await gatewayWith('key-100', gatewayFetch).fetchProviders(); + expect(gatewayFetch).toHaveBeenCalledTimes(101); + await gatewayWith('key-0', gatewayFetch).fetchProviders(); + expect(gatewayFetch).toHaveBeenCalledTimes(102); + }); + + it('remembers a failure with no earlier catalog for 15 seconds', async () => { + const now = vi.spyOn(Date, 'now').mockReturnValue(1_000_000); + const gatewayFetch = vi.fn(async () => new Response('unavailable', { status: 503 })); + const gateway = gatewayWith('deployment-key', gatewayFetch); + + await expect(gateway.fetchProviders()).rejects.toThrow('Gateway model catalog unavailable (503)'); + expect(gatewayFetch).toHaveBeenCalledOnce(); + + now.mockReturnValue(1_000_000 + 14_000); + await expect(gateway.fetchProviders()).rejects.toThrow('Gateway model catalog unavailable (503)'); + expect(gatewayFetch).toHaveBeenCalledOnce(); + + now.mockReturnValue(1_000_000 + 16_000); + await expect(gateway.fetchProviders()).rejects.toThrow('Gateway model catalog unavailable (503)'); + expect(gatewayFetch).toHaveBeenCalledTimes(2); + }); + + it('remembers a network error with no earlier catalog', async () => { + const now = vi.spyOn(Date, 'now').mockReturnValue(1_000_000); + const gatewayFetch = vi.fn(async () => { + throw new TypeError('fetch failed'); + }); + const gateway = gatewayWith('deployment-key', gatewayFetch); + + await expect(gateway.fetchProviders()).rejects.toThrow('fetch failed'); + now.mockReturnValue(1_000_000 + 5_000); + await expect(gateway.fetchProviders()).rejects.toThrow('fetch failed'); + expect(gatewayFetch).toHaveBeenCalledOnce(); + }); + + it('clears a remembered failure on success, and the next failure starts a new window', async () => { + const now = vi.spyOn(Date, 'now').mockReturnValue(1_000_000); + const gatewayFetch = vi.fn(async () => new Response('unavailable', { status: 503 })); + const gateway = gatewayWith('deployment-key', gatewayFetch); + await expect(gateway.fetchProviders()).rejects.toThrow('(503)'); + + gatewayFetch.mockImplementation(async () => catalog(['openai/chat-model'])); + now.mockReturnValue(1_000_000 + 16_000); + expect((await gateway.fetchProviders()).mastra?.models).toEqual(['openai/chat-model']); + expect(gatewayFetch).toHaveBeenCalledTimes(2); + + // Past the stale window there is no list to fall back on, so a failure is remembered again. + gatewayFetch.mockImplementation(async () => new Response('unavailable', { status: 503 })); + now.mockReturnValue(1_000_000 + 16_000 + 11 * 60_000); + await expect(gateway.fetchProviders()).rejects.toThrow('(503)'); + expect(gatewayFetch).toHaveBeenCalledTimes(3); + now.mockReturnValue(1_000_000 + 16_000 + 11 * 60_000 + 14_000); + await expect(gateway.fetchProviders()).rejects.toThrow('(503)'); + expect(gatewayFetch).toHaveBeenCalledTimes(3); + now.mockReturnValue(1_000_000 + 16_000 + 11 * 60_000 + 16_000); + await expect(gateway.fetchProviders()).rejects.toThrow('(503)'); + expect(gatewayFetch).toHaveBeenCalledTimes(4); + }); + + it.each([401, 403])('asks the gateway again on every call after a %i', async status => { + vi.spyOn(Date, 'now').mockReturnValue(1_000_000); + const gatewayFetch = vi.fn(async () => new Response('rejected', { status })); + const gateway = gatewayWith('deployment-key', gatewayFetch); + + await expect(gateway.fetchProviders()).rejects.toThrow(`(${status})`); + await expect(gateway.fetchProviders()).rejects.toThrow(`(${status})`); + expect(gatewayFetch).toHaveBeenCalledTimes(2); + + gatewayFetch.mockImplementation(async () => catalog(['openai/chat-model'])); + expect((await gateway.fetchProviders()).mastra?.models).toEqual(['openai/chat-model']); + }); + + it('shares one failing request between concurrent callers', async () => { + let respond!: (response: Response) => void; + const gatewayFetch = vi.fn(() => new Promise(resolve => (respond = resolve))); + const gateway = gatewayWith('deployment-key', gatewayFetch); + + const first = gateway.fetchProviders(); + const second = gateway.fetchProviders(); + await vi.waitFor(() => expect(gatewayFetch).toHaveBeenCalledOnce()); + respond(new Response('unavailable', { status: 503 })); + + await expect(first).rejects.toThrow('(503)'); + await expect(second).rejects.toThrow('(503)'); + expect(gatewayFetch).toHaveBeenCalledOnce(); + }); + + it('counts remembered failures toward the 100 catalog limit', async () => { + const gatewayFetch = vi.fn(async () => new Response('unavailable', { status: 503 })); + for (let i = 0; i <= 100; i++) { + await expect(gatewayWith(`key-${i}`, gatewayFetch).fetchProviders()).rejects.toThrow('(503)'); + } + expect(gatewayFetch).toHaveBeenCalledTimes(101); + + await expect(gatewayWith('key-100', gatewayFetch).fetchProviders()).rejects.toThrow('(503)'); + expect(gatewayFetch).toHaveBeenCalledTimes(101); + await expect(gatewayWith('key-0', gatewayFetch).fetchProviders()).rejects.toThrow('(503)'); + expect(gatewayFetch).toHaveBeenCalledTimes(102); + }); + + it('forgets remembered failures when the cache is reset', async () => { + const gatewayFetch = vi.fn(async () => new Response('unavailable', { status: 503 })); + const gateway = gatewayWith('deployment-key', gatewayFetch); + await expect(gateway.fetchProviders()).rejects.toThrow('(503)'); + + _resetMastraGatewayCatalogCache(); + await expect(gateway.fetchProviders()).rejects.toThrow('(503)'); + expect(gatewayFetch).toHaveBeenCalledTimes(2); + }); + }); + + describe('resolveLanguageModel streaming usage', () => { + const prompt = [{ role: 'user', content: [{ type: 'text', text: 'hi' }] }]; + + function captureBodies() { + const bodies: Record[] = []; + const fetchImpl = vi.fn(async (_input: any, init: any) => { + bodies.push(JSON.parse(String(init?.body))); + return new Response('{}', { status: 200, headers: { 'content-type': 'application/json' } }); + }); + return { bodies, fetchImpl }; + } + + async function send(model: any) { + await model.doStream({ prompt }).catch(() => {}); + await model.doGenerate({ prompt }).catch(() => {}); + } + + it('asks for usage on streaming requests only with an API key', async () => { + const { bodies, fetchImpl } = captureBodies(); + vi.stubGlobal('fetch', fetchImpl); + try { + const model = new MastraGateway({ apiKey: 'gateway-key' }).resolveLanguageModel({ + modelId: 'gpt-4o', + providerId: 'openai', + apiKey: 'gateway-key', + }); + await send(model); + } finally { + vi.unstubAllGlobals(); + } + + expect(bodies).toHaveLength(2); + expect(bodies[0]).toMatchObject({ stream: true, stream_options: { include_usage: true } }); + expect(bodies[1]).not.toHaveProperty('stream_options'); + expect(bodies[1]).not.toHaveProperty('usage'); + }); + + it('reports usage sent in a separate final chunk', async () => { + const frames = [ + { id: 'c1', choices: [{ index: 0, delta: { role: 'assistant', content: 'ok' } }] }, + { id: 'c1', choices: [{ index: 0, delta: {}, finish_reason: 'stop' }] }, + { id: 'c1', choices: [], usage: { prompt_tokens: 7, completion_tokens: 3, total_tokens: 10 } }, + ]; + const body = frames.map(frame => `data: ${JSON.stringify(frame)}\n\n`).join('') + 'data: [DONE]\n\n'; + const customFetch = vi.fn( + async () => new Response(body, { status: 200, headers: { 'content-type': 'text/event-stream' } }), + ); + const model: any = new MastraGateway({ + apiKey: 'gateway-key', + customFetch: customFetch as any, + }).resolveLanguageModel({ modelId: 'gpt-4o', providerId: 'openai', apiKey: 'gateway-key' }); + + const { stream } = await model.doStream({ prompt }); + const parts: any[] = []; + for await (const part of stream) parts.push(part); + + const finish = parts.find(part => part.type === 'finish'); + expect(finish?.usage).toMatchObject({ inputTokens: { total: 7 }, outputTokens: { total: 3 } }); + }); + + it('asks for usage on streaming requests only with a custom fetch', async () => { + const { bodies, fetchImpl } = captureBodies(); + const model = new MastraGateway({ apiKey: 'gateway-key', customFetch: fetchImpl as any }).resolveLanguageModel({ + modelId: 'gpt-4o', + providerId: 'openai', + apiKey: 'gateway-key', + }); + await send(model); + + expect(bodies).toHaveLength(2); + expect(bodies[0]).toMatchObject({ stream: true, stream_options: { include_usage: true } }); + expect(bodies[1]).not.toHaveProperty('stream_options'); + expect(bodies[1]).not.toHaveProperty('usage'); + }); + }); + describe('resolveLanguageModel gateway authorization header', () => { const GATEWAY_AUTH_HEADER = 'X-Memory-Gateway-Authorization'; @@ -248,4 +594,86 @@ describe('MastraGateway', () => { expect(getHeaders()?.get(GATEWAY_AUTH_HEADER)).toBe('Bearer gateway-key'); }); }); + + describe('resolveLanguageModel with a provider sign-in fetch', () => { + function stubResponse() { + return new Response('{}', { status: 200, headers: { 'content-type': 'application/json' } }); + } + + async function firstRequest( + gatewayKey: string, + providerId: string, + modelId: string, + constructorKey: string | undefined = gatewayKey, + ) { + const customFetch = vi.fn(async () => stubResponse()); + const globalFetch = vi.fn(async (_input: RequestInfo | URL, _init?: RequestInit) => stubResponse()); + vi.stubGlobal('fetch', globalFetch); + try { + const gateway = new MastraGateway({ apiKey: constructorKey, baseUrl: 'https://gateway.example', customFetch }); + const model: any = gateway.resolveLanguageModel({ modelId, providerId, apiKey: gatewayKey }); + await model.doGenerate({ prompt: [{ role: 'user', content: [{ type: 'text', text: 'hi' }] }] }).catch(() => { + // the stubbed response is not a valid provider payload; only the request matters + }); + } finally { + vi.unstubAllGlobals(); + } + const call = customFetch.mock.calls[0] ?? globalFetch.mock.calls[0]; + const [input, init] = call as unknown as [RequestInfo | URL, RequestInit | undefined]; + return { + usedCustomFetch: customFetch.mock.calls.length > 0, + url: String(input), + headers: new Headers(init?.headers), + body: JSON.parse(String(init?.body)), + }; + } + + it.each([ + ['msk-gateway-us_org-key', 'anthropic', 'claude-sonnet-4-5'], + ['msk-gateway-eu_org-key', 'openai', 'gpt-5.5'], + ['mskv2_org-key', 'anthropic', 'claude-sonnet-4-5'], + ])( + 'sends the %s organization key in Authorization to the chat endpoint for %s', + async (key, providerId, modelId) => { + const request = await firstRequest(key, providerId, modelId); + + expect(request.usedCustomFetch).toBe(false); + expect(request.url).toBe('https://gateway.example/v1/chat/completions'); + expect(request.body.model).toBe(`${providerId}/${modelId}`); + expect(request.headers.get('Authorization')).toBe(`Bearer ${key}`); + expect(request.headers.get('X-Memory-Gateway-Authorization')).toBeNull(); + }, + ); + + it('classifies the key the request carries, not the key the gateway was built with', async () => { + const request = await firstRequest( + 'msk-gateway-eu_override-key', + 'anthropic', + 'claude-sonnet-4-5', + 'msk_project-key', + ); + + expect(request.usedCustomFetch).toBe(false); + expect(request.url).toBe('https://gateway.example/v1/chat/completions'); + expect(request.headers.get('Authorization')).toBe('Bearer msk-gateway-eu_override-key'); + }); + + it('keeps the provider sign-in pass-through for a project key when MASTRA_GATEWAY_VERSION is v2', async () => { + process.env.MASTRA_GATEWAY_VERSION = 'v2'; + + const request = await firstRequest('msk_project-key', 'anthropic', 'claude-sonnet-4-5'); + + expect(request.usedCustomFetch).toBe(true); + expect(request.url).toBe('https://gateway.example/v1/messages'); + expect(request.headers.get('X-Memory-Gateway-Authorization')).toBe('Bearer msk_project-key'); + }); + + it('keeps the provider sign-in pass-through for a project key', async () => { + const request = await firstRequest('msk_project-key', 'anthropic', 'claude-sonnet-4-5'); + + expect(request.usedCustomFetch).toBe(true); + expect(request.url).toBe('https://gateway.example/v1/messages'); + expect(request.headers.get('X-Memory-Gateway-Authorization')).toBe('Bearer msk_project-key'); + }); + }); }); diff --git a/packages/core/src/llm/model/gateways/mastra.ts b/packages/core/src/llm/model/gateways/mastra.ts index c1a84285d0d..eb60ee2fb16 100644 --- a/packages/core/src/llm/model/gateways/mastra.ts +++ b/packages/core/src/llm/model/gateways/mastra.ts @@ -1,3 +1,4 @@ +import { createHash } from 'node:crypto'; import { createAnthropic } from '@ai-sdk/anthropic-v6'; import { createOpenRouter } from '@openrouter/ai-sdk-provider-v6'; import { MastraError } from '../../../error/index.js'; @@ -41,10 +42,15 @@ function withGatewayAuthHeader(apiKey: string, headers?: Record) */ export function gatewayServesObservationalMemory(apiKey: string | undefined): boolean { if (typeof apiKey !== 'string' || apiKey.length === 0) return false; - if (apiKey.startsWith('msk-gateway-') || apiKey.startsWith('mskv2_')) return false; + if (isOrganizationKey(apiKey)) return false; return apiKey.startsWith('msk_'); } +/** Organization keys (`msk-gateway-…`, including regional ones such as `msk-gateway-eu_…`, and `mskv2_…`) belong to Gateway v2. */ +function isOrganizationKey(apiKey: string): boolean { + return apiKey.startsWith('msk-gateway-') || apiKey.startsWith('mskv2_'); +} + export class MastraGateway extends MastraModelGateway { readonly id = 'mastra'; readonly name = 'Gateway'; @@ -67,16 +73,20 @@ export class MastraGateway extends MastraModelGateway { /** * Gateway v2 deployments are signaled by MASTRA_GATEWAY_VERSION=v2 (set by the platform) - * or by a v2 (organization) API key prefix, which predates the env var signal. + * or by a v2 (organization) API key prefix, which predates the env var signal. A v1 project + * key (`msk_`) is always served by v1, so it wins over the env var. * Keep in sync with isGatewayV2 in @mastra/server's gateway-memory-client. * Public so the server can ask a registered gateway instance for its - * effective v2 state (the instance may carry a v2 key via config.apiKey + * effective v2 state (the instance may carry a key via config.apiKey * that env-only checks cannot see). */ isV2(): boolean { - if (process.env['MASTRA_GATEWAY_VERSION'] === 'v2') return true; const key = this.config?.apiKey ?? process.env['MASTRA_GATEWAY_API_KEY']; - return !!key && (key.startsWith('msk-gateway-') || key.startsWith('mskv2_')); + // A v1 project key (`msk_`) is always served by v1, so it wins over the env var. + // The v2 prefixes (`msk-gateway-`, `mskv2_`) do not start with `msk_`. + if (key?.startsWith('msk_')) return false; + if (process.env['MASTRA_GATEWAY_VERSION'] === 'v2') return true; + return !!key && isOrganizationKey(key); } async fetchProviders(): Promise> { @@ -85,27 +95,12 @@ export class MastraGateway extends MastraModelGateway { let models = openrouterConfig?.models ?? []; if (isV2) { const apiKey = await this.getApiKey(); - const response = await (this.config?.customFetch ?? fetch)(`${this.getBaseUrl()}/v1/models`, { - headers: this.config?.customFetch ? withGatewayAuthHeader(apiKey) : { Authorization: `Bearer ${apiKey}` }, - signal: AbortSignal.timeout(10_000), - }); - if (!response.ok) throw new Error(`Gateway model catalog unavailable (${response.status})`); - const body: unknown = await response.json(); - if (!body || typeof body !== 'object' || !('data' in body) || !Array.isArray(body.data)) { - throw new Error('Gateway model catalog has an invalid response'); - } - // The Mastra gateway model uses the OpenAI-compatible chat transport. - // Models offered only through another endpoint cannot be selected here. - models = body.data - .filter( - (item): item is { id: string; endpoints: string[] } => - !!item && - typeof item === 'object' && - typeof item.id === 'string' && - Array.isArray(item.endpoints) && - item.endpoints.includes('chat'), - ) - .map(item => item.id); + // Gateway v2 reads the key only from Authorization or x-api-key. A custom fetch is a v1 + // pass-through that owns Authorization, so the catalogue request uses the plain global fetch. + const fetchImpl = globalThis.fetch; + models = await loadCachedV2Catalog(fetchImpl, `${this.getBaseUrl()}\n${hashApiKey(apiKey)}`, () => + this.fetchV2Catalog(fetchImpl, apiKey), + ); } const providers = { @@ -124,6 +119,32 @@ export class MastraGateway extends MastraModelGateway { return providers; } + private async fetchV2Catalog(fetchImpl: typeof fetch, apiKey: string): Promise { + const response = await fetchImpl(`${this.getBaseUrl()}/v1/models`, { + headers: { Authorization: `Bearer ${apiKey}` }, + signal: AbortSignal.timeout(10_000), + }); + if (!response.ok) { + throw new GatewayCatalogError(`Gateway model catalog unavailable (${response.status})`, response.status); + } + const body: unknown = await response.json(); + if (!body || typeof body !== 'object' || !('data' in body) || !Array.isArray(body.data)) { + throw new Error('Gateway model catalog has an invalid response'); + } + // The Mastra gateway model uses the OpenAI-compatible chat transport. + // Models offered only through another endpoint cannot be selected here. + return body.data + .filter( + (item): item is { id: string; endpoints: string[] } => + !!item && + typeof item === 'object' && + typeof item.id === 'string' && + Array.isArray(item.endpoints) && + item.endpoints.includes('chat'), + ) + .map(item => item.id); + } + async buildUrl(_modelId: string): Promise { return `${this.getBaseUrl()}/v1`; } @@ -155,23 +176,32 @@ export class MastraGateway extends MastraModelGateway { const baseURL = `${this.getBaseUrl()}/v1`; const fullModelId = `${providerId}/${modelId}`; - if (this.config?.customFetch && providerId === 'anthropic') { + // Passing a provider sign-in through the gateway (customFetch sets Authorization, the gateway key + // travels in GATEWAY_AUTH_HEADER) only works on the v1 gateway. The gateway picks v1 or v2 from the + // key the request carries, and v2 reads its key only from Authorization, so a request whose own key + // is an organization key always takes the API key path. + const passThroughFetch = isOrganizationKey(apiKey) ? undefined : this.config?.customFetch; + + if (passThroughFetch && providerId === 'anthropic') { // Anthropic OAuth path: use native Anthropic SDK (sends /messages, not /chat/completions) return createAnthropic({ apiKey: 'oauth-gateway-placeholder', baseURL, headers: withGatewayAuthHeader(apiKey, headers), - fetch: this.config.customFetch as any, + fetch: passThroughFetch as any, })(modelId) as unknown as GatewayLanguageModel; } - if (this.config?.customFetch) { + if (passThroughFetch) { // Non-Anthropic OAuth path: gateway key in GATEWAY_AUTH_HEADER, customFetch owns Authorization return createOpenRouter({ apiKey: 'oauth-gateway-placeholder', baseURL, headers: withGatewayAuthHeader(apiKey, headers), - fetch: this.config.customFetch, + fetch: passThroughFetch, + // Strict mode adds stream_options.include_usage to streaming requests only, + // so routes that report usage in a separate final chunk still return it. + compatibility: 'strict', }).chat(fullModelId) as unknown as GatewayLanguageModel; } @@ -183,6 +213,146 @@ export class MastraGateway extends MastraModelGateway { 'User-Agent': MASTRA_USER_AGENT, ...headers, }, + // Strict mode adds stream_options.include_usage to streaming requests only. + compatibility: 'strict', }).chat(fullModelId) as unknown as GatewayLanguageModel; } } + +/** How long a v2 model catalogue is reused before it is fetched again. */ +const V2_CATALOG_TTL_MS = 60_000; +/** How long the last good v2 catalogue is still served while refreshes fail. */ +const V2_CATALOG_STALE_IF_ERROR_MS = 10 * 60_000; +/** + * How long to wait after a failed request before trying again: the last good catalogue is served + * meanwhile, or, when there is none, the failure is thrown again. + */ +const V2_CATALOG_RETRY_AFTER_ERROR_MS = 15_000; +/** Most catalogues kept per fetch implementation. */ +const V2_CATALOG_MAX_ENTRIES = 100; + +/** A catalogue request that got an HTTP error status. */ +class GatewayCatalogError extends Error { + constructor( + message: string, + readonly status: number, + ) { + super(message); + this.name = 'GatewayCatalogError'; + } +} + +interface V2CatalogEntry { + models?: string[]; + fetchedAt?: number; + retryAt?: number; + pending?: Promise; + /** A failure with no list to fall back on, rethrown until `failedUntil` without a new request. */ + failure?: unknown; + failedUntil?: number; +} + +/** + * v2 model catalogues, first by the global fetch in effect when the request was made, then by + * base URL and API key hash. A replaced global fetch (instrumentation, test doubles) gets its own + * entries, so a list from one is never served through another. The raw key is never stored. + */ +let v2CatalogCaches = new WeakMap>(); + +/** Only an in-memory cache key, so the raw key is not held. Not a password hash. */ +function hashApiKey(apiKey: string): string { + return createHash('sha256').update(apiKey).digest('hex'); +} + +/** A 401 or 403 means the key was rejected, so a list cached for it must not be served. */ +function isRejectedKey(error: unknown): boolean { + return error instanceof GatewayCatalogError && (error.status === 401 || error.status === 403); +} + +/** + * Drops settled entries past the stale window (keeping a failure that is still remembered), then + * the oldest settled ones over the size cap. + */ +function pruneV2Catalogs(cache: Map, now: number): void { + for (const [key, entry] of cache) { + const failureRemembered = entry.failedUntil !== undefined && now < entry.failedUntil; + const listExpired = entry.fetchedAt === undefined || now - entry.fetchedAt >= V2_CATALOG_STALE_IF_ERROR_MS; + if (!entry.pending && !failureRemembered && listExpired) cache.delete(key); + } + for (const [key, entry] of cache) { + if (cache.size <= V2_CATALOG_MAX_ENTRIES) break; + if (!entry.pending) cache.delete(key); + } +} + +/** + * Returns the cached v2 catalogue for `cacheKey`, fetching it when it is missing or older than + * the TTL. Concurrent callers share one request. When a refresh fails with a timeout, a network + * error or a server error, the last good catalogue is returned for a bounded time, and the next + * attempt waits a short while. Without a catalogue to fall back on, the error is thrown, and the + * same error is thrown again without a new request for that short while, so callers during an + * outage do not each wait for the request timeout. A rejected key (401 or 403) drops the cached + * catalogue and is never remembered, so every call asks the gateway again. + */ +function loadCachedV2Catalog( + fetchImpl: typeof fetch, + cacheKey: string, + load: () => Promise, +): Promise { + let cache = v2CatalogCaches.get(fetchImpl); + if (!cache) { + cache = new Map(); + v2CatalogCaches.set(fetchImpl, cache); + } + const entries = cache; + const entry = entries.get(cacheKey); + const now = Date.now(); + if (entry?.models && entry.fetchedAt !== undefined) { + const age = now - entry.fetchedAt; + const fresh = age < V2_CATALOG_TTL_MS; + const backingOff = entry.retryAt !== undefined && now < entry.retryAt && age < V2_CATALOG_STALE_IF_ERROR_MS; + if (fresh || backingOff) return Promise.resolve(entry.models); + } + if (entry?.failedUntil !== undefined && now < entry.failedUntil) return Promise.reject(entry.failure); + if (entry?.pending) return entry.pending; + + const pending = load().then( + models => { + const settledAt = Date.now(); + entries.delete(cacheKey); + entries.set(cacheKey, { models, fetchedAt: settledAt }); + pruneV2Catalogs(entries, settledAt); + return models; + }, + error => { + const previous = entries.get(cacheKey); + const settledAt = Date.now(); + if ( + !isRejectedKey(error) && + previous?.models && + previous.fetchedAt !== undefined && + settledAt - previous.fetchedAt < V2_CATALOG_STALE_IF_ERROR_MS + ) { + entries.set(cacheKey, { + models: previous.models, + fetchedAt: previous.fetchedAt, + retryAt: settledAt + V2_CATALOG_RETRY_AFTER_ERROR_MS, + }); + return previous.models; + } + entries.delete(cacheKey); + if (!isRejectedKey(error)) { + entries.set(cacheKey, { failure: error, failedUntil: settledAt + V2_CATALOG_RETRY_AFTER_ERROR_MS }); + pruneV2Catalogs(entries, settledAt); + } + throw error; + }, + ); + entries.set(cacheKey, { ...entry, pending }); + return pending; +} + +/** @internal Clear the cached v2 model catalogues. For testing only. */ +export function _resetMastraGatewayCatalogCache(): void { + v2CatalogCaches = new WeakMap(); +} diff --git a/packages/core/src/llm/model/provider-registry.test.ts b/packages/core/src/llm/model/provider-registry.test.ts index 3af1f3b27aa..1674bf3271c 100644 --- a/packages/core/src/llm/model/provider-registry.test.ts +++ b/packages/core/src/llm/model/provider-registry.test.ts @@ -933,6 +933,95 @@ describe('GatewayRegistry Auto-Refresh', () => { fs.rmSync(tmpDir, { recursive: true, force: true }); }); + describe('Mastra gateway in the shared cache', () => { + const envKeys = ['MASTRA_GATEWAY_VERSION', 'MASTRA_GATEWAY_API_KEY'] as const; + let savedEnv: Record = {}; + + beforeEach(() => { + savedEnv = Object.fromEntries(envKeys.map(key => [key, process.env[key]])); + for (const key of envKeys) delete process.env[key]; + }); + + afterEach(() => { + for (const key of envKeys) { + if (savedEnv[key] === undefined) delete process.env[key]; + else process.env[key] = savedEnv[key]; + } + GatewayRegistry.getInstance({ useDynamicLoading: true }).registerCustomGateways([]); + }); + + async function sync(mastraCatalog: () => Promise>) { + const registry = GatewayRegistry.getInstance({ useDynamicLoading: true }); + const writtenFiles: string[] = []; + vi.spyOn(fs.promises, 'writeFile').mockImplementation(async (filePath: any, data: any) => { + writtenFiles.push(`${filePath.toString()}\n${String(data)}`); + }); + vi.spyOn(fs.promises, 'rename').mockResolvedValue(); + vi.spyOn(fs.promises, 'copyFile').mockResolvedValue(); + vi.spyOn(fs, 'mkdirSync').mockReturnValue(undefined); + vi.spyOn(ModelsDevGateway.prototype, 'fetchProviders').mockResolvedValue({ + 'test-provider': { + name: 'Test Provider', + models: ['model-a'], + apiKeyEnvVar: 'TEST_API_KEY', + gateway: 'models-dev', + } as ProviderConfig, + }); + vi.spyOn(NetlifyGateway.prototype, 'fetchProviders').mockResolvedValue({}); + const mastraFetchProvidersSpy = vi + .spyOn(MastraGateway.prototype, 'fetchProviders') + .mockImplementation(mastraCatalog); + + await registry.syncGateways(true); + + const registryWrites = writtenFiles.filter(entry => entry.includes('provider-registry.json')); + expect(registryWrites.length).toBeGreaterThan(0); + for (const entry of registryWrites) expect(entry).toContain('test-provider'); + return { mastraFetchProvidersSpy, registryWrites }; + } + + const failingCatalog = async () => { + throw new Error('Gateway model catalog unavailable (401)'); + }; + + it('does not sync the default Mastra gateway in v2 mode', async () => { + process.env.MASTRA_GATEWAY_VERSION = 'v2'; + process.env.MASTRA_GATEWAY_API_KEY = 'deployment-key'; + + const { mastraFetchProvidersSpy, registryWrites } = await sync(failingCatalog); + + expect(mastraFetchProvidersSpy).not.toHaveBeenCalled(); + for (const entry of registryWrites) expect(entry).not.toContain('"mastra"'); + }); + + it('does not sync a registered Mastra gateway that holds an organization key', async () => { + GatewayRegistry.getInstance({ useDynamicLoading: true }).registerCustomGateways([ + new MastraGateway({ apiKey: 'mskv2_org-key' }), + ]); + + const { mastraFetchProvidersSpy, registryWrites } = await sync(failingCatalog); + + expect(mastraFetchProvidersSpy).not.toHaveBeenCalled(); + for (const entry of registryWrites) expect(entry).not.toContain('"mastra"'); + }); + + it('still syncs the Mastra gateway with a v1 project key', async () => { + process.env.MASTRA_GATEWAY_API_KEY = 'msk_0123456789abcdef0123456789abcdef'; + + const { mastraFetchProvidersSpy, registryWrites } = await sync(async () => ({ + mastra: { + name: 'Gateway', + models: ['openai/chat-model'], + apiKeyEnvVar: 'MASTRA_GATEWAY_API_KEY', + gateway: 'mastra', + } as ProviderConfig, + })); + + expect(mastraFetchProvidersSpy).toHaveBeenCalled(); + for (const entry of registryWrites) expect(entry).toContain('"mastra"'); + }); + }); + it('should skip syncGateways when MASTRA_OFFLINE is set to true', async () => { process.env.MASTRA_OFFLINE = 'true'; diff --git a/packages/core/src/llm/model/provider-registry.ts b/packages/core/src/llm/model/provider-registry.ts index fd8b562ff60..1ed63815f1a 100644 --- a/packages/core/src/llm/model/provider-registry.ts +++ b/packages/core/src/llm/model/provider-registry.ts @@ -866,8 +866,13 @@ export class GatewayRegistry { ...(writeToSrc ? [] : [new MastraGateway()]), ]; - // Combine default and custom gateways - const gateways = [...defaultGateways, ...this.customGateways]; + // Combine default and custom gateways. A Mastra gateway in v2 mode, default or registered, + // lists models for one organization's key, so it is not written to the machine-wide cache, + // and its failures cannot stop the rest of the refresh. + const gateways = [...defaultGateways, ...this.customGateways].filter(gateway => { + const candidate = gateway as { id?: string; isV2?: () => boolean }; + return !(candidate.id === 'mastra' && typeof candidate.isV2 === 'function' && candidate.isV2()); + }); // Fetch provider data const { diff --git a/packages/server/src/server/handlers/gateway-memory-client.ts b/packages/server/src/server/handlers/gateway-memory-client.ts index 271a6ec90c9..011958b3142 100644 --- a/packages/server/src/server/handlers/gateway-memory-client.ts +++ b/packages/server/src/server/handlers/gateway-memory-client.ts @@ -293,24 +293,29 @@ function isV2GatewayKey(key: string): boolean { } /** - * Whether this deployment targets Gateway v2. Signaled either by the - * MASTRA_GATEWAY_VERSION=v2 runtime env var (set by the platform), by a v2 - * (organization) API key prefix (which predates the env var signal), or by a - * registered Mastra gateway instance that carries a v2 key via config.apiKey - * (pass `mastra` so instance-level config is consulted too). + * Whether this deployment targets Gateway v2. Signaled by the MASTRA_GATEWAY_VERSION=v2 runtime + * env var (set by the platform) or by a v2 (organization) API key prefix, which predates the env + * var signal. A v1 project key (`msk_`) is always served by v1, so it wins over the env var. + * + * Pass `mastra` so the registered Mastra gateway answers: it holds the key model calls use, which + * may be set in its config rather than the environment, and its `isV2()` applies the same rules. + * Keep the environment rules below in sync with `MastraGateway.isV2` in `@mastra/core`. */ export function isGatewayV2(mastra?: { listGateways?: () => Record | undefined }): boolean { - if (process.env.MASTRA_GATEWAY_VERSION === 'v2') return true; - const key = process.env.MASTRA_GATEWAY_API_KEY; - if (!!key && isV2GatewayKey(key)) return true; - // A user-registered MastraGateway can hold a v2 key in its config rather - // than the environment; ask the instance for its effective v2 state. const gateways = mastra?.listGateways?.() ?? {}; for (const gateway of Object.values(gateways)) { const candidate = gateway as { id?: string; isV2?: () => boolean }; - if (candidate?.id === 'mastra' && typeof candidate.isV2 === 'function' && candidate.isV2()) return true; + if (candidate?.id === 'mastra' && typeof candidate.isV2 === 'function') return candidate.isV2(); } - return false; + return isGatewayV2ByEnvironment(); +} + +/** Whether the environment alone (MASTRA_GATEWAY_API_KEY and MASTRA_GATEWAY_VERSION) says v2. */ +function isGatewayV2ByEnvironment(): boolean { + const key = process.env.MASTRA_GATEWAY_API_KEY; + if (key?.startsWith('msk_')) return false; + if (process.env.MASTRA_GATEWAY_VERSION === 'v2') return true; + return !!key && isV2GatewayKey(key); } // Singleton-ish lazy instance @@ -320,10 +325,14 @@ let _gatewayClient: GatewayMemoryClient | null | undefined; * Get a GatewayMemoryClient instance, or null if gateway is not configured. * Pass `mastra` so a registered v2 gateway configuration takes precedence * over a v1 environment key. + * + * The client is built from MASTRA_GATEWAY_API_KEY and MASTRA_GATEWAY_URL, so it is built only when + * the environment itself says v1. A registered gateway that is v1 only because of a key in its own + * config, which the server cannot use here, does not make the environment key a v1 key. */ export function getGatewayClient(mastra?: Parameters[0]): GatewayMemoryClient | null { // Gateway v2 has no memory REST API; Studio must use the agent's own memory configuration. - if (isGatewayV2(mastra)) return null; + if (isGatewayV2(mastra) || isGatewayV2ByEnvironment()) return null; if (_gatewayClient !== undefined) return _gatewayClient; const url = process.env.MASTRA_GATEWAY_URL || 'https://gateway-api.mastra.ai'; const key = process.env.MASTRA_GATEWAY_API_KEY; diff --git a/packages/server/src/server/handlers/gateway-memory-client.v2.test.ts b/packages/server/src/server/handlers/gateway-memory-client.v2.test.ts index 9cb5398dd88..d5222583dd0 100644 --- a/packages/server/src/server/handlers/gateway-memory-client.v2.test.ts +++ b/packages/server/src/server/handlers/gateway-memory-client.v2.test.ts @@ -1,3 +1,4 @@ +import { MastraGateway } from '@mastra/core/llm'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { getGatewayClient, isGatewayV2 } from './gateway-memory-client.js'; @@ -21,6 +22,18 @@ describe('Gateway memory client in v2 deployments', () => { expect(isGatewayV2()).toBe(false); }); + it('is not v2 for a v1 key even when MASTRA_GATEWAY_VERSION is v2', () => { + vi.stubEnv('MASTRA_GATEWAY_VERSION', 'v2'); + vi.stubEnv('MASTRA_GATEWAY_API_KEY', 'msk_v1-key'); + expect(isGatewayV2()).toBe(false); + }); + + it('is v2 when MASTRA_GATEWAY_VERSION is v2 and the key has no v1 prefix', () => { + vi.stubEnv('MASTRA_GATEWAY_VERSION', 'v2'); + vi.stubEnv('MASTRA_GATEWAY_API_KEY', 'deployment-key'); + expect(isGatewayV2()).toBe(true); + }); + it('detects v2 from a registered Mastra gateway instance carrying a v2 config key', () => { vi.stubEnv('MASTRA_GATEWAY_VERSION', ''); vi.stubEnv('MASTRA_GATEWAY_API_KEY', ''); @@ -35,6 +48,24 @@ describe('Gateway memory client in v2 deployments', () => { expect(getGatewayClient(mastra)).toBeNull(); }); + it('agrees with a registered gateway whose v1 config key wins over MASTRA_GATEWAY_VERSION', () => { + vi.stubEnv('MASTRA_GATEWAY_VERSION', 'v2'); + vi.stubEnv('MASTRA_GATEWAY_API_KEY', ''); + const gateway = new MastraGateway({ apiKey: 'msk_0123456789abcdef0123456789abcdef' }); + const mastra = { listGateways: () => ({ mastra: gateway }) }; + expect(gateway.isV2()).toBe(false); + expect(isGatewayV2(mastra)).toBe(false); + }); + + it('does not build a v1 memory client from an organization env key', () => { + vi.stubEnv('MASTRA_GATEWAY_VERSION', ''); + vi.stubEnv('MASTRA_GATEWAY_API_KEY', 'mskv2_org-key'); + const gateway = new MastraGateway({ apiKey: 'msk_0123456789abcdef0123456789abcdef' }); + const mastra = { listGateways: () => ({ mastra: gateway }) }; + expect(isGatewayV2(mastra)).toBe(false); + expect(getGatewayClient(mastra)).toBeNull(); + }); + it('is not v2 when the registered Mastra gateway instance reports v1', () => { vi.stubEnv('MASTRA_GATEWAY_VERSION', ''); vi.stubEnv('MASTRA_GATEWAY_API_KEY', ''); @@ -42,3 +73,67 @@ describe('Gateway memory client in v2 deployments', () => { expect(isGatewayV2(mastra)).toBe(false); }); }); + +describe('v2 detection and the v1 memory client across configurations', () => { + afterEach(() => vi.unstubAllEnvs()); + + const envKeys = { + none: '', + project: 'msk_0123456789abcdef0123456789abcdef', + organization: 'mskv2_org-key', + unprefixed: 'deployment-key', + } as const; + const registeredKeys = { + none: undefined, + v1: 'msk_fedcba9876543210fedcba9876543210', + v2: 'mskv2_registered', + } as const; + + // [env key, MASTRA_GATEWAY_VERSION, registered gateway key, isGatewayV2, v1 memory client built] + const table: Array<[keyof typeof envKeys, 'unset' | 'v2', keyof typeof registeredKeys, boolean, boolean]> = [ + ['none', 'unset', 'none', false, false], + ['none', 'v2', 'none', true, false], + ['none', 'unset', 'v1', false, false], + ['none', 'v2', 'v1', false, false], + ['none', 'unset', 'v2', true, false], + ['none', 'v2', 'v2', true, false], + ['project', 'unset', 'none', false, true], + ['project', 'v2', 'none', false, true], + ['project', 'unset', 'v1', false, true], + ['project', 'v2', 'v1', false, true], + ['project', 'unset', 'v2', true, false], + ['project', 'v2', 'v2', true, false], + ['organization', 'unset', 'none', true, false], + ['organization', 'v2', 'none', true, false], + ['organization', 'unset', 'v1', false, false], + ['organization', 'v2', 'v1', false, false], + ['organization', 'unset', 'v2', true, false], + ['organization', 'v2', 'v2', true, false], + ['unprefixed', 'unset', 'none', false, true], + ['unprefixed', 'v2', 'none', true, false], + ['unprefixed', 'unset', 'v1', false, true], + ['unprefixed', 'v2', 'v1', false, false], + ['unprefixed', 'unset', 'v2', true, false], + ['unprefixed', 'v2', 'v2', true, false], + ]; + + it.each(table)( + 'env key %s, MASTRA_GATEWAY_VERSION %s, registered key %s: v2 is %s, v1 memory client built is %s', + async (envKey, version, registeredKey, expectedV2, expectedClient) => { + vi.stubEnv('MASTRA_GATEWAY_API_KEY', envKeys[envKey]); + vi.stubEnv('MASTRA_GATEWAY_VERSION', version === 'v2' ? 'v2' : ''); + const registered = registeredKeys[registeredKey]; + // Mastra always lists a Mastra gateway: the registered one, or a default one without a config key. + const gateway = registered ? new MastraGateway({ apiKey: registered }) : new MastraGateway(); + const mastra = { listGateways: () => ({ mastra: gateway }) }; + + // A fresh module, so the cached client from one case cannot answer for another. + vi.resetModules(); + const client = await import('./gateway-memory-client.js'); + + expect(client.isGatewayV2(mastra)).toBe(expectedV2); + expect(client.getGatewayClient(mastra) !== null).toBe(expectedClient); + if (!registered) expect(client.isGatewayV2()).toBe(expectedV2); + }, + ); +});