From ea3bea4c50a1844ae329f97eda46fca4ea99388e Mon Sep 17 00:00:00 2001 From: Alex Booker Date: Wed, 12 Aug 2026 14:17:06 +0100 Subject: [PATCH] docs: retire guides section (#21329) --- .github/workflows/lint-docs.yml | 2 +- docs/docusaurus.config.ts | 14 +- docs/feature-versioning.json | 1 - docs/lint-staged.config.mjs | 2 +- docs/package.json | 8 +- docs/scripts/validate-sidebar-docs.ts | 6 - docs/scripts/validate-sidebar-new-tags.ts | 1 - docs/src/components/mobile-docs-dropdown.tsx | 6 - docs/src/content/en/docs/agents/overview.mdx | 4 +- .../content/en/docs/deployment/workers.mdx | 250 ++++++- docs/src/content/en/docs/index.mdx | 2 +- .../en/docs/mastra-platform/deploy.mdx | 2 +- .../exporters/mastra-platform.mdx | 2 +- .../content/en/docs/observability/logging.mdx | 2 +- .../content/en/docs/server/auth/workers.mdx | 2 +- .../en/guides/deployment/mastra-workers.mdx | 622 ------------------ .../en/guides/getting-started/quickstart.mdx | 89 --- docs/src/content/en/guides/index.mdx | 38 -- docs/src/content/en/guides/sidebars.js | 90 --- .../en/integrations/deploy/amazon-ec2.mdx | 2 +- .../en/integrations/deploy/aws-lambda.mdx | 2 +- .../deploy/azure-app-services.mdx | 2 +- .../en/integrations/deploy/cloudflare.mdx | 2 +- .../en/integrations/deploy/digital-ocean.mdx | 2 +- .../en/integrations/deploy/kubernetes.mdx | 3 +- .../en/integrations/deploy/netlify.mdx | 2 +- .../content/en/integrations/deploy/vercel.mdx | 2 +- .../en/integrations/tools/firecrawl.mdx | 2 +- .../content/en/integrations/voice/livekit.mdx | 4 +- .../content/en/reference/agents/getVoice.mdx | 2 +- .../ai-sdk/overview.mdx} | 2 +- .../content/en/reference/configuration.mdx | 2 +- .../content/en/reference/manual-install.mdx | 2 +- docs/src/content/en/reference/sidebars.js | 5 + .../{guides => reference}/voice/overview.mdx | 0 .../voice/speech-to-speech.mdx | 2 +- .../voice/speech-to-text.mdx | 2 +- .../voice/text-to-speech.mdx | 2 +- .../sidebars-handler.ts | 5 - .../source-resolver.ts | 1 - docs/src/theme/Navbar/tab-switcher.tsx | 6 - docs/tests/helpers/pages.ts | 3 +- docs/tests/navigation.spec.ts | 1 - docs/tests/og-image.spec.ts | 2 +- docs/vercel.json | 90 ++- docs/vercel.redirects.json | 45 +- 46 files changed, 407 insertions(+), 931 deletions(-) delete mode 100644 docs/src/content/en/guides/deployment/mastra-workers.mdx delete mode 100644 docs/src/content/en/guides/getting-started/quickstart.mdx delete mode 100644 docs/src/content/en/guides/index.mdx delete mode 100644 docs/src/content/en/guides/sidebars.js rename docs/src/content/en/{guides/agent-frameworks/ai-sdk.mdx => reference/ai-sdk/overview.mdx} (99%) rename docs/src/content/en/{guides => reference}/voice/overview.mdx (100%) rename docs/src/content/en/{guides => reference}/voice/speech-to-speech.mdx (99%) rename docs/src/content/en/{guides => reference}/voice/speech-to-text.mdx (99%) rename docs/src/content/en/{guides => reference}/voice/text-to-speech.mdx (99%) diff --git a/.github/workflows/lint-docs.yml b/.github/workflows/lint-docs.yml index 7644b1eee9b..618400e8de3 100644 --- a/.github/workflows/lint-docs.yml +++ b/.github/workflows/lint-docs.yml @@ -230,7 +230,7 @@ jobs: "${{ github.event.pull_request.head.sha }}" \ -- \ docs/src/content/en/docs \ - docs/src/content/en/guides \ + docs/src/content/en/integrations \ docs/src/content/en/reference \ | grep '\.mdx$' || true ) diff --git a/docs/docusaurus.config.ts b/docs/docusaurus.config.ts index 54db5e23e43..be6c98d672c 100644 --- a/docs/docusaurus.config.ts +++ b/docs/docusaurus.config.ts @@ -104,18 +104,6 @@ const config: Config = { remarkPlugins: [...SHARED_REMARK_PLUGINS], }, ], - [ - '@docusaurus/plugin-content-docs', - { - id: 'guides', - path: 'src/content/en/guides', - routeBasePath: 'guides', - sidebarPath: './src/content/en/guides/sidebars.js', - editUrl: 'https://github.com/mastra-ai/mastra/tree/main/docs', - admonitions: ADMONITIONS_CONFIG, - remarkPlugins: [...SHARED_REMARK_PLUGINS], - }, - ], [ '@docusaurus/plugin-content-docs', { @@ -147,7 +135,7 @@ const config: Config = { { label: 'Quickstart', description: 'Get up and running with Mastra', - link: '/guides/getting-started/quickstart', + link: '/docs', }, { label: 'Studio', description: 'Test your agents, workflows, and tools', link: '/docs/studio/overview' }, { diff --git a/docs/feature-versioning.json b/docs/feature-versioning.json index af8edd8c2f5..66b6cc52f8e 100644 --- a/docs/feature-versioning.json +++ b/docs/feature-versioning.json @@ -13,7 +13,6 @@ "/integrations/frameworks/express": "v1", "/reference/manual-install": "v1", "/integrations/frameworks/next-js": "v1", - "/guides/getting-started/quickstart": "v1", "/integrations/frameworks/sveltekit": "v1", "/integrations/frameworks/vite-react": "v1", "/reference/migrations/ai-sdk-v4-to-v5": "v1", diff --git a/docs/lint-staged.config.mjs b/docs/lint-staged.config.mjs index 7ce1798efeb..7cf021ed302 100644 --- a/docs/lint-staged.config.mjs +++ b/docs/lint-staged.config.mjs @@ -1,4 +1,4 @@ export default { '*.{ts,tsx,js,jsx,json,md,yml,yaml,css}': ['oxfmt --no-error-on-unmatched-pattern'], - 'src/content/en/{docs,guides,reference}/**/*.mdx': ['oxfmt-mdx --write'], + 'src/content/en/{docs,integrations,reference}/**/*.mdx': ['oxfmt-mdx --write'], } diff --git a/docs/package.json b/docs/package.json index fec03ef8a75..e36b0f4d221 100644 --- a/docs/package.json +++ b/docs/package.json @@ -25,9 +25,9 @@ "vale:download": "node scripts/vale/download.mjs", "vale": "scripts/vale/bin/vale", "vale:sync": "scripts/vale/bin/vale sync", - "lint:vale": "scripts/vale/bin/vale src/content/en/docs src/content/en/guides src/content/en/integrations src/content/en/reference", + "lint:vale": "scripts/vale/bin/vale src/content/en/docs src/content/en/integrations src/content/en/reference", "lint:vale:ai": "pnpm lint:vale --minAlertLevel=error --output=line", - "lint:remark": "remark --no-stdout --frail --quiet --ext mdx src/content/en/docs src/content/en/guides src/content/en/integrations src/content/en/reference", + "lint:remark": "remark --no-stdout --frail --quiet --ext mdx src/content/en/docs src/content/en/integrations src/content/en/reference", "lint:prose": "npm-run-all --parallel lint:vale:ai lint:remark", "test": "vitest run", "test:watch": "vitest watch", @@ -38,8 +38,8 @@ "format:check": "pnpm format:oxfmt:check && pnpm format:mdx:check", "format:oxfmt": "oxfmt .", "format:oxfmt:check": "oxfmt --check .", - "format:mdx": "oxfmt-mdx --write src/content/en/docs src/content/en/guides src/content/en/integrations src/content/en/reference", - "format:mdx:check": "oxfmt-mdx --check src/content/en/docs src/content/en/guides src/content/en/integrations src/content/en/reference" + "format:mdx": "oxfmt-mdx --write src/content/en/docs src/content/en/integrations src/content/en/reference", + "format:mdx:check": "oxfmt-mdx --check src/content/en/docs src/content/en/integrations src/content/en/reference" }, "dependencies": { "@docusaurus/core": "^3.10.1", diff --git a/docs/scripts/validate-sidebar-docs.ts b/docs/scripts/validate-sidebar-docs.ts index 4811eef0761..fdb924a688b 100644 --- a/docs/scripts/validate-sidebar-docs.ts +++ b/docs/scripts/validate-sidebar-docs.ts @@ -25,12 +25,6 @@ const SECTIONS: SectionConfig[] = [ sidebarPath: 'src/content/en/docs/sidebars.js', sidebarKey: 'docsSidebar', }, - { - name: 'guides', - contentDir: 'src/content/en/guides', - sidebarPath: 'src/content/en/guides/sidebars.js', - sidebarKey: 'guidesSidebar', - }, { name: 'integrations', contentDir: 'src/content/en/integrations', diff --git a/docs/scripts/validate-sidebar-new-tags.ts b/docs/scripts/validate-sidebar-new-tags.ts index f75fa566456..aada9be2785 100644 --- a/docs/scripts/validate-sidebar-new-tags.ts +++ b/docs/scripts/validate-sidebar-new-tags.ts @@ -30,7 +30,6 @@ interface StaleNewTag extends NewTagEntry { const SIDEBAR_FILES = [ 'src/content/en/docs/sidebars.js', - 'src/content/en/guides/sidebars.js', 'src/content/en/integrations/sidebars.js', 'src/content/en/reference/sidebars.js', 'src/content/en/models/sidebars.js', diff --git a/docs/src/components/mobile-docs-dropdown.tsx b/docs/src/components/mobile-docs-dropdown.tsx index fa4153b1ad8..53fc2562767 100644 --- a/docs/src/components/mobile-docs-dropdown.tsx +++ b/docs/src/components/mobile-docs-dropdown.tsx @@ -25,12 +25,6 @@ const docsTabs = [ href: '/integrations/', basePath: '/integrations', }, - { - id: 'Guides', - label: 'Guides', - href: '/guides/', - basePath: '/guides', - }, { id: 'Reference', label: 'Reference', diff --git a/docs/src/content/en/docs/agents/overview.mdx b/docs/src/content/en/docs/agents/overview.mdx index 97ca5eab7c4..5e871d155d0 100644 --- a/docs/src/content/en/docs/agents/overview.mdx +++ b/docs/src/content/en/docs/agents/overview.mdx @@ -69,7 +69,7 @@ Use [Studio](/docs/studio/overview) to test your agent with different messages a ## Use your agent -After registration, retrieve your agent with [`mastra.getAgentById()`](/reference/core/getAgentById). Call `.generate()` for a complete response or `.stream()` to deliver tokens in real time. You can call agents from [workflow steps](/docs/workflows/agents-and-tools), [tools](/docs/agents/using-tools), the [Mastra Client](/reference/client-js/mastra-client), route handlers, [server adapters](/docs/server/server-adapters), or the command line. Visit the [guides section](/guides) to learn how to use agents in your framework of choice. +After registration, retrieve your agent with [`mastra.getAgentById()`](/reference/core/getAgentById). Call `.generate()` for a complete response or `.stream()` to deliver tokens in real time. You can call agents from [workflow steps](/docs/workflows/agents-and-tools), [tools](/docs/agents/using-tools), the [Mastra Client](/reference/client-js/mastra-client), route handlers, [server adapters](/docs/server/server-adapters), or the command line. See [Integrations](/integrations) to use agents in your application framework. When referencing an agent from your Mastra instance, use `mastra.getAgentById()` to ensure it has access to shared services such as instance-level storage, logging, and agent registry. A directly imported agent can still work with its own local configuration, but it won't have access to those shared services. @@ -118,7 +118,7 @@ Once your agent is running, use this table to find the right page for what you w | Keep your agent safe | [Guardrails](/docs/agents/guardrails) | | Build agents that correct their work | [Rubric scorer](/docs/capabilities/subagents#rubric-scorer) | | Swap instructions or models based on request context | [Dynamic configuration](/docs/server/request-context) | -| Add speech-to-text or text-to-speech | [Voice](/guides/voice/overview) | +| Add speech-to-text or text-to-speech | [Voice](/reference/voice/overview) | | Connect to Slack, Discord, or Telegram | [Channels](/docs/capabilities/channels) | ## Multi-agent systems diff --git a/docs/src/content/en/docs/deployment/workers.mdx b/docs/src/content/en/docs/deployment/workers.mdx index ce36b6643e5..2b3becd306e 100644 --- a/docs/src/content/en/docs/deployment/workers.mdx +++ b/docs/src/content/en/docs/deployment/workers.mdx @@ -124,7 +124,23 @@ Run the same build artifact in multiple containers, each with a different [`MAST Split deployments require a distributed PubSub backend ([`RedisStreamsPubSub`](/reference/pubsub/redis-streams) or [`GoogleCloudPubSub`](/reference/pubsub/google-cloud-pubsub)), a shared [storage backend](/reference/workers/overview#supported-storage-backends), and network connectivity between the orchestration worker and the API. -The [worker deployment guide](/guides/deployment/mastra-workers) walks through this setup with Docker Compose and Kubernetes examples. +### Select workers + +Set [`MASTRA_WORKERS`](/reference/workers/overview#mastra_workers) to control which workers run in each process: + +| Value | Behavior | +| - | - | +| `false` | Disable all workers. Use this for the API process in a fully split deployment. | +| `orchestration` | Start the orchestration worker. | +| `scheduler` | Start the scheduler worker. | +| `backgroundTasks` | Start the background task worker. | +| `orchestration,backgroundTasks` | Start multiple workers from a comma-separated allowlist. | + +You can also pass a worker name to the CLI. The command sets `MASTRA_WORKERS` in the spawned process: + +```bash +mastra worker start orchestration +``` ## Network architecture @@ -138,6 +154,237 @@ In a split deployment: All three worker types (orchestration, scheduler, background task) sit behind the API on a private network. They share access to the PubSub backend and storage database but never receive traffic directly from clients. If a worker-related feature needs an HTTP route (for example, token minting for a voice integration), that route runs on the API server, not on the worker process. +## Deploy split workers + +Build the API and worker artifacts: + +```bash +mastra build +mastra worker build --output-dir .mastra/worker +``` + +`mastra build` creates the API artifact in `.mastra/output/`. [`mastra worker build`](/reference/cli/mastra#mastra-worker-build) creates a worker artifact in `.mastra/worker/`. The following Dockerfile accepts either directory: + +```dockerfile title="Dockerfile" +FROM node:22-alpine + +ARG MASTRA_OUTPUT=.mastra/output + +WORKDIR /app + +COPY ${MASTRA_OUTPUT}/package.json ${MASTRA_OUTPUT}/.npmrc* ./ +RUN npm install --omit=dev + +COPY ${MASTRA_OUTPUT}/ . + +EXPOSE 4111 +CMD ["node", "index.mjs"] +``` + +See [Deploy a Mastra server](/docs/deployment/mastra-server) for more information about the build output. + +### Docker Compose + +The following configuration runs PostgreSQL, Redis, the API, and one process for each worker type. Every process uses shared infrastructure, and the worker processes use the worker artifact. + +```yaml title="docker-compose.yml" +x-worker: &worker + build: + context: . + args: + MASTRA_OUTPUT: .mastra/worker + +x-mastra-environment: &shared-environment + DATABASE_URL: postgres://mastra:${POSTGRES_PASSWORD}@postgres:5432/mastra + REDIS_URL: redis://redis:6379 + +services: + postgres: + image: postgres:16-alpine + environment: + POSTGRES_USER: mastra + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: mastra + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ['CMD-SHELL', 'pg_isready -U mastra'] + interval: 5s + timeout: 3s + retries: 5 + + redis: + image: redis:7-alpine + healthcheck: + test: ['CMD', 'redis-cli', 'ping'] + interval: 5s + timeout: 3s + retries: 5 + + api: + build: + context: . + args: + MASTRA_OUTPUT: .mastra/output + ports: + - '4111:4111' + environment: + <<: *shared-environment + WORKER_TOKEN: ${WORKER_TOKEN} + MASTRA_WORKERS: 'false' + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + healthcheck: + test: ['CMD', 'wget', '-qO-', 'http://localhost:4111/api/agents'] + interval: 5s + timeout: 3s + retries: 5 + + orchestration-worker: + <<: *worker + environment: + <<: *shared-environment + MASTRA_WORKERS: orchestration + MASTRA_STEP_EXECUTION_URL: http://api:4111/api + MASTRA_WORKER_AUTH_TOKEN: ${WORKER_TOKEN} + depends_on: + api: + condition: service_healthy + + scheduler-worker: + <<: *worker + environment: + <<: *shared-environment + MASTRA_WORKERS: scheduler + depends_on: + api: + condition: service_healthy + + background-task-worker: + <<: *worker + environment: + <<: *shared-environment + MASTRA_WORKERS: backgroundTasks + depends_on: + api: + condition: service_healthy + +volumes: + pgdata: +``` + +Set the secrets next to `docker-compose.yml`, along with any model provider credentials your application needs: + +```bash title=".env" +POSTGRES_PASSWORD=your-secure-password +WORKER_TOKEN=your-shared-secret-token +``` + +Configure the API auth provider to accept `WORKER_TOKEN` before exposing the deployment. The orchestration worker sends the same value through `MASTRA_WORKER_AUTH_TOKEN`. The scheduler and background task workers don't call the step execution endpoint in this pull-based topology, so they don't need that variable. + +Start the stack and verify that the containers and API are available: + +```bash +docker compose up -d +docker compose ps +curl http://localhost:4111/api/agents +``` + +### Kubernetes + +Create separate Deployments for the API, orchestration worker, scheduler worker, and background task worker. Use the same image and Secret for each Deployment. Set only the role-specific environment variables directly on each container. + +The orchestration worker Deployment has the following shape: + +```yaml title="k8s/orchestration-worker.yaml" +apiVersion: apps/v1 +kind: Deployment +metadata: + name: orchestration-worker +spec: + replicas: 1 + selector: + matchLabels: + app: orchestration-worker + template: + metadata: + labels: + app: orchestration-worker + spec: + containers: + - name: worker + image: your-registry/mastra-workers:latest + env: + - name: MASTRA_WORKERS + value: orchestration + - name: MASTRA_STEP_EXECUTION_URL + value: http://api:4111/api + envFrom: + - secretRef: + name: mastra-secrets + resources: + requests: + cpu: 250m + memory: 256Mi +``` + +Use `MASTRA_WORKERS: scheduler` and `MASTRA_WORKERS: backgroundTasks` for the other worker Deployments. Set `MASTRA_WORKERS: 'false'` on the API Deployment and expose the API with a Service. Give every process access to the same database and PubSub backend. Configure the API auth provider with a worker token, then expose that token to the orchestration worker as `MASTRA_WORKER_AUTH_TOKEN`. See [Deploy Mastra to Kubernetes](/integrations/deploy/kubernetes) for the base Kubernetes resources. + +Apply the manifests, then verify the pods and API: + +```bash +kubectl apply -f k8s/ +kubectl get pods +kubectl port-forward svc/api 4111:4111 +``` + +In a separate terminal, request an API route: + +```bash +curl http://localhost:4111/api/agents +``` + +### Step execution URL + +In a fully split deployment, the orchestration worker delegates workflow step execution to the API over HTTP. Set `MASTRA_STEP_EXECUTION_URL` to the API's internal URL, including the `/api` prefix: + +```bash +MASTRA_STEP_EXECUTION_URL=http://api:4111/api +``` + +Without this variable, the orchestration worker attempts to execute steps in its own process. That process doesn't have access to the full Mastra runtime in a split deployment. + +The endpoint uses the server's normal auth pipeline. If the API has an auth provider, set `MASTRA_WORKER_AUTH_TOKEN` to a bearer token that provider accepts. Mastra forwards the value as an `Authorization: Bearer` credential. The configured auth provider validates the token. See [Worker authentication](/docs/server/auth/workers) for server configuration and other credential formats. + +### Scale workers + +The orchestration and background task workers can scale horizontally. PubSub consumer groups distribute events across their instances: + +```bash +docker compose up -d --scale orchestration-worker=3 +docker compose up -d --scale background-task-worker=2 +``` + +For Kubernetes, change the Deployment replica count manually or use a HorizontalPodAutoscaler: + +```bash +kubectl scale deployment/orchestration-worker --replicas=3 +kubectl scale deployment/background-task-worker --replicas=2 +``` + +Run exactly one scheduler worker. Multiple schedulers polling the same storage can publish duplicate events for a schedule. + +### Crash recovery + +A distributed PubSub backend persists unacknowledged events so orchestration and background task workers can resume processing after a restart. If the API is unavailable when the orchestration worker requests step execution, the request fails and the event is delivered again. An event can be processed more than once, so handlers should be idempotent when possible. + +The scheduler calculates the next fire time from the current time after it restarts. It doesn't replay schedules that elapsed while it was unavailable. + +If the API crashes while a step is executing, that work can be lost and the workflow run can remain in a `running` state. See [known limitations](#known-limitations) and [durable agent crash recovery](/docs/long-running-agents/durable-agents#crash-recovery). + ## Known limitations - **No dead-letter queue**: Failed events are nacked and retried, but there's no DLQ for events that fail after all retries. @@ -147,7 +394,6 @@ All three worker types (orchestration, scheduler, background task) sit behind th ## Related -- [Worker deployment guide](/guides/deployment/mastra-workers): Docker Compose and Kubernetes examples - [Worker authentication](/docs/server/auth/workers): Secure worker-to-API communication - [Workers reference](/reference/workers/overview): Details about worker environment variables and types, with a list of supported storage backends - [CLI reference](/reference/cli/mastra#mastra-worker-build): `mastra worker build` and `mastra worker start` diff --git a/docs/src/content/en/docs/index.mdx b/docs/src/content/en/docs/index.mdx index 605da5869a6..e4e90365079 100644 --- a/docs/src/content/en/docs/index.mdx +++ b/docs/src/content/en/docs/index.mdx @@ -51,7 +51,7 @@ Run this command to create a general-purpose agent harness with a local workspac -You can open [Studio](/docs/studio/overview) immediately, which is an interactive UI for your Mastra project. See the [quickstart guide](/guides/getting-started/quickstart) for a full walkthrough. +You can open [Studio](/docs/studio/overview) immediately, which is an interactive UI for your Mastra project. ## Integrate with your framework diff --git a/docs/src/content/en/docs/mastra-platform/deploy.mdx b/docs/src/content/en/docs/mastra-platform/deploy.mdx index 7ced87a6ee4..5084514deee 100644 --- a/docs/src/content/en/docs/mastra-platform/deploy.mdx +++ b/docs/src/content/en/docs/mastra-platform/deploy.mdx @@ -22,7 +22,7 @@ This page covers the unified deploy flow. The earlier split commands, [`mastra s ## Before you begin -You'll need a [Mastra application](/guides/getting-started/quickstart) and a [Mastra platform](https://projects.mastra.ai) account. If you're not authenticated, the CLI prompts you to log in on first use. +You'll need a [Mastra application](/docs) and a [Mastra platform](https://projects.mastra.ai) account. If you're not authenticated, the CLI prompts you to log in on first use. A local `.env` file is optional. Environment variables stored on the platform are used as-is at deploy time, and managed resources like [hosted databases](/docs/mastra-platform/database) inject their own variables. Pass `--env-file` only when you want to layer local values on top. diff --git a/docs/src/content/en/docs/observability/integrations/exporters/mastra-platform.mdx b/docs/src/content/en/docs/observability/integrations/exporters/mastra-platform.mdx index 99141b36f55..4b517a75dee 100644 --- a/docs/src/content/en/docs/observability/integrations/exporters/mastra-platform.mdx +++ b/docs/src/content/en/docs/observability/integrations/exporters/mastra-platform.mdx @@ -19,7 +19,7 @@ The `MastraPlatformExporter` sends traces, logs, metrics, scores, and feedback t If you host your Mastra application on your own infrastructure (not on Mastra platform), you still need a deployed Studio project to view traces, logs, and metrics. `MastraPlatformExporter` sends data to a Studio project, so one must exist before you can use it. -1. [Create a Mastra project](/guides/getting-started/quickstart) if you don't have one yet. +1. [Create a Mastra project](/docs) if you don't have one yet. 1. [Deploy Studio](/docs/studio/deployment#mastra-platform) to the Mastra platform with `mastra studio deploy`. 1. Follow the [quickstart steps below](#quickstart) to create an access token and find your project ID. diff --git a/docs/src/content/en/docs/observability/logging.mdx b/docs/src/content/en/docs/observability/logging.mdx index 1affde07225..bec327c70f1 100644 --- a/docs/src/content/en/docs/observability/logging.mdx +++ b/docs/src/content/en/docs/observability/logging.mdx @@ -18,7 +18,7 @@ When deploying to the Mastra platform, logs are shown in the dashboard. In self- ## Configuring logs with `PinoLogger` -When [initializing a new Mastra project](/guides/getting-started/quickstart) using the CLI, `PinoLogger` is included by default. +When [initializing a new Mastra project](/docs) using the CLI, `PinoLogger` is included by default. ```typescript title="src/mastra/index.ts" import { Mastra } from '@mastra/core/mastra' diff --git a/docs/src/content/en/docs/server/auth/workers.mdx b/docs/src/content/en/docs/server/auth/workers.mdx index 8821f9abe20..b57cf19f0fd 100644 --- a/docs/src/content/en/docs/server/auth/workers.mdx +++ b/docs/src/content/en/docs/server/auth/workers.mdx @@ -145,6 +145,6 @@ Your auth provider's `authenticateToken` callback must recognize whatever creden - [Auth overview](/docs/server/auth): Available auth providers and how they work - [Token-based auth](/docs/server/auth/simple-auth): Token-to-user mapping authentication -- [Worker deployment](/guides/deployment/mastra-workers): Set up split worker processes +- [Worker deployment](/docs/deployment/workers): Set up split worker processes - [Workers reference](/reference/workers/overview): Configuration details for all worker types - [CLI reference](/reference/cli/mastra#mastra-worker-build): `mastra worker build` and `mastra worker start` diff --git a/docs/src/content/en/guides/deployment/mastra-workers.mdx b/docs/src/content/en/guides/deployment/mastra-workers.mdx deleted file mode 100644 index b5a2c9335a9..00000000000 --- a/docs/src/content/en/guides/deployment/mastra-workers.mdx +++ /dev/null @@ -1,622 +0,0 @@ ---- -title: "Deploy Mastra workers | Deployment" -description: "Learn how to deploy Mastra workers as separate processes for independent scaling, event durability, and process isolation." ---- - -import Steps from "@site/src/components/Steps"; -import StepItem from "@site/src/components/StepItem"; -import Tabs from '@theme/Tabs' -import TabItem from '@theme/TabItem' - -# Deploy Mastra workers - -Run [Mastra workers](/docs/deployment/workers) as separate processes so you can scale orchestration, scheduling, and background tasks independently from the API. This guide walks through a fully split deployment using Docker Compose or Kubernetes. - -:::beta - -Breaking changes may occur without a major version bump until the API is stable. - -::: - -:::note -This guide covers splitting workers into their own containers. If you only need workers to run in-process alongside the API, see [Workers](/docs/deployment/workers). No extra setup is required. -::: - -## Before you begin - -You'll need: - -- A [Mastra application](/guides/getting-started/quickstart) -- [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/), or a [Kubernetes](https://kubernetes.io/docs/setup/) cluster with [`kubectl`](https://kubernetes.io/docs/tasks/tools/) -- A distributed PubSub backend: [Redis](https://redis.io/) for [`RedisStreamsPubSub`](/reference/pubsub/redis-streams), or a [Google Cloud](https://cloud.google.com/) project for [`GoogleCloudPubSub`](/reference/pubsub/google-cloud-pubsub) -- A shared database reachable from every container. See [supported storage backends](/reference/workers/overview#supported-storage-backends) for the full list. - -:::warning -The default in-memory PubSub can't deliver events across processes. You must configure a distributed PubSub backend before splitting workers into separate containers. -::: - -## Configure shared infrastructure - -Point the `Mastra` instance at a distributed PubSub backend and a shared database. Use environment variables so the same image runs in every container. - - - - ```typescript title="src/mastra/index.ts" - import { Mastra } from '@mastra/core/mastra' - import { RedisStreamsPubSub } from '@mastra/redis-streams' - import { PostgresStore } from '@mastra/pg' - - export const mastra = new Mastra({ - storage: new PostgresStore({ - connectionString: process.env.DATABASE_URL!, - }), - pubsub: new RedisStreamsPubSub({ - url: process.env.REDIS_URL!, - }), - }) - ``` - - - - ```typescript title="src/mastra/index.ts" - import { Mastra } from '@mastra/core/mastra' - import { GoogleCloudPubSub } from '@mastra/google-cloud-pubsub' - import { LibSQLStore } from '@mastra/libsql' - - export const mastra = new Mastra({ - storage: new LibSQLStore({ - url: process.env.DATABASE_URL!, - }), - pubsub: new GoogleCloudPubSub({ - projectId: process.env.GCP_PROJECT_ID!, - }), - }) - ``` - - - -Any [supported storage backend](/reference/workers/overview#supported-storage-backends) works. Swap the storage adapter for your preferred database. - -## Deploy - - - - Build your Mastra application. The output runs in every container. - - ```bash - mastra build - ``` - - This produces a self-contained `.mastra/output/` directory. See [Deploy a Mastra server](/docs/deployment/mastra-server) for details on the build output. - - - - Create a Dockerfile that copies the pre-built output and installs production dependencies: - - ```dockerfile title="app/Dockerfile" - FROM node:22-alpine - - WORKDIR /app - - COPY .mastra/output/package.json .mastra/output/.npmrc* ./ - RUN npm install --omit=dev - - COPY .mastra/output/ . - - EXPOSE 4111 - CMD ["node", "index.mjs"] - ``` - - - - Define the fully split topology. The setup runs six services: a database, a PubSub backend, the API server, and three workers. Each worker runs the same image with a different `MASTRA_WORKERS` value to control which worker starts. - - The API sets `MASTRA_WORKERS: "false"` to disable all event processing. The orchestration worker sets `MASTRA_STEP_EXECUTION_URL` to point step execution requests at the API's internal URL. See [step execution URL](#step-execution-url) for details. - - All services share a `MASTRA_WORKER_AUTH_TOKEN`. Workers include this token in requests to the API so the API can verify the caller is a trusted internal service. See [worker authentication](/docs/server/auth/workers) for details. - - - - ```yaml title="docker-compose.yml" - services: - postgres: - image: postgres:16-alpine - environment: - POSTGRES_USER: mastra - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_DB: mastra - ports: - - '5432:5432' - volumes: - - pgdata:/var/lib/postgresql/data - healthcheck: - test: ['CMD-SHELL', 'pg_isready -U mastra'] - interval: 5s - timeout: 3s - retries: 5 - - redis: - image: redis:7-alpine - ports: - - '6379:6379' - healthcheck: - test: ['CMD', 'redis-cli', 'ping'] - interval: 5s - timeout: 3s - retries: 5 - - api: - build: ./app - ports: - - '4111:4111' - environment: - DATABASE_URL: postgres://mastra:${POSTGRES_PASSWORD}@postgres:5432/mastra - REDIS_URL: redis://redis:6379 - MASTRA_WORKERS: 'false' - MASTRA_WORKER_AUTH_TOKEN: ${MASTRA_WORKER_AUTH_TOKEN} - depends_on: - postgres: - condition: service_healthy - redis: - condition: service_healthy - healthcheck: - test: ['CMD', 'wget', '-qO-', 'http://localhost:4111/api/agents'] - interval: 5s - timeout: 3s - retries: 5 - - orchestration-worker: - build: ./app - environment: - DATABASE_URL: postgres://mastra:${POSTGRES_PASSWORD}@postgres:5432/mastra - REDIS_URL: redis://redis:6379 - MASTRA_WORKERS: orchestration - MASTRA_STEP_EXECUTION_URL: http://api:4111/api - MASTRA_WORKER_AUTH_TOKEN: ${MASTRA_WORKER_AUTH_TOKEN} - depends_on: - api: - condition: service_healthy - - scheduler-worker: - build: ./app - environment: - DATABASE_URL: postgres://mastra:${POSTGRES_PASSWORD}@postgres:5432/mastra - REDIS_URL: redis://redis:6379 - MASTRA_WORKERS: scheduler - MASTRA_WORKER_AUTH_TOKEN: ${MASTRA_WORKER_AUTH_TOKEN} - depends_on: - api: - condition: service_healthy - - background-task-worker: - build: ./app - environment: - DATABASE_URL: postgres://mastra:${POSTGRES_PASSWORD}@postgres:5432/mastra - REDIS_URL: redis://redis:6379 - MASTRA_WORKERS: backgroundTasks - MASTRA_WORKER_AUTH_TOKEN: ${MASTRA_WORKER_AUTH_TOKEN} - depends_on: - api: - condition: service_healthy - - volumes: - pgdata: - ``` - - Create a `.env` file next to your `docker-compose.yml`: - - ```bash title=".env" - POSTGRES_PASSWORD=your-secure-password - MASTRA_WORKER_AUTH_TOKEN=your-shared-secret-token - ``` - - :::note - Remember to set any other environment variables your application needs (e.g., your [model provider](/models/providers) API key). - ::: - - - - Create a namespace and a Secret with your connection strings: - - ```yaml title="k8s/namespace.yaml" - apiVersion: v1 - kind: Namespace - metadata: - name: mastra-workers - ``` - - ```bash - kubectl apply -f k8s/namespace.yaml - kubectl create secret generic mastra-secrets -n mastra-workers \ - --from-literal=POSTGRES_PASSWORD='your-password' \ - --from-literal=DATABASE_URL='postgresql://mastra:your-password@postgres:5432/mastra' \ - --from-literal=REDIS_URL='redis://redis:6379' \ - --from-literal=MASTRA_WORKER_AUTH_TOKEN='your-shared-token' - ``` - - :::note - Add any other environment variables your application needs (e.g., your [model provider](/models/providers) API key) to the Secret or as additional `--from-literal` entries. - ::: - - Build and push the Docker image to a registry your cluster can pull from: - - ```bash - docker build -t your-registry/mastra-workers:latest ./app - docker push your-registry/mastra-workers:latest - ``` - - Apply Deployments and Services for the database, PubSub backend, API, and three workers. The example below uses in-cluster Postgres and Redis. In production, use managed services (e.g., Amazon RDS, Cloud SQL, ElastiCache, Memorystore). - - ```yaml title="k8s/postgres.yaml" - apiVersion: apps/v1 - kind: Deployment - metadata: - name: postgres - namespace: mastra-workers - spec: - replicas: 1 - selector: - matchLabels: - app: postgres - template: - metadata: - labels: - app: postgres - spec: - containers: - - name: postgres - image: postgres:16-alpine - ports: - - containerPort: 5432 - env: - - name: POSTGRES_USER - value: mastra - - name: POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - name: mastra-secrets - key: POSTGRES_PASSWORD - - name: POSTGRES_DB - value: mastra - volumeMounts: - - name: pgdata - mountPath: /var/lib/postgresql/data - volumes: - - name: pgdata - emptyDir: {} - --- - apiVersion: v1 - kind: Service - metadata: - name: postgres - namespace: mastra-workers - spec: - selector: - app: postgres - ports: - - port: 5432 - targetPort: 5432 - ``` - - :::caution - The Postgres example above uses `emptyDir` for storage, which means data is lost when the pod restarts. In production, replace it with a `PersistentVolumeClaim` or use a managed database service. - ::: - - ```yaml title="k8s/redis.yaml" - apiVersion: apps/v1 - kind: Deployment - metadata: - name: redis - namespace: mastra-workers - spec: - replicas: 1 - selector: - matchLabels: - app: redis - template: - metadata: - labels: - app: redis - spec: - containers: - - name: redis - image: redis:7-alpine - args: ['--appendonly', 'yes'] - ports: - - containerPort: 6379 - --- - apiVersion: v1 - kind: Service - metadata: - name: redis - namespace: mastra-workers - spec: - selector: - app: redis - ports: - - port: 6379 - targetPort: 6379 - ``` - - ```yaml title="k8s/api.yaml" - apiVersion: apps/v1 - kind: Deployment - metadata: - name: api - namespace: mastra-workers - spec: - replicas: 1 - selector: - matchLabels: - app: api - template: - metadata: - labels: - app: api - spec: - containers: - - name: api - image: your-registry/mastra-workers:latest - ports: - - containerPort: 4111 - env: - - name: MASTRA_WORKERS - value: 'false' - envFrom: - - secretRef: - name: mastra-secrets - readinessProbe: - httpGet: - path: /api/agents - port: 4111 - initialDelaySeconds: 10 - periodSeconds: 5 - livenessProbe: - httpGet: - path: /api/agents - port: 4111 - initialDelaySeconds: 15 - periodSeconds: 10 - resources: - requests: - cpu: 500m - memory: 512Mi - --- - apiVersion: v1 - kind: Service - metadata: - name: api - namespace: mastra-workers - spec: - selector: - app: api - ports: - - port: 4111 - targetPort: 4111 - ``` - - ```yaml title="k8s/orchestration-worker.yaml" - apiVersion: apps/v1 - kind: Deployment - metadata: - name: orchestration-worker - namespace: mastra-workers - spec: - replicas: 1 - selector: - matchLabels: - app: orchestration-worker - template: - metadata: - labels: - app: orchestration-worker - spec: - containers: - - name: worker - image: your-registry/mastra-workers:latest - env: - - name: MASTRA_WORKERS - value: orchestration - - name: MASTRA_STEP_EXECUTION_URL - value: http://api:4111/api - envFrom: - - secretRef: - name: mastra-secrets - resources: - requests: - cpu: 250m - memory: 256Mi - ``` - - ```yaml title="k8s/scheduler-worker.yaml" - apiVersion: apps/v1 - kind: Deployment - metadata: - name: scheduler-worker - namespace: mastra-workers - spec: - replicas: 1 - selector: - matchLabels: - app: scheduler-worker - template: - metadata: - labels: - app: scheduler-worker - spec: - containers: - - name: worker - image: your-registry/mastra-workers:latest - env: - - name: MASTRA_WORKERS - value: scheduler - envFrom: - - secretRef: - name: mastra-secrets - resources: - requests: - cpu: 250m - memory: 256Mi - ``` - - ```yaml title="k8s/background-task-worker.yaml" - apiVersion: apps/v1 - kind: Deployment - metadata: - name: background-task-worker - namespace: mastra-workers - spec: - replicas: 1 - selector: - matchLabels: - app: background-task-worker - template: - metadata: - labels: - app: background-task-worker - spec: - containers: - - name: worker - image: your-registry/mastra-workers:latest - env: - - name: MASTRA_WORKERS - value: backgroundTasks - envFrom: - - secretRef: - name: mastra-secrets - resources: - requests: - cpu: 250m - memory: 256Mi - ``` - - Apply all manifests and wait for the API to become ready: - - ```bash - kubectl apply -f k8s/ - kubectl wait -n mastra-workers --for=condition=ready pod -l app=api --timeout=90s - kubectl wait -n mastra-workers --for=condition=ready pod -l app=orchestration-worker --timeout=60s - kubectl wait -n mastra-workers --for=condition=ready pod -l app=scheduler-worker --timeout=60s - kubectl wait -n mastra-workers --for=condition=ready pod -l app=background-task-worker --timeout=60s - ``` - - - - - - Verify the stack is running and the API responds: - - - - ```bash - docker compose up -d - docker compose ps - curl http://localhost:4111/api/agents - ``` - - - - ```bash - kubectl get pods -n mastra-workers - kubectl port-forward -n mastra-workers svc/api 4111:4111 - ``` - - In a separate terminal: - - ```bash - curl http://localhost:4111/api/agents - ``` - - A JSON list of your agents confirms the API and workers are running. - - - - - -## Step execution URL - -In a fully split deployment, the orchestration worker runs in a separate container from the API. When it processes a workflow event, it delegates step execution to the API over HTTP. - -Set `MASTRA_STEP_EXECUTION_URL` to the API's internal URL, including the `/api` prefix: - -```bash -MASTRA_STEP_EXECUTION_URL=http://api:4111/api -``` - -The orchestration worker sends a `POST` request to `${MASTRA_STEP_EXECUTION_URL}/workflows/:workflowId/runs/:runId/steps/execute` for each step. The API resolves the workflow and executes the step locally. - -Without this variable, the orchestration worker attempts to execute steps in-process. That works when the worker runs alongside the API, but fails in split deployments where the worker doesn't have access to the full Mastra runtime. - -## Scaling - -The orchestration and background task workers are safe to scale horizontally. PubSub consumer groups distribute events across instances, so each event is processed once: - - - - ```bash - docker compose up -d --scale orchestration-worker=3 - docker compose up -d --scale background-task-worker=2 - ``` - - - - ```bash - kubectl scale deployment/orchestration-worker -n mastra-workers --replicas=3 - kubectl scale deployment/background-task-worker -n mastra-workers --replicas=2 - ``` - - For automatic scaling, add a HorizontalPodAutoscaler: - - ```yaml title="k8s/hpa.yaml" - apiVersion: autoscaling/v2 - kind: HorizontalPodAutoscaler - metadata: - name: orchestration-worker - namespace: mastra-workers - spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: orchestration-worker - minReplicas: 1 - maxReplicas: 10 - metrics: - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: 70 - ``` - - :::note - CPU-based autoscaling needs the [metrics-server](https://github.com/kubernetes-sigs/metrics-server) running in the cluster. Managed clusters like GKE, EKS, and AKS include it by default. - ::: - - - -The API can also scale horizontally behind a load balancer. - -**Don't scale the scheduler worker.** Run exactly one instance. Multiple schedulers polling the same storage fire duplicate events for the same schedule. - -## Crash recovery - -Workers recover from crashes because the distributed PubSub backend persists unacknowledged events: - -- **Orchestration worker**: Pending events stay in the PubSub backend. When the worker restarts, it picks up where it left off. -- **Scheduler worker**: No events are missed permanently. The scheduler computes the next fire time from the current time on restart, not from where it left off. -- **API during step execution**: The orchestration worker's HTTP request fails. The event is nacked and redelivered on the next attempt. - -:::warning -If the API crashes while a step is already executing (e.g., mid-sleep), that step's work is lost. The workflow run may remain stuck in a `running` state. Mastra doesn't yet have automatic timeout-based recovery for this scenario. -::: - -## Related - -- [Workers](/docs/deployment/workers): What workers are and when to use them -- [Worker authentication](/docs/server/auth/workers): Secure worker-to-API communication -- [Workers reference](/reference/workers/overview): Configuration details for all worker types -- [CLI reference](/reference/cli/mastra#mastra-worker-build): `mastra worker build` and `mastra worker start` -- [PubSub](/docs/server/pubsub): Event delivery backends -- [Deploy a Mastra server](/docs/deployment/mastra-server): Build output and server configuration -- [Deploy Mastra to Kubernetes](/integrations/deploy/kubernetes): Multi-pod deployment with durable agents diff --git a/docs/src/content/en/guides/getting-started/quickstart.mdx b/docs/src/content/en/guides/getting-started/quickstart.mdx deleted file mode 100644 index 0c66557cdd4..00000000000 --- a/docs/src/content/en/guides/getting-started/quickstart.mdx +++ /dev/null @@ -1,89 +0,0 @@ ---- -title: "Quickstart" -description: Get started with Mastra by scaffolding an agent harness project in minutes. ---- - -import QuickstartPrompt from "@site/src/content/en/docs/getting-started/_partial-quickstart-prompt.mdx"; -import Tabs from "@theme/Tabs"; -import TabItem from "@theme/TabItem"; - -# Mastra Quickstart - - - **For AI agents:** Create a new Mastra project non-interactively with `npm create mastra@latest -- --llm `, replacing `` with `openai`, `anthropic`, `google`, or `xai`. For a minimal project without an agent or model provider, use `npm create mastra@latest -- --empty`. To build a Mastra project from scratch, see the [manual installation guide](/reference/manual-install). - - -The `create-mastra` CLI is the quickest way to start a Mastra project. It creates an agent harness with workspace tools, memory, task tracking, web access, schedules, storage, and observability. - -For more control, see the [manual installation guide](/reference/manual-install). To add Mastra to an existing project, use [`mastra init`](/reference/cli/mastra#mastra-init). - - - -:::tip[πŸ“Ή Watch] - -Watch the [Mastra AI agent course](https://www.youtube.com/watch?v=lCmf_qrGfGA) for a guided introduction to building agents with Mastra. - -::: - -## Before you begin - -You'll need an API key from OpenAI, Anthropic, Google Gemini, or xAI. You can skip entering it during setup and add it to `.env` later. - -## Create the project - -Run the interactive setup: - - - - ```bash - npm create mastra@latest - ``` - - - - ```bash - pnpm create mastra@latest - ``` - - - - ```bash - yarn create mastra - ``` - - - - ```bash - bunx create-mastra - ``` - - - -The command asks for a project name, model provider, and optional API key. It then installs dependencies, installs Mastra skills for your coding assistants, and creates an initial Git commit when you aren't already inside a repository. - -The generated `src/mastra` directory contains the Mastra entry point, agent, workspace tools, and supporting code. See the [project structure reference](/reference/project-structure) for the standard Mastra layout. - -:::tip - -Use `--template