From c17bc951ca783d8023bf54d69035c19c0c72ea2f Mon Sep 17 00:00:00 2001 From: Tom Beckenham <34339192+tombeckenham@users.noreply.github.com> Date: Tue, 8 Sep 2026 16:23:59 +1000 Subject: [PATCH] feat: add generateWorld() and generateLiveVideo() for prompt-steerable sessions (#1321) * feat: add generateWorld() activity and Reactor adapter Add a session-based world generation activity, a first-party @tanstack/ai-reactor adapter that mints scoped tokens for Orbis and other Reactor world models, and a ts-react-world example app. * feat: add reactorVideo adapter for live generateVideo sessions Mint a scoped Reactor token from generateVideo() for Helios, FastH3, Orbis, LongLive, and LTX. Live sessions return a token instead of a download URL, so the browser connects with the js-sdk. * feat: use BYOK in the ts-react-world example Paste a Reactor key in the browser. The relay reads x-byok-reactor, then REACTOR_API_KEY, and mints the session token. * fix: unlock saved BYOK keys in the world example Passkey storage locks the keyring after a reload. Show Unlock, treat locked keys as saved, and call byok.prepare() before minting. * fix: load Reactor wasm in the world example Vite was pre-bundling @reactor-team/js-sdk, which broke the relative import of dist/wasm/reactor_wasm.js. Exclude the SDK from optimizeDeps and resolve that glue file from the package. * fix: hydrate BYOK env and passkey peek like ts-react-chat Call byok.ready() and getEnvKeyStatus() on mount so a refresh shows a locked passkey key or REACTOR_API_KEY on the relay. * fix: pre-bundle awaitqueue when the Reactor SDK is unbundled Excluding @reactor-team/js-sdk from optimizeDeps left its CJS dependency awaitqueue as a raw named import. Include it so Vite converts AwaitQueue. * feat: add generateLive() and fold world into the media example Mint live session tokens with generateLive(). reactorVideo and falLive (H3 Max Director) use that activity. generateVideo stays the job path. The media example now has Live and World tabs. The ts-react-world example is removed. * fix: play live video and open the key dialog on a missing key Helios uses set_sr_scale, not set_resolution. Mute the video and re-attach on unmute so Chrome shows frames. A missing key now returns byokMissing and opens the header dialog. Seedance is a tab on the media example. Sherif ignores the dual @fal-ai/client versions. * fix: proxy fal Director auth and keep Orbis on World WMA rejects a /tokens/ JWT as Key credentials. The media example opens H3 Max Director through /api/fal/proxy with the real FAL_KEY. Orbis stays on generateWorld only. Live and World share seed-image upload. * ci: apply automated fixes * fix: return the WMA app id from falLive and lock the proxy generateLive() now puts fal-ai/minimax-h3-max-director on result.model so wma(live.model) works. The example proxy only forwards WMA session paths and Director /ice. Doc snippets type-check under kiira. * refactor: rename generateLive to generateLiveVideo The activity, adapter kind, fal factory, and createLive method now use the LiveVideo name so they match generateLiveVideo(). * ci: apply automated fixes --------- Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> --- .changeset/generate-world.md | 8 + CLAUDE.md | 1 + README.md | 31 +- docs/adapters/fal.md | 21 +- docs/adapters/reactor.md | 178 ++++++ docs/advanced/otel.md | 4 +- docs/advanced/tree-shaking.md | 4 + docs/advanced/typed-options.md | 2 + docs/config.json | 36 +- docs/getting-started/overview.md | 1 + docs/media/generation-hooks.md | 2 +- docs/media/generations.md | 8 +- docs/media/live-generation.md | 172 ++++++ docs/media/video-generation.md | 11 +- docs/media/world-generation.md | 166 ++++++ examples/README.md | 17 + examples/ts-react-media/package.json | 4 + .../src/components/ByokKeyDialog.tsx | 289 +++++++++ .../ts-react-media/src/components/Header.tsx | 36 +- .../src/components/ImageGenerator.tsx | 14 +- .../src/components/LiveVideoStudio.tsx | 564 ++++++++++++++++++ .../src/components/OmniStudio.tsx | 28 +- .../src/components/SeedImageField.tsx | 68 +++ .../src/components/SeedanceStudio.tsx | 22 +- .../src/components/VideoGenerator.tsx | 26 +- .../src/components/WorldStudio.tsx | 413 +++++++++++++ .../ts-react-media/src/lib/attach-stream.ts | 32 + examples/ts-react-media/src/lib/byok.ts | 126 ++++ examples/ts-react-media/src/lib/models.ts | 99 +++ .../ts-react-media/src/lib/reactor-session.ts | 54 ++ .../src/lib/server-functions.ts | 251 ++++++-- examples/ts-react-media/src/routeTree.gen.ts | 42 +- .../src/routes/api.fal.proxy.ts | 54 ++ examples/ts-react-media/src/routes/index.tsx | 96 ++- .../ts-react-media/src/routes/seedance.tsx | 61 -- examples/ts-react-media/vite.config.ts | 34 +- kiira.config.ts | 1 + package.json | 2 +- packages/ai-event-client/src/index.ts | 112 ++++ packages/ai-fal/package.json | 3 +- packages/ai-fal/src/adapters/live.ts | 212 +++++++ packages/ai-fal/src/index.ts | 19 + packages/ai-fal/tests/live-adapter.test.ts | 188 ++++++ packages/ai-reactor/LICENSE | 21 + packages/ai-reactor/README.md | 61 ++ packages/ai-reactor/package.json | 71 +++ packages/ai-reactor/src/adapters/video.ts | 113 ++++ packages/ai-reactor/src/adapters/world.ts | 115 ++++ packages/ai-reactor/src/byok.ts | 7 + packages/ai-reactor/src/index.ts | 37 ++ packages/ai-reactor/src/model-meta.ts | 102 ++++ packages/ai-reactor/src/utils/client.ts | 85 +++ .../ai-reactor/tests/video-adapter.test.ts | 127 ++++ .../ai-reactor/tests/world-adapter.test.ts | 157 +++++ packages/ai-reactor/tsconfig.json | 8 + packages/ai-reactor/vite.config.ts | 36 ++ .../activities/generateLiveVideo/adapter.ts | 99 +++ .../src/activities/generateLiveVideo/index.ts | 339 +++++++++++ .../ai/src/activities/generateVideo/index.ts | 7 +- .../src/activities/generateWorld/adapter.ts | 96 +++ .../ai/src/activities/generateWorld/index.ts | 339 +++++++++++ packages/ai/src/activities/index.ts | 44 ++ .../ai/src/activities/middleware/types.ts | 2 + packages/ai/src/client.ts | 8 + packages/ai/src/index.ts | 8 + packages/ai/src/middlewares/otel.ts | 2 + packages/ai/src/types.ts | 128 ++++ packages/ai/tests/generate-live-video.test.ts | 84 +++ packages/ai/tests/generate-world.test.ts | 99 +++ .../middlewares/generation-middleware.test.ts | 92 +++ packages/ai/tests/middlewares/otel.test.ts | 2 + pnpm-lock.yaml | 175 +++--- pnpm-workspace.yaml | 4 + testing/e2e/package.json | 2 + testing/e2e/src/routeTree.gen.ts | 42 ++ testing/e2e/src/routes/api.video-live.ts | 86 +++ testing/e2e/src/routes/api.world.ts | 72 +++ testing/e2e/tests/video-live.spec.ts | 61 ++ testing/e2e/tests/world-generation.spec.ts | 40 ++ tsconfig.docs.json | 4 + 80 files changed, 6002 insertions(+), 285 deletions(-) create mode 100644 .changeset/generate-world.md create mode 100644 docs/adapters/reactor.md create mode 100644 docs/media/live-generation.md create mode 100644 docs/media/world-generation.md create mode 100644 examples/ts-react-media/src/components/ByokKeyDialog.tsx create mode 100644 examples/ts-react-media/src/components/LiveVideoStudio.tsx create mode 100644 examples/ts-react-media/src/components/SeedImageField.tsx create mode 100644 examples/ts-react-media/src/components/WorldStudio.tsx create mode 100644 examples/ts-react-media/src/lib/attach-stream.ts create mode 100644 examples/ts-react-media/src/lib/byok.ts create mode 100644 examples/ts-react-media/src/lib/reactor-session.ts create mode 100644 examples/ts-react-media/src/routes/api.fal.proxy.ts delete mode 100644 examples/ts-react-media/src/routes/seedance.tsx create mode 100644 packages/ai-fal/src/adapters/live.ts create mode 100644 packages/ai-fal/tests/live-adapter.test.ts create mode 100644 packages/ai-reactor/LICENSE create mode 100644 packages/ai-reactor/README.md create mode 100644 packages/ai-reactor/package.json create mode 100644 packages/ai-reactor/src/adapters/video.ts create mode 100644 packages/ai-reactor/src/adapters/world.ts create mode 100644 packages/ai-reactor/src/byok.ts create mode 100644 packages/ai-reactor/src/index.ts create mode 100644 packages/ai-reactor/src/model-meta.ts create mode 100644 packages/ai-reactor/src/utils/client.ts create mode 100644 packages/ai-reactor/tests/video-adapter.test.ts create mode 100644 packages/ai-reactor/tests/world-adapter.test.ts create mode 100644 packages/ai-reactor/tsconfig.json create mode 100644 packages/ai-reactor/vite.config.ts create mode 100644 packages/ai/src/activities/generateLiveVideo/adapter.ts create mode 100644 packages/ai/src/activities/generateLiveVideo/index.ts create mode 100644 packages/ai/src/activities/generateWorld/adapter.ts create mode 100644 packages/ai/src/activities/generateWorld/index.ts create mode 100644 packages/ai/tests/generate-live-video.test.ts create mode 100644 packages/ai/tests/generate-world.test.ts create mode 100644 testing/e2e/src/routes/api.video-live.ts create mode 100644 testing/e2e/src/routes/api.world.ts create mode 100644 testing/e2e/tests/video-live.spec.ts create mode 100644 testing/e2e/tests/world-generation.spec.ts diff --git a/.changeset/generate-world.md b/.changeset/generate-world.md new file mode 100644 index 000000000..55a74e056 --- /dev/null +++ b/.changeset/generate-world.md @@ -0,0 +1,8 @@ +--- +'@tanstack/ai': minor +'@tanstack/ai-event-client': patch +'@tanstack/ai-reactor': minor +'@tanstack/ai-fal': minor +--- + +Add `generateWorld()` and `generateLiveVideo()` for prompt-steerable sessions, plus a first-party Reactor adapter (`reactorWorld`, `reactorVideo`) and fal `falLiveVideo()` for H3 Max Director. Reactor returns a session JWT. falLiveVideo returns the WMA app id on `result.model` so the browser can call `wma(live.model)`. `generateVideo()` stays the job path that polls for a file URL. diff --git a/CLAUDE.md b/CLAUDE.md index 94bcf38ae..34b12b185 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -130,6 +130,7 @@ testing/ examples/ # Example applications ├── ts-react-chat/ # React chat example +├── ts-react-media/ # Image, video, live, and world generation ├── ts-solid-chat/ # Solid chat example ├── ts-vue-chat/ # Vue chat example ├── ts-svelte-chat/ # Svelte chat example diff --git a/README.md b/README.md index ffde43b16..ca4e4eccf 100644 --- a/README.md +++ b/README.md @@ -190,7 +190,7 @@ Learn more in the - [Generations](https://tanstack.com/ai/latest/docs/media/generations) - one pattern for image generation, text-to-speech, transcription, summarization, - audio generation, and video generation. + audio generation, video generation, and world generation. - [Realtime Voice Chat](https://tanstack.com/ai/latest/docs/media/realtime-chat) - build low-latency realtime voice experiences. - [Code Mode](https://tanstack.com/ai/latest/docs/code-mode/code-mode) - let @@ -202,20 +202,21 @@ Learn more in the Official adapters include: -| Package | Use it for | -| ------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------ | -| [`@tanstack/ai-openrouter`](https://tanstack.com/ai/latest/docs/adapters/openrouter) | 300+ models through one OpenRouter API, with per-request cost tracking | -| [`@tanstack/ai-openai`](https://tanstack.com/ai/latest/docs/adapters/openai) | OpenAI chat, image, video, speech, transcription, realtime, and provider tools | -| [`@tanstack/ai-anthropic`](https://tanstack.com/ai/latest/docs/adapters/anthropic) | Anthropic Claude chat, thinking, tools, structured outputs, and Vertex Claude | -| [`@tanstack/ai-gemini`](https://tanstack.com/ai/latest/docs/adapters/gemini) | Google Gemini chat, image, speech, and audio generation | -| [`@tanstack/ai-vertex`](https://tanstack.com/ai/latest/docs/adapters/vertex) | Gemini on Vertex AI with regional endpoints and Google Cloud credentials | -| [`@tanstack/ai-ollama`](https://tanstack.com/ai/latest/docs/adapters/ollama) | Local Ollama models | -| [`@tanstack/ai-grok`](https://tanstack.com/ai/latest/docs/adapters/grok) | xAI Grok chat, images, and realtime | -| [`@tanstack/ai-groq`](https://tanstack.com/ai/latest/docs/adapters/groq) | Groq low-latency inference | -| [`@tanstack/ai-elevenlabs`](https://tanstack.com/ai/latest/docs/adapters/elevenlabs) | ElevenLabs realtime voice, speech, transcription, music, and sound effects | -| [`@tanstack/ai-byteplus`](https://tanstack.com/ai/latest/docs/adapters/byteplus) | BytePlus Seed chat, Seedance video, Seedream image, and Seed Speech TTS/ASR | -| [`@tanstack/ai-fal`](https://tanstack.com/ai/latest/docs/adapters/fal) | fal.ai image, video, audio, speech, and transcription models | -| [`@tanstack/ai-cloudflare`](https://tanstack.com/ai/latest/docs/adapters/cloudflare) | Cloudflare Workers AI chat, embeddings, image, speech, transcription, and AI Gateway | +| Package | Use it for | +| ------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------- | +| [`@tanstack/ai-openrouter`](https://tanstack.com/ai/latest/docs/adapters/openrouter) | 300+ models through one OpenRouter API, with per-request cost tracking | +| [`@tanstack/ai-openai`](https://tanstack.com/ai/latest/docs/adapters/openai) | OpenAI chat, image, video, speech, transcription, realtime, and provider tools | +| [`@tanstack/ai-anthropic`](https://tanstack.com/ai/latest/docs/adapters/anthropic) | Anthropic Claude chat, thinking, tools, structured outputs, and Vertex Claude | +| [`@tanstack/ai-gemini`](https://tanstack.com/ai/latest/docs/adapters/gemini) | Google Gemini chat, image, speech, and audio generation | +| [`@tanstack/ai-vertex`](https://tanstack.com/ai/latest/docs/adapters/vertex) | Gemini on Vertex AI with regional endpoints and Google Cloud credentials | +| [`@tanstack/ai-ollama`](https://tanstack.com/ai/latest/docs/adapters/ollama) | Local Ollama models | +| [`@tanstack/ai-grok`](https://tanstack.com/ai/latest/docs/adapters/grok) | xAI Grok chat, images, and realtime | +| [`@tanstack/ai-groq`](https://tanstack.com/ai/latest/docs/adapters/groq) | Groq low-latency inference | +| [`@tanstack/ai-elevenlabs`](https://tanstack.com/ai/latest/docs/adapters/elevenlabs) | ElevenLabs realtime voice, speech, transcription, music, and sound effects | +| [`@tanstack/ai-byteplus`](https://tanstack.com/ai/latest/docs/adapters/byteplus) | BytePlus Seed chat, Seedance video, Seedream image, and Seed Speech TTS/ASR | +| [`@tanstack/ai-fal`](https://tanstack.com/ai/latest/docs/adapters/fal) | fal.ai image, video, audio, speech, and transcription models | +| [`@tanstack/ai-reactor`](https://tanstack.com/ai/latest/docs/adapters/reactor) | Reactor live world and video generation (Orbis, Happy Oyster, LingBot, Helios, FastH3) | +| [`@tanstack/ai-cloudflare`](https://tanstack.com/ai/latest/docs/adapters/cloudflare) | Cloudflare Workers AI chat, embeddings, image, speech, transcription, and AI Gateway | The adapter system is tree-shakeable by activity. Import `openaiText` for chat, `openaiImage` for images, `falVideo` for video, `geminiSpeech` for TTS, and so diff --git a/docs/adapters/fal.md b/docs/adapters/fal.md index c93fbaffe..c8fc2503e 100644 --- a/docs/adapters/fal.md +++ b/docs/adapters/fal.md @@ -13,7 +13,7 @@ keywords: - adapter --- -The fal.ai adapter provides access to 600+ models on the fal.ai platform for image, video, audio, speech, and transcription. Unlike text-focused adapters, the fal adapter is **media-focused** — it supports `generateImage()`, `generateVideo()`, `generateAudio()`, `generateSpeech()`, and `generateTranscription()` but does not support `chat()` or tools. +The fal.ai adapter provides access to 600+ models on the fal.ai platform for image, video, live, audio, speech, and transcription. Unlike text-focused adapters, the fal adapter is **media-focused**. It supports `generateImage()`, `generateVideo()`, `generateLiveVideo()`, `generateAudio()`, `generateSpeech()`, and `generateTranscription()`. It does not support `chat()` or tools. For a full working example, see the [fal.ai example app](https://github.com/TanStack/ai/tree/main/examples/ts-react-media). @@ -396,6 +396,12 @@ const sfx = await generateAudio({ | `fal-ai/ltx-2/text-to-video/fast` | Text-to-Video | Fast text-to-video | | `fal-ai/ltx-2/image-to-video/fast` | Image-to-Video | Fast image-to-video animation | +### Live Models + +| Model | Mode | Description | +|-------|------|-------------| +| `minimax/h3-max/director` | Live | Steerable live stream. Use `falLiveVideo()` with `generateLiveVideo()`. | + ### Text-to-Speech Models | Model | Description | @@ -473,6 +479,19 @@ Creates a fal.ai video adapter using the `FAL_KEY` environment variable or an ex **Returns:** A `FalVideoAdapter` instance for use with `generateVideo()` and `getVideoJobStatus()`. +### `falLiveVideo(model, config?)` + +Creates a fal.ai live-video adapter for H3 Max Director. `generateLiveVideo()` returns the WMA app id on `result.model` (`fal-ai/minimax-h3-max-director`). Open that id with `wma(live.model)` through a server proxy that attaches `FAL_KEY`. Do not send `live.token` as `Key` credentials. Call `allowedFalLiveVideoProxyTarget()` in the proxy so it forwards only WMA `/ice`, `/session`, `/session/heartbeat`, and Director `/ice`. + +**Parameters:** + +- `model` - `"minimax/h3-max/director"` +- `config.apiKey?` - Your fal.ai API key (falls back to `FAL_KEY` env var) + +**Returns:** A `FalLiveVideoAdapter` instance for use with `generateLiveVideo()`. + +See [Live Generation](../media/live-generation) for the browser connect step. + ### `falSpeech(model, config?)` Creates a fal.ai text-to-speech adapter. diff --git a/docs/adapters/reactor.md b/docs/adapters/reactor.md new file mode 100644 index 000000000..f89ff1993 --- /dev/null +++ b/docs/adapters/reactor.md @@ -0,0 +1,178 @@ +--- +title: Reactor +id: reactor-adapter +description: "Generate live worlds and video with Reactor models in TanStack AI via the @tanstack/ai-reactor adapter." +keywords: + - tanstack ai + - reactor + - world generation + - live generation + - orbis + - helios + - generateWorld + - generateLiveVideo + - adapter +--- + +Reactor hosts live world and video models. You describe a scene. Then you open a session and stream video. You can steer the stream with a new prompt. + +Use `reactorWorld()` with `generateWorld()` for navigable worlds. Use `reactorVideo()` with `generateLiveVideo()` for live video models. Both mint a session token. Neither supports `chat()`. + +## Installation + + + +react: @tanstack/ai-reactor +vue: @tanstack/ai-reactor +solid: @tanstack/ai-reactor +svelte: @tanstack/ai-reactor +preact: @tanstack/ai-reactor +angular: @tanstack/ai-reactor +vanilla: @tanstack/ai-reactor +octane: @tanstack/ai-reactor + + + +Peer dependency: `@tanstack/ai`. The browser also needs `@reactor-team/js-sdk` to connect and play the stream. See [World Generation](../media/world-generation) and [Live Generation](../media/live-generation). + +A full working app is in [`examples/ts-react-media`](https://github.com/TanStack/ai/tree/main/examples/ts-react-media). Open the World or Live tab. + +## API key + +Create a key in the [Reactor dashboard](https://www.reactor.inc/dashboard). Keys start with `rk_`. + +The example app uses [Bring Your Own Key](../advanced/byok). The browser pastes the key. The relay reads `x-byok-reactor`, then `REACTOR_API_KEY`. + +```ts +import { generateWorld } from '@tanstack/ai' +import { byokMissing, getByokKey } from '@tanstack/ai/byok/server' +import { reactorWorld } from '@tanstack/ai-reactor' +import { reactorByok } from '@tanstack/ai-reactor/byok' + +export async function POST(request: Request) { + const apiKey = getByokKey(request, reactorByok) + if (!apiKey) return byokMissing(reactorByok) + + const world = await generateWorld({ + adapter: reactorWorld('visko-orbis-stable', { apiKey }), + prompt: 'A neon cyberpunk city at night, slow aerial drift', + }) + + return Response.json({ + token: world.token, + model: world.model, + prompt: world.prompt, + expiresAt: world.expiresAt, + }) +} +``` + +The adapter mints a **session-scoped** token for that model only. Hand `world.token`, `world.model`, and `world.prompt` to the browser. Do not put the API key in the JSON body. + +## Models + +Pass a string literal so TypeScript can narrow options. + +```ts +import { reactorWorld } from '@tanstack/ai-reactor' + +const adapter = reactorWorld('visko-orbis-stable') +``` + +| Id | Connect slug | +| --- | --- | +| `visko-orbis-stable` | `reactor/visko-orbis-stable` | +| `visko-orbis-dynamic` | `reactor/visko-orbis-dynamic` | +| `happy-oyster-adventure` | `reactor/happy-oyster-adventure` | +| `happy-oyster-director` | `reactor/happy-oyster-director` | +| `lingbot-world-2` | `reactor/lingbot-world-2` | +| `lingbot` | `reactor/lingbot` | +| `helios` | `reactor/helios` | + +`world.model` is the connect slug. Pass it to `new Reactor({ modelName })`. + +## Live video + +Reactor video is a live stream, not a finished file. `generateLiveVideo()` returns a token. The browser connects, sets the prompt, and plays the track. + +```ts +import { generateLiveVideo } from '@tanstack/ai' +import { reactorVideo } from '@tanstack/ai-reactor' + +const apiKey = process.env.REACTOR_API_KEY ?? '' +const live = await generateLiveVideo({ + adapter: reactorVideo('helios', { apiKey }), + prompt: 'A neon cyberpunk city at night, slow aerial drift', +}) +``` + +Hand `live.token`, `live.model`, and `live.prompt` to the browser. Connect as shown in [Live Generation](../media/live-generation). + +| Id | Connect slug | +| --- | --- | +| `helios` | `reactor/helios` | +| `fast-h3` | `reactor/fast-h3` | +| `longlive-v2` | `reactor/longlive-v2` | +| `ltx2` | `reactor/ltx2` | + +`helios` also works with `reactorWorld()`. Pick `generateLiveVideo()` when you want a video session. Pick `generateWorld()` when you want a navigable world. + +Pass a text prompt to `generateWorld()` / `generateLiveVideo()`. Those calls mint a token. They do not send resolution or a seed image. + +After `connect`, send browser commands. LingBot starts from a seed image. Pass a `File` from ``. The SDK uploads it and returns a `FileRef`. Do not send base64. `start` still needs `set_prompt`. Send a short default, then steer after the first frame. + +```ts +import { Reactor } from '@reactor-team/js-sdk' + +const reactor = new Reactor({ modelName: 'reactor/lingbot' }) +const picker = document.querySelector('input[type="file"]') +if (!(picker instanceof HTMLInputElement)) { + throw new Error('Pick a seed image') +} +const file = picker.files?.[0] +if (file === undefined) { + throw new Error('Pick a seed image') +} +const image = await reactor.uploadFile(file) +await reactor.sendCommand('set_image', { image }) +await reactor.sendCommand('set_prompt', { prompt: 'Follow the seed image.' }) +await reactor.sendCommand('start', {}) +``` + +Helios can take the same `File` with `set_conditioning` so prompt and image land together. + +## Browser session options + +Orbis reads these on the next `start`. Keep them in client state. Send them with `sendCommand` after `connect`. They are not token-mint fields. + +```ts +import { Reactor } from '@reactor-team/js-sdk' + +const reactor = new Reactor({ modelName: 'reactor/visko-orbis-stable' }) +await reactor.sendCommand('set_resolution', { resolution: '2k' }) +await reactor.sendCommand('set_seed', { seed: 42 }) +await reactor.sendCommand('set_audio_enabled', { enabled: true }) +``` + +| Option | Command | Meaning | +| --- | --- | --- | +| `resolution` | `set_resolution` | `1080p`, `2k`, or `4k` delivery tier | +| `seed` | `set_seed` | RNG seed for the next run | +| `audioEnabled` | `set_audio_enabled` | When `false`, skip audio compute | +| `audioPrompt` | `set_audio_prompt` | Sound description, or `""` for picture-driven audio | + +## Custom endpoint + +```ts +import { reactorWorld } from '@tanstack/ai-reactor' + +const apiKey = process.env.REACTOR_API_KEY ?? '' +const adapter = reactorWorld('visko-orbis-stable', { + apiKey, + baseUrl: 'https://api.reactor.inc', +}) +``` + +## What you have now + +A server call that mints a scoped Reactor token for one world or video model. Next: connect in the browser as shown in [World Generation](../media/world-generation) or [Live Generation](../media/live-generation). diff --git a/docs/advanced/otel.md b/docs/advanced/otel.md index e49f94488..f56a27ccb 100644 --- a/docs/advanced/otel.md +++ b/docs/advanced/otel.md @@ -215,7 +215,7 @@ otelMiddleware({ ## Beyond chat: media activities -`otelMiddleware` is not chat-only. The media activities — `generateImage`, `generateVideo`, `generateAudio`, `generateSpeech`, and `generateTranscription` — accept the **same** `otelMiddleware` value on their `middleware` option. Each is a single request → response (or submit → poll for video), so the middleware emits one span per call instead of the chat span tree: +`otelMiddleware` is not chat-only. The media activities (`generateImage`, `generateVideo`, `generateLiveVideo`, `generateWorld`, `generateAudio`, `generateSpeech`, and `generateTranscription`) accept the **same** `otelMiddleware` value on their `middleware` option. Each is a single request to response (or submit then poll for video), so the middleware emits one span per call instead of the chat span tree: ```ts import { generateImage } from '@tanstack/ai' @@ -246,6 +246,8 @@ Each media call produces one `CLIENT` span tagged with the activity's `gen_ai.op | `generateAudio` | `audio_generation` | | `generateSpeech` | `text_to_speech` | | `generateTranscription` | `transcription` | +| `generateWorld` | `world_generation` | +| `generateLiveVideo` | `live_video_generation` | | `summarize` | `summarize` | The span carries `gen_ai.system` and `gen_ai.request.model` at start and, on finish, the same `gen_ai.usage.*` / `tanstack.ai.usage.*` attributes documented above — including the `tanstack.ai.usage.billed_quantity` / `tanstack.ai.usage.billed_unit` pair for unit-billed media. When a `Meter` is supplied it records the `gen_ai.client.operation.duration` histogram, tagged per activity. For streaming video the span covers the full create → poll → complete lifecycle. Non-streaming video is two calls, so the submit itself emits no span — the run opens once the provider accepts the job, and the `getVideoJobStatus()` poll that observes a terminal state ends it. If a streaming video consumer abandons the stream before completion, the span is ended via `onAbort` (status `ERROR`, `tanstack.ai.completion.reason = cancelled`) rather than leaked. diff --git a/docs/advanced/tree-shaking.md b/docs/advanced/tree-shaking.md index 950380d71..67cdea3f8 100644 --- a/docs/advanced/tree-shaking.md +++ b/docs/advanced/tree-shaking.md @@ -39,6 +39,8 @@ import { generateImage } from '@tanstack/ai' // Image generation import { generateSpeech } from '@tanstack/ai' // Text-to-speech import { generateTranscription } from '@tanstack/ai' // Audio transcription import { generateVideo } from '@tanstack/ai' // Video generation +import { generateLiveVideo } from '@tanstack/ai' // Live video sessions +import { generateWorld } from '@tanstack/ai' // Live world sessions ``` ### Example: Chat Only @@ -274,6 +276,8 @@ Each adapter type implements a specific interface: - `TTSAdapter` - Provides `generateSpeech()` method for text-to-speech - `TranscriptionAdapter` - Provides `generateTranscription()` method for audio transcription - `VideoAdapter` - Provides `generateVideo()` method for video generation +- `LiveVideoAdapter` - Provides `createLiveVideo()` method for live video sessions +- `WorldAdapter` - Provides `createWorld()` method for live world sessions All adapters have a `kind` property that indicates their type: diff --git a/docs/advanced/typed-options.md b/docs/advanced/typed-options.md index cafa8a904..315457d8b 100644 --- a/docs/advanced/typed-options.md +++ b/docs/advanced/typed-options.md @@ -64,6 +64,8 @@ Each helper mirrors the activity it pairs with. Same options, same return type. | `createImageOptions` | `generateImage()` | image adapter (e.g. `openaiImage`, `falImage`) | | `createAudioOptions` | `generateAudio()` | audio adapter (e.g. `falAudio`, `geminiAudio`) | | `createVideoOptions` | `generateVideo()` / `getVideoJobStatus()` | video adapter (e.g. `falVideo`, `openaiVideo`) | +| `createLiveVideoOptions` | `generateLiveVideo()` | live adapter (e.g. `reactorVideo`, `falLiveVideo`) | +| `createWorldOptions` | `generateWorld()` | world adapter (e.g. `reactorWorld`) | | `createSpeechOptions` | `generateSpeech()` | speech adapter (e.g. `openaiSpeech`, `elevenlabsSpeech`) | | `createTranscriptionOptions` | `generateTranscription()` | transcription adapter (e.g. `openaiTranscription`, `falTranscription`) | diff --git a/docs/config.json b/docs/config.json index 95cb68b51..8e5d2a9f9 100644 --- a/docs/config.json +++ b/docs/config.json @@ -13,7 +13,7 @@ "label": "Overview", "to": "getting-started/overview", "addedAt": "2026-04-15", - "updatedAt": "2026-09-03" + "updatedAt": "2026-09-04" }, { "label": "Quick Start", @@ -223,7 +223,7 @@ "label": "OpenTelemetry", "to": "advanced/otel", "addedAt": "2026-05-08", - "updatedAt": "2026-08-08" + "updatedAt": "2026-09-04" }, { "label": "Compaction", @@ -567,7 +567,7 @@ "label": "Generations", "to": "media/generations", "addedAt": "2026-04-15", - "updatedAt": "2026-08-19" + "updatedAt": "2026-09-04" }, { "label": "Realtime Voice Chat", @@ -609,13 +609,25 @@ "label": "Video Generation", "to": "media/video-generation", "addedAt": "2026-04-15", - "updatedAt": "2026-08-31" + "updatedAt": "2026-09-04" + }, + { + "label": "Live Generation", + "to": "media/live-generation", + "addedAt": "2026-09-04", + "updatedAt": "2026-09-04" + }, + { + "label": "World Generation", + "to": "media/world-generation", + "addedAt": "2026-09-04", + "updatedAt": "2026-09-04" }, { "label": "Generation Hooks", "to": "media/generation-hooks", "addedAt": "2026-04-15", - "updatedAt": "2026-09-03" + "updatedAt": "2026-09-04" } ], "tab": "guides" @@ -912,7 +924,8 @@ { "label": "Tree-Shaking", "to": "advanced/tree-shaking", - "addedAt": "2026-04-15" + "addedAt": "2026-04-15", + "updatedAt": "2026-09-04" }, { "label": "Extend Adapter", @@ -922,7 +935,8 @@ { "label": "Typed Pre-Configured Options", "to": "advanced/typed-options", - "addedAt": "2026-05-25" + "addedAt": "2026-05-25", + "updatedAt": "2026-09-04" }, { "label": "Approval Flow Processing", @@ -1000,7 +1014,13 @@ "label": "fal.ai", "to": "adapters/fal", "addedAt": "2026-04-15", - "updatedAt": "2026-08-20" + "updatedAt": "2026-09-04" + }, + { + "label": "Reactor", + "to": "adapters/reactor", + "addedAt": "2026-09-04", + "updatedAt": "2026-09-04" }, { "label": "OpenRouter Adapter", diff --git a/docs/getting-started/overview.md b/docs/getting-started/overview.md index b7dba57b0..526063064 100644 --- a/docs/getting-started/overview.md +++ b/docs/getting-started/overview.md @@ -125,6 +125,7 @@ With the help of adapters, TanStack AI can connect to various LLM providers. Ava - **@tanstack/ai-bedrock** - Amazon Bedrock (Claude, Nova, Llama, and more via AWS) - **@tanstack/ai-byteplus** - BytePlus (Seed chat, Seedance video, Seedream image, Seed Speech) - **@tanstack/ai-fal** - fal (image & video generation) +- **@tanstack/ai-reactor** - Reactor (live world and video generation: Orbis, Happy Oyster, LingBot, Helios, FastH3) - **@tanstack/ai-llmgateway** - LLM Gateway (hundreds of models via one OpenAI-compatible endpoint, self-hostable) - **@tanstack/ai-cloudflare** - Cloudflare Workers AI (chat, embeddings, image, speech, transcription) and AI Gateway routing - **@tanstack/ai-lovable** - Lovable AI Gateway (Google and OpenAI chat, image, video, embeddings, and speech via one project key) diff --git a/docs/media/generation-hooks.md b/docs/media/generation-hooks.md index bd42e0171..7b231fcfb 100644 --- a/docs/media/generation-hooks.md +++ b/docs/media/generation-hooks.md @@ -67,7 +67,7 @@ export async function POST(req: Request) { } ``` -The same pattern applies to all generation types -- swap `generateImage` for `generateSpeech`, `generateTranscription`, `summarize`, or `generateVideo`. See the individual media guides for server-side details. +The same pattern applies to image, speech, transcription, summarize, and video. Live and world generation mint a session token. Connect in the browser. See [Live Generation](./live-generation) and [World Generation](./world-generation). ## useGenerateImage diff --git a/docs/media/generations.md b/docs/media/generations.md index ccc269691..57c7bee89 100644 --- a/docs/media/generations.md +++ b/docs/media/generations.md @@ -15,10 +15,12 @@ keywords: # Generations -You want an image, some speech, a transcript or a video, not a conversation. Every one +You want an image, some speech, a transcript, a video, or a live stream, not a conversation. Every one of those is a **generation**: one request, one result. They all share the same shape, so learning one teaches you the rest. +For a live, prompt-steerable stream that does not end as a file, see [Live Generation](./live-generation) or [World Generation](./world-generation). + ## The fastest path A server route that streams the result: @@ -87,8 +89,10 @@ streaming is handled for you. All three are written out in | Transcription | `generateTranscription()` | `useTranscription()` | [Transcription](./transcription) | | Summarization | `summarize()` | `useSummarize()` | - | | Video generation | `generateVideo()` | `useGenerateVideo()` | [Video Generation](./video-generation) | +| Live generation | `generateLiveVideo()` | - | [Live Generation](./live-generation) | +| World generation | `generateWorld()` | - | [World Generation](./world-generation) | -> **Note:** Video generation uses a jobs/polling architecture. The `useGenerateVideo` hook additionally exposes `jobId`, `videoStatus`, `onJobCreated`, and `onStatusUpdate` for tracking the polling lifecycle. See the [Video Generation](./video-generation) guide for details. +> **Note:** Video adapters use a jobs/polling architecture. The `useGenerateVideo` hook additionally exposes `jobId`, `videoStatus`, `onJobCreated`, and `onStatusUpdate` for tracking the polling lifecycle. Live and world sessions mint a token. There is no download URL. ## Advanced diff --git a/docs/media/live-generation.md b/docs/media/live-generation.md new file mode 100644 index 000000000..4cccee713 --- /dev/null +++ b/docs/media/live-generation.md @@ -0,0 +1,172 @@ +--- +title: Live Generation +id: live-generation +order: 6.5 +description: "Open a live, prompt-steerable video session with generateLiveVideo(). Mint a connect payload on the server, connect in the browser, and steer the stream with new prompts." +keywords: + - tanstack ai + - live generation + - generateLiveVideo + - reactor + - fal + - h3 max director + - helios + - live video + - experimental +--- + +# Live Generation (Experimental) + +You want a video that plays while it generates. You also want to change the prompt mid-run. A finite video job stops with a file. `generateLiveVideo()` opens a session instead. + +Call `generateLiveVideo()` on the server. It returns a connect payload: a model id, a prompt, and a token. The browser uses that payload to open the stream. + +> **Experimental.** The API can change. Live models bill per session-second while a GPU is held. + +## 1. Mint a session on the server + +Pick one adapter. They do not return a download URL. + +```ts group=live-reactor +import { generateLiveVideo } from '@tanstack/ai' +import { reactorVideo } from '@tanstack/ai-reactor' + +const live = await generateLiveVideo({ + adapter: reactorVideo('helios'), + prompt: 'A chef tosses noodles in a steel wok, flames leaping', +}) + +// Hand live.token, live.model, and live.prompt to the browser. +``` + +```ts group=live-fal +import { generateLiveVideo } from '@tanstack/ai' +import { falLiveVideo } from '@tanstack/ai-fal' + +const live = await generateLiveVideo({ + adapter: falLiveVideo('minimax/h3-max/director'), + prompt: 'Live shopping stream: a host holds up a gold watch to camera', +}) + +// live.model is 'fal-ai/minimax-h3-max-director'. Hand it to the browser. +``` + +Set `REACTOR_API_KEY` or `FAL_KEY`, or pass `apiKey` in the adapter config. Do not put the key in the JSON body. + +A full app lives in [`examples/ts-react-media`](https://github.com/TanStack/ai/tree/main/examples/ts-react-media). + +## 2. Connect in the browser + +The server half is the same for every live adapter. The browser client is not. + +### Reactor + +Install `@reactor-team/js-sdk`. Connect with the token. Helios uses `set_sr_scale`, then `set_prompt`, then `start`. + +```ts group=live-reactor +import { Reactor } from '@reactor-team/js-sdk' + +const reactor = new Reactor({ modelName: live.model }) +const video = document.querySelector('video') + +reactor.on('trackReceived', (name, _track, stream) => { + if (name !== 'main_video') return + if (!video) return + video.muted = true + const attach = () => { + video.srcObject = null + video.srcObject = stream + void video.play().catch((error: unknown) => { + if (error instanceof DOMException && error.name === 'AbortError') return + }) + } + attach() + for (const track of stream.getTracks()) { + track.addEventListener('unmute', attach) + } +}) + +await reactor.connect(live.token) +await reactor.sendCommand('set_sr_scale', { sr_scale: '2x' }) +await reactor.sendCommand('set_prompt', { prompt: live.prompt }) +await reactor.sendCommand('start', {}) +``` + +FastH3 uses `enqueue` only. LongLive uses `set_shot` then `start`. + +Helios can also take a seed image. Pass a `File` from ``. Do not send base64. + +```ts group=live-reactor +const picker = document.querySelector('input[type="file"]') +if (!(picker instanceof HTMLInputElement)) { + throw new Error('Pick a seed image') +} +const file = picker.files?.[0] +if (file === undefined) { + throw new Error('Pick a seed image') +} +const image = await reactor.uploadFile(file) +await reactor.sendCommand('set_conditioning', { prompt: live.prompt, image }) +await reactor.sendCommand('start', {}) +``` + +A later `set_prompt` morphs the shot at the next chunk. + +See the [Reactor adapter](../adapters/reactor) for model ids. + +### fal H3 Max Director + +Install `@fal-ai/client@alpha`. Keep `FAL_KEY` on the server. `live.model` is the WMA app id. Open WMA through a proxy that attaches the key. Do not send `live.token` as `Key` credentials. + +The example proxy is `src/routes/api.fal.proxy.ts` in [`examples/ts-react-media`](https://github.com/TanStack/ai/tree/main/examples/ts-react-media). It attaches the key and forwards only to `wma.fal.run` (`/ice`, `/session`, `/session/heartbeat`) and Director `/ice`. + +```ts group=live-fal +import { createFalClient } from '@fal-ai/client' +import { wma } from '@fal-ai/client/realtime' + +const fal = createFalClient({ proxyUrl: '/api/fal/proxy' }) +const video = document.querySelector('video') + +const session = fal.realtime.open(wma(live.model), { + receive: ['video', 'audio'], + onError: (error) => { + console.error(error) + }, + onMedia: (stream) => { + if (!video) return + video.muted = true + video.srcObject = stream + void video.play().catch((error: unknown) => { + if (error instanceof DOMException && error.name === 'AbortError') return + }) + }, +}) + +session.send({ + type: 'configure', + prompt: live.prompt, + prompt_version: 1, + protocol_version: 1, +}) + +await session.ready +``` + +To steer, send `{ type: 'prompt', prompt, prompt_version }` and increase `prompt_version` each time. To stop, send `{ type: 'stop' }` and close the session. + +Director bills a 60 second minimum. Resolution is `480p` or `768p`. See the [fal adapter](../adapters/fal). + +## Models + +| Adapter | Id | What it does | +| --- | --- | --- | +| `reactorVideo()` | `helios` | Interactive realtime video | +| `reactorVideo()` | `fast-h3` | Fast live clips on a live track | +| `reactorVideo()` | `longlive-v2` | Shot-based live clips | +| `falLiveVideo()` | `minimax/h3-max/director` | Steerable live stream over WMA | + +For Orbis and LingBot, use [World Generation](./world-generation). For a file that finishes, use [Video Generation](./video-generation). + +## What you have now + +A server call that returns a connect payload. The browser opens the stream and steers it until you stop. diff --git a/docs/media/video-generation.md b/docs/media/video-generation.md index f751d5d63..32409efec 100644 --- a/docs/media/video-generation.md +++ b/docs/media/video-generation.md @@ -2,7 +2,7 @@ title: Video Generation id: video-generation order: 6 -description: "Generate video from text prompts with OpenAI Sora, Google Veo, Gemini Omni Flash, xAI Grok Imagine, BytePlus Seedance, OpenRouter, or fal.ai using TanStack AI's experimental generateVideo() jobs/polling API." +description: "Generate video from text prompts with OpenAI Sora, Google Veo, Gemini Omni Flash, xAI Grok Imagine, BytePlus Seedance, OpenRouter, or fal.ai using TanStack AI's experimental generateVideo() API." keywords: - tanstack ai - video generation @@ -39,12 +39,14 @@ keywords: ## Overview -TanStack AI provides experimental support for video generation through dedicated video adapters. Unlike image generation, video generation is an **asynchronous operation** that uses a jobs/polling pattern: +TanStack AI provides experimental support for video generation through dedicated video adapters. Most providers are **asynchronous** and use a jobs/polling pattern: 1. **Create a job** - Submit a prompt and receive a job ID 2. **Poll for status** - Check the job status until it's complete 3. **Retrieve the video** - Get the URL to download/view the generated video +For a prompt-steerable live stream (no download URL), use [Live Generation](./live-generation) or [World Generation](./world-generation). + Currently supported: - **OpenAI**: Sora-2 and Sora-2-Pro models (when available) @@ -80,6 +82,8 @@ const { jobId, model } = await generateVideo({ console.log("Job started:", jobId); ``` +For a stream that plays while it generates, and that you can steer with a new prompt, use [Live Generation](./live-generation). `generateVideo()` is the job path: create, poll, then fetch a file URL. + ### Polling for Status ```typescript @@ -925,9 +929,12 @@ await generateVideo({ #### VideoJobResult (from create) ```typescript +import type { PersistedArtifactRef } from '@tanstack/ai/client' + interface VideoJobResult { jobId: string; // Unique job identifier for polling model: string; // Model used for generation + artifacts?: Array } ``` diff --git a/docs/media/world-generation.md b/docs/media/world-generation.md new file mode 100644 index 000000000..2b7169731 --- /dev/null +++ b/docs/media/world-generation.md @@ -0,0 +1,166 @@ +--- +title: World Generation +id: world-generation +order: 7 +description: "Open a live, prompt-steerable world session with generateWorld(). Mint a token on the server, connect in the browser, and steer the stream with new prompts." +keywords: + - tanstack ai + - world generation + - generateWorld + - reactor + - orbis + - infinite world + - live video + - experimental +--- + +# World Generation (Experimental) + +You want a world that generates while the viewer watches or changes the prompt. A finite video job stops. `generateWorld()` opens a live session instead. + +Call `generateWorld()` on the server. It returns a short-lived token, a model slug, and the prompt. The browser connects, sets the prompt, and starts the stream. LingBot also needs a seed image after connect. + +> **Experimental.** The API can change. World models bill per session-second while a GPU is held. + +## 1. Mint a session on the server + +The browser can paste a Reactor key. The relay reads `x-byok-reactor`, then `REACTOR_API_KEY`. Do not put the key in the JSON body. + +```ts +import { generateWorld } from '@tanstack/ai' +import { byokMissing, getByokKey } from '@tanstack/ai/byok/server' +import { reactorWorld } from '@tanstack/ai-reactor' +import { reactorByok } from '@tanstack/ai-reactor/byok' + +export async function POST(request: Request) { + const apiKey = getByokKey(request, reactorByok) + if (!apiKey) return byokMissing(reactorByok) + + const body = await request.json() + const prompt = typeof body.prompt === 'string' ? body.prompt : '' + if (prompt.length === 0) { + return Response.json({ error: 'prompt is required' }, { status: 400 }) + } + + const world = await generateWorld({ + adapter: reactorWorld('visko-orbis-stable', { apiKey }), + prompt, + }) + + return Response.json({ + token: world.token, + model: world.model, + prompt: world.prompt, + expiresAt: world.expiresAt, + }) +} +``` + +See [Bring Your Own Key](../advanced/byok) for the client store. + +## 2. Connect in the browser + +Install `@reactor-team/js-sdk`. Connect with the token. Then set the prompt and start. + +```ts group=world-browser +import { Reactor } from '@reactor-team/js-sdk' +import { defineByok, defaultByokStorage } from '@tanstack/ai-client/byok' +import { reactorByok } from '@tanstack/ai-reactor/byok' + +const byok = defineByok({ + storage: defaultByokStorage(), + providers: [reactorByok], +}) +byok.setServerCoverage(true) + +const video = document.querySelector('video') +if (!video) { + throw new Error('Missing video element') +} + +const world = await fetch('/api/world', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + ...byok.headers(reactorByok.id), + }, + body: JSON.stringify({ + prompt: + 'A dramatic coastline of black volcanic cliffs at golden hour, a single unbroken take.', + }), +}).then(async (response) => { + if (!response.ok) { + throw new Error('World session failed') + } + return response.json() +}) + +const token = typeof world.token === 'string' ? world.token : '' +const model = typeof world.model === 'string' ? world.model : '' +const prompt = typeof world.prompt === 'string' ? world.prompt : '' +if (token.length === 0 || model.length === 0 || prompt.length === 0) { + throw new Error('World payload is incomplete') +} + +const reactor = new Reactor({ modelName: model }) + +reactor.on('trackReceived', (name, _track, stream) => { + if (name !== 'main_video') return + video.muted = true + const attach = () => { + video.srcObject = null + video.srcObject = stream + void video.play().catch((error: unknown) => { + if (error instanceof DOMException && error.name === 'AbortError') return + }) + } + attach() + for (const track of stream.getTracks()) { + track.addEventListener('unmute', attach) + } +}) + +await reactor.connect(token) +await reactor.sendCommand('set_resolution', { resolution: '1080p' }) +await reactor.sendCommand('set_prompt', { prompt }) +await reactor.sendCommand('start', {}) +``` + +The video element now plays a live world. A new `set_prompt` during the run morphs the scene at the next chunk. + +LingBot and LingBot World 2 start from a seed image. Pass a `File` from ``. The SDK uploads the file. Do not send base64. `start` still needs `set_prompt`. Send a short default, then steer after the first frame. + +```ts group=world-browser +const picker = document.querySelector('input[type="file"]') +if (!(picker instanceof HTMLInputElement)) { + throw new Error('Pick a seed image') +} +const file = picker.files?.[0] +if (file === undefined) { + throw new Error('Pick a seed image') +} +const image = await reactor.uploadFile(file) +await reactor.sendCommand('set_image', { image }) +await reactor.sendCommand('set_prompt', { prompt: 'Follow the seed image.' }) +await reactor.sendCommand('start', {}) +``` + +## Models + +`reactorWorld()` accepts these ids. The result `model` field is the Reactor connect slug. + +| Id | Connect slug | What it does | +| --- | --- | --- | +| `visko-orbis-stable` | `reactor/visko-orbis-stable` | Steerable video with realtime audio | +| `visko-orbis-dynamic` | `reactor/visko-orbis-dynamic` | Same family, live resolution switch | +| `lingbot-world-2` | `reactor/lingbot-world-2` | Image-anchored navigable world | +| `lingbot` | `reactor/lingbot` | Image-anchored navigable video | +| `helios` | `reactor/helios` | Interactive realtime video | + +Happy Oyster (`happy-oyster-adventure`, `happy-oyster-director`) uses `createWorld` and `startTravel` after connect. See the [Reactor adapter](../adapters/reactor) for every id. Helios also works with `generateLiveVideo()` and `reactorVideo()`. See [Live Generation](./live-generation). + +A full app lives in [`examples/ts-react-media`](https://github.com/TanStack/ai/tree/main/examples/ts-react-media). Open the World tab. + +## What you have now + +A server route that mints a world session, and a browser that streams it. Change the prompt while the video plays to steer the world. diff --git a/examples/README.md b/examples/README.md index 2e03103fd..324106712 100644 --- a/examples/README.md +++ b/examples/README.md @@ -7,6 +7,7 @@ This directory contains comprehensive examples demonstrating TanStack AI across Choose an example based on your use case: - **Want a full-stack TypeScript app?** → [TanStack Chat (ts-react-chat)](#tanstack-chat-ts-react-chat) +- **Need a live world stream?** → [World generation (ts-react-media)](#world-generation-ts-react-media) - **Need a vanilla JS frontend?** → [Vanilla Chat](#vanilla-chat) - **Multi-User TypeScript chat app?** → [Group Chat (ts-group-chat)](#group-chat-ts-group-chat) - **Polyglot AG-UI backends (Go/Rust/PHP/Zig/Bash/Python)?** → [AG-UI Polyglot Echo (ag-ui)](#ag-ui-polyglot-echo-ag-ui) @@ -128,6 +129,22 @@ pnpm dev --- +### World generation (ts-react-media) + +The media example has a **World** tab. Paste a Reactor key in the header +dialog. The relay calls `generateWorld()`. The page connects and can steer +the scene mid-run. + +```bash +cd examples/ts-react-media +pnpm install +pnpm dev +``` + +Open the World tab. Paste a Reactor key, or set `REACTOR_API_KEY`. + +--- + ### Vanilla Chat A framework-free chat application using pure JavaScript and `@tanstack/ai-client`. diff --git a/examples/ts-react-media/package.json b/examples/ts-react-media/package.json index 89f5d0b74..7d054923a 100644 --- a/examples/ts-react-media/package.json +++ b/examples/ts-react-media/package.json @@ -10,14 +10,18 @@ "test:types": "tsc" }, "dependencies": { + "@fal-ai/client": "1.11.0-alpha.2", + "@reactor-team/js-sdk": "^3.0.1", "@tailwindcss/vite": "^4.1.18", "@tanstack/ai": "workspace:*", "@tanstack/ai-byteplus": "workspace:*", + "@tanstack/ai-client": "workspace:*", "@tanstack/ai-fal": "workspace:*", "@tanstack/ai-gemini": "workspace:*", "@tanstack/ai-grok": "workspace:*", "@tanstack/ai-openrouter": "workspace:*", "@tanstack/ai-react": "workspace:*", + "@tanstack/ai-reactor": "workspace:*", "@tanstack/react-router": "^1.158.4", "@tanstack/react-start": "^1.159.0", "@tanstack/router-plugin": "^1.158.4", diff --git a/examples/ts-react-media/src/components/ByokKeyDialog.tsx b/examples/ts-react-media/src/components/ByokKeyDialog.tsx new file mode 100644 index 000000000..a1d64c3b0 --- /dev/null +++ b/examples/ts-react-media/src/components/ByokKeyDialog.tsx @@ -0,0 +1,289 @@ +import { useEffect, useState } from 'react' +import { KeyRound, X } from 'lucide-react' +import { useByok } from '@tanstack/ai-react' +import { byok, KEYED_PROVIDERS } from '@/lib/byok' +import type { KeyStatus } from '@tanstack/ai-client/byok' +import type { ProviderId } from '@tanstack/ai/byok' + +export type ByokKeyDialogProps = { + open: boolean + onOpenChange: (open: boolean) => void + envStatus?: Record +} + +const STATUS_LABEL: Record< + KeyStatus['state'], + { label: string; className: string } +> = { + empty: { label: 'Not set', className: 'text-gray-500' }, + set: { label: 'Saved', className: 'text-gray-400' }, + locked: { label: 'Locked', className: 'text-amber-400' }, + error: { label: 'Check failed', className: 'text-red-400' }, +} + +export function ByokKeyDialog({ + open, + onOpenChange, + envStatus, +}: ByokKeyDialogProps) { + const snapshot = useByok(byok) + const [unlocking, setUnlocking] = useState(false) + const [actionError, setActionError] = useState('') + + useEffect(() => { + if (!open) return + const onKey = (event: KeyboardEvent) => { + if (event.key === 'Escape') onOpenChange(false) + } + window.addEventListener('keydown', onKey) + return () => window.removeEventListener('keydown', onKey) + }, [open, onOpenChange]) + + const locked = snapshot.locked + const needsAttention = + locked || + KEYED_PROVIDERS.every((provider) => { + const status = snapshot.status[provider.id] + const empty = !status || status.state === 'empty' + return empty && !envStatus?.[provider.id] + }) + + return ( + <> + + + {open ? ( +
onOpenChange(false)} + > +
+
event.stopPropagation()} + > +
+

API keys

+ +
+

+ Keys stay in this browser and are sent per request in a header. + Never stored on the server. Providers with a server env key + already work without one. +

+ + {snapshot.locked ? ( +
+ Saved keys are locked ({byok.storage.label}). + +
+ ) : null} + + {byok.storage.warning ? ( +

+ {byok.storage.warning} +

+ ) : null} + {snapshot.storageError ? ( +

+ {snapshot.storageError} +

+ ) : null} + {actionError ? ( +

{actionError}

+ ) : null} + +
+ {KEYED_PROVIDERS.map((provider) => ( + + ))} +
+
+
+
+ ) : null} + + ) +} + +function ProviderRow({ + id, + label, + status, + locked, + hasEnvKey, +}: { + id: ProviderId + label: string + status: KeyStatus | undefined + locked: boolean + hasEnvKey: boolean +}) { + const [draft, setDraft] = useState('') + const [rowError, setRowError] = useState('') + const state = status?.state ?? 'empty' + const masked = status && 'masked' in status ? status.masked : undefined + const isLocked = locked || state === 'locked' + const hasKey = state !== 'empty' + + return ( +
+
+ {label} + +
+ + {hasKey && masked ? ( +
+ + {masked} + + +
+ ) : null} + + {state === 'error' && status && 'message' in status ? ( +

{status.message}

+ ) : null} + {rowError ?

{rowError}

: null} + +
{ + event.preventDefault() + const next = draft.trim() + if (!next || isLocked) return + setRowError('') + void byok + .update(id, next) + .then(() => setDraft('')) + .catch((error: unknown) => + setRowError( + error instanceof Error ? error.message : 'Could not save key', + ), + ) + }} + > + setDraft(event.target.value)} + className="flex-1 rounded-md border border-gray-600 bg-gray-950 px-2 py-1.5 text-sm text-white placeholder-gray-500 disabled:opacity-50" + /> + +
+
+ ) +} + +function PresenceBadge({ + state, + hasKey, + isLocked, + hasEnvKey, +}: { + state: KeyStatus['state'] + hasKey: boolean + isLocked: boolean + hasEnvKey: boolean +}) { + if (hasKey && !isLocked) { + return ( + Your key + ) + } + if (isLocked) { + return Locked + } + if (hasEnvKey) { + return ( + Server key + ) + } + const { label, className } = STATUS_LABEL[state] + return {label} +} diff --git a/examples/ts-react-media/src/components/Header.tsx b/examples/ts-react-media/src/components/Header.tsx index ee8fa53c8..687e5b93a 100644 --- a/examples/ts-react-media/src/components/Header.tsx +++ b/examples/ts-react-media/src/components/Header.tsx @@ -1,10 +1,32 @@ +import { useEffect, useState } from 'react' import { Link } from '@tanstack/react-router' +import { useByok } from '@tanstack/ai-react' +import { ByokKeyDialog } from '@/components/ByokKeyDialog' +import { byok, getEnvKeyStatus } from '@/lib/byok' const navLinkClass = 'px-3 py-1.5 rounded-lg text-sm font-medium text-gray-300 hover:bg-gray-700 hover:text-white transition-colors' const navLinkActiveClass = 'bg-gray-700 text-white' export default function Header() { + const [keyDialogOpen, setKeyDialogOpen] = useState(false) + const [envStatus, setEnvStatus] = useState>({}) + const snapshot = useByok(byok) + + useEffect(() => { + void byok.ready() + }, []) + + useEffect(() => { + void getEnvKeyStatus() + .then(setEnvStatus) + .catch(() => setEnvStatus({})) + }, []) + + useEffect(() => { + if (snapshot.prompt) setKeyDialogOpen(true) + }, [snapshot.prompt]) + return (

@@ -16,7 +38,7 @@ export default function Header() { Image & Video Generation -

) diff --git a/examples/ts-react-media/src/components/ImageGenerator.tsx b/examples/ts-react-media/src/components/ImageGenerator.tsx index af18858fb..921d2adb6 100644 --- a/examples/ts-react-media/src/components/ImageGenerator.tsx +++ b/examples/ts-react-media/src/components/ImageGenerator.tsx @@ -4,6 +4,7 @@ import { useGenerateImage } from '@tanstack/ai-react' import type { MediaPrompt } from '@tanstack/ai/client' import { generateImageFn } from '@/lib/server-functions' +import { byok, callWithByok, toByokProvider } from '@/lib/byok' import { getRandomImagePrompt } from '@/lib/prompts' import { IMAGE_MODELS } from '@/lib/models' import type { ImageModel } from '@/lib/models' @@ -292,11 +293,16 @@ function ImageModelCard({ // switch is picked here rather than being sent as a request field. // `options.signal` is the hook's abort signal — forwarding it lets an // unmount or a `stop()` cancel the request rather than orphan it. + byok, + byokProvider: () => toByokProvider(model.provider), fetcher: (input, options) => - generateImageFn({ - data: { prompt: input.prompt, model: model.id }, - signal: options?.signal, - }), + callWithByok( + generateImageFn({ + data: { prompt: input.prompt, model: model.id }, + signal: options?.signal, + headers: options?.headers, + }), + ), onResult: (generated) => { const image = generated.images[0] if (image) onImageGenerated?.(getImageSrc(image)) diff --git a/examples/ts-react-media/src/components/LiveVideoStudio.tsx b/examples/ts-react-media/src/components/LiveVideoStudio.tsx new file mode 100644 index 000000000..fc4aa164d --- /dev/null +++ b/examples/ts-react-media/src/components/LiveVideoStudio.tsx @@ -0,0 +1,564 @@ +import { useEffect, useRef, useState } from 'react' +import { Loader2, Square, TriangleAlert } from 'lucide-react' +import { generateLiveVideoFn } from '@/lib/server-functions' +import { attachStream } from '@/lib/attach-stream' +import { SeedImageField } from '@/components/SeedImageField' +import { + byok, + callWithByok, + falByok, + reactorByok, + requestByokFromError, +} from '@/lib/byok' +import { + FAL_LIVE_VIDEO_APP, + LIVE_VIDEO_MODEL_LABELS, + LIVE_VIDEO_MODELS, + LIVE_VIDEO_PROMPTS, + isFalLiveVideoModel, + isLiveVideoModelId, + liveVideoProvider, + liveVideoResolutions, +} from '@/lib/models' +import { + liveAcceptsSeedImage, + watchReactorFailure, +} from '@/lib/reactor-session' +import type { Reactor } from '@reactor-team/js-sdk' +import type { + LiveVideoModelId, + LiveVideoProvider, + LiveVideoResolution, +} from '@/lib/models' + +type SessionStatus = 'idle' | 'connecting' | 'live' | 'error' + +type FalLiveSession = { + send: (message: Record) => void + close: () => void | Promise +} + +type LiveHandle = + | { provider: 'reactor'; reactor: Reactor; model: LiveVideoModelId } + | { provider: 'fal'; session: FalLiveSession } + +async function startReactorLive( + reactor: Reactor, + model: LiveVideoModelId, + prompt: string, + resolution: LiveVideoResolution, + seedFile: File | null, +): Promise { + if (model === 'fast-h3') { + await reactor.sendCommand('enqueue', { prompt }) + return + } + if (model === 'longlive-v2') { + await reactor.sendCommand('set_shot', { prompt }) + await reactor.sendCommand('start', {}) + return + } + if (model === 'helios') { + await reactor.sendCommand('set_sr_scale', { + sr_scale: resolution === '2k' || resolution === '4k' ? '4x' : '2x', + }) + if (seedFile) { + const image = await reactor.uploadFile(seedFile) + await reactor.sendCommand('set_conditioning', { prompt, image }) + } else { + await reactor.sendCommand('set_prompt', { prompt }) + } + await reactor.sendCommand('start', {}) + return + } + if (resolution === '1080p' || resolution === '2k' || resolution === '4k') { + await reactor.sendCommand('set_resolution', { resolution }) + } + await reactor.sendCommand('set_prompt', { prompt }) + await reactor.sendCommand('start', {}) +} + +async function steerReactorLive( + reactor: Reactor, + model: LiveVideoModelId, + prompt: string, +): Promise { + if (model === 'fast-h3') { + await reactor.sendCommand('enqueue', { prompt }) + return + } + if (model === 'longlive-v2') { + await reactor.sendCommand('set_shot', { prompt }) + return + } + await reactor.sendCommand('set_prompt', { prompt }) +} + +function errorMessage(error: unknown): string { + if (!(error instanceof Error)) return String(error) + const cause = error.cause + if (cause instanceof Error && cause.message.length > 0) { + return `${error.message} ${cause.message}` + } + return error.message +} + +function readLivePayload(value: unknown): { + token: string + model: string + prompt: string + provider: LiveVideoProvider +} { + if (typeof value !== 'object' || value === null) { + throw new Error('Live video payload is incomplete') + } + const token = + 'token' in value && typeof value.token === 'string' ? value.token : '' + const model = + 'model' in value && typeof value.model === 'string' ? value.model : '' + const prompt = + 'prompt' in value && typeof value.prompt === 'string' ? value.prompt : '' + const provider = + 'provider' in value && + (value.provider === 'reactor' || value.provider === 'fal') + ? value.provider + : null + if ( + token.length === 0 || + model.length === 0 || + prompt.length === 0 || + provider === null + ) { + throw new Error('Live video payload is incomplete') + } + return { token, model, prompt, provider } +} + +function defaultResolution(provider: LiveVideoProvider): LiveVideoResolution { + return provider === 'fal' ? '768p' : '1080p' +} + +async function openFalSession(args: { + model: string + prompt: string + resolution: LiveVideoResolution + onMedia: (stream: MediaStream) => void + onError: (error: unknown) => void +}): Promise { + const { createFalClient } = await import('@fal-ai/client') + const { wma } = await import('@fal-ai/client/realtime') + const fal = createFalClient({ + proxyUrl: '/api/fal/proxy', + fetch: (input, init) => { + const headers = new Headers(init?.headers) + for (const [name, value] of Object.entries(byok.headers(falByok.id))) { + headers.set(name, value) + } + return fetch(input, { ...init, headers }) + }, + }) + const wmaApp = isFalLiveVideoModel(args.model) + ? FAL_LIVE_VIDEO_APP[args.model] + : args.model + const session = fal.realtime.open(wma(wmaApp), { + receive: ['video', 'audio'], + onMedia: args.onMedia, + onError: args.onError, + onData: (raw) => { + let parsed: unknown + try { + parsed = JSON.parse(String(raw)) + } catch { + return + } + if ( + typeof parsed === 'object' && + parsed !== null && + 'type' in parsed && + parsed.type === 'error' && + 'error' in parsed && + typeof parsed.error === 'string' + ) { + args.onError(new Error(parsed.error)) + } + }, + onState: (state) => { + if (state === 'failed') args.onError(new Error('Live session failed')) + }, + }) + session.send({ + type: 'configure', + prompt: args.prompt, + prompt_version: 1, + protocol_version: 1, + ...(args.resolution === '480p' || args.resolution === '768p' + ? { resolution: args.resolution } + : {}), + }) + await session.ready + return session +} + +export default function LiveVideoStudio() { + const [prompt, setPrompt] = useState(LIVE_VIDEO_PROMPTS[0] ?? '') + const [steerPrompt, setSteerPrompt] = useState('') + const [model, setModel] = useState('helios') + const [resolution, setResolution] = useState('1080p') + const [seedFile, setSeedFile] = useState(null) + const [status, setStatus] = useState('idle') + const [error, setError] = useState(null) + const [playing, setPlaying] = useState(false) + const videoRef = useRef(null) + const handleRef = useRef(null) + const detachStreamRef = useRef<(() => void) | null>(null) + const unwatchRef = useRef<(() => void) | null>(null) + const falPromptVersionRef = useRef(1) + const statusRef = useRef('idle') + statusRef.current = status + + const provider = liveVideoProvider(model) + const resolutions = liveVideoResolutions(provider) + const keyProvider = provider === 'fal' ? falByok : reactorByok + const showSeedField = liveAcceptsSeedImage(model) + + async function teardown() { + unwatchRef.current?.() + unwatchRef.current = null + const handle = handleRef.current + handleRef.current = null + detachStreamRef.current?.() + detachStreamRef.current = null + if (handle?.provider === 'reactor') { + try { + await handle.reactor.disconnect() + } catch { + // Disconnect can fail if the session already ended. + } + } else if (handle?.provider === 'fal') { + try { + handle.session.send({ type: 'stop' }) + await handle.session.close() + } catch { + // Close can fail if the session already ended. + } + } + const video = videoRef.current + if (video) { + video.srcObject = null + } + } + + async function stop() { + await teardown() + setPlaying(false) + setStatus('idle') + } + + useEffect(() => { + return () => { + void teardown() + } + }, []) + + async function start() { + setError(null) + setPlaying(false) + setStatus('connecting') + try { + await byok.prepare(keyProvider.id) + const live = readLivePayload( + await callWithByok( + generateLiveVideoFn({ + data: { prompt, model, resolution }, + headers: byok.headers(keyProvider.id), + }), + ), + ) + const video = videoRef.current + if (!video) throw new Error('Video element is missing') + + if (live.provider === 'fal') { + falPromptVersionRef.current = 1 + const session = await openFalSession({ + model: live.model, + prompt: live.prompt, + resolution, + onMedia: (stream) => { + detachStreamRef.current?.() + detachStreamRef.current = attachStream( + video, + stream, + (playError) => { + setError(errorMessage(playError)) + }, + ) + }, + onError: (openError) => { + setError(errorMessage(openError)) + setStatus('error') + }, + }) + handleRef.current = { provider: 'fal', session } + } else { + const { Reactor: ReactorClient } = await import('@reactor-team/js-sdk') + const reactor = new ReactorClient({ + modelName: live.model, + }) + handleRef.current = { provider: 'reactor', reactor, model } + + let rejectStart: ((error: Error) => void) | undefined + const startFailed = new Promise((_, reject) => { + rejectStart = reject + }) + unwatchRef.current = watchReactorFailure(reactor, (message) => { + setError(message) + rejectStart?.(new Error(message)) + if (statusRef.current === 'live') setStatus('error') + }) + reactor.on('trackReceived', (name, _track, stream) => { + if (name !== 'main_video') return + detachStreamRef.current?.() + detachStreamRef.current = attachStream(video, stream, (playError) => { + setError(errorMessage(playError)) + }) + }) + + await reactor.connect(live.token) + try { + await Promise.race([ + startReactorLive(reactor, model, live.prompt, resolution, seedFile), + startFailed, + ]) + } finally { + rejectStart = undefined + } + } + + setSteerPrompt('') + setStatus('live') + } catch (caught) { + await stop() + requestByokFromError(caught) + setError(errorMessage(caught)) + setStatus('error') + } + } + + async function steer() { + const handle = handleRef.current + const next = steerPrompt.trim() + if (!handle || next.length === 0) return + setError(null) + try { + if (handle.provider === 'fal') { + falPromptVersionRef.current += 1 + handle.session.send({ + type: 'prompt', + prompt: next, + prompt_version: falPromptVersionRef.current, + }) + } else { + await steerReactorLive(handle.reactor, handle.model, next) + } + setPrompt(next) + setSteerPrompt('') + } catch (caught) { + setError(errorMessage(caught)) + } + } + + const isLive = status === 'live' + const isBusy = status === 'connecting' + + return ( +
+

+ Live video. Paste a key in the header dialog, or set{' '} + + {provider === 'fal' ? 'FAL_KEY' : 'REACTOR_API_KEY'} + {' '} + on the server. Then start a session and type under the view to steer. +

+ +
+
+ +
{ + event.preventDefault() + if (isLive) { + void steer() + return + } + if (!isBusy && prompt.trim().length > 0) void start() + }} + > +