diff --git a/.changeset/forty-rabbits-melt.md b/.changeset/forty-rabbits-melt.md new file mode 100644 index 00000000000..480870e8f60 --- /dev/null +++ b/.changeset/forty-rabbits-melt.md @@ -0,0 +1,6 @@ +--- +'@mastra/clickhouse': patch +'@mastra/core': patch +--- + +Added clickhouse storage diff --git a/packages/core/src/storage/base.ts b/packages/core/src/storage/base.ts index f7ed41aced0..78fe499148e 100644 --- a/packages/core/src/storage/base.ts +++ b/packages/core/src/storage/base.ts @@ -2,7 +2,14 @@ import { MastraBase } from '../base'; import type { MessageType, StorageThreadType } from '../memory/types'; import type { WorkflowRunState } from '../workflows'; -import { TABLE_WORKFLOW_SNAPSHOT, TABLE_EVALS, TABLE_MESSAGES, TABLE_THREADS, TABLE_TRACES } from './constants'; +import { + TABLE_WORKFLOW_SNAPSHOT, + TABLE_EVALS, + TABLE_MESSAGES, + TABLE_THREADS, + TABLE_TRACES, + TABLE_SCHEMAS, +} from './constants'; import type { TABLE_NAMES } from './constants'; import type { EvalRow, StorageColumn, StorageGetMessagesArg } from './types'; @@ -158,104 +165,27 @@ export abstract class MastraStorage extends MastraBase { this.hasInitialized = Promise.all([ this.createTable({ tableName: TABLE_WORKFLOW_SNAPSHOT, - schema: { - workflow_name: { - type: 'text', - }, - run_id: { - type: 'text', - }, - snapshot: { - type: 'text', - }, - createdAt: { - type: 'timestamp', - }, - updatedAt: { - type: 'timestamp', - }, - }, + schema: TABLE_SCHEMAS[TABLE_WORKFLOW_SNAPSHOT], }), this.createTable({ tableName: TABLE_EVALS, - schema: { - input: { - type: 'text', - }, - output: { - type: 'text', - }, - result: { - type: 'jsonb', - }, - agent_name: { - type: 'text', - }, - metric_name: { - type: 'text', - }, - instructions: { - type: 'text', - }, - test_info: { - type: 'jsonb', - nullable: true, - }, - global_run_id: { - type: 'text', - }, - run_id: { - type: 'text', - }, - created_at: { - type: 'timestamp', - }, - }, + schema: TABLE_SCHEMAS[TABLE_EVALS], }), this.createTable({ tableName: TABLE_THREADS, - schema: { - id: { type: 'text', nullable: false, primaryKey: true }, - resourceId: { type: 'text', nullable: false }, - title: { type: 'text', nullable: false }, - metadata: { type: 'text', nullable: true }, - createdAt: { type: 'timestamp', nullable: false }, - updatedAt: { type: 'timestamp', nullable: false }, - }, + schema: TABLE_SCHEMAS[TABLE_THREADS], }), this.createTable({ tableName: TABLE_MESSAGES, - schema: { - id: { type: 'text', nullable: false, primaryKey: true }, - thread_id: { type: 'text', nullable: false }, - content: { type: 'text', nullable: false }, - role: { type: 'text', nullable: false }, - type: { type: 'text', nullable: false }, - createdAt: { type: 'timestamp', nullable: false }, - }, + schema: TABLE_SCHEMAS[TABLE_MESSAGES], }), this.createTable({ tableName: TABLE_TRACES, - schema: { - id: { type: 'text', nullable: false, primaryKey: true }, - parentSpanId: { type: 'text', nullable: true }, - name: { type: 'text', nullable: false }, - traceId: { type: 'text', nullable: false }, - scope: { type: 'text', nullable: false }, - kind: { type: 'integer', nullable: false }, - attributes: { type: 'jsonb', nullable: true }, - status: { type: 'jsonb', nullable: true }, - events: { type: 'jsonb', nullable: true }, - links: { type: 'jsonb', nullable: true }, - other: { type: 'text', nullable: true }, - startTime: { type: 'bigint', nullable: false }, - endTime: { type: 'bigint', nullable: false }, - createdAt: { type: 'timestamp', nullable: false }, - }, + schema: TABLE_SCHEMAS[TABLE_TRACES], }), ]).then(() => true); diff --git a/packages/core/src/storage/constants.ts b/packages/core/src/storage/constants.ts index a3abeeb3013..1d20cd46e06 100644 --- a/packages/core/src/storage/constants.ts +++ b/packages/core/src/storage/constants.ts @@ -1,3 +1,5 @@ +import type { StorageColumn } from './types'; + export const TABLE_WORKFLOW_SNAPSHOT = 'mastra_workflow_snapshot'; export const TABLE_EVALS = 'mastra_evals'; export const TABLE_MESSAGES = 'mastra_messages'; @@ -10,3 +12,96 @@ export type TABLE_NAMES = | typeof TABLE_MESSAGES | typeof TABLE_THREADS | typeof TABLE_TRACES; + +export const TABLE_SCHEMAS: Record> = { + [TABLE_WORKFLOW_SNAPSHOT]: { + workflow_name: { + type: 'text', + }, + run_id: { + type: 'text', + }, + snapshot: { + type: 'text', + }, + createdAt: { + type: 'timestamp', + }, + updatedAt: { + type: 'timestamp', + }, + }, + [TABLE_EVALS]: { + id: { + type: 'text', + nullable: false, + primaryKey: true, + }, + input: { + type: 'text', + }, + output: { + type: 'text', + }, + result: { + type: 'jsonb', + }, + agent_name: { + type: 'text', + }, + metric_name: { + type: 'text', + }, + instructions: { + type: 'text', + }, + test_info: { + type: 'jsonb', + nullable: true, + }, + global_run_id: { + type: 'text', + }, + run_id: { + type: 'text', + }, + created_at: { + type: 'timestamp', + }, + createdAt: { + type: 'timestamp', + }, + }, + [TABLE_THREADS]: { + id: { type: 'text', nullable: false, primaryKey: true }, + resourceId: { type: 'text', nullable: false }, + title: { type: 'text', nullable: false }, + metadata: { type: 'text', nullable: true }, + createdAt: { type: 'timestamp', nullable: false }, + updatedAt: { type: 'timestamp', nullable: false }, + }, + [TABLE_MESSAGES]: { + id: { type: 'text', nullable: false, primaryKey: true }, + thread_id: { type: 'text', nullable: false }, + content: { type: 'text', nullable: false }, + role: { type: 'text', nullable: false }, + type: { type: 'text', nullable: false }, + createdAt: { type: 'timestamp', nullable: false }, + }, + [TABLE_TRACES]: { + id: { type: 'text', nullable: false, primaryKey: true }, + parentSpanId: { type: 'text', nullable: true }, + name: { type: 'text', nullable: false }, + traceId: { type: 'text', nullable: false }, + scope: { type: 'text', nullable: false }, + kind: { type: 'integer', nullable: false }, + attributes: { type: 'jsonb', nullable: true }, + status: { type: 'jsonb', nullable: true }, + events: { type: 'jsonb', nullable: true }, + links: { type: 'jsonb', nullable: true }, + other: { type: 'text', nullable: true }, + startTime: { type: 'bigint', nullable: false }, + endTime: { type: 'bigint', nullable: false }, + createdAt: { type: 'timestamp', nullable: false }, + }, +}; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ea90c5f8036..a0e2507f220 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -3926,6 +3926,37 @@ importers: specifier: ^3.0.9 version: 3.0.9(@edge-runtime/vm@3.2.0)(@types/debug@4.1.12)(@types/node@20.17.27)(jiti@2.4.2)(jsdom@26.0.0(bufferutil@4.0.9)(utf-8-validate@6.0.5))(lightningcss@1.29.2)(terser@5.39.0)(tsx@4.19.3)(yaml@2.7.0) + stores/clickhouse: + dependencies: + '@clickhouse/client': + specifier: ^1.11.0 + version: 1.11.0 + '@mastra/core': + specifier: workspace:^ + version: link:../../packages/core + devDependencies: + '@internal/lint': + specifier: workspace:* + version: link:../../packages/_config + '@microsoft/api-extractor': + specifier: ^7.52.1 + version: 7.52.2(@types/node@20.17.27) + '@types/node': + specifier: ^20.17.27 + version: 20.17.27 + eslint: + specifier: ^9.23.0 + version: 9.23.0(jiti@2.4.2) + tsup: + specifier: ^8.4.0 + version: 8.4.0(@microsoft/api-extractor@7.52.2(@types/node@20.17.27))(@swc/core@1.11.10(@swc/helpers@0.5.15))(jiti@2.4.2)(postcss@8.5.3)(tsx@4.19.3)(typescript@5.8.2)(yaml@2.7.0) + typescript: + specifier: ^5.7.3 + version: 5.8.2 + vitest: + specifier: ^3.0.9 + version: 3.0.9(@edge-runtime/vm@3.2.0)(@types/debug@4.1.12)(@types/node@20.17.27)(jiti@2.4.2)(jsdom@26.0.0(bufferutil@4.0.9)(utf-8-validate@6.0.5))(lightningcss@1.29.2)(terser@5.39.0)(tsx@4.19.3)(yaml@2.7.0) + stores/pg: dependencies: '@mastra/core': @@ -5719,6 +5750,13 @@ packages: '@clack/prompts@0.8.2': resolution: {integrity: sha512-6b9Ab2UiZwJYA9iMyboYyW9yJvAO9V753ZhS+DHKEjZRKAxPPOb7MXXu84lsPFG+vZt6FRFniZ8rXi+zCIw4yQ==} + '@clickhouse/client-common@1.11.0': + resolution: {integrity: sha512-O0xbwv7HiMXayokrf5dYIBpjBnYekcOXWz60T1cXLmiZ8vgrfNRCiOpybJkrMXKnw9D0mWCgPUu/rgMY7U1f4g==} + + '@clickhouse/client@1.11.0': + resolution: {integrity: sha512-VYTQfR0y/BtrIDEjuSce1zv85OvHak5sUhZVyNYJzbAgWHy3jFf8Os7FdUSeqyKav0xGGy+2X+dRanTFjI5Oug==} + engines: {node: '>=16'} + '@cloudflare/kv-asset-handler@0.4.0': resolution: {integrity: sha512-+tv3z+SPp+gqTIcImN9o0hqE9xyfQjI1XD9pL6NuKjua9B1y7mNYv0S9cP+QEbA4ppVgGZEmKOvHX5G5Ei1CVA==} engines: {node: '>=18.0.0'} @@ -22412,6 +22450,12 @@ snapshots: picocolors: 1.1.1 sisteransi: 1.0.5 + '@clickhouse/client-common@1.11.0': {} + + '@clickhouse/client@1.11.0': + dependencies: + '@clickhouse/client-common': 1.11.0 + '@cloudflare/kv-asset-handler@0.4.0': dependencies: mime: 3.0.0 diff --git a/stores/clickhouse/.gitignore b/stores/clickhouse/.gitignore new file mode 100644 index 00000000000..60569e82bd6 --- /dev/null +++ b/stores/clickhouse/.gitignore @@ -0,0 +1,3 @@ +.env +dist/ +node_modules/ diff --git a/stores/clickhouse/CHANGELOG.md b/stores/clickhouse/CHANGELOG.md new file mode 100644 index 00000000000..e69de29bb2d diff --git a/stores/clickhouse/README.md b/stores/clickhouse/README.md new file mode 100644 index 00000000000..9cd90850227 --- /dev/null +++ b/stores/clickhouse/README.md @@ -0,0 +1,122 @@ +# @mastra/clickhouse + +Clickhouse implementation for Mastra, providing efficient storage capabilities with support for threads, messages, and workflow snapshots. + +## Installation + +```bash +npm install @mastra/clickhouse +``` + +## Prerequisites + +- Clickhouse server (version 21.8 or higher recommended) +- Node.js 16 or higher + +## Usage + +```typescript +import { ClickhouseStore } from '@mastra/clickhouse'; + +const store = new ClickhouseStore({ + url: 'http://localhost:8123', + username: 'default', + password: 'password', +}); + +// Create a thread +await store.saveThread({ + id: 'thread-123', + resourceId: 'resource-456', + title: 'My Thread', + metadata: { key: 'value' }, + createdAt: new Date(), + updatedAt: new Date(), +}); + +// Add messages to thread +await store.saveMessages([ + { + id: 'msg-789', + threadId: 'thread-123', + role: 'user', + type: 'text', + content: [{ type: 'text', text: 'Hello' }], + createdAt: new Date(), + }, +]); + +// Query threads and messages +const savedThread = await store.getThreadById({ threadId: 'thread-123' }); +const messages = await store.getMessages({ threadId: 'thread-123' }); + +// Clean up +await store.close(); +``` + +## Configuration + +The Clickhouse store can be initialized with the following configuration: + +```typescript +type ClickhouseConfig = { + url: string; // Clickhouse HTTP interface URL + username: string; // Database username + password: string; // Database password +}; +``` + +## Features + +### Storage Features + +- Thread and message storage with JSON support +- Efficient batch operations +- Rich metadata support +- Timestamp tracking +- Workflow snapshot persistence +- Optimized for high-volume data ingestion +- Uses Clickhouse's MergeTree and ReplacingMergeTree engines for optimal performance + +### Table Engines + +The store uses different table engines for different types of data: + +- `MergeTree()`: Used for messages, traces, and evals +- `ReplacingMergeTree()`: Used for threads and workflow snapshots + +## Storage Methods + +### Thread Operations + +- `saveThread(thread)`: Create or update a thread +- `getThreadById({ threadId })`: Get a thread by ID +- `updateThread({ id, title, metadata })`: Update thread title and metadata +- `deleteThread({ threadId })`: Delete a thread and its messages + +### Message Operations + +- `saveMessages(messages)`: Save multiple messages +- `getMessages({ threadId, selectBy? })`: Get messages for a thread with optional filtering +- `deleteMessages(messageIds)`: Delete specific messages + +### Workflow Operations + +- `persistWorkflowSnapshot({ workflowName, runId, snapshot })`: Save workflow state +- `loadWorkflowSnapshot({ workflowName, runId })`: Load workflow state + +## Data Types + +The store supports the following data types: + +- `text`: String +- `timestamp`: DateTime64(3) +- `uuid`: String +- `jsonb`: String (JSON serialized) +- `integer`: Int64 +- `bigint`: Int64 + +## Related Links + +- [Clickhouse Documentation](https://clickhouse.com/docs) +- [Clickhouse Node.js Client](https://github.com/clickhouse/clickhouse-js) diff --git a/stores/clickhouse/docker-compose.yaml b/stores/clickhouse/docker-compose.yaml new file mode 100644 index 00000000000..6a3269cefa0 --- /dev/null +++ b/stores/clickhouse/docker-compose.yaml @@ -0,0 +1,15 @@ +services: + db: + image: clickhouse/clickhouse-server:latest + container_name: 'clickhouse-test-db' + ports: + - '8123:8123' + - '9000:9000' + environment: + CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1 + CLICKHOUSE_USER: ${CLICKHOUSE_USER:-default} + CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-password} + volumes: + - clickhouse_data:/var/lib/clickhouse +volumes: + clickhouse_data: diff --git a/stores/clickhouse/eslint.config.js b/stores/clickhouse/eslint.config.js new file mode 100644 index 00000000000..3435bb3d51b --- /dev/null +++ b/stores/clickhouse/eslint.config.js @@ -0,0 +1,6 @@ +import { createConfig } from '@internal/lint/eslint'; + +const config = await createConfig(); + +/** @type {import("eslint").Linter.Config[]} */ +export default [...config.map(conf => ({ ...conf, ignores: [...(conf.ignores || []), '**/vitest.perf.config.ts'] }))]; diff --git a/stores/clickhouse/package.json b/stores/clickhouse/package.json new file mode 100644 index 00000000000..bb395ec4594 --- /dev/null +++ b/stores/clickhouse/package.json @@ -0,0 +1,45 @@ +{ + "name": "@mastra/clickhouse", + "version": "0.2.7-alpha.0", + "description": "Clickhouse provider for Mastra - includes db storage capabilities", + "type": "module", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "exports": { + ".": { + "import": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + }, + "require": { + "types": "./dist/index.d.cts", + "default": "./dist/index.cjs" + } + }, + "./package.json": "./package.json" + }, + "scripts": { + "build": "tsup src/index.ts --format esm,cjs --experimental-dts --clean --treeshake=smallest --splitting", + "build:watch": "pnpm build --watch", + "pretest": "docker compose up -d && (for i in $(seq 1 30); do docker compose exec -T db clickhouse-client --query 'SELECT 1' && break || (sleep 1; [ $i -eq 30 ] && exit 1); done)", + "test": "vitest run", + "posttest": "docker compose down -v", + "pretest:watch": "docker compose up -d", + "test:watch": "vitest watch", + "posttest:watch": "docker compose down -v", + "lint": "eslint ." + }, + "dependencies": { + "@mastra/core": "workspace:^", + "@clickhouse/client": "^1.11.0" + }, + "devDependencies": { + "@internal/lint": "workspace:*", + "@microsoft/api-extractor": "^7.52.1", + "@types/node": "^20.17.27", + "eslint": "^9.23.0", + "tsup": "^8.4.0", + "typescript": "^5.8.2", + "vitest": "^3.0.9" + } +} diff --git a/stores/clickhouse/src/index.ts b/stores/clickhouse/src/index.ts new file mode 100644 index 00000000000..85674ee7cdb --- /dev/null +++ b/stores/clickhouse/src/index.ts @@ -0,0 +1 @@ +export * from './storage'; diff --git a/stores/clickhouse/src/storage/index.test.ts b/stores/clickhouse/src/storage/index.test.ts new file mode 100644 index 00000000000..c68670f0877 --- /dev/null +++ b/stores/clickhouse/src/storage/index.test.ts @@ -0,0 +1,391 @@ +import { randomUUID } from 'crypto'; +import type { WorkflowRunState } from '@mastra/core/workflows'; +import { describe, it, expect, beforeAll, beforeEach, afterAll } from 'vitest'; + +import { ClickhouseStore } from '.'; +import type { ClickhouseConfig } from '.'; + +const TEST_CONFIG: ClickhouseConfig = { + url: process.env.CLICKHOUSE_URL || 'http://localhost:8123', + username: process.env.CLICKHOUSE_USERNAME || 'default', + password: process.env.CLICKHOUSE_PASSWORD || 'password', +}; + +// Sample test data factory functions +const createSampleThread = () => ({ + id: `thread-${randomUUID()}`, + resourceId: `resource-${randomUUID()}`, + title: 'Test Thread', + createdAt: new Date(), + updatedAt: new Date(), + metadata: { key: 'value' }, +}); + +const createSampleMessage = (threadId: string, createdAt: Date = new Date()) => + ({ + id: `msg-${randomUUID()}`, + role: 'user', + type: 'text', + threadId, + content: [{ type: 'text', text: 'Hello' }], + createdAt, + }) as any; + +describe('ClickhouseStore', () => { + let store: ClickhouseStore; + + beforeAll(async () => { + store = new ClickhouseStore(TEST_CONFIG); + await store.init(); + }); + + beforeEach(async () => { + // Clear tables before each test + await store.clearTable({ tableName: 'mastra_workflow_snapshot' }); + await store.clearTable({ tableName: 'mastra_messages' }); + await store.clearTable({ tableName: 'mastra_threads' }); + }); + + describe('Thread Operations', () => { + it('should create and retrieve a thread', async () => { + const thread = createSampleThread(); + console.log('Saving thread:', thread); + + // Save thread + const savedThread = await store.__saveThread({ thread }); + expect(savedThread).toEqual(thread); + + // Retrieve thread + const retrievedThread = await store.__getThreadById({ threadId: thread.id }); + expect(retrievedThread?.title).toEqual(thread.title); + }, 10e3); + + it('should return null for non-existent thread', async () => { + const result = await store.__getThreadById({ threadId: 'non-existent' }); + expect(result).toBeNull(); + }, 10e3); + + it('should get threads by resource ID', async () => { + const thread1 = createSampleThread(); + const thread2 = { ...createSampleThread(), resourceId: thread1.resourceId }; + + await store.__saveThread({ thread: thread1 }); + await store.__saveThread({ thread: thread2 }); + + const threads = await store.__getThreadsByResourceId({ resourceId: thread1.resourceId }); + expect(threads).toHaveLength(2); + expect(threads.map(t => t.id)).toEqual(expect.arrayContaining([thread1.id, thread2.id])); + }, 10e3); + + it('should update thread title and metadata', async () => { + const thread = createSampleThread(); + await store.__saveThread({ thread }); + + const newMetadata = { newKey: 'newValue' }; + const updatedThread = await store.__updateThread({ + id: thread.id, + title: 'Updated Title', + metadata: newMetadata, + }); + + expect(updatedThread.title).toBe('Updated Title'); + expect(updatedThread.metadata).toEqual({ + ...thread.metadata, + ...newMetadata, + }); + + // Verify persistence + const retrievedThread = await store.__getThreadById({ threadId: thread.id }); + expect(retrievedThread).toEqual(updatedThread); + }, 10e3); + + it('should delete thread and its messages', async () => { + const thread = createSampleThread(); + await store.__saveThread({ thread }); + + // Add some messages + const messages = [createSampleMessage(thread.id), createSampleMessage(thread.id)]; + await store.__saveMessages({ messages }); + + await store.__deleteThread({ threadId: thread.id }); + + const retrievedThread = await store.__getThreadById({ threadId: thread.id }); + expect(retrievedThread).toBeNull(); + + // Verify messages were also deleted + const retrievedMessages = await store.__getMessages({ threadId: thread.id }); + expect(retrievedMessages).toHaveLength(0); + }, 10e3); + }); + + describe('Message Operations', () => { + it('should save and retrieve messages', async () => { + const thread = createSampleThread(); + await store.__saveThread({ thread }); + + const messages = [ + createSampleMessage(thread.id, new Date(Date.now() - 1000 * 60 * 60 * 24)), + createSampleMessage(thread.id), + ]; + + // Save messages + const savedMessages = await store.__saveMessages({ messages }); + expect(savedMessages).toEqual(messages); + + // Retrieve messages + const retrievedMessages = await store.__getMessages({ threadId: thread.id }); + expect(retrievedMessages).toHaveLength(2); + console.log('Messages:', messages); + console.log('Retrieved messages:', retrievedMessages); + expect(retrievedMessages).toEqual(expect.arrayContaining(messages)); + }, 10e3); + + it('should handle empty message array', async () => { + const result = await store.__saveMessages({ messages: [] }); + expect(result).toEqual([]); + }, 10e3); + + it('should maintain message order', async () => { + const thread = createSampleThread(); + await store.__saveThread({ thread }); + + const messages = [ + { + ...createSampleMessage(thread.id, new Date(Date.now() - 1000 * 3)), + content: [{ type: 'text', text: 'First' }], + }, + { + ...createSampleMessage(thread.id, new Date(Date.now() - 1000 * 2)), + content: [{ type: 'text', text: 'Second' }], + }, + { + ...createSampleMessage(thread.id, new Date(Date.now() - 1000 * 1)), + content: [{ type: 'text', text: 'Third' }], + }, + ]; + + await store.__saveMessages({ messages }); + + const retrievedMessages = await store.__getMessages({ threadId: thread.id }); + expect(retrievedMessages).toHaveLength(3); + + // Verify order is maintained + retrievedMessages.forEach((msg, idx) => { + expect(msg.content[0].text).toBe(messages[idx].content[0].text); + }); + }, 10e3); + + // it('should rollback on error during message save', async () => { + // const thread = createSampleThread(); + // await store.__saveThread({ thread }); + + // const messages = [ + // createSampleMessage(thread.id), + // { ...createSampleMessage(thread.id), id: null }, // This will cause an error + // ]; + + // await expect(store.__saveMessages({ messages })).rejects.toThrow(); + + // // Verify no messages were saved + // const savedMessages = await store.__getMessages({ threadId: thread.id }); + // expect(savedMessages).toHaveLength(0); + // }); + }); + + describe('Edge Cases and Error Handling', () => { + it('should handle large metadata objects', async () => { + const thread = createSampleThread(); + const largeMetadata = { + ...thread.metadata, + largeArray: Array.from({ length: 1000 }, (_, i) => ({ index: i, data: 'test'.repeat(100) })), + }; + + const threadWithLargeMetadata = { + ...thread, + metadata: largeMetadata, + }; + + await store.__saveThread({ thread: threadWithLargeMetadata }); + const retrieved = await store.__getThreadById({ threadId: thread.id }); + + expect(retrieved?.metadata).toEqual(largeMetadata); + }, 10e3); + + it('should handle special characters in thread titles', async () => { + const thread = { + ...createSampleThread(), + title: 'Special \'quotes\' and "double quotes" and emoji 🎉', + }; + + await store.__saveThread({ thread }); + const retrieved = await store.__getThreadById({ threadId: thread.id }); + + expect(retrieved?.title).toBe(thread.title); + }, 10e3); + + it('should handle concurrent thread updates', async () => { + const thread = createSampleThread(); + await store.__saveThread({ thread }); + + // Perform multiple updates concurrently + const updates = Array.from({ length: 5 }, (_, i) => + store.__updateThread({ + id: thread.id, + title: `Update ${i}`, + metadata: { update: i }, + }), + ); + + await expect(Promise.all(updates)).resolves.toBeDefined(); + + // Verify final state + const finalThread = await store.__getThreadById({ threadId: thread.id }); + expect(finalThread).toBeDefined(); + }, 10e3); + }); + + describe('Workflow Snapshots', () => { + it('should persist and load workflow snapshots', async () => { + const workflowName = 'test-workflow'; + const runId = `run-${randomUUID()}`; + const snapshot = { + status: 'running', + context: { + stepResults: {}, + attempts: {}, + triggerData: { type: 'manual' }, + }, + } as any; + + await store.persistWorkflowSnapshot({ + workflowName, + runId, + snapshot, + }); + + const loadedSnapshot = await store.loadWorkflowSnapshot({ + workflowName, + runId, + }); + + expect(loadedSnapshot).toEqual(snapshot); + }, 10e3); + + it('should return null for non-existent workflow snapshot', async () => { + const result = await store.loadWorkflowSnapshot({ + workflowName: 'non-existent', + runId: 'non-existent', + }); + + expect(result).toBeNull(); + }, 10e3); + + it('should update existing workflow snapshot', async () => { + const workflowName = 'test-workflow'; + const runId = `run-${randomUUID()}`; + const initialSnapshot = { + status: 'running', + context: { + stepResults: {}, + attempts: {}, + triggerData: { type: 'manual' }, + }, + }; + + await store.persistWorkflowSnapshot({ + workflowName, + runId, + snapshot: initialSnapshot as any, + }); + + const updatedSnapshot = { + status: 'completed', + context: { + stepResults: { + 'step-1': { status: 'success', result: { data: 'test' } }, + }, + attempts: { 'step-1': 1 }, + triggerData: { type: 'manual' }, + }, + } as any; + + await store.persistWorkflowSnapshot({ + workflowName, + runId, + snapshot: updatedSnapshot, + }); + + const loadedSnapshot = await store.loadWorkflowSnapshot({ + workflowName, + runId, + }); + + expect(loadedSnapshot).toEqual(updatedSnapshot); + }, 10e3); + + it('should handle complex workflow state', async () => { + const workflowName = 'complex-workflow'; + const runId = `run-${randomUUID()}`; + const complexSnapshot = { + value: { currentState: 'running' }, + context: { + stepResults: { + 'step-1': { + status: 'success', + result: { + nestedData: { + array: [1, 2, 3], + object: { key: 'value' }, + date: new Date().toISOString(), + }, + }, + }, + 'step-2': { + status: 'waiting', + dependencies: ['step-3', 'step-4'], + }, + }, + attempts: { 'step-1': 1, 'step-2': 0 }, + triggerData: { + type: 'scheduled', + metadata: { + schedule: '0 0 * * *', + timezone: 'UTC', + }, + }, + }, + activePaths: [ + { + stepPath: ['step-1'], + stepId: 'step-1', + status: 'success', + }, + { + stepPath: ['step-2'], + stepId: 'step-2', + status: 'waiting', + }, + ], + runId: runId, + timestamp: Date.now(), + }; + + await store.persistWorkflowSnapshot({ + workflowName, + runId, + snapshot: complexSnapshot as WorkflowRunState, + }); + + const loadedSnapshot = await store.loadWorkflowSnapshot({ + workflowName, + runId, + }); + + expect(loadedSnapshot).toEqual(complexSnapshot); + }, 10e3); + }); + + afterAll(async () => { + await store.close(); + }); +}); diff --git a/stores/clickhouse/src/storage/index.ts b/stores/clickhouse/src/storage/index.ts new file mode 100644 index 00000000000..c8c6bdbf377 --- /dev/null +++ b/stores/clickhouse/src/storage/index.ts @@ -0,0 +1,714 @@ +import type { MessageType, StorageThreadType } from '@mastra/core/memory'; +import { + MastraStorage, + TABLE_EVALS, + TABLE_MESSAGES, + TABLE_SCHEMAS, + TABLE_THREADS, + TABLE_TRACES, + TABLE_WORKFLOW_SNAPSHOT, +} from '@mastra/core/storage'; +import type { EvalRow, StorageColumn, StorageGetMessagesArg, TABLE_NAMES } from '@mastra/core/storage'; +import type { WorkflowRunState } from '@mastra/core/workflows'; +import { createClient, ClickHouseClient } from '@clickhouse/client'; + +export type ClickhouseConfig = { + url: string; + username: string; + password: string; +}; + +const TABLE_ENGINES: Record = { + [TABLE_MESSAGES]: `MergeTree()`, + [TABLE_WORKFLOW_SNAPSHOT]: `ReplacingMergeTree()`, + [TABLE_TRACES]: `MergeTree()`, + [TABLE_THREADS]: `ReplacingMergeTree()`, + [TABLE_EVALS]: `MergeTree()`, +}; + +const COLUMN_TYPES: Record = { + text: 'String', + timestamp: 'DateTime64(3)', + uuid: 'String', + jsonb: 'String', + integer: 'Int64', + bigint: 'Int64', +}; + +function transformRows(rows: any[]): R[] { + return rows.map((row: any) => transformRow(row)); +} + +function transformRow(row: any): R { + if (!row) { + return row; + } + + if (row.createdAt) { + row.createdAt = new Date(row.createdAt); + } + if (row.updatedAt) { + row.updatedAt = new Date(row.updatedAt); + } + return row; +} + +export class ClickhouseStore extends MastraStorage { + private db: ClickHouseClient; + + constructor(config: ClickhouseConfig) { + super({ name: 'ClickhouseStore' }); + this.db = createClient({ + url: config.url, + username: config.username, + password: config.password, + clickhouse_settings: { + date_time_input_format: 'best_effort', + date_time_output_format: 'iso', // This is crucial + use_client_time_zone: 1, + }, + }); + } + + getEvalsByAgentName(_agentName: string, _type?: 'test' | 'live'): Promise { + throw new Error('Method not implemented.'); + } + + async batchInsert({ tableName, records }: { tableName: TABLE_NAMES; records: Record[] }): Promise { + try { + await this.db.insert({ + table: tableName, + values: records.map(record => ({ + ...record, + createdAt: record.createdAt.toISOString(), + updatedAt: record.updatedAt.toISOString(), + })), + format: 'JSONEachRow', + clickhouse_settings: { + // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z') + date_time_input_format: 'best_effort', + use_client_time_zone: 1, + }, + }); + } catch (error) { + console.error(`Error inserting into ${tableName}:`, error); + throw error; + } + } + + async getTraces({ + name, + scope, + page, + perPage, + attributes, + }: { + name?: string; + scope?: string; + page: number; + perPage: number; + attributes?: Record; + }): Promise { + let idx = 1; + const limit = perPage; + const offset = page * perPage; + + const args: Record = {}; + + const conditions: string[] = []; + if (name) { + conditions.push(`name LIKE CONCAT(\$${idx++}, '%')`); + } + if (scope) { + conditions.push(`scope = \$${idx++}`); + } + if (attributes) { + Object.keys(attributes).forEach(key => { + conditions.push(`attributes->>'${key}' = \$${idx++}`); + }); + } + + const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : ''; + + if (name) { + args[name] = 'String'; + } + + if (scope) { + args[scope] = 'String'; + } + + if (attributes) { + for (const [_key, value] of Object.entries(attributes)) { + args[value] = 'String'; + } + } + + const result = await this.db.query({ + query: `SELECT toDateTime64(createdAt, 3) as createdAt, toDateTime64(updatedAt, 3) as updatedAt FROM ${TABLE_TRACES} ${whereClause} ORDER BY "createdAt" DESC LIMIT ${limit} OFFSET ${offset}`, + query_params: args, + clickhouse_settings: { + // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z') + date_time_input_format: 'best_effort', + date_time_output_format: 'iso', + use_client_time_zone: 1, + }, + }); + + if (!result) { + return []; + } + + const rows = await result.json(); + return transformRows(rows.data); + } + + async createTable({ + tableName, + schema, + }: { + tableName: TABLE_NAMES; + schema: Record; + }): Promise { + try { + const columns = Object.entries(schema) + .map(([name, def]) => { + const constraints = []; + if (!def.nullable) constraints.push('NOT NULL'); + return `"${name}" ${COLUMN_TYPES[def.type]} ${constraints.join(' ')}`; + }) + .join(',\n'); + + const sql = + tableName === TABLE_WORKFLOW_SNAPSHOT + ? ` + CREATE TABLE IF NOT EXISTS ${tableName} ( + ${['id String'].concat(columns)} + ) + ENGINE = ${TABLE_ENGINES[tableName]} + PARTITION BY "createdAt" + PRIMARY KEY (createdAt, run_id, workflow_name) + ORDER BY (createdAt, run_id, workflow_name) + SETTINGS index_granularity = 8192; + ` + : ` + CREATE TABLE IF NOT EXISTS ${tableName} ( + ${columns} + ) + ENGINE = ${TABLE_ENGINES[tableName]} + PARTITION BY "createdAt" + PRIMARY KEY (createdAt, id) + ORDER BY (createdAt, id) + SETTINGS index_granularity = 8192; + `; + + await this.db.query({ + query: sql, + clickhouse_settings: { + // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z') + date_time_input_format: 'best_effort', + date_time_output_format: 'iso', + use_client_time_zone: 1, + }, + }); + } catch (error) { + console.error(`Error creating table ${tableName}:`, error); + throw error; + } + } + + async clearTable({ tableName }: { tableName: TABLE_NAMES }): Promise { + try { + await this.db.query({ + query: `TRUNCATE TABLE ${tableName}`, + clickhouse_settings: { + // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z') + date_time_input_format: 'best_effort', + date_time_output_format: 'iso', + use_client_time_zone: 1, + }, + }); + } catch (error) { + console.error(`Error clearing table ${tableName}:`, error); + throw error; + } + } + + async insert({ tableName, record }: { tableName: TABLE_NAMES; record: Record }): Promise { + try { + await this.db.insert({ + table: tableName, + values: [ + { + ...record, + createdAt: record.createdAt.toISOString(), + updatedAt: record.updatedAt.toISOString(), + }, + ], + format: 'JSONEachRow', + clickhouse_settings: { + // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z') + date_time_input_format: 'best_effort', + use_client_time_zone: 1, + }, + }); + } catch (error) { + console.error(`Error inserting into ${tableName}:`, error); + throw error; + } + } + + async load({ tableName, keys }: { tableName: TABLE_NAMES; keys: Record }): Promise { + try { + const keyEntries = Object.entries(keys); + const conditions = keyEntries + .map( + ([key], index) => + `"${key}" = {var_${key}:${COLUMN_TYPES[TABLE_SCHEMAS[tableName as TABLE_NAMES]?.[key]?.type ?? 'text']}}`, + ) + .join(' AND '); + const values = keyEntries.reduce((acc, [key, value]) => { + return { ...acc, [`var_${key}`]: value }; + }, {}); + + const result = await this.db.query({ + query: `SELECT *, toDateTime64(createdAt, 3) as createdAt, toDateTime64(updatedAt, 3) as updatedAt FROM ${tableName} ${TABLE_ENGINES[tableName as TABLE_NAMES].startsWith('ReplacingMergeTree') ? 'FINAL' : ''} WHERE ${conditions}`, + query_params: values, + clickhouse_settings: { + // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z') + date_time_input_format: 'best_effort', + date_time_output_format: 'iso', + use_client_time_zone: 1, + }, + }); + + if (!result) { + return null; + } + + const rows = await result.json(); + // If this is a workflow snapshot, parse the snapshot field + if (tableName === TABLE_WORKFLOW_SNAPSHOT) { + const snapshot = rows.data[0] as any; + if (!snapshot) { + return null; + } + if (typeof snapshot.snapshot === 'string') { + snapshot.snapshot = JSON.parse(snapshot.snapshot); + } + return transformRow(snapshot); + } + + const data: R = transformRow(rows.data[0]); + return data; + } catch (error) { + console.error(`Error loading from ${tableName}:`, error); + throw error; + } + } + + async getThreadById({ threadId }: { threadId: string }): Promise { + try { + const result = await this.db.query({ + query: `SELECT + id, + "resourceId", + title, + metadata, + toDateTime64(createdAt, 3) as createdAt, + toDateTime64(updatedAt, 3) as updatedAt + FROM "${TABLE_THREADS}" + FINAL + WHERE id = {var_id:String}`, + query_params: { var_id: threadId }, + clickhouse_settings: { + // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z') + date_time_input_format: 'best_effort', + date_time_output_format: 'iso', + use_client_time_zone: 1, + }, + }); + + const rows = await result.json(); + const thread = transformRow(rows.data[0]) as StorageThreadType; + + if (!thread) { + return null; + } + + return { + ...thread, + metadata: typeof thread.metadata === 'string' ? JSON.parse(thread.metadata) : thread.metadata, + createdAt: thread.createdAt, + updatedAt: thread.updatedAt, + }; + } catch (error) { + console.error(`Error getting thread ${threadId}:`, error); + throw error; + } + } + + async getThreadsByResourceId({ resourceId }: { resourceId: string }): Promise { + try { + const result = await this.db.query({ + query: `SELECT + id, + "resourceId", + title, + metadata, + toDateTime64(createdAt, 3) as createdAt, + toDateTime64(updatedAt, 3) as updatedAt + FROM "${TABLE_THREADS}" + WHERE "resourceId" = {var_resourceId:String}`, + query_params: { var_resourceId: resourceId }, + clickhouse_settings: { + // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z') + date_time_input_format: 'best_effort', + date_time_output_format: 'iso', + use_client_time_zone: 1, + }, + }); + + const rows = await result.json(); + const threads = transformRows(rows.data) as StorageThreadType[]; + + return threads.map((thread: StorageThreadType) => ({ + ...thread, + metadata: typeof thread.metadata === 'string' ? JSON.parse(thread.metadata) : thread.metadata, + createdAt: thread.createdAt, + updatedAt: thread.updatedAt, + })); + } catch (error) { + console.error(`Error getting threads for resource ${resourceId}:`, error); + throw error; + } + } + + async saveThread({ thread }: { thread: StorageThreadType }): Promise { + try { + await this.db.insert({ + table: TABLE_THREADS, + values: [ + { + ...thread, + createdAt: thread.createdAt.toISOString(), + updatedAt: thread.updatedAt.toISOString(), + }, + ], + format: 'JSONEachRow', + clickhouse_settings: { + // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z') + date_time_input_format: 'best_effort', + use_client_time_zone: 1, + }, + }); + + return thread; + } catch (error) { + console.error('Error saving thread:', error); + throw error; + } + } + + async updateThread({ + id, + title, + metadata, + }: { + id: string; + title: string; + metadata: Record; + }): Promise { + try { + // First get the existing thread to merge metadata + const existingThread = await this.getThreadById({ threadId: id }); + if (!existingThread) { + throw new Error(`Thread ${id} not found`); + } + + // Merge the existing metadata with the new metadata + const mergedMetadata = { + ...existingThread.metadata, + ...metadata, + }; + + const updatedThread = { + ...existingThread, + title, + metadata: mergedMetadata, + updatedAt: new Date(), + }; + + await this.db.insert({ + table: TABLE_THREADS, + values: [ + { + ...updatedThread, + updatedAt: updatedThread.updatedAt.toISOString(), + }, + ], + format: 'JSONEachRow', + clickhouse_settings: { + // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z') + date_time_input_format: 'best_effort', + use_client_time_zone: 1, + }, + }); + + return updatedThread; + } catch (error) { + console.error('Error updating thread:', error); + throw error; + } + } + + async deleteThread({ threadId }: { threadId: string }): Promise { + try { + // First delete all messages associated with this thread + await this.db.command({ + query: `DELETE FROM "${TABLE_MESSAGES}" WHERE thread_id = '${threadId}';`, + query_params: { var_thread_id: threadId }, + clickhouse_settings: {}, + }); + + // Then delete the thread + await this.db.command({ + query: `DELETE FROM "${TABLE_THREADS}" WHERE id = {var_id:String};`, + query_params: { var_id: threadId }, + clickhouse_settings: {}, + }); + } catch (error) { + console.error('Error deleting thread:', error); + throw error; + } + } + + async getMessages({ threadId, selectBy }: StorageGetMessagesArg): Promise { + try { + const messages: any[] = []; + const limit = typeof selectBy?.last === `number` ? selectBy.last : 40; + const include = selectBy?.include || []; + + if (include.length) { + const includeResult = await this.db.query({ + query: ` + WITH ordered_messages AS ( + SELECT + *, + toDateTime64(createdAt, 3) as createdAt, + toDateTime64(updatedAt, 3) as updatedAt, + ROW_NUMBER() OVER (ORDER BY "createdAt" DESC) as row_num + FROM "${TABLE_MESSAGES}" + WHERE thread_id = {var_thread_id:String} + ) + SELECT + m.id AS id, + m.content as content, + m.role as role, + m.type as type, + m.createdAt as createdAt, + m.updatedAt as updatedAt, + m.thread_id AS "threadId" + FROM ordered_messages m + WHERE m.id = ANY({var_include:Array(String)}) + OR EXISTS ( + SELECT 1 FROM ordered_messages target + WHERE target.id = ANY({var_include:Array(String)}) + AND ( + -- Get previous messages based on the max withPreviousMessages + (m.row_num <= target.row_num + {var_withPreviousMessages:Int64} AND m.row_num > target.row_num) + OR + -- Get next messages based on the max withNextMessages + (m.row_num >= target.row_num - {var_withNextMessages:Int64} AND m.row_num < target.row_num) + ) + ) + ORDER BY m."createdAt" DESC + `, + query_params: { + var_thread_id: threadId, + var_include: include.map(i => i.id), + var_withPreviousMessages: Math.max(...include.map(i => i.withPreviousMessages || 0)), + var_withNextMessages: Math.max(...include.map(i => i.withNextMessages || 0)), + }, + clickhouse_settings: { + // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z') + date_time_input_format: 'best_effort', + date_time_output_format: 'iso', + use_client_time_zone: 1, + }, + }); + + const rows = await includeResult.json(); + messages.push(...transformRows(rows.data)); + } + + // Then get the remaining messages, excluding the ids we just fetched + const result = await this.db.query({ + query: ` + SELECT + id, + content, + role, + type, + toDateTime64(createdAt, 3) as createdAt, + thread_id AS "threadId" + FROM "${TABLE_MESSAGES}" + WHERE thread_id = {threadId:String} + AND id NOT IN ({exclude:Array(String)}) + ORDER BY "createdAt" DESC + LIMIT {limit:Int64} + `, + query_params: { + threadId, + exclude: messages.map(m => m.id), + limit, + }, + clickhouse_settings: { + // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z') + date_time_input_format: 'best_effort', + date_time_output_format: 'iso', + use_client_time_zone: 1, + }, + }); + + const rows = await result.json(); + messages.push(...transformRows(rows.data)); + + // Sort all messages by creation date + messages.sort((a, b) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime()); + + // Parse message content + messages.forEach(message => { + if (typeof message.content === 'string') { + try { + message.content = JSON.parse(message.content); + } catch { + // If parsing fails, leave as string + } + } + }); + + return messages as T; + } catch (error) { + console.error('Error getting messages:', error); + throw error; + } + } + + async saveMessages({ messages }: { messages: MessageType[] }): Promise { + if (messages.length === 0) return messages; + + try { + const threadId = messages[0]?.threadId; + if (!threadId) { + throw new Error('Thread ID is required'); + } + + // Check if thread exists + const thread = await this.getThreadById({ threadId }); + if (!thread) { + throw new Error(`Thread ${threadId} not found`); + } + + await this.db.insert({ + table: TABLE_MESSAGES, + format: 'JSONEachRow', + values: messages.map(message => ({ + id: message.id, + thread_id: threadId, + content: typeof message.content === 'string' ? message.content : JSON.stringify(message.content), + createdAt: message.createdAt.toISOString(), + role: message.role, + type: message.type, + })), + clickhouse_settings: { + // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z') + date_time_input_format: 'best_effort', + use_client_time_zone: 1, + }, + }); + + return messages; + } catch (error) { + console.error('Error saving messages:', error); + throw error; + } + } + + async persistWorkflowSnapshot({ + workflowName, + runId, + snapshot, + }: { + workflowName: string; + runId: string; + snapshot: WorkflowRunState; + }): Promise { + try { + const currentSnapshot = await this.load({ + tableName: TABLE_WORKFLOW_SNAPSHOT, + keys: { workflow_name: workflowName, run_id: runId }, + }); + + const now = new Date(); + const persisting = currentSnapshot + ? { + ...currentSnapshot, + snapshot: JSON.stringify(snapshot), + updatedAt: now.toISOString(), + } + : { + workflow_name: workflowName, + run_id: runId, + snapshot: JSON.stringify(snapshot), + createdAt: now.toISOString(), + updatedAt: now.toISOString(), + }; + + await this.db.insert({ + table: TABLE_WORKFLOW_SNAPSHOT, + format: 'JSONEachRow', + values: [persisting], + clickhouse_settings: { + // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z') + date_time_input_format: 'best_effort', + use_client_time_zone: 1, + }, + }); + } catch (error) { + console.error('Error persisting workflow snapshot:', error); + throw error; + } + } + + async loadWorkflowSnapshot({ + workflowName, + runId, + }: { + workflowName: string; + runId: string; + }): Promise { + try { + const result = await this.load({ + tableName: TABLE_WORKFLOW_SNAPSHOT, + keys: { + workflow_name: workflowName, + run_id: runId, + }, + }); + + if (!result) { + return null; + } + + return (result as any).snapshot; + } catch (error) { + console.error('Error loading workflow snapshot:', error); + throw error; + } + } + + async close(): Promise { + await this.db.close(); + } +} diff --git a/stores/clickhouse/tsconfig.json b/stores/clickhouse/tsconfig.json new file mode 100644 index 00000000000..6750fddcd4d --- /dev/null +++ b/stores/clickhouse/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "../../tsconfig.node.json", + "include": ["src/**/*"], + "exclude": ["node_modules", "**/*.test.ts"] +} diff --git a/stores/clickhouse/vitest.config.ts b/stores/clickhouse/vitest.config.ts new file mode 100644 index 00000000000..b293c3a1936 --- /dev/null +++ b/stores/clickhouse/vitest.config.ts @@ -0,0 +1,12 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + environment: 'node', + include: ['src/**/*.test.ts'], + exclude: ['src/**/*.performance.test.ts'], + coverage: { + reporter: ['text', 'json', 'html'], + }, + }, +});