From 9b1ac993eedac331486d94c3f8e560297f523e02 Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Mon, 16 Nov 2026 08:08:53 -0800 Subject: [PATCH] [miniflare] Use workerd's built-in D1 for local D1 databases Local D1 bindings now go to workerd's D1 Worker, the bundled module /workerd/d1.js, with {"database": ""} in their props. Its Durable Objects keep Miniflare's uniqueKey (miniflare-D1DatabaseObject) and disk, so existing local databases are the same files; a database written by the previous Miniflare reads back unchanged. Miniflare's own D1 object (database.worker.ts) and its helpers are removed. The entry worker, used by the local explorer and shared storage to reach a database named at runtime, now forwards to workerd's D1 through the debug port, as the KV entry worker does. Tests: served_by is "workerd"; query() errors are named Error, not D1_ERROR. Removed are three regression tests for failures writing the session bookmark, which no longer writes. PRAGMA miniflare_d1_export, for wrangler d1 export --local, is answered by workerd's D1, and the export tests pass against it. Needs a workerd release with workerd#8486, and generated/workerd.ts regenerated from it so the config can carry Worker.Module.bundled; until then the D1 tests fail. --- .changeset/miniflare-workerd-d1.md | 9 + packages/miniflare/src/plugins/d1/index.ts | 152 +++++----- .../miniflare/src/runtime/config/workerd.ts | 1 + .../miniflare/src/workers/d1/constants.ts | 7 + .../src/workers/d1/database.worker.ts | 266 ----------------- packages/miniflare/src/workers/d1/dumpSql.ts | 278 ------------------ .../miniflare/src/workers/d1/entry.worker.ts | 113 ++----- packages/miniflare/src/workers/d1/export.ts | 64 ---- packages/miniflare/src/workers/d1/schemas.ts | 93 ------ .../miniflare/test/plugins/d1/index.spec.ts | 83 ------ .../miniflare/test/plugins/d1/jsrpc.spec.ts | 9 +- packages/miniflare/test/plugins/d1/suite.ts | 12 +- 12 files changed, 126 insertions(+), 961 deletions(-) create mode 100644 .changeset/miniflare-workerd-d1.md delete mode 100644 packages/miniflare/src/workers/d1/database.worker.ts delete mode 100644 packages/miniflare/src/workers/d1/dumpSql.ts delete mode 100644 packages/miniflare/src/workers/d1/export.ts delete mode 100644 packages/miniflare/src/workers/d1/schemas.ts diff --git a/.changeset/miniflare-workerd-d1.md b/.changeset/miniflare-workerd-d1.md new file mode 100644 index 0000000000..c2e2ed8a6c --- /dev/null +++ b/.changeset/miniflare-workerd-d1.md @@ -0,0 +1,9 @@ +--- +"miniflare": minor +--- + +Use workerd's built-in D1 for local D1 databases + +Local D1 databases are now run by workerd's own D1 Worker rather than simulated by Miniflare. Existing local databases keep their data: they stay in the same directory under the persistence path. + +Differences: `meta.served_by` is `"workerd"` and errors thrown by `query()` are named `Error` rather than `D1_ERROR`. diff --git a/packages/miniflare/src/plugins/d1/index.ts b/packages/miniflare/src/plugins/d1/index.ts index 04fbdb9dd4..d9460b05a1 100644 --- a/packages/miniflare/src/plugins/d1/index.ts +++ b/packages/miniflare/src/plugins/d1/index.ts @@ -1,38 +1,31 @@ import fs from "node:fs/promises"; -import SCRIPT_D1_DATABASE_OBJECT from "worker:d1/database"; import SCRIPT_D1_ENTRY from "worker:d1/entry"; -import { SharedBindings } from "../../workers"; -import { D1_LOCAL_ENTRY_SERVICE_NAME } from "../../workers/d1/constants"; +import { kVoid } from "../../runtime"; +import { + D1Bindings, + D1_LOCAL_ENTRY_SERVICE_NAME, + D1_WORKERD_SERVICE_NAME, +} from "../../workers/d1/constants"; import { buildObjectEntryProps, buildRemoteProxyProps, getEnvBindingsOfType, - getMiniflareObjectBindings, getPersistPath, getRemoteProxyConnectionString, getStorageService, - objectEntryWorker, ProxyNodeBinding, remoteProxyClientWorker, - SERVICE_LOOPBACK, } from "../shared"; -import type { - Service, - Worker_Binding, - Worker_Binding_DurableObjectNamespaceDesignator, -} from "../../runtime"; +import type { Service, Worker_Binding } from "../../runtime"; import type { Plugin } from "../shared"; export const D1_PLUGIN_NAME = "d1"; const D1_STORAGE_SERVICE_NAME = `${D1_PLUGIN_NAME}:storage`; -const D1_DATABASE_SERVICE_PREFIX = `${D1_PLUGIN_NAME}:db`; // One shared remote-proxy service for all remote D1 databases (config via props). const D1_REMOTE_SERVICE_NAME = `${D1_PLUGIN_NAME}:db:remote`; -const D1_DATABASE_OBJECT_CLASS_NAME = "D1DatabaseObject"; -const D1_DATABASE_OBJECT: Worker_Binding_DurableObjectNamespaceDesignator = { - serviceName: D1_DATABASE_SERVICE_PREFIX, - className: D1_DATABASE_OBJECT_CLASS_NAME, -}; +// The `uniqueKey` of Miniflare's own D1 objects before workerd's D1, kept so +// that existing local databases are the same files. +const D1_UNIQUE_KEY = "miniflare-D1DatabaseObject"; export const D1_PLUGIN: Plugin = { bindingTypeDescription: "D1 database", @@ -45,18 +38,24 @@ export const D1_PLUGIN: Plugin = { options.dev ); - // Remote databases share one proxy service (config via props); local - // databases share one entry service with the id supplied via props. + // Remote databases share one proxy service (config via props). Local + // databases are workerd's D1, or with shared storage the storage + // owner's entry service. const serviceDesignator = remoteProxyConnectionString ? { name: D1_REMOTE_SERVICE_NAME, props: buildRemoteProxyProps(remoteProxyConnectionString, name), } - : getStorageService( - D1_LOCAL_ENTRY_SERVICE_NAME, - buildObjectEntryProps(id), - sharedOptions - ); + : sharedOptions.unsafeEnableSharedStorage + ? getStorageService( + D1_LOCAL_ENTRY_SERVICE_NAME, + buildObjectEntryProps(id), + sharedOptions + ) + : { + name: D1_WORKERD_SERVICE_NAME, + props: { json: JSON.stringify({ database: id }) }, + }; return { name, @@ -86,7 +85,8 @@ export const D1_PLUGIN: Plugin = { const services: Service[] = []; - // One shared entry service for all local databases (id supplied via props). + // workerd's D1, and the entry service that reaches a database named at + // runtime, for the local explorer and shared storage. const hasLocal = databases.some( ([, db]) => @@ -95,18 +95,50 @@ export const D1_PLUGIN: Plugin = { sharedOptions.unsafeEnableSharedStorage || sharedOptions.unsafeLocalExplorer; if (hasLocal) { - services.push({ - name: D1_LOCAL_ENTRY_SERVICE_NAME, - worker: { - ...objectEntryWorker(D1_DATABASE_OBJECT), - // Preserve D1 error names across RPC without changing the helper's - // compatibility date (internal workers don't inherit the user's date). - compatibilityFlags: ["enhanced_error_serialization"], - // Reuse the DO binding configuration, but replace the HTTP-only router - // with D1's entrypoint, which also implements the query RPC method. - modules: [{ name: "entry.worker.js", esModule: SCRIPT_D1_ENTRY() }], + const persistPath = getPersistPath( + D1_PLUGIN_NAME, + tmpPath, + sharedOptions.resourcePersistencePath + ); + await fs.mkdir(persistPath, { recursive: true }); + services.push( + { + name: D1_LOCAL_ENTRY_SERVICE_NAME, + worker: { + compatibilityDate: "2025-01-01", + modules: [{ name: "entry.worker.js", esModule: SCRIPT_D1_ENTRY() }], + bindings: [ + { name: D1Bindings.DEBUG_PORT, workerdDebugPort: kVoid }, + ], + }, }, - }); + // workerd's D1 module and its config, as d1.js documents it. + { + name: D1_WORKERD_SERVICE_NAME, + worker: { + compatibilityDate: "2026-01-01", + modules: [{ name: "d1.js", bundled: "/workerd/d1.js" }], + bindings: [ + { + name: "DATABASE", + durableObjectNamespace: { className: "D1Database" }, + }, + ], + durableObjectNamespaces: [ + { + className: "D1Database", + uniqueKey: D1_UNIQUE_KEY, + enableSql: true, + }, + ], + durableObjectStorage: { localDisk: D1_STORAGE_SERVICE_NAME }, + }, + }, + { + name: D1_STORAGE_SERVICE_NAME, + disk: { path: persistPath, writable: true }, + } + ); } // One shared proxy service for all remote (mixed-mode) databases. @@ -120,54 +152,6 @@ export const D1_PLUGIN: Plugin = { }); } - if (hasLocal) { - const uniqueKey = `miniflare-${D1_DATABASE_OBJECT_CLASS_NAME}`; - const persistPath = getPersistPath( - D1_PLUGIN_NAME, - tmpPath, - sharedOptions.resourcePersistencePath - ); - await fs.mkdir(persistPath, { recursive: true }); - - const storageService: Service = { - name: D1_STORAGE_SERVICE_NAME, - disk: { path: persistPath, writable: true }, - }; - const objectService: Service = { - name: D1_DATABASE_SERVICE_PREFIX, - worker: { - compatibilityDate: "2023-07-24", - compatibilityFlags: ["nodejs_compat", "experimental"], - modules: [ - { - name: "database.worker.js", - esModule: SCRIPT_D1_DATABASE_OBJECT(), - }, - ], - durableObjectNamespaces: [ - { - className: D1_DATABASE_OBJECT_CLASS_NAME, - uniqueKey, - }, - ], - // Store Durable Object SQL databases in persist path - durableObjectStorage: { localDisk: D1_STORAGE_SERVICE_NAME }, - // Bind blob disk directory service to object - bindings: [ - { - name: SharedBindings.MAYBE_SERVICE_BLOBS, - service: { name: D1_STORAGE_SERVICE_NAME }, - }, - { - name: SharedBindings.MAYBE_SERVICE_LOOPBACK, - service: { name: SERVICE_LOOPBACK }, - }, - ...getMiniflareObjectBindings(), - ], - }, - }; - services.push(storageService, objectService); - } return services; }, }; diff --git a/packages/miniflare/src/runtime/config/workerd.ts b/packages/miniflare/src/runtime/config/workerd.ts index 402dc1f740..606cccfd00 100644 --- a/packages/miniflare/src/runtime/config/workerd.ts +++ b/packages/miniflare/src/runtime/config/workerd.ts @@ -116,6 +116,7 @@ export type Worker_Module = { name: string; } & ( | { esModule?: string } + | { bundled?: string } | { commonJsModule?: string } | { text?: string } | { data?: Uint8Array } diff --git a/packages/miniflare/src/workers/d1/constants.ts b/packages/miniflare/src/workers/d1/constants.ts index b98d0e7484..6e614f6666 100644 --- a/packages/miniflare/src/workers/d1/constants.ts +++ b/packages/miniflare/src/workers/d1/constants.ts @@ -1 +1,8 @@ export const D1_LOCAL_ENTRY_SERVICE_NAME = "d1:db:entry"; +// The Worker running workerd's D1 (the bundled module `/workerd/d1.js`), +// reached with props `{ database }`. +export const D1_WORKERD_SERVICE_NAME = "d1:workerd"; + +export const D1Bindings = { + DEBUG_PORT: "MINIFLARE_DEBUG_PORT", +} as const; diff --git a/packages/miniflare/src/workers/d1/database.worker.ts b/packages/miniflare/src/workers/d1/database.worker.ts deleted file mode 100644 index 5cd3caf3bc..0000000000 --- a/packages/miniflare/src/workers/d1/database.worker.ts +++ /dev/null @@ -1,266 +0,0 @@ -import assert from "node:assert"; -import { - get, - HttpError, - MiniflareDurableObject, - POST, - viewToBuffer, -} from "miniflare:shared"; -import { z } from "miniflare:zod"; -import { dumpSql } from "./dumpSql"; -import { isD1ExportPragma } from "./export"; -import { - D1QueriesSchema, - D1_SESSION_COMMIT_TOKEN_HTTP_HEADER, -} from "./schemas"; -import type { D1ExportPragma } from "./export"; -import type { D1Meta, D1Query, D1RowsAndColumns, D1Value } from "./schemas"; -import type { - MiniflareDurableObjectEnv, - RouteHandler, - TypedSqlStorage, - TypedValue, -} from "miniflare:shared"; - -const D1ResultsFormatSchema = z - .enum(["ARRAY_OF_OBJECTS", "ROWS_AND_COLUMNS", "NONE"]) - .catch("ARRAY_OF_OBJECTS"); - -type D1ResultsFormat = z.infer; - -type D1Results = D1RowsAndColumns | Record[] | undefined; - -const served_by = "miniflare.db"; -interface D1SuccessResponse { - success: true; - results: D1Results; - meta: D1Meta; -} -interface D1FailureResponse { - success: false; - error: string; -} - -export class D1Error extends HttpError { - constructor(readonly cause: unknown) { - super(500); - } - - toResponse(): Response { - const error = - typeof this.cause === "object" && - this.cause !== null && - "message" in this.cause && - typeof this.cause.message === "string" - ? this.cause.message - : String(this.cause); - const response: D1FailureResponse = { success: false, error }; - return Response.json(response); - } -} - -function convertParams(params: D1Query["params"]): TypedValue[] { - return (params ?? []).map((param) => - // If `param` is an array, assume it's a byte array - Array.isArray(param) ? viewToBuffer(new Uint8Array(param)) : param - ); -} - -function convertRows(rows: TypedValue[][]): D1Value[][] { - return rows.map((row) => - row.map((value) => { - let normalised: D1Value; - if (value instanceof ArrayBuffer) { - // If `value` is an array, convert it to a regular numeric array - normalised = Array.from(new Uint8Array(value)); - } else { - normalised = value; - } - return normalised; - }) - ); -} - -function rowsToObjects( - columns: string[], - rows: D1Value[][] -): Record[] { - return rows.map((row) => - Object.fromEntries(columns.map((name, i) => [name, row[i]])) - ); -} - -function sqlStmts(db: TypedSqlStorage) { - return { - getChanges: db.prepare<[], { totalChanges: number; lastRowId: number }>( - "SELECT total_changes() AS totalChanges, last_insert_rowid() AS lastRowId" - ), - }; -} - -export class D1DatabaseObject extends MiniflareDurableObject { - readonly #stmts: ReturnType; - - constructor(state: DurableObjectState, env: MiniflareDurableObjectEnv) { - super(state, env); - this.#stmts = sqlStmts(this.db); - } - - #changes() { - const changes = get(this.#stmts.getChanges()); - assert(changes !== undefined); - return changes; - } - - #query = (format: D1ResultsFormat, query: D1Query): D1SuccessResponse => { - const beforeSize = this.state.storage.sql.databaseSize; - const beforeChanges = this.#changes(); - - const params = convertParams(query.params ?? []); - // Include cursor consumption in SQL time, but exclude metadata queries and - // blob/object conversion so duration has the same meaning as in D1. - const beforeTime = performance.now(); - const cursor = this.db.prepare(query.sql)(...params); - const columns = cursor.columnNames; - const rawRows = Array.from(cursor.raw()); - const duration = performance.now() - beforeTime; - const rows = convertRows(rawRows); - - let results = undefined; - if (format === "ROWS_AND_COLUMNS") { - results = { columns, rows }; - } else { - results = rowsToObjects(columns, rows); - } - // Note that the "NONE" format behaviour here is inconsistent with workerd. - // See comment: https://github.com/cloudflare/workers-sdk/pull/5917#issuecomment-2133313156 - - const afterSize = this.state.storage.sql.databaseSize; - const afterChanges = this.#changes(); - - const changes = afterChanges.totalChanges - beforeChanges.totalChanges; - - const hasChanges = changes !== 0; - const lastRowChanged = afterChanges.lastRowId !== beforeChanges.lastRowId; - const sizeChanged = afterSize !== beforeSize; - const changed = hasChanges || lastRowChanged || sizeChanged; - - return { - success: true, - results, - meta: { - served_by, - served_by_primary: true, - total_attempts: 1, - timings: { sql_duration_ms: duration }, - duration, - changes, - last_row_id: afterChanges.lastRowId, - changed_db: changed, - size_after: afterSize, - rows_read: cursor.rowsRead, - rows_written: cursor.rowsWritten, - }, - }; - }; - - /** - * Runs all queries in one storage transaction. - * - * @param queries The queries to run in order. - * @param format The format for the result rows. - * @param includeCommitToken Whether to get the current session bookmark. - * @returns The query results, and the session bookmark if requested. - */ - async #txn( - queries: D1Query[], - format: D1ResultsFormat, - includeCommitToken: boolean - ): Promise<{ results: D1SuccessResponse[]; commitToken?: string }> { - // SQLite rejects statements that are just comments, so filter them out. - // Match remote D1, which also filters `--` comments at offset 0. - queries = queries.filter( - (query) => query.sql.replace(/^\s*--.*/gm, "").trim().length > 0 - ); - if (queries.length === 0) { - const error = new Error("No SQL statements detected."); - throw new D1Error(error); - } - - try { - // Roll back the queries if bookmark retrieval fails, so a catchable - // D1_ERROR never reports these writes as failed after committing them. - return await this.state.storage.transaction(async () => { - const results = queries.map(this.#query.bind(this, format)); - // In local development, workerd stores a new bookmark each time one is - // requested. Only sessions use bookmarks, so other requests skip this - // write and read-only queries commit nothing, as in production D1. - if (!includeCommitToken) { - return { results }; - } - let commitToken: string; - try { - commitToken = await this.state.storage.getCurrentBookmark(); - } catch (e) { - const message = e instanceof Error ? e.message : String(e); - throw new Error(`Failed to get session commit token: ${message}`, { - cause: e, - }); - } - return { results, commitToken }; - }); - } catch (e) { - throw new D1Error(e); - } - } - - @POST("/query") - @POST("/execute") - queryExecute: RouteHandler = async (req) => { - let queries = D1QueriesSchema.parse(await req.json()); - if (!Array.isArray(queries)) { - queries = [queries]; - } - - // Special local-mode-only handlers - if (isD1ExportPragma(queries)) { - return this.#doExportData(queries); - } - - const { searchParams } = new URL(req.url); - const resultsFormat = D1ResultsFormatSchema.parse( - searchParams.get("resultsFormat") - ); - - // The binding sends this header only for `withSession()` sessions. - const includeCommitToken = Boolean( - req.headers.get(D1_SESSION_COMMIT_TOKEN_HTTP_HEADER)?.trim() - ); - const { results, commitToken } = await this.#txn( - queries, - resultsFormat, - includeCommitToken - ); - - return Response.json(results, { - headers: - commitToken === undefined - ? undefined - : { [D1_SESSION_COMMIT_TOKEN_HTTP_HEADER]: commitToken }, - }); - }; - - #doExportData(queries: D1ExportPragma) { - const [noSchema, noData, ...tables] = queries[0].params; - const options = { - noSchema: Boolean(noSchema), - noData: Boolean(noData), - tables, - }; - return Response.json({ - success: true, - results: [Array.from(dumpSql(this.state.storage.sql, options))], - meta: {}, - }); - } -} diff --git a/packages/miniflare/src/workers/d1/dumpSql.ts b/packages/miniflare/src/workers/d1/dumpSql.ts deleted file mode 100644 index 58d2220b68..0000000000 --- a/packages/miniflare/src/workers/d1/dumpSql.ts +++ /dev/null @@ -1,278 +0,0 @@ -// NOTE: this file is duplicated between miniflare and d1-workers, and should be kept in sync by hand. -// Please tag someone from the D1 team as a reviewer and they will do this for you. -// -// NOTE: this function duplicates the logic inside SQLite's shell.c.in as close -// as possible, with any deviations noted. -import type { SqlStorage } from "@cloudflare/workers-types/experimental"; - -/** Stats tracking for dumpSql. Will be mutated in place if provided. */ -export interface DumpSqlStats { - rows_read: number; - rows_written: number; - // Stats for tracking INSERT statement sizes (column names are always included) - /** Number of INSERT statements over 100KB (current size, with column names) */ - inserts_over_100kb_with_column_names: number; - /** Number of INSERT statements that would be over 100KB without column names (for backward comparison) */ - inserts_already_over_100kb: number; - /** Total number of INSERT statements generated */ - total_inserts: number; - /** Maximum INSERT statement size without column names (hypothetical, for backward comparison) */ - max_insert_size: number; - /** Maximum INSERT statement size (current size, with column names) */ - max_insert_size_with_column_names: number; -} - -export function* dumpSql( - db: SqlStorage, - options?: { - noSchema?: boolean; - noData?: boolean; - tables?: string[]; - }, - /** Optional stats tracking. Will be mutated in place if provided */ - stats?: DumpSqlStats -) { - // WARNING: the caller in D1 assumes non-empty exports, so think carefully before removing this initial yield. - yield `PRAGMA defer_foreign_keys=TRUE;`; - - // Empty set means include all tables - const filterTables = new Set(options?.tables || []); - const { noData, noSchema } = options || {}; - - // Taken from SQLite shell.c.in https://github.com/sqlite/sqlite/blob/105c20648e1b05839fd0638686b95f2e3998abcb/src/shell.c.in#L8463-L8469 - // @ts-expect-error -- SqlStorageStatement needs to be callable - const tables_cursor = db.prepare(` - SELECT name, type, sql - FROM sqlite_schema AS o - WHERE (true) AND type=='table' - AND sql NOT NULL - ORDER BY tbl_name='sqlite_sequence', rowid; - `)(); - const tables: { name: string; type: string; sql: string }[] = - Array.from(tables_cursor); - if (stats) { - stats.rows_read += tables_cursor.rowsRead; - stats.rows_written += tables_cursor.rowsWritten; - } - - for (const { name: table, sql } of tables) { - if (filterTables.size > 0 && !filterTables.has(table)) { - continue; - } - - if (table === "sqlite_sequence") { - if (!noSchema) { - yield `DELETE FROM sqlite_sequence;`; - } - } else if (table.match(/^sqlite_stat./)) { - // This feels like it should really appear _after_ the contents of sqlite_stat[1,4] but I'm choosing - // to match SQLite's dump output exactly so writing it immediately like they do. - if (!noSchema) { - yield `ANALYZE sqlite_schema;`; - } - } else if (sql.startsWith(`CREATE VIRTUAL TABLE`)) { - throw new Error( - `D1 Export error: cannot export databases with Virtual Tables (fts5)` - ); - } else if (table.startsWith("_cf_") || table.startsWith("sqlite_")) { - continue; - } else { - // SQLite dump has an extremely weird behaviour where, if the table was explicitly - // quoted i.e. "Table", then in the dump it has `IF NOT EXISTS` injected. I don't understand - // why, but on the off chance there's a good reason to I am following suit. - if (sql.match(/CREATE TABLE ['"].*/)) { - if (!noSchema) { - yield `CREATE TABLE IF NOT EXISTS ${sql.substring(13)};`; - } - } else { - if (!noSchema) { - yield `${sql};`; - } - } - } - - if (noData) { - continue; - } - // eslint-disable-next-line workers-sdk/no-unsafe-command-execution -- input is escaped via escapeId() so this PRAGMA call is safe - const columns_cursor = db.exec(`PRAGMA table_info=${escapeId(table)}`); - - const columns = Array.from(columns_cursor) as { - cid: string; - name: string; - type: string; - notnull: number; - dflt_val: string | null; - pk: number; - }[]; - if (stats) { - stats.rows_read += columns_cursor.rowsRead; - stats.rows_written += columns_cursor.rowsWritten; - } - - const select = `SELECT ${columns.map((c) => escapeId(c.name)).join(", ")} FROM ${escapeId(table)};`; - const rows_cursor = db.exec(select); - const columnNames = columns.map((c) => escapeId(c.name)).join(","); - // The column names portion is: " (" + columnNames + ")" = 3 + columnNames.length - const columnNamesOverhead = 3 + columnNames.length; - for (const dataRow of rows_cursor.raw()) { - const formattedCells = dataRow.map((cell: unknown, i: number) => { - const colType = columns[i].type; - const cellType = typeof cell; - if (cell === null) { - return "NULL"; - } else if (cellType === "number") { - if (!Number.isFinite(cell)) { - return (cell as number) > 0 ? "9e999" : "-9e999"; - } - return cell; - } else if (cellType === "string") { - return outputQuotedEscapedString(cell); - } else if (cell instanceof ArrayBuffer) { - return `X'${Array.prototype.map - .call(new Uint8Array(cell), (b) => b.toString(16).padStart(2, "0")) - .join("")}'`; - } else { - console.error({ - message: "dumpSql: unexpected cell type", - colType, - cellType, - cell, - column: columns[i], - }); - return "ERROR"; - } - }); - - const insertStmt = `INSERT INTO ${escapeId(table)} (${columnNames}) VALUES(${formattedCells.join(",")});`; - - // Track stats for INSERT statement sizes - if (stats) { - const currentSize = insertStmt.length; - // Calculate what the size would be without column names (for comparison) - const sizeWithoutColumnNames = currentSize - columnNamesOverhead; - const LIMIT = 100 * 1024; // 100KB - - stats.total_inserts++; - if (sizeWithoutColumnNames > LIMIT) { - stats.inserts_already_over_100kb++; - } - if (currentSize > LIMIT) { - stats.inserts_over_100kb_with_column_names++; - } - if (sizeWithoutColumnNames > stats.max_insert_size) { - stats.max_insert_size = sizeWithoutColumnNames; - } - if (currentSize > stats.max_insert_size_with_column_names) { - stats.max_insert_size_with_column_names = currentSize; - } - } - - yield insertStmt; - } - if (stats) { - stats.rows_read += rows_cursor.rowsRead; - stats.rows_written += rows_cursor.rowsWritten; - } - } - - if (!noSchema) { - // Taken from SQLite shell.c.in https://github.com/sqlite/sqlite/blob/105c20648e1b05839fd0638686b95f2e3998abcb/src/shell.c.in#L8473-L8478 - const rest_of_schema = db.exec( - [ - "SELECT name, sql", - "FROM sqlite_schema AS o", - "WHERE (true) AND sql NOT NULL", - " AND type IN ('index', 'trigger', 'view')", - // 'DESC' appears in the code linked above but the observed behaviour of SQLite appears otherwise - "ORDER BY type COLLATE NOCASE", - ].join(" ") - ); - for (const { name, sql } of rest_of_schema) { - if (filterTables.size > 0 && !filterTables.has(name as string)) { - continue; - } - yield `${sql};`; - } - if (stats) { - stats.rows_read += rest_of_schema.rowsRead; - stats.rows_written += rest_of_schema.rowsWritten; - } - } -} - -// Ported `output_quoted_escaped_string` from https://github.com/sqlite/sqlite/blob/master/src/shell.c.in#L1799-L1862 -function outputQuotedEscapedString(cell: unknown) { - let lfs = false; - let crs = false; - - const quotesOrNewlinesRegexp = /'|(\n)|(\r)/g; - - // Function to replace ' with '', while also tracking whether the string contains any \r or \n chars - const escapeQuotesDetectingNewlines = (_: string, lf: string, cr: string) => { - if (lf) { - lfs = true; - return `\\n`; - } - if (cr) { - crs = true; - return `\\r`; - } - return `''`; - }; - - const escaped_string = (cell as string).replace( - quotesOrNewlinesRegexp, - escapeQuotesDetectingNewlines - ); - let output_string = `'${escaped_string}'`; - if (crs) { - output_string = `replace(${output_string},'\\r',char(13))`; - } - if (lfs) { - output_string = `replace(${output_string},'\\n',char(10))`; - } - return output_string; -} - -// Ported from quoteChar: https://github.com/sqlite/sqlite/blob/378bf82e2bc09734b8c5869f9b148efe37d29527/src/shell.c.in#L990 -export function sqliteQuote(token: string) { - // Empty input - return token.length === 0 || - // Doesn't start with alpha or underscore - !token.match(/^[a-zA-Z_]/) || - token.match(/\W/) || - SQLITE_KEYWORDS.has(token.toUpperCase()) - ? `"${token}"` - : token; -} - -/** - * Escape an identifier for use in SQL statements. - * @param id - The identifier to escape. - * @returns - */ -function escapeId(id: string) { - return `"${id.replace(/"/g, '""')}"`; -} - -// List taken from `aKeywordTable` inhttps://github.com/sqlite/sqlite/blob/378bf82e2bc09734b8c5869f9b148efe37d29527/tool/mkkeywordhash.c#L172 -// prettier-ignore -export const SQLITE_KEYWORDS = new Set([ - "ABORT", "ACTION", "ADD", "AFTER", "ALL", "ALTER", "ALWAYS", "ANALYZE", "AND", "AS", "ASC", - "ATTACH", "AUTOINCREMENT", "BEFORE", "BEGIN", "BETWEEN", "BY", "CASCADE", "CASE", "CAST", - "CHECK", "COLLATE", "COLUMN", "COMMIT", "CONFLICT", "CONSTRAINT", "CREATE", "CROSS", "CURRENT", - "CURRENT_DATE", "CURRENT_TIME", "CURRENT_TIMESTAMP", "DATABASE", "DEFAULT", "DEFERRED", - "DEFERRABLE", "DELETE", "DESC", "DETACH", "DISTINCT", "DO", "DROP", "END", "EACH", "ELSE", - "ESCAPE", "EXCEPT", "EXCLUSIVE", "EXCLUDE", "EXISTS", "EXPLAIN", "FAIL", "FILTER", "FIRST", - "FOLLOWING", "FOR", "FOREIGN", "FROM", "FULL", "GENERATED", "GLOB", "GROUP", "GROUPS", "HAVING", - "IF", "IGNORE", "IMMEDIATE", "IN", "INDEX", "INDEXED", "INITIALLY", "INNER", "INSERT", "INSTEAD", - "INTERSECT", "INTO", "IS", "ISNULL", "JOIN", "KEY", "LAST", "LEFT", "LIKE", "LIMIT", "MATCH", - "MATERIALIZED", "NATURAL", "NO", "NOT", "NOTHING", "NOTNULL", "NULL", "NULLS", "OF", "OFFSET", - "ON", "OR", "ORDER", "OTHERS", "OUTER", "OVER", "PARTITION", "PLAN", "PRAGMA", "PRECEDING", - "PRIMARY", "QUERY", "RAISE", "RANGE", "RECURSIVE", "REFERENCES", "REGEXP", "REINDEX", "RELEASE", - "RENAME", "REPLACE", "RESTRICT", "RETURNING", "RIGHT", "ROLLBACK", "ROW", "ROWS", "SAVEPOINT", - "SELECT", "SET", "TABLE", "TEMP", "TEMPORARY", "THEN", "TIES", "TO", "TRANSACTION", "TRIGGER", - "UNBOUNDED", "UNION", "UNIQUE", "UPDATE", "USING", "VACUUM", "VALUES", "VIEW", "VIRTUAL", "WHEN", - "WHERE", "WINDOW", "WITH", "WITHOUT" -]); diff --git a/packages/miniflare/src/workers/d1/entry.worker.ts b/packages/miniflare/src/workers/d1/entry.worker.ts index c2c41c5e47..4d38544b62 100644 --- a/packages/miniflare/src/workers/d1/entry.worker.ts +++ b/packages/miniflare/src/workers/d1/entry.worker.ts @@ -1,97 +1,42 @@ import { WorkerEntrypoint } from "cloudflare:workers"; import { SharedBindings, SharedHeaders } from "../shared/constants"; -import { isD1ExportPragma, parseD1RpcExportResponse } from "./export"; -import { - D1RowsAndColumnsResponseSchema, - D1RpcRequestSchema, - D1_SESSION_COMMIT_TOKEN_HTTP_HEADER, -} from "./schemas"; -import type { D1RpcExportResponse } from "./export"; -import type { D1RpcRequest, D1RpcResponse } from "./schemas"; +import { D1Bindings, D1_WORKERD_SERVICE_NAME } from "./constants"; +import type { WorkerdDebugPortConnector } from "../core/dev-registry-proxy-shared.worker"; -interface D1EntryProps { - // The shared routing key holds this binding's database ID, not a DO namespace. +interface Props { [SharedBindings.TEXT_NAMESPACE]?: string; } -interface D1EntryEnv { - [SharedBindings.TEXT_NAMESPACE]?: string; - // All local databases live in this namespace; the ID in props selects one. - [SharedBindings.DURABLE_OBJECT_NAMESPACE_OBJECT]: DurableObjectNamespace; +interface Env { + [D1Bindings.DEBUG_PORT]: WorkerdDebugPortConnector; } +type D1Fetcher = Fetcher & { query(request: unknown): Promise }; -/** Routes D1 HTTP and RPC requests to the same local SQLite database. */ -export default class D1Entry extends WorkerEntrypoint< - D1EntryEnv, - D1EntryProps -> { - /** Preserves the HTTP protocol used by bindings without the RPC flag. */ - async fetch(request: Request): Promise { - // Local Explorer addresses unbound databases through the internal header; - // preserve the shared entry worker's props/binding/header precedence. - const requestNamespace = request.headers.get(SharedHeaders.NAMESPACE); - const name = - this.ctx.props[SharedBindings.TEXT_NAMESPACE] ?? - this.env[SharedBindings.TEXT_NAMESPACE] ?? - requestNamespace; - if (name === null) { - throw new Error( - "D1 entry worker: no database ID provided via props, binding, or request header" - ); +// Routes D1's HTTP and RPC protocols to workerd's D1 for a database named at +// runtime, by props or the `MF-Namespace` header, which a binding's fixed +// props cannot do: the local explorer and shared storage call this. +export default class D1Entry extends WorkerEntrypoint { + #d1(request?: Request): D1Fetcher { + const database = + this.ctx.props?.[SharedBindings.TEXT_NAMESPACE] ?? + request?.headers.get(SharedHeaders.NAMESPACE) ?? + null; + if (database === null) { + throw new Error("d1:db:entry: no database in props or header"); } - const namespace = this.env[SharedBindings.DURABLE_OBJECT_NAMESPACE_OBJECT]; - const stub = namespace.get(namespace.idFromName(name)); - if (requestNamespace !== null) { - const headers = new Headers(request.headers); - headers.delete(SharedHeaders.NAMESPACE); - request = new Request(request, { headers }); - } - // The shared DO base uses this name to initialise its storage namespace. - return await stub.fetch(request, { cf: { miniflare: { name } } }); + return this.env[D1Bindings.DEBUG_PORT] + .current() + .getEntrypoint(D1_WORKERD_SERVICE_NAME, undefined, { + database, + }) as D1Fetcher; } - /** Returns raw transactional results; query failures are thrown over RPC. */ - async query( - request: D1RpcRequest - ): Promise { - const { queries, bookmark } = D1RpcRequestSchema.parse(request); - const headers = new Headers({ "Content-Type": "application/json" }); - if (bookmark) { - headers.set(D1_SESSION_COMMIT_TOKEN_HTTP_HEADER, bookmark); - } + fetch(request: Request): Promise { + const headers = new Headers(request.headers); + headers.delete(SharedHeaders.NAMESPACE); + return this.#d1(request).fetch(new Request(request, { headers })); + } - // Reuse the HTTP execution path so transactions, validation, and metadata - // remain owned by the database worker rather than duplicated for RPC. - const response = await this.fetch( - new Request("http://d1/query?resultsFormat=ROWS_AND_COLUMNS", { - method: "POST", - headers, - body: JSON.stringify(queries), - }) - ); - const responseBody: unknown = await response.json(); - if (isD1ExportPragma(queries)) { - return parseD1RpcExportResponse(responseBody); - } - - const body = D1RowsAndColumnsResponseSchema.parse(responseBody); - if (!Array.isArray(body)) { - const error = new Error(body.error); - error.name = "D1_ERROR"; - throw error; - } - - const result: D1RpcResponse = { - results: body.map(({ meta, results }) => ({ - meta, - data: { kind: "raw", ...results }, - })), - }; - const responseBookmark = response.headers.get( - D1_SESSION_COMMIT_TOKEN_HTTP_HEADER - ); - if (responseBookmark) { - result.bookmark = responseBookmark; - } - return result; + query(request: unknown): Promise { + return this.#d1().query(request); } } diff --git a/packages/miniflare/src/workers/d1/export.ts b/packages/miniflare/src/workers/d1/export.ts deleted file mode 100644 index 9f6cc38038..0000000000 --- a/packages/miniflare/src/workers/d1/export.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { z } from "miniflare:zod"; -import { D1FailureResponseSchema } from "./schemas"; -import type { D1Query, D1RpcResultData } from "./schemas"; - -/** Local-only pragma used to export a database's SQL schema and data. */ -export const D1_EXPORT_PRAGMA = "PRAGMA miniflare_d1_export(?,?,?);"; - -/** Export flags followed by an optional list of table names. */ -export type D1ExportPragma = [ - { sql: typeof D1_EXPORT_PRAGMA; params: [number, number, ...string[]] }, -]; - -/** Matches the database worker's existing local export dispatch rules. */ -export function isD1ExportPragma( - queries: D1Query[] -): queries is D1ExportPragma { - return ( - queries.length === 1 && - queries[0].sql === D1_EXPORT_PRAGMA && - (queries[0].params?.length || 0) >= 2 - ); -} - -const D1ExportMetaSchema = z.record(z.string(), z.never()); -type D1ExportMeta = z.infer; - -const D1ExportResponseSchema = z.union([ - z.object({ - success: z.literal(true), - results: z.tuple([z.array(z.string())]), - meta: D1ExportMetaSchema, - }), - D1FailureResponseSchema, -]); - -/** Local exports return one dump row and do not report execution metadata. */ -export interface D1RpcExportResponse { - results: [{ meta: D1ExportMeta; data: D1RpcResultData }]; -} - -/** Adapts the legacy local export response, throwing backend failures over RPC. */ -export function parseD1RpcExportResponse(input: unknown): D1RpcExportResponse { - const body = D1ExportResponseSchema.parse(input); - if (!body.success) { - const error = new Error(body.error); - error.name = "D1_ERROR"; - throw error; - } - - return { - results: [ - { - meta: body.meta, - data: { - kind: "raw", - // The HTTP wrapper exposes the export row's array indices - // as column names; preserve that convention over RPC. - columns: body.results[0].map((_, index) => String(index)), - rows: body.results, - }, - }, - ], - }; -} diff --git a/packages/miniflare/src/workers/d1/schemas.ts b/packages/miniflare/src/workers/d1/schemas.ts deleted file mode 100644 index 1f0b2fcdcb..0000000000 --- a/packages/miniflare/src/workers/d1/schemas.ts +++ /dev/null @@ -1,93 +0,0 @@ -import { z } from "miniflare:zod"; - -/** Internal bookmark header shared by the workerd wrapper and D1 backend. */ -export const D1_SESSION_COMMIT_TOKEN_HTTP_HEADER = - "x-cf-d1-session-commit-token"; - -/** JSON representation of SQLite values used by the D1 HTTP protocol. */ -export const D1ValueSchema = z.union([ - z.number(), - z.string(), - z.null(), - z.number().array(), -]); -/** Blobs use numeric byte arrays rather than ArrayBuffers on the wire. */ -export type D1Value = z.infer; - -/** SQL and positional parameters accepted by the local database worker. */ -export const D1QuerySchema = z.object({ - sql: z.string(), - params: z.array(D1ValueSchema).nullable().optional(), -}); -/** Parameters are converted to SQLite values by the database worker. */ -export type D1Query = z.infer; - -/** The HTTP protocol accepts either one query or a transactional batch. */ -export const D1QueriesSchema = z.union([D1QuerySchema, z.array(D1QuerySchema)]); - -/** RPC queries always contain a batch and may specify a session bookmark. */ -export const D1RpcRequestSchema = z.object({ - queries: z.array(D1QuerySchema), - bookmark: z.string().optional(), -}); -/** The bookmark may also be a first-query routing constraint. */ -export type D1RpcRequest = z.infer; - -/** D1 query metadata; geography is optional because local execution has no colo. */ -export const D1MetaSchema = z.object({ - served_by: z.string(), - served_by_primary: z.boolean().optional(), - served_by_region: z.string().optional(), - served_by_colo: z.string().optional(), - timings: z.object({ sql_duration_ms: z.number() }).optional(), - duration: z.number(), - changes: z.number(), - last_row_id: z.number(), - changed_db: z.boolean(), - size_after: z.number(), - rows_read: z.number(), - rows_written: z.number(), - total_attempts: z.number().optional(), -}); -/** SQL duration excludes request handling and result serialization. */ -export type D1Meta = z.infer; - -/** Rows-and-columns results preserve duplicate column names and empty rows. */ -export const D1RowsAndColumnsSchema = z.object({ - columns: z.array(z.string()), - rows: z.array(z.array(D1ValueSchema)), -}); -/** Column names are positional and need not be unique. */ -export type D1RowsAndColumns = z.infer; - -const D1RowsAndColumnsSuccessSchema = z.object({ - success: z.literal(true), - results: D1RowsAndColumnsSchema, - meta: D1MetaSchema, -}); - -/** A backend failure contains no successful query or export results. */ -export const D1FailureResponseSchema = z.object({ - success: z.literal(false), - error: z.string(), -}); - -/** A query failure is returned instead of any successful HTTP batch results. */ -export const D1RowsAndColumnsResponseSchema = z.union([ - z.array(D1RowsAndColumnsSuccessSchema), - D1FailureResponseSchema, -]); - -/** The RPC discriminator identifies the rows-and-columns payload to workerd. */ -export interface D1RpcResultData extends D1RowsAndColumns { - kind: "raw"; -} - -/** RPC successes return results directly; failures are thrown instead. */ -export interface D1RpcResponse { - results: { - meta: D1Meta; - data: D1RpcResultData; - }[]; - bookmark?: string; -} diff --git a/packages/miniflare/test/plugins/d1/index.spec.ts b/packages/miniflare/test/plugins/d1/index.spec.ts index a0b5ad3ce0..3afcdf0f9c 100644 --- a/packages/miniflare/test/plugins/d1/index.spec.ts +++ b/packages/miniflare/test/plugins/d1/index.spec.ts @@ -63,35 +63,6 @@ async function lockForWrites(external: DatabaseSync) { } } -// Regression test for https://github.com/cloudflare/workers-sdk/issues/14916. -test("surfaces recoverable SQLite errors as catchable D1 query errors", async ({ - expect, -}) => { - const { db, external } = await createPersistedDatabase(); - const session = db.withSession(); - await lockForWrites(external); - - try { - // A read succeeds while another connection holds the write lock, but - // writing the session bookmark fails. The error must stay catchable. - await expect( - session.prepare("SELECT * FROM entries").all() - ).rejects.toMatchObject({ - message: expect.stringMatching( - /^D1_ERROR: Failed to get session commit token:/ - ), - cause: { - message: expect.stringMatching(/^Failed to get session commit token:/), - }, - }); - } finally { - external.exec("ROLLBACK"); - } - - const { results } = await db.prepare("SELECT * FROM entries").all(); - expect(results).toEqual([]); -}); - // Regression test for https://github.com/cloudflare/workers-sdk/issues/16549. test("does not write for read-only queries outside a session", async ({ expect, @@ -117,57 +88,3 @@ test("does not write for read-only queries outside a session", async ({ external.exec("ROLLBACK"); } }); - -// Regression test for https://github.com/cloudflare/workers-sdk/pull/14921#discussion_r4047630401. -test.for(["run", "batch"] as const)( - "rolls back %s writes when session bookmark retrieval fails", - async (method, { expect }) => { - const { db, external } = await createPersistedDatabase(); - const session = db.withSession("first-primary"); - await session.prepare("SELECT * FROM entries").all(); - const bookmark = session.getBookmark(); - assert(bookmark !== null); - - // A trigger on workerd's metadata table makes bookmark updates fail - // in its trusted SQLite scope, after the user SQL has succeeded. - // The local session bookmark is stored under metadata key 2. - external.exec(` - CREATE TRIGGER fail_bookmark BEFORE INSERT ON _cf_METADATA - WHEN NEW.key = 2 - BEGIN - SELECT RAISE(ABORT, 'bookmark lookup failed'); - END; - `); - const insert = session.prepare( - "INSERT INTO entries (value) VALUES ('order')" - ); - function execute() { - return method === "run" ? insert.run() : session.batch([insert, insert]); - } - - await expect(execute()).rejects.toMatchObject({ - message: expect.stringMatching( - /^D1_ERROR: Failed to get session commit token:/ - ), - cause: { - message: expect.stringMatching(/^Failed to get session commit token:/), - }, - }); - expect(session.getBookmark()).toBe(bookmark); - // Inspect through the other connection while bookmark lookup still fails. - expect(external.prepare("SELECT * FROM entries").all()).toEqual([]); - - external.exec("DROP TRIGGER fail_bookmark"); - await execute(); - const { results } = await session.prepare("SELECT * FROM entries").all(); - expect(results).toEqual( - method === "run" - ? [{ id: 1, value: "order" }] - : [ - { id: 1, value: "order" }, - { id: 2, value: "order" }, - ] - ); - expect(session.getBookmark()).not.toBe(bookmark); - } -); diff --git a/packages/miniflare/test/plugins/d1/jsrpc.spec.ts b/packages/miniflare/test/plugins/d1/jsrpc.spec.ts index 8d75b2d24c..e0b24b479f 100644 --- a/packages/miniflare/test/plugins/d1/jsrpc.spec.ts +++ b/packages/miniflare/test/plugins/d1/jsrpc.spec.ts @@ -117,7 +117,7 @@ test.for([ const response = await mf.dispatchFetch("http://localhost"); expect(await response.json()).toMatchObject({ meta: { - served_by: "miniflare.db", + served_by: "workerd", served_by_primary: true, total_attempts: 1, duration: expect.any(Number), @@ -161,8 +161,11 @@ test("SQL errors propagate directly and convenience methods preserve their prefi return Response.json(errors); `); const response = await mf.dispatchFetch("http://localhost"); + // workerd's D1 leaves its errors named "Error": a caller without + // enhanced_error_serialization would see a "D1_ERROR" name folded into + // the message, which the binding prefixes again. const backendError = { - name: "D1_ERROR", + name: "Error", message: expect.stringMatching(/^no such table: missing_table/), }; expect(await response.json()).toEqual([ @@ -321,7 +324,7 @@ test.for([ `); const response = await mf.dispatchFetch("http://localhost"); expect(await response.json()).toEqual({ - name: "D1_ERROR", + name: "Error", message, }); expect(response.status).toBe(500); diff --git a/packages/miniflare/test/plugins/d1/suite.ts b/packages/miniflare/test/plugins/d1/suite.ts index 38ad066857..a238c366e7 100644 --- a/packages/miniflare/test/plugins/d1/suite.ts +++ b/packages/miniflare/test/plugins/d1/suite.ts @@ -275,7 +275,7 @@ test("D1PreparedStatement: run", async ({ expect }) => { duration: result.meta.duration, // Not an `INSERT`, so `last_row_id` non-deterministic last_row_id: result.meta.last_row_id, - served_by: "miniflare.db", + served_by: "workerd", served_by_primary: true, total_attempts: 1, timings: { sql_duration_ms: result.meta.duration }, @@ -302,7 +302,7 @@ test("D1PreparedStatement: run", async ({ expect }) => { // Don't know duration, so just match on returned value asserted > 0 duration: result.meta.duration, last_row_id: 4, - served_by: "miniflare.db", + served_by: "workerd", served_by_primary: true, total_attempts: 1, timings: { sql_duration_ms: result.meta.duration }, @@ -331,7 +331,7 @@ test("D1PreparedStatement: run", async ({ expect }) => { last_row_id: result.meta.last_row_id, rows_read: 1, rows_written: 1, - served_by: "miniflare.db", + served_by: "workerd", served_by_primary: true, total_attempts: 1, timings: { sql_duration_ms: result.meta.duration }, @@ -364,7 +364,7 @@ test("D1PreparedStatement: run", async ({ expect }) => { // Don't know duration, so just match on returned value asserted > 0 duration: result.meta.duration, last_row_id: 5, - served_by: "miniflare.db", + served_by: "workerd", served_by_primary: true, total_attempts: 1, timings: { sql_duration_ms: result.meta.duration }, @@ -396,7 +396,7 @@ test("D1PreparedStatement: all", async ({ expect }) => { duration: result.meta.duration, // Not an `INSERT`, so `last_row_id` non-deterministic last_row_id: result.meta.last_row_id, - served_by: "miniflare.db", + served_by: "workerd", served_by_primary: true, total_attempts: 1, timings: { sql_duration_ms: result.meta.duration }, @@ -425,7 +425,7 @@ test("D1PreparedStatement: all", async ({ expect }) => { last_row_id: result.meta.last_row_id, rows_read: 1, rows_written: 0, - served_by: "miniflare.db", + served_by: "workerd", served_by_primary: true, total_attempts: 1, timings: { sql_duration_ms: result.meta.duration },