diff --git a/.changeset/miniflare-workerd-d1.md b/.changeset/miniflare-workerd-d1.md new file mode 100644 index 0000000000..c2e2ed8a6c --- /dev/null +++ b/.changeset/miniflare-workerd-d1.md @@ -0,0 +1,9 @@ +--- +"miniflare": minor +--- + +Use workerd's built-in D1 for local D1 databases + +Local D1 databases are now run by workerd's own D1 Worker rather than simulated by Miniflare. Existing local databases keep their data: they stay in the same directory under the persistence path. + +Differences: `meta.served_by` is `"workerd"` and errors thrown by `query()` are named `Error` rather than `D1_ERROR`. diff --git a/packages/miniflare/src/plugins/d1/index.ts b/packages/miniflare/src/plugins/d1/index.ts index 04fbdb9dd4..d9460b05a1 100644 --- a/packages/miniflare/src/plugins/d1/index.ts +++ b/packages/miniflare/src/plugins/d1/index.ts @@ -1,38 +1,31 @@ import fs from "node:fs/promises"; -import SCRIPT_D1_DATABASE_OBJECT from "worker:d1/database"; import SCRIPT_D1_ENTRY from "worker:d1/entry"; -import { SharedBindings } from "../../workers"; -import { D1_LOCAL_ENTRY_SERVICE_NAME } from "../../workers/d1/constants"; +import { kVoid } from "../../runtime"; +import { + D1Bindings, + D1_LOCAL_ENTRY_SERVICE_NAME, + D1_WORKERD_SERVICE_NAME, +} from "../../workers/d1/constants"; import { buildObjectEntryProps, buildRemoteProxyProps, getEnvBindingsOfType, - getMiniflareObjectBindings, getPersistPath, getRemoteProxyConnectionString, getStorageService, - objectEntryWorker, ProxyNodeBinding, remoteProxyClientWorker, - SERVICE_LOOPBACK, } from "../shared"; -import type { - Service, - Worker_Binding, - Worker_Binding_DurableObjectNamespaceDesignator, -} from "../../runtime"; +import type { Service, Worker_Binding } from "../../runtime"; import type { Plugin } from "../shared"; export const D1_PLUGIN_NAME = "d1"; const D1_STORAGE_SERVICE_NAME = `${D1_PLUGIN_NAME}:storage`; -const D1_DATABASE_SERVICE_PREFIX = `${D1_PLUGIN_NAME}:db`; // One shared remote-proxy service for all remote D1 databases (config via props). const D1_REMOTE_SERVICE_NAME = `${D1_PLUGIN_NAME}:db:remote`; -const D1_DATABASE_OBJECT_CLASS_NAME = "D1DatabaseObject"; -const D1_DATABASE_OBJECT: Worker_Binding_DurableObjectNamespaceDesignator = { - serviceName: D1_DATABASE_SERVICE_PREFIX, - className: D1_DATABASE_OBJECT_CLASS_NAME, -}; +// The `uniqueKey` of Miniflare's own D1 objects before workerd's D1, kept so +// that existing local databases are the same files. +const D1_UNIQUE_KEY = "miniflare-D1DatabaseObject"; export const D1_PLUGIN: Plugin = { bindingTypeDescription: "D1 database", @@ -45,18 +38,24 @@ export const D1_PLUGIN: Plugin = { options.dev ); - // Remote databases share one proxy service (config via props); local - // databases share one entry service with the id supplied via props. + // Remote databases share one proxy service (config via props). Local + // databases are workerd's D1, or with shared storage the storage + // owner's entry service. const serviceDesignator = remoteProxyConnectionString ? { name: D1_REMOTE_SERVICE_NAME, props: buildRemoteProxyProps(remoteProxyConnectionString, name), } - : getStorageService( - D1_LOCAL_ENTRY_SERVICE_NAME, - buildObjectEntryProps(id), - sharedOptions - ); + : sharedOptions.unsafeEnableSharedStorage + ? getStorageService( + D1_LOCAL_ENTRY_SERVICE_NAME, + buildObjectEntryProps(id), + sharedOptions + ) + : { + name: D1_WORKERD_SERVICE_NAME, + props: { json: JSON.stringify({ database: id }) }, + }; return { name, @@ -86,7 +85,8 @@ export const D1_PLUGIN: Plugin = { const services: Service[] = []; - // One shared entry service for all local databases (id supplied via props). + // workerd's D1, and the entry service that reaches a database named at + // runtime, for the local explorer and shared storage. const hasLocal = databases.some( ([, db]) => @@ -95,18 +95,50 @@ export const D1_PLUGIN: Plugin = { sharedOptions.unsafeEnableSharedStorage || sharedOptions.unsafeLocalExplorer; if (hasLocal) { - services.push({ - name: D1_LOCAL_ENTRY_SERVICE_NAME, - worker: { - ...objectEntryWorker(D1_DATABASE_OBJECT), - // Preserve D1 error names across RPC without changing the helper's - // compatibility date (internal workers don't inherit the user's date). - compatibilityFlags: ["enhanced_error_serialization"], - // Reuse the DO binding configuration, but replace the HTTP-only router - // with D1's entrypoint, which also implements the query RPC method. - modules: [{ name: "entry.worker.js", esModule: SCRIPT_D1_ENTRY() }], + const persistPath = getPersistPath( + D1_PLUGIN_NAME, + tmpPath, + sharedOptions.resourcePersistencePath + ); + await fs.mkdir(persistPath, { recursive: true }); + services.push( + { + name: D1_LOCAL_ENTRY_SERVICE_NAME, + worker: { + compatibilityDate: "2025-01-01", + modules: [{ name: "entry.worker.js", esModule: SCRIPT_D1_ENTRY() }], + bindings: [ + { name: D1Bindings.DEBUG_PORT, workerdDebugPort: kVoid }, + ], + }, }, - }); + // workerd's D1 module and its config, as d1.js documents it. + { + name: D1_WORKERD_SERVICE_NAME, + worker: { + compatibilityDate: "2026-01-01", + modules: [{ name: "d1.js", bundled: "/workerd/d1.js" }], + bindings: [ + { + name: "DATABASE", + durableObjectNamespace: { className: "D1Database" }, + }, + ], + durableObjectNamespaces: [ + { + className: "D1Database", + uniqueKey: D1_UNIQUE_KEY, + enableSql: true, + }, + ], + durableObjectStorage: { localDisk: D1_STORAGE_SERVICE_NAME }, + }, + }, + { + name: D1_STORAGE_SERVICE_NAME, + disk: { path: persistPath, writable: true }, + } + ); } // One shared proxy service for all remote (mixed-mode) databases. @@ -120,54 +152,6 @@ export const D1_PLUGIN: Plugin = { }); } - if (hasLocal) { - const uniqueKey = `miniflare-${D1_DATABASE_OBJECT_CLASS_NAME}`; - const persistPath = getPersistPath( - D1_PLUGIN_NAME, - tmpPath, - sharedOptions.resourcePersistencePath - ); - await fs.mkdir(persistPath, { recursive: true }); - - const storageService: Service = { - name: D1_STORAGE_SERVICE_NAME, - disk: { path: persistPath, writable: true }, - }; - const objectService: Service = { - name: D1_DATABASE_SERVICE_PREFIX, - worker: { - compatibilityDate: "2023-07-24", - compatibilityFlags: ["nodejs_compat", "experimental"], - modules: [ - { - name: "database.worker.js", - esModule: SCRIPT_D1_DATABASE_OBJECT(), - }, - ], - durableObjectNamespaces: [ - { - className: D1_DATABASE_OBJECT_CLASS_NAME, - uniqueKey, - }, - ], - // Store Durable Object SQL databases in persist path - durableObjectStorage: { localDisk: D1_STORAGE_SERVICE_NAME }, - // Bind blob disk directory service to object - bindings: [ - { - name: SharedBindings.MAYBE_SERVICE_BLOBS, - service: { name: D1_STORAGE_SERVICE_NAME }, - }, - { - name: SharedBindings.MAYBE_SERVICE_LOOPBACK, - service: { name: SERVICE_LOOPBACK }, - }, - ...getMiniflareObjectBindings(), - ], - }, - }; - services.push(storageService, objectService); - } return services; }, }; diff --git a/packages/miniflare/src/runtime/config/workerd.ts b/packages/miniflare/src/runtime/config/workerd.ts index 402dc1f740..606cccfd00 100644 --- a/packages/miniflare/src/runtime/config/workerd.ts +++ b/packages/miniflare/src/runtime/config/workerd.ts @@ -116,6 +116,7 @@ export type Worker_Module = { name: string; } & ( | { esModule?: string } + | { bundled?: string } | { commonJsModule?: string } | { text?: string } | { data?: Uint8Array } diff --git a/packages/miniflare/src/workers/d1/constants.ts b/packages/miniflare/src/workers/d1/constants.ts index b98d0e7484..6e614f6666 100644 --- a/packages/miniflare/src/workers/d1/constants.ts +++ b/packages/miniflare/src/workers/d1/constants.ts @@ -1 +1,8 @@ export const D1_LOCAL_ENTRY_SERVICE_NAME = "d1:db:entry"; +// The Worker running workerd's D1 (the bundled module `/workerd/d1.js`), +// reached with props `{ database }`. +export const D1_WORKERD_SERVICE_NAME = "d1:workerd"; + +export const D1Bindings = { + DEBUG_PORT: "MINIFLARE_DEBUG_PORT", +} as const; diff --git a/packages/miniflare/src/workers/d1/database.worker.ts b/packages/miniflare/src/workers/d1/database.worker.ts deleted file mode 100644 index 5cd3caf3bc..0000000000 --- a/packages/miniflare/src/workers/d1/database.worker.ts +++ /dev/null @@ -1,266 +0,0 @@ -import assert from "node:assert"; -import { - get, - HttpError, - MiniflareDurableObject, - POST, - viewToBuffer, -} from "miniflare:shared"; -import { z } from "miniflare:zod"; -import { dumpSql } from "./dumpSql"; -import { isD1ExportPragma } from "./export"; -import { - D1QueriesSchema, - D1_SESSION_COMMIT_TOKEN_HTTP_HEADER, -} from "./schemas"; -import type { D1ExportPragma } from "./export"; -import type { D1Meta, D1Query, D1RowsAndColumns, D1Value } from "./schemas"; -import type { - MiniflareDurableObjectEnv, - RouteHandler, - TypedSqlStorage, - TypedValue, -} from "miniflare:shared"; - -const D1ResultsFormatSchema = z - .enum(["ARRAY_OF_OBJECTS", "ROWS_AND_COLUMNS", "NONE"]) - .catch("ARRAY_OF_OBJECTS"); - -type D1ResultsFormat = z.infer; - -type D1Results = D1RowsAndColumns | Record[] | undefined; - -const served_by = "miniflare.db"; -interface D1SuccessResponse { - success: true; - results: D1Results; - meta: D1Meta; -} -interface D1FailureResponse { - success: false; - error: string; -} - -export class D1Error extends HttpError { - constructor(readonly cause: unknown) { - super(500); - } - - toResponse(): Response { - const error = - typeof this.cause === "object" && - this.cause !== null && - "message" in this.cause && - typeof this.cause.message === "string" - ? this.cause.message - : String(this.cause); - const response: D1FailureResponse = { success: false, error }; - return Response.json(response); - } -} - -function convertParams(params: D1Query["params"]): TypedValue[] { - return (params ?? []).map((param) => - // If `param` is an array, assume it's a byte array - Array.isArray(param) ? viewToBuffer(new Uint8Array(param)) : param - ); -} - -function convertRows(rows: TypedValue[][]): D1Value[][] { - return rows.map((row) => - row.map((value) => { - let normalised: D1Value; - if (value instanceof ArrayBuffer) { - // If `value` is an array, convert it to a regular numeric array - normalised = Array.from(new Uint8Array(value)); - } else { - normalised = value; - } - return normalised; - }) - ); -} - -function rowsToObjects( - columns: string[], - rows: D1Value[][] -): Record[] { - return rows.map((row) => - Object.fromEntries(columns.map((name, i) => [name, row[i]])) - ); -} - -function sqlStmts(db: TypedSqlStorage) { - return { - getChanges: db.prepare<[], { totalChanges: number; lastRowId: number }>( - "SELECT total_changes() AS totalChanges, last_insert_rowid() AS lastRowId" - ), - }; -} - -export class D1DatabaseObject extends MiniflareDurableObject { - readonly #stmts: ReturnType; - - constructor(state: DurableObjectState, env: MiniflareDurableObjectEnv) { - super(state, env); - this.#stmts = sqlStmts(this.db); - } - - #changes() { - const changes = get(this.#stmts.getChanges()); - assert(changes !== undefined); - return changes; - } - - #query = (format: D1ResultsFormat, query: D1Query): D1SuccessResponse => { - const beforeSize = this.state.storage.sql.databaseSize; - const beforeChanges = this.#changes(); - - const params = convertParams(query.params ?? []); - // Include cursor consumption in SQL time, but exclude metadata queries and - // blob/object conversion so duration has the same meaning as in D1. - const beforeTime = performance.now(); - const cursor = this.db.prepare(query.sql)(...params); - const columns = cursor.columnNames; - const rawRows = Array.from(cursor.raw()); - const duration = performance.now() - beforeTime; - const rows = convertRows(rawRows); - - let results = undefined; - if (format === "ROWS_AND_COLUMNS") { - results = { columns, rows }; - } else { - results = rowsToObjects(columns, rows); - } - // Note that the "NONE" format behaviour here is inconsistent with workerd. - // See comment: https://github.com/cloudflare/workers-sdk/pull/5917#issuecomment-2133313156 - - const afterSize = this.state.storage.sql.databaseSize; - const afterChanges = this.#changes(); - - const changes = afterChanges.totalChanges - beforeChanges.totalChanges; - - const hasChanges = changes !== 0; - const lastRowChanged = afterChanges.lastRowId !== beforeChanges.lastRowId; - const sizeChanged = afterSize !== beforeSize; - const changed = hasChanges || lastRowChanged || sizeChanged; - - return { - success: true, - results, - meta: { - served_by, - served_by_primary: true, - total_attempts: 1, - timings: { sql_duration_ms: duration }, - duration, - changes, - last_row_id: afterChanges.lastRowId, - changed_db: changed, - size_after: afterSize, - rows_read: cursor.rowsRead, - rows_written: cursor.rowsWritten, - }, - }; - }; - - /** - * Runs all queries in one storage transaction. - * - * @param queries The queries to run in order. - * @param format The format for the result rows. - * @param includeCommitToken Whether to get the current session bookmark. - * @returns The query results, and the session bookmark if requested. - */ - async #txn( - queries: D1Query[], - format: D1ResultsFormat, - includeCommitToken: boolean - ): Promise<{ results: D1SuccessResponse[]; commitToken?: string }> { - // SQLite rejects statements that are just comments, so filter them out. - // Match remote D1, which also filters `--` comments at offset 0. - queries = queries.filter( - (query) => query.sql.replace(/^\s*--.*/gm, "").trim().length > 0 - ); - if (queries.length === 0) { - const error = new Error("No SQL statements detected."); - throw new D1Error(error); - } - - try { - // Roll back the queries if bookmark retrieval fails, so a catchable - // D1_ERROR never reports these writes as failed after committing them. - return await this.state.storage.transaction(async () => { - const results = queries.map(this.#query.bind(this, format)); - // In local development, workerd stores a new bookmark each time one is - // requested. Only sessions use bookmarks, so other requests skip this - // write and read-only queries commit nothing, as in production D1. - if (!includeCommitToken) { - return { results }; - } - let commitToken: string; - try { - commitToken = await this.state.storage.getCurrentBookmark(); - } catch (e) { - const message = e instanceof Error ? e.message : String(e); - throw new Error(`Failed to get session commit token: ${message}`, { - cause: e, - }); - } - return { results, commitToken }; - }); - } catch (e) { - throw new D1Error(e); - } - } - - @POST("/query") - @POST("/execute") - queryExecute: RouteHandler = async (req) => { - let queries = D1QueriesSchema.parse(await req.json()); - if (!Array.isArray(queries)) { - queries = [queries]; - } - - // Special local-mode-only handlers - if (isD1ExportPragma(queries)) { - return this.#doExportData(queries); - } - - const { searchParams } = new URL(req.url); - const resultsFormat = D1ResultsFormatSchema.parse( - searchParams.get("resultsFormat") - ); - - // The binding sends this header only for `withSession()` sessions. - const includeCommitToken = Boolean( - req.headers.get(D1_SESSION_COMMIT_TOKEN_HTTP_HEADER)?.trim() - ); - const { results, commitToken } = await this.#txn( - queries, - resultsFormat, - includeCommitToken - ); - - return Response.json(results, { - headers: - commitToken === undefined - ? undefined - : { [D1_SESSION_COMMIT_TOKEN_HTTP_HEADER]: commitToken }, - }); - }; - - #doExportData(queries: D1ExportPragma) { - const [noSchema, noData, ...tables] = queries[0].params; - const options = { - noSchema: Boolean(noSchema), - noData: Boolean(noData), - tables, - }; - return Response.json({ - success: true, - results: [Array.from(dumpSql(this.state.storage.sql, options))], - meta: {}, - }); - } -} diff --git a/packages/miniflare/src/workers/d1/dumpSql.ts b/packages/miniflare/src/workers/d1/dumpSql.ts deleted file mode 100644 index 58d2220b68..0000000000 --- a/packages/miniflare/src/workers/d1/dumpSql.ts +++ /dev/null @@ -1,278 +0,0 @@ -// NOTE: this file is duplicated between miniflare and d1-workers, and should be kept in sync by hand. -// Please tag someone from the D1 team as a reviewer and they will do this for you. -// -// NOTE: this function duplicates the logic inside SQLite's shell.c.in as close -// as possible, with any deviations noted. -import type { SqlStorage } from "@cloudflare/workers-types/experimental"; - -/** Stats tracking for dumpSql. Will be mutated in place if provided. */ -export interface DumpSqlStats { - rows_read: number; - rows_written: number; - // Stats for tracking INSERT statement sizes (column names are always included) - /** Number of INSERT statements over 100KB (current size, with column names) */ - inserts_over_100kb_with_column_names: number; - /** Number of INSERT statements that would be over 100KB without column names (for backward comparison) */ - inserts_already_over_100kb: number; - /** Total number of INSERT statements generated */ - total_inserts: number; - /** Maximum INSERT statement size without column names (hypothetical, for backward comparison) */ - max_insert_size: number; - /** Maximum INSERT statement size (current size, with column names) */ - max_insert_size_with_column_names: number; -} - -export function* dumpSql( - db: SqlStorage, - options?: { - noSchema?: boolean; - noData?: boolean; - tables?: string[]; - }, - /** Optional stats tracking. Will be mutated in place if provided */ - stats?: DumpSqlStats -) { - // WARNING: the caller in D1 assumes non-empty exports, so think carefully before removing this initial yield. - yield `PRAGMA defer_foreign_keys=TRUE;`; - - // Empty set means include all tables - const filterTables = new Set(options?.tables || []); - const { noData, noSchema } = options || {}; - - // Taken from SQLite shell.c.in https://github.com/sqlite/sqlite/blob/105c20648e1b05839fd0638686b95f2e3998abcb/src/shell.c.in#L8463-L8469 - // @ts-expect-error -- SqlStorageStatement needs to be callable - const tables_cursor = db.prepare(` - SELECT name, type, sql - FROM sqlite_schema AS o - WHERE (true) AND type=='table' - AND sql NOT NULL - ORDER BY tbl_name='sqlite_sequence', rowid; - `)(); - const tables: { name: string; type: string; sql: string }[] = - Array.from(tables_cursor); - if (stats) { - stats.rows_read += tables_cursor.rowsRead; - stats.rows_written += tables_cursor.rowsWritten; - } - - for (const { name: table, sql } of tables) { - if (filterTables.size > 0 && !filterTables.has(table)) { - continue; - } - - if (table === "sqlite_sequence") { - if (!noSchema) { - yield `DELETE FROM sqlite_sequence;`; - } - } else if (table.match(/^sqlite_stat./)) { - // This feels like it should really appear _after_ the contents of sqlite_stat[1,4] but I'm choosing - // to match SQLite's dump output exactly so writing it immediately like they do. - if (!noSchema) { - yield `ANALYZE sqlite_schema;`; - } - } else if (sql.startsWith(`CREATE VIRTUAL TABLE`)) { - throw new Error( - `D1 Export error: cannot export databases with Virtual Tables (fts5)` - ); - } else if (table.startsWith("_cf_") || table.startsWith("sqlite_")) { - continue; - } else { - // SQLite dump has an extremely weird behaviour where, if the table was explicitly - // quoted i.e. "Table", then in the dump it has `IF NOT EXISTS` injected. I don't understand - // why, but on the off chance there's a good reason to I am following suit. - if (sql.match(/CREATE TABLE ['"].*/)) { - if (!noSchema) { - yield `CREATE TABLE IF NOT EXISTS ${sql.substring(13)};`; - } - } else { - if (!noSchema) { - yield `${sql};`; - } - } - } - - if (noData) { - continue; - } - // eslint-disable-next-line workers-sdk/no-unsafe-command-execution -- input is escaped via escapeId() so this PRAGMA call is safe - const columns_cursor = db.exec(`PRAGMA table_info=${escapeId(table)}`); - - const columns = Array.from(columns_cursor) as { - cid: string; - name: string; - type: string; - notnull: number; - dflt_val: string | null; - pk: number; - }[]; - if (stats) { - stats.rows_read += columns_cursor.rowsRead; - stats.rows_written += columns_cursor.rowsWritten; - } - - const select = `SELECT ${columns.map((c) => escapeId(c.name)).join(", ")} FROM ${escapeId(table)};`; - const rows_cursor = db.exec(select); - const columnNames = columns.map((c) => escapeId(c.name)).join(","); - // The column names portion is: " (" + columnNames + ")" = 3 + columnNames.length - const columnNamesOverhead = 3 + columnNames.length; - for (const dataRow of rows_cursor.raw()) { - const formattedCells = dataRow.map((cell: unknown, i: number) => { - const colType = columns[i].type; - const cellType = typeof cell; - if (cell === null) { - return "NULL"; - } else if (cellType === "number") { - if (!Number.isFinite(cell)) { - return (cell as number) > 0 ? "9e999" : "-9e999"; - } - return cell; - } else if (cellType === "string") { - return outputQuotedEscapedString(cell); - } else if (cell instanceof ArrayBuffer) { - return `X'${Array.prototype.map - .call(new Uint8Array(cell), (b) => b.toString(16).padStart(2, "0")) - .join("")}'`; - } else { - console.error({ - message: "dumpSql: unexpected cell type", - colType, - cellType, - cell, - column: columns[i], - }); - return "ERROR"; - } - }); - - const insertStmt = `INSERT INTO ${escapeId(table)} (${columnNames}) VALUES(${formattedCells.join(",")});`; - - // Track stats for INSERT statement sizes - if (stats) { - const currentSize = insertStmt.length; - // Calculate what the size would be without column names (for comparison) - const sizeWithoutColumnNames = currentSize - columnNamesOverhead; - const LIMIT = 100 * 1024; // 100KB - - stats.total_inserts++; - if (sizeWithoutColumnNames > LIMIT) { - stats.inserts_already_over_100kb++; - } - if (currentSize > LIMIT) { - stats.inserts_over_100kb_with_column_names++; - } - if (sizeWithoutColumnNames > stats.max_insert_size) { - stats.max_insert_size = sizeWithoutColumnNames; - } - if (currentSize > stats.max_insert_size_with_column_names) { - stats.max_insert_size_with_column_names = currentSize; - } - } - - yield insertStmt; - } - if (stats) { - stats.rows_read += rows_cursor.rowsRead; - stats.rows_written += rows_cursor.rowsWritten; - } - } - - if (!noSchema) { - // Taken from SQLite shell.c.in https://github.com/sqlite/sqlite/blob/105c20648e1b05839fd0638686b95f2e3998abcb/src/shell.c.in#L8473-L8478 - const rest_of_schema = db.exec( - [ - "SELECT name, sql", - "FROM sqlite_schema AS o", - "WHERE (true) AND sql NOT NULL", - " AND type IN ('index', 'trigger', 'view')", - // 'DESC' appears in the code linked above but the observed behaviour of SQLite appears otherwise - "ORDER BY type COLLATE NOCASE", - ].join(" ") - ); - for (const { name, sql } of rest_of_schema) { - if (filterTables.size > 0 && !filterTables.has(name as string)) { - continue; - } - yield `${sql};`; - } - if (stats) { - stats.rows_read += rest_of_schema.rowsRead; - stats.rows_written += rest_of_schema.rowsWritten; - } - } -} - -// Ported `output_quoted_escaped_string` from https://github.com/sqlite/sqlite/blob/master/src/shell.c.in#L1799-L1862 -function outputQuotedEscapedString(cell: unknown) { - let lfs = false; - let crs = false; - - const quotesOrNewlinesRegexp = /'|(\n)|(\r)/g; - - // Function to replace ' with '', while also tracking whether the string contains any \r or \n chars - const escapeQuotesDetectingNewlines = (_: string, lf: string, cr: string) => { - if (lf) { - lfs = true; - return `\\n`; - } - if (cr) { - crs = true; - return `\\r`; - } - return `''`; - }; - - const escaped_string = (cell as string).replace( - quotesOrNewlinesRegexp, - escapeQuotesDetectingNewlines - ); - let output_string = `'${escaped_string}'`; - if (crs) { - output_string = `replace(${output_string},'\\r',char(13))`; - } - if (lfs) { - output_string = `replace(${output_string},'\\n',char(10))`; - } - return output_string; -} - -// Ported from quoteChar: https://github.com/sqlite/sqlite/blob/378bf82e2bc09734b8c5869f9b148efe37d29527/src/shell.c.in#L990 -export function sqliteQuote(token: string) { - // Empty input - return token.length === 0 || - // Doesn't start with alpha or underscore - !token.match(/^[a-zA-Z_]/) || - token.match(/\W/) || - SQLITE_KEYWORDS.has(token.toUpperCase()) - ? `"${token}"` - : token; -} - -/** - * Escape an identifier for use in SQL statements. - * @param id - The identifier to escape. - * @returns - */ -function escapeId(id: string) { - return `"${id.replace(/"/g, '""')}"`; -} - -// List taken from `aKeywordTable` inhttps://github.com/sqlite/sqlite/blob/378bf82e2bc09734b8c5869f9b148efe37d29527/tool/mkkeywordhash.c#L172 -// prettier-ignore -export const SQLITE_KEYWORDS = new Set([ - "ABORT", "ACTION", "ADD", "AFTER", "ALL", "ALTER", "ALWAYS", "ANALYZE", "AND", "AS", "ASC", - "ATTACH", "AUTOINCREMENT", "BEFORE", "BEGIN", "BETWEEN", "BY", "CASCADE", "CASE", "CAST", - "CHECK", "COLLATE", "COLUMN", "COMMIT", "CONFLICT", "CONSTRAINT", "CREATE", "CROSS", "CURRENT", - "CURRENT_DATE", "CURRENT_TIME", "CURRENT_TIMESTAMP", "DATABASE", "DEFAULT", "DEFERRED", - "DEFERRABLE", "DELETE", "DESC", "DETACH", "DISTINCT", "DO", "DROP", "END", "EACH", "ELSE", - "ESCAPE", "EXCEPT", "EXCLUSIVE", "EXCLUDE", "EXISTS", "EXPLAIN", "FAIL", "FILTER", "FIRST", - "FOLLOWING", "FOR", "FOREIGN", "FROM", "FULL", "GENERATED", "GLOB", "GROUP", "GROUPS", "HAVING", - "IF", "IGNORE", "IMMEDIATE", "IN", "INDEX", "INDEXED", "INITIALLY", "INNER", "INSERT", "INSTEAD", - "INTERSECT", "INTO", "IS", "ISNULL", "JOIN", "KEY", "LAST", "LEFT", "LIKE", "LIMIT", "MATCH", - "MATERIALIZED", "NATURAL", "NO", "NOT", "NOTHING", "NOTNULL", "NULL", "NULLS", "OF", "OFFSET", - "ON", "OR", "ORDER", "OTHERS", "OUTER", "OVER", "PARTITION", "PLAN", "PRAGMA", "PRECEDING", - "PRIMARY", "QUERY", "RAISE", "RANGE", "RECURSIVE", "REFERENCES", "REGEXP", "REINDEX", "RELEASE", - "RENAME", "REPLACE", "RESTRICT", "RETURNING", "RIGHT", "ROLLBACK", "ROW", "ROWS", "SAVEPOINT", - "SELECT", "SET", "TABLE", "TEMP", "TEMPORARY", "THEN", "TIES", "TO", "TRANSACTION", "TRIGGER", - "UNBOUNDED", "UNION", "UNIQUE", "UPDATE", "USING", "VACUUM", "VALUES", "VIEW", "VIRTUAL", "WHEN", - "WHERE", "WINDOW", "WITH", "WITHOUT" -]); diff --git a/packages/miniflare/src/workers/d1/entry.worker.ts b/packages/miniflare/src/workers/d1/entry.worker.ts index c2c41c5e47..4d38544b62 100644 --- a/packages/miniflare/src/workers/d1/entry.worker.ts +++ b/packages/miniflare/src/workers/d1/entry.worker.ts @@ -1,97 +1,42 @@ import { WorkerEntrypoint } from "cloudflare:workers"; import { SharedBindings, SharedHeaders } from "../shared/constants"; -import { isD1ExportPragma, parseD1RpcExportResponse } from "./export"; -import { - D1RowsAndColumnsResponseSchema, - D1RpcRequestSchema, - D1_SESSION_COMMIT_TOKEN_HTTP_HEADER, -} from "./schemas"; -import type { D1RpcExportResponse } from "./export"; -import type { D1RpcRequest, D1RpcResponse } from "./schemas"; +import { D1Bindings, D1_WORKERD_SERVICE_NAME } from "./constants"; +import type { WorkerdDebugPortConnector } from "../core/dev-registry-proxy-shared.worker"; -interface D1EntryProps { - // The shared routing key holds this binding's database ID, not a DO namespace. +interface Props { [SharedBindings.TEXT_NAMESPACE]?: string; } -interface D1EntryEnv { - [SharedBindings.TEXT_NAMESPACE]?: string; - // All local databases live in this namespace; the ID in props selects one. - [SharedBindings.DURABLE_OBJECT_NAMESPACE_OBJECT]: DurableObjectNamespace; +interface Env { + [D1Bindings.DEBUG_PORT]: WorkerdDebugPortConnector; } +type D1Fetcher = Fetcher & { query(request: unknown): Promise }; -/** Routes D1 HTTP and RPC requests to the same local SQLite database. */ -export default class D1Entry extends WorkerEntrypoint< - D1EntryEnv, - D1EntryProps -> { - /** Preserves the HTTP protocol used by bindings without the RPC flag. */ - async fetch(request: Request): Promise { - // Local Explorer addresses unbound databases through the internal header; - // preserve the shared entry worker's props/binding/header precedence. - const requestNamespace = request.headers.get(SharedHeaders.NAMESPACE); - const name = - this.ctx.props[SharedBindings.TEXT_NAMESPACE] ?? - this.env[SharedBindings.TEXT_NAMESPACE] ?? - requestNamespace; - if (name === null) { - throw new Error( - "D1 entry worker: no database ID provided via props, binding, or request header" - ); +// Routes D1's HTTP and RPC protocols to workerd's D1 for a database named at +// runtime, by props or the `MF-Namespace` header, which a binding's fixed +// props cannot do: the local explorer and shared storage call this. +export default class D1Entry extends WorkerEntrypoint { + #d1(request?: Request): D1Fetcher { + const database = + this.ctx.props?.[SharedBindings.TEXT_NAMESPACE] ?? + request?.headers.get(SharedHeaders.NAMESPACE) ?? + null; + if (database === null) { + throw new Error("d1:db:entry: no database in props or header"); } - const namespace = this.env[SharedBindings.DURABLE_OBJECT_NAMESPACE_OBJECT]; - const stub = namespace.get(namespace.idFromName(name)); - if (requestNamespace !== null) { - const headers = new Headers(request.headers); - headers.delete(SharedHeaders.NAMESPACE); - request = new Request(request, { headers }); - } - // The shared DO base uses this name to initialise its storage namespace. - return await stub.fetch(request, { cf: { miniflare: { name } } }); + return this.env[D1Bindings.DEBUG_PORT] + .current() + .getEntrypoint(D1_WORKERD_SERVICE_NAME, undefined, { + database, + }) as D1Fetcher; } - /** Returns raw transactional results; query failures are thrown over RPC. */ - async query( - request: D1RpcRequest - ): Promise { - const { queries, bookmark } = D1RpcRequestSchema.parse(request); - const headers = new Headers({ "Content-Type": "application/json" }); - if (bookmark) { - headers.set(D1_SESSION_COMMIT_TOKEN_HTTP_HEADER, bookmark); - } + fetch(request: Request): Promise { + const headers = new Headers(request.headers); + headers.delete(SharedHeaders.NAMESPACE); + return this.#d1(request).fetch(new Request(request, { headers })); + } - // Reuse the HTTP execution path so transactions, validation, and metadata - // remain owned by the database worker rather than duplicated for RPC. - const response = await this.fetch( - new Request("http://d1/query?resultsFormat=ROWS_AND_COLUMNS", { - method: "POST", - headers, - body: JSON.stringify(queries), - }) - ); - const responseBody: unknown = await response.json(); - if (isD1ExportPragma(queries)) { - return parseD1RpcExportResponse(responseBody); - } - - const body = D1RowsAndColumnsResponseSchema.parse(responseBody); - if (!Array.isArray(body)) { - const error = new Error(body.error); - error.name = "D1_ERROR"; - throw error; - } - - const result: D1RpcResponse = { - results: body.map(({ meta, results }) => ({ - meta, - data: { kind: "raw", ...results }, - })), - }; - const responseBookmark = response.headers.get( - D1_SESSION_COMMIT_TOKEN_HTTP_HEADER - ); - if (responseBookmark) { - result.bookmark = responseBookmark; - } - return result; + query(request: unknown): Promise { + return this.#d1().query(request); } } diff --git a/packages/miniflare/src/workers/d1/export.ts b/packages/miniflare/src/workers/d1/export.ts deleted file mode 100644 index 9f6cc38038..0000000000 --- a/packages/miniflare/src/workers/d1/export.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { z } from "miniflare:zod"; -import { D1FailureResponseSchema } from "./schemas"; -import type { D1Query, D1RpcResultData } from "./schemas"; - -/** Local-only pragma used to export a database's SQL schema and data. */ -export const D1_EXPORT_PRAGMA = "PRAGMA miniflare_d1_export(?,?,?);"; - -/** Export flags followed by an optional list of table names. */ -export type D1ExportPragma = [ - { sql: typeof D1_EXPORT_PRAGMA; params: [number, number, ...string[]] }, -]; - -/** Matches the database worker's existing local export dispatch rules. */ -export function isD1ExportPragma( - queries: D1Query[] -): queries is D1ExportPragma { - return ( - queries.length === 1 && - queries[0].sql === D1_EXPORT_PRAGMA && - (queries[0].params?.length || 0) >= 2 - ); -} - -const D1ExportMetaSchema = z.record(z.string(), z.never()); -type D1ExportMeta = z.infer; - -const D1ExportResponseSchema = z.union([ - z.object({ - success: z.literal(true), - results: z.tuple([z.array(z.string())]), - meta: D1ExportMetaSchema, - }), - D1FailureResponseSchema, -]); - -/** Local exports return one dump row and do not report execution metadata. */ -export interface D1RpcExportResponse { - results: [{ meta: D1ExportMeta; data: D1RpcResultData }]; -} - -/** Adapts the legacy local export response, throwing backend failures over RPC. */ -export function parseD1RpcExportResponse(input: unknown): D1RpcExportResponse { - const body = D1ExportResponseSchema.parse(input); - if (!body.success) { - const error = new Error(body.error); - error.name = "D1_ERROR"; - throw error; - } - - return { - results: [ - { - meta: body.meta, - data: { - kind: "raw", - // The HTTP wrapper exposes the export row's array indices - // as column names; preserve that convention over RPC. - columns: body.results[0].map((_, index) => String(index)), - rows: body.results, - }, - }, - ], - }; -} diff --git a/packages/miniflare/src/workers/d1/schemas.ts b/packages/miniflare/src/workers/d1/schemas.ts deleted file mode 100644 index 1f0b2fcdcb..0000000000 --- a/packages/miniflare/src/workers/d1/schemas.ts +++ /dev/null @@ -1,93 +0,0 @@ -import { z } from "miniflare:zod"; - -/** Internal bookmark header shared by the workerd wrapper and D1 backend. */ -export const D1_SESSION_COMMIT_TOKEN_HTTP_HEADER = - "x-cf-d1-session-commit-token"; - -/** JSON representation of SQLite values used by the D1 HTTP protocol. */ -export const D1ValueSchema = z.union([ - z.number(), - z.string(), - z.null(), - z.number().array(), -]); -/** Blobs use numeric byte arrays rather than ArrayBuffers on the wire. */ -export type D1Value = z.infer; - -/** SQL and positional parameters accepted by the local database worker. */ -export const D1QuerySchema = z.object({ - sql: z.string(), - params: z.array(D1ValueSchema).nullable().optional(), -}); -/** Parameters are converted to SQLite values by the database worker. */ -export type D1Query = z.infer; - -/** The HTTP protocol accepts either one query or a transactional batch. */ -export const D1QueriesSchema = z.union([D1QuerySchema, z.array(D1QuerySchema)]); - -/** RPC queries always contain a batch and may specify a session bookmark. */ -export const D1RpcRequestSchema = z.object({ - queries: z.array(D1QuerySchema), - bookmark: z.string().optional(), -}); -/** The bookmark may also be a first-query routing constraint. */ -export type D1RpcRequest = z.infer; - -/** D1 query metadata; geography is optional because local execution has no colo. */ -export const D1MetaSchema = z.object({ - served_by: z.string(), - served_by_primary: z.boolean().optional(), - served_by_region: z.string().optional(), - served_by_colo: z.string().optional(), - timings: z.object({ sql_duration_ms: z.number() }).optional(), - duration: z.number(), - changes: z.number(), - last_row_id: z.number(), - changed_db: z.boolean(), - size_after: z.number(), - rows_read: z.number(), - rows_written: z.number(), - total_attempts: z.number().optional(), -}); -/** SQL duration excludes request handling and result serialization. */ -export type D1Meta = z.infer; - -/** Rows-and-columns results preserve duplicate column names and empty rows. */ -export const D1RowsAndColumnsSchema = z.object({ - columns: z.array(z.string()), - rows: z.array(z.array(D1ValueSchema)), -}); -/** Column names are positional and need not be unique. */ -export type D1RowsAndColumns = z.infer; - -const D1RowsAndColumnsSuccessSchema = z.object({ - success: z.literal(true), - results: D1RowsAndColumnsSchema, - meta: D1MetaSchema, -}); - -/** A backend failure contains no successful query or export results. */ -export const D1FailureResponseSchema = z.object({ - success: z.literal(false), - error: z.string(), -}); - -/** A query failure is returned instead of any successful HTTP batch results. */ -export const D1RowsAndColumnsResponseSchema = z.union([ - z.array(D1RowsAndColumnsSuccessSchema), - D1FailureResponseSchema, -]); - -/** The RPC discriminator identifies the rows-and-columns payload to workerd. */ -export interface D1RpcResultData extends D1RowsAndColumns { - kind: "raw"; -} - -/** RPC successes return results directly; failures are thrown instead. */ -export interface D1RpcResponse { - results: { - meta: D1Meta; - data: D1RpcResultData; - }[]; - bookmark?: string; -} diff --git a/packages/miniflare/test/plugins/d1/index.spec.ts b/packages/miniflare/test/plugins/d1/index.spec.ts index a0b5ad3ce0..3afcdf0f9c 100644 --- a/packages/miniflare/test/plugins/d1/index.spec.ts +++ b/packages/miniflare/test/plugins/d1/index.spec.ts @@ -63,35 +63,6 @@ async function lockForWrites(external: DatabaseSync) { } } -// Regression test for https://github.com/cloudflare/workers-sdk/issues/14916. -test("surfaces recoverable SQLite errors as catchable D1 query errors", async ({ - expect, -}) => { - const { db, external } = await createPersistedDatabase(); - const session = db.withSession(); - await lockForWrites(external); - - try { - // A read succeeds while another connection holds the write lock, but - // writing the session bookmark fails. The error must stay catchable. - await expect( - session.prepare("SELECT * FROM entries").all() - ).rejects.toMatchObject({ - message: expect.stringMatching( - /^D1_ERROR: Failed to get session commit token:/ - ), - cause: { - message: expect.stringMatching(/^Failed to get session commit token:/), - }, - }); - } finally { - external.exec("ROLLBACK"); - } - - const { results } = await db.prepare("SELECT * FROM entries").all(); - expect(results).toEqual([]); -}); - // Regression test for https://github.com/cloudflare/workers-sdk/issues/16549. test("does not write for read-only queries outside a session", async ({ expect, @@ -117,57 +88,3 @@ test("does not write for read-only queries outside a session", async ({ external.exec("ROLLBACK"); } }); - -// Regression test for https://github.com/cloudflare/workers-sdk/pull/14921#discussion_r4047630401. -test.for(["run", "batch"] as const)( - "rolls back %s writes when session bookmark retrieval fails", - async (method, { expect }) => { - const { db, external } = await createPersistedDatabase(); - const session = db.withSession("first-primary"); - await session.prepare("SELECT * FROM entries").all(); - const bookmark = session.getBookmark(); - assert(bookmark !== null); - - // A trigger on workerd's metadata table makes bookmark updates fail - // in its trusted SQLite scope, after the user SQL has succeeded. - // The local session bookmark is stored under metadata key 2. - external.exec(` - CREATE TRIGGER fail_bookmark BEFORE INSERT ON _cf_METADATA - WHEN NEW.key = 2 - BEGIN - SELECT RAISE(ABORT, 'bookmark lookup failed'); - END; - `); - const insert = session.prepare( - "INSERT INTO entries (value) VALUES ('order')" - ); - function execute() { - return method === "run" ? insert.run() : session.batch([insert, insert]); - } - - await expect(execute()).rejects.toMatchObject({ - message: expect.stringMatching( - /^D1_ERROR: Failed to get session commit token:/ - ), - cause: { - message: expect.stringMatching(/^Failed to get session commit token:/), - }, - }); - expect(session.getBookmark()).toBe(bookmark); - // Inspect through the other connection while bookmark lookup still fails. - expect(external.prepare("SELECT * FROM entries").all()).toEqual([]); - - external.exec("DROP TRIGGER fail_bookmark"); - await execute(); - const { results } = await session.prepare("SELECT * FROM entries").all(); - expect(results).toEqual( - method === "run" - ? [{ id: 1, value: "order" }] - : [ - { id: 1, value: "order" }, - { id: 2, value: "order" }, - ] - ); - expect(session.getBookmark()).not.toBe(bookmark); - } -); diff --git a/packages/miniflare/test/plugins/d1/jsrpc.spec.ts b/packages/miniflare/test/plugins/d1/jsrpc.spec.ts index 8d75b2d24c..e0b24b479f 100644 --- a/packages/miniflare/test/plugins/d1/jsrpc.spec.ts +++ b/packages/miniflare/test/plugins/d1/jsrpc.spec.ts @@ -117,7 +117,7 @@ test.for([ const response = await mf.dispatchFetch("http://localhost"); expect(await response.json()).toMatchObject({ meta: { - served_by: "miniflare.db", + served_by: "workerd", served_by_primary: true, total_attempts: 1, duration: expect.any(Number), @@ -161,8 +161,11 @@ test("SQL errors propagate directly and convenience methods preserve their prefi return Response.json(errors); `); const response = await mf.dispatchFetch("http://localhost"); + // workerd's D1 leaves its errors named "Error": a caller without + // enhanced_error_serialization would see a "D1_ERROR" name folded into + // the message, which the binding prefixes again. const backendError = { - name: "D1_ERROR", + name: "Error", message: expect.stringMatching(/^no such table: missing_table/), }; expect(await response.json()).toEqual([ @@ -321,7 +324,7 @@ test.for([ `); const response = await mf.dispatchFetch("http://localhost"); expect(await response.json()).toEqual({ - name: "D1_ERROR", + name: "Error", message, }); expect(response.status).toBe(500); diff --git a/packages/miniflare/test/plugins/d1/suite.ts b/packages/miniflare/test/plugins/d1/suite.ts index 38ad066857..a238c366e7 100644 --- a/packages/miniflare/test/plugins/d1/suite.ts +++ b/packages/miniflare/test/plugins/d1/suite.ts @@ -275,7 +275,7 @@ test("D1PreparedStatement: run", async ({ expect }) => { duration: result.meta.duration, // Not an `INSERT`, so `last_row_id` non-deterministic last_row_id: result.meta.last_row_id, - served_by: "miniflare.db", + served_by: "workerd", served_by_primary: true, total_attempts: 1, timings: { sql_duration_ms: result.meta.duration }, @@ -302,7 +302,7 @@ test("D1PreparedStatement: run", async ({ expect }) => { // Don't know duration, so just match on returned value asserted > 0 duration: result.meta.duration, last_row_id: 4, - served_by: "miniflare.db", + served_by: "workerd", served_by_primary: true, total_attempts: 1, timings: { sql_duration_ms: result.meta.duration }, @@ -331,7 +331,7 @@ test("D1PreparedStatement: run", async ({ expect }) => { last_row_id: result.meta.last_row_id, rows_read: 1, rows_written: 1, - served_by: "miniflare.db", + served_by: "workerd", served_by_primary: true, total_attempts: 1, timings: { sql_duration_ms: result.meta.duration }, @@ -364,7 +364,7 @@ test("D1PreparedStatement: run", async ({ expect }) => { // Don't know duration, so just match on returned value asserted > 0 duration: result.meta.duration, last_row_id: 5, - served_by: "miniflare.db", + served_by: "workerd", served_by_primary: true, total_attempts: 1, timings: { sql_duration_ms: result.meta.duration }, @@ -396,7 +396,7 @@ test("D1PreparedStatement: all", async ({ expect }) => { duration: result.meta.duration, // Not an `INSERT`, so `last_row_id` non-deterministic last_row_id: result.meta.last_row_id, - served_by: "miniflare.db", + served_by: "workerd", served_by_primary: true, total_attempts: 1, timings: { sql_duration_ms: result.meta.duration }, @@ -425,7 +425,7 @@ test("D1PreparedStatement: all", async ({ expect }) => { last_row_id: result.meta.last_row_id, rows_read: 1, rows_written: 0, - served_by: "miniflare.db", + served_by: "workerd", served_by_primary: true, total_attempts: 1, timings: { sql_duration_ms: result.meta.duration },