diff --git a/.changeset/miniflare-workerd-assets.md b/.changeset/miniflare-workerd-assets.md new file mode 100644 index 0000000000..45443d247e --- /dev/null +++ b/.changeset/miniflare-workerd-assets.md @@ -0,0 +1,9 @@ +--- +"miniflare": minor +--- + +Serve local static assets with workerd's native assets service + +Local Workers with `assets` are now served by workerd's `assets` service, which routes each request to the User Worker or to a file as Cloudflare does, and sends the User Worker every other event (RPC, scheduled, tail). It reads files from the assets directory as requests arrive, so Miniflare no longer walks the directory to build a manifest at startup, and files added or changed while it runs are served at once. `_headers`, `_redirects`, `.assetsignore`, `html_handling`, `not_found_handling`, `base_path` and `run_worker_first` behave as before. + +Miniflare no longer checks the asset count and size limits locally; `wrangler deploy` still does. diff --git a/packages/miniflare/scripts/build.mjs b/packages/miniflare/scripts/build.mjs index f7beeee031..142a714ce6 100644 --- a/packages/miniflare/scripts/build.mjs +++ b/packages/miniflare/scripts/build.mjs @@ -99,19 +99,6 @@ const localExplorerWorkerPath = path.join( "local-explorer", "explorer.worker.ts" ); -const assetWorkerPaths = new Set([ - path.join(workersRoot, "assets", "assets.worker.ts"), - path.join(workersRoot, "assets", "router.worker.ts"), -]); -const workersSharedSentryPath = path.join( - pkgRoot, - "..", - "workers-shared", - "utils", - "sentry.ts" -); -const noopSentryPath = path.join(workersRoot, "assets", "sentry-noop.ts"); - /** * Test fixtures that need to be transpiled by esbuild as part of the build. * These are loaded dynamically by tests and must be pre-compiled. @@ -141,28 +128,6 @@ const rewriteNodeToInternalPlugin = { }, }; -/** - * Replaces the production Sentry setup used by the asset and router Workers. - * Miniflare never binds the required Sentry credentials, so the production - * implementation always returns before initialising Toucan. - * - * @type {esbuild.Plugin} - */ -const noopSentryPlugin = { - name: "noop-sentry", - setup(build) { - build.onResolve({ filter: /sentry(?:\.ts)?$/ }, (args) => { - const importPath = path.resolve(args.resolveDir, args.path); - if ( - importPath === workersSharedSentryPath || - `${importPath}.ts` === workersSharedSentryPath - ) { - return { path: noopSentryPath }; - } - }); - }, -}; - /** * esbuild plugin that redirects Zod imports to the shared workerd extension. * @@ -259,7 +224,6 @@ const embedWorkersPlugin = { args.path === miniflareZodExtensionPath ? [rewriteNodeToInternalPlugin] : []), - ...(assetWorkerPaths.has(args.path) ? [noopSentryPlugin] : []), // The Zod extension must bundle the real package, while all other // workers import that single shared copy. ...(args.path === miniflareZodExtensionPath diff --git a/packages/miniflare/src/index.ts b/packages/miniflare/src/index.ts index 25ee1e48a4..6f0d5fae71 100644 --- a/packages/miniflare/src/index.ts +++ b/packages/miniflare/src/index.ts @@ -79,7 +79,7 @@ import { import { getLocalArtifactsNamespaceNames } from "./plugins/artifacts"; import { ArtifactsController } from "./plugins/artifacts/controller"; import { getArtifactsEventSubscriptions } from "./plugins/artifacts/events"; -import { RPC_PROXY_SERVICE_NAME } from "./plugins/assets/constants"; +import { ROUTER_SERVICE_NAME } from "./plugins/assets/constants"; import { BROWSER_VERSION } from "./plugins/browser-rendering/browser-version"; import { closeBrowserProcess } from "./plugins/browser-rendering/process"; import { @@ -2241,7 +2241,7 @@ export class Miniflare { ); if (maybeAssetTargetService && !binding.service?.entrypoint) { assert(binding.service?.name); - binding.service.name = `${RPC_PROXY_SERVICE_NAME}:${targetWorkerName}`; + binding.service.name = `${ROUTER_SERVICE_NAME}:${targetWorkerName}`; } } } @@ -2339,7 +2339,7 @@ export class Miniflare { const service = workerOpts.config.assets && entrypoint === "default" ? { - name: `${RPC_PROXY_SERVICE_NAME}:${workerOpts.config.name}`, + name: `${ROUTER_SERVICE_NAME}:${workerOpts.config.name}`, } : { name: getUserServiceName(serviceName), @@ -2528,7 +2528,7 @@ export class Miniflare { fallbackWorkerName: this.#workerOpts[0].config.assets && !this.#workerOpts[0].config.name.startsWith("vitest-plugin-runner-") - ? `${RPC_PROXY_SERVICE_NAME}:${this.#workerOpts[0].config.name}` + ? `${ROUTER_SERVICE_NAME}:${this.#workerOpts[0].config.name}` : getUserServiceName(this.#workerOpts[0].config.name), tmpPath: this.#tmpPath, log: this.#log, @@ -3022,7 +3022,7 @@ export class Miniflare { workerOpts.dev.unsafeOverrideFetchWorker ); } else if (workerOpts.config.assets) { - defaultEntrypointService = `${RPC_PROXY_SERVICE_NAME}:${workerName}`; + defaultEntrypointService = `${ROUTER_SERVICE_NAME}:${workerName}`; } else { defaultEntrypointService = getUserServiceName(workerName); } diff --git a/packages/miniflare/src/plugins/assets/constants.ts b/packages/miniflare/src/plugins/assets/constants.ts index 376a246156..be7b6bb954 100644 --- a/packages/miniflare/src/plugins/assets/constants.ts +++ b/packages/miniflare/src/plugins/assets/constants.ts @@ -1,5 +1,2 @@ export const ASSETS_PLUGIN_NAME = "assets"; -export const ASSETS_SERVICE_NAME = `${ASSETS_PLUGIN_NAME}:assets-service`; export const ROUTER_SERVICE_NAME = `${ASSETS_PLUGIN_NAME}:router`; -export const RPC_PROXY_SERVICE_NAME = `${ASSETS_PLUGIN_NAME}:rpc-proxy`; -export const ASSETS_KV_SERVICE_NAME = `${ASSETS_PLUGIN_NAME}:kv`; diff --git a/packages/miniflare/src/plugins/assets/index.ts b/packages/miniflare/src/plugins/assets/index.ts index 0719a9d6ec..f979f02f48 100644 --- a/packages/miniflare/src/plugins/assets/index.ts +++ b/packages/miniflare/src/plugins/assets/index.ts @@ -1,25 +1,8 @@ -import crypto from "node:crypto"; import fs from "node:fs/promises"; import path, { join } from "node:path"; -import { - CONTENT_HASH_OFFSET, - ENTRY_SIZE, - getContentType, - HEADER_SIZE, - MAX_ASSET_COUNT, - MAX_ASSET_SIZE, - normalizeFilePath, - PATH_HASH_OFFSET, - PATH_HASH_SIZE, -} from "@cloudflare/workers-shared"; -import { - constructHeaders, - constructRedirects, -} from "@cloudflare/workers-shared/utils/configuration/constructConfiguration"; -import { parseHeaders } from "@cloudflare/workers-shared/utils/configuration/parseHeaders"; -import { parseRedirects } from "@cloudflare/workers-shared/utils/configuration/parseRedirects"; import { parseStaticRouting } from "@cloudflare/workers-shared/utils/configuration/parseStaticRouting"; import { + CF_ASSETS_IGNORE_FILENAME, HEADERS_FILENAME, REDIRECTS_FILENAME, } from "@cloudflare/workers-shared/utils/constants"; @@ -28,31 +11,12 @@ import { InvalidAssetsIgnoreError, maybeGetFile, } from "@cloudflare/workers-shared/utils/helpers"; -import { - HeadersSchema, - RedirectsSchema, -} from "@cloudflare/workers-shared/utils/types"; -import prettyBytes from "pretty-bytes"; -import SCRIPT_ASSETS from "worker:assets/assets"; -import SCRIPT_ASSETS_KV from "worker:assets/assets-kv"; -import SCRIPT_ROUTER from "worker:assets/router"; -import SCRIPT_RPC_PROXY from "worker:assets/rpc-proxy"; import { MiniflareCoreError } from "../../shared"; -import { SharedBindings } from "../../workers"; -import { getUserServiceName } from "../core"; +import { FALLBACK_COMPATIBILITY_DATE, getUserServiceName } from "../core"; import { getEnvBindingsOfType, ProxyNodeBinding } from "../shared"; -import { - ASSETS_KV_SERVICE_NAME, - ASSETS_PLUGIN_NAME, - ASSETS_SERVICE_NAME, - ROUTER_SERVICE_NAME, - RPC_PROXY_SERVICE_NAME, -} from "./constants"; -import type { ParsedMiniflareWorkerConfig } from "../../config/schema"; +import { ASSETS_PLUGIN_NAME, ROUTER_SERVICE_NAME } from "./constants"; import type { Service } from "../../runtime"; import type { Plugin } from "../shared"; -import type { Logger, RouterConfig } from "@cloudflare/workers-shared"; -import type { AssetConfig } from "@cloudflare/workers-shared/utils/types"; function resolveAssetDirectory( rootPath: string | undefined, @@ -64,32 +28,6 @@ function resolveAssetDirectory( return directory; } -/** - * Builds the router worker's `RouterConfig` from the parsed worker config, - * mirroring `resolveAssetOptions` in `@cloudflare/deploy-helpers`: - * - `has_user_worker` comes from the explicit `assets.hasUserWorker` flag - * (defaults to false). It can't be inferred from manifest presence because - * wrangler injects a placeholder script for assets-only workers. - * - `run_worker_first: boolean` → `invoke_user_worker_ahead_of_assets`. - * - `run_worker_first: string[]` → parsed `static_routing` rules. - */ -function resolveRouterConfig( - config: ParsedMiniflareWorkerConfig -): RouterConfig { - const routerConfig: RouterConfig = { - has_user_worker: config.assets?.hasUserWorker ?? false, - }; - - const runWorkerFirst = config.assets?.runWorkerFirst; - if (typeof runWorkerFirst === "boolean") { - routerConfig.invoke_user_worker_ahead_of_assets = runWorkerFirst; - } else if (Array.isArray(runWorkerFirst)) { - routerConfig.static_routing = parseStaticRouting(runWorkerFirst); - } - - return routerConfig; -} - export const ASSETS_PLUGIN: Plugin = { bindingTypeDescription: "Assets", async getBindings(options) { @@ -103,7 +41,8 @@ export const ASSETS_PLUGIN: Plugin = { // binding between User Worker and Asset Worker name: bindingName, service: { - name: `${ASSETS_SERVICE_NAME}:${options.config.name}`, + name: `${ROUTER_SERVICE_NAME}:${options.config.name}`, + entrypoint: "AssetWorker", }, }, ]; @@ -119,7 +58,7 @@ export const ASSETS_PLUGIN: Plugin = { }; }, - async getServices({ options, log, tmpPath }) { + async getServices({ options, tmpPath }) { const assets = options.config.assets; if (!assets) { return []; @@ -141,389 +80,54 @@ export const ASSETS_PLUGIN: Plugin = { // It lives inside this instance's temporary directory so `dispose()` // and the exit hook remove it, and its fixed path lets repeated // `getServices()` calls reuse it. Workers can share it because it is - // always empty and the disk service is read-only. + // always empty and the assets service only reads it. assetDirectory = path.join(tmpPath, ASSETS_PLUGIN_NAME, "empty"); await fs.mkdir(assetDirectory, { recursive: true }); } - const storageServiceName = `${ASSETS_PLUGIN_NAME}:storage:${options.config.name}`; - const storageService: Service = { - name: storageServiceName, - disk: { - path: assetDirectory, - // Reject writes even when callers bypass the read-only namespace Worker. - writable: false, - allowDotfiles: true, - }, - }; - - const { encodedAssetManifest, assetsReverseMap } = - await buildAssetManifest(assetDirectory); - - const redirectsFile = join(assetDirectory, REDIRECTS_FILENAME); - const headersFile = join(assetDirectory, HEADERS_FILENAME); - - const redirectsContents = maybeGetFile(redirectsFile); - const headersContents = maybeGetFile(headersFile); - - const assetParserLogger = { - debug: (message) => log.debug(message), - log: (message) => log.info(message), - info: (message) => log.info(message), - warn: (message) => log.warn(message), - error: (error) => log.error(error), - } satisfies Logger; - - let parsedRedirects: AssetConfig["redirects"] | undefined; - if (redirectsContents !== undefined) { - const redirects = parseRedirects(redirectsContents, { - htmlHandling: assets.htmlHandling, - }); - parsedRedirects = RedirectsSchema.parse( - constructRedirects({ - redirects, - redirectsFile, - logger: assetParserLogger, - }).redirects - ); + // Rejected here, as wrangler would, rather than when workerd loads. + if (Array.isArray(assets.runWorkerFirst)) { + parseStaticRouting(assets.runWorkerFirst); } - - let parsedHeaders: AssetConfig["headers"] | undefined; - if (headersContents !== undefined) { - const headers = parseHeaders(headersContents); - parsedHeaders = HeadersSchema.parse( - constructHeaders({ - headers, - headersFile, - logger: assetParserLogger, - }).headers - ); - } - - const routerConfig = resolveRouterConfig(options.config); - - const assetConfig: AssetConfig = { - compatibility_date: options.config.compatibilityDate, - compatibility_flags: options.config.compatibilityFlags, - html_handling: assets.htmlHandling, - not_found_handling: assets.notFoundHandling, - base_path: assets.basePath, - redirects: parsedRedirects, - headers: parsedHeaders, - debug: true, - has_static_routing: Boolean(routerConfig.static_routing), - }; - - const id = options.config.name; - - const namespaceService: Service = { - name: `${ASSETS_KV_SERVICE_NAME}:${id}`, - worker: { - compatibilityDate: "2023-07-24", - compatibilityFlags: ["nodejs_compat"], - modules: [ - { - name: "assets-kv-worker.mjs", - esModule: SCRIPT_ASSETS_KV(), - }, - ], - bindings: [ - { - name: SharedBindings.MAYBE_SERVICE_BLOBS, - service: { name: storageServiceName }, - }, - { - name: "ASSETS_REVERSE_MAP", - json: JSON.stringify(assetsReverseMap), - }, - ], - }, - }; - - const assetService: Service = { - name: `${ASSETS_SERVICE_NAME}:${id}`, - worker: { - // TODO: read these from the wrangler.toml - compatibilityDate: "2024-07-31", - compatibilityFlags: ["nodejs_compat", "enable_ctx_exports"], - modules: [ - { - name: "asset-worker.mjs", - esModule: SCRIPT_ASSETS(), - }, - ], - bindings: [ - { - name: "ASSETS_KV_NAMESPACE", - kvNamespace: { - name: `${ASSETS_KV_SERVICE_NAME}:${id}`, - }, - }, - { - name: "ASSETS_MANIFEST", - data: encodedAssetManifest, - }, - { - name: "CONFIG", - json: JSON.stringify(assetConfig), - }, - ], - }, - }; - - const routerService: Service = { - name: `${ROUTER_SERVICE_NAME}:${id}`, - worker: { - // TODO: read these from the wrangler.toml - compatibilityDate: "2024-07-31", - compatibilityFlags: [ - "nodejs_compat", - "no_nodejs_compat_v2", - "enable_ctx_exports", - ], - modules: [ - { - name: "router-worker.mjs", - esModule: SCRIPT_ROUTER(), - }, - ], - bindings: [ - { - name: "ASSET_WORKER", - service: { - name: `${ASSETS_SERVICE_NAME}:${id}`, - }, - }, - { - name: "USER_WORKER", - service: { name: getUserServiceName(id) }, - }, - { - name: "CONFIG", - json: JSON.stringify(routerConfig), - }, - ], - }, - }; - - const assetsProxyService: Service = { - name: `${RPC_PROXY_SERVICE_NAME}:${id}`, - worker: { - compatibilityDate: "2024-08-01", - compatibilityFlags: ["service_binding_extra_handlers"], - modules: [ - { - name: "assets-proxy-worker.mjs", - esModule: SCRIPT_RPC_PROXY(), - }, - ], - bindings: [ - { - name: "ROUTER_WORKER", - service: { - name: `${ROUTER_SERVICE_NAME}:${id}`, - }, - }, - { - name: "USER_WORKER", - service: { - name: getUserServiceName(id), - }, - }, - ], - }, - }; - - return [ - storageService, - namespaceService, - assetService, - routerService, - assetsProxyService, - ]; - }, -}; - -/** - * The Asset Manifest and Asset Reverse Map are used to map a request path to an asset. - * 1. Hash path of request - * 2. Use this path hash to find the manifest entry - * 3. Get content hash from manifest entry - * 4a. In prod, use content hash to get asset from KV - * 4b. In dev, we fake out the KV store and use the file system instead. - * Use content hash to get file path from asset reverse map. - */ - -export const buildAssetManifest = async (dir: string) => { - const { manifest, assetsReverseMap } = await walk(dir); - const sortedAssetManifest = sortManifest(manifest); - const encodedAssetManifest = encodeManifest(sortedAssetManifest); - return { encodedAssetManifest, assetsReverseMap }; -}; - -export type ManifestEntry = { - pathHash: Uint8Array; - contentHash: Uint8Array; -}; - -export type AssetReverseMap = { - [pathHash: string]: { filePath: string; contentType: string | null }; -}; - -/** - * Traverses the asset directory to create an asset manifest and asset reverse map. - * These are available to the Asset Worker as a binding. - * NB: This runs every time the dev server restarts. - */ -const walk = async (dir: string) => { - const files = await fs.readdir(dir, { recursive: true }); - const manifest: ManifestEntry[] = []; - const assetsReverseMap: AssetReverseMap = {}; - const { assetsIgnoreFunction } = await createAssetsIgnoreFunction(dir).catch( - (error: unknown) => { + // Only to reject an `.assetsignore` that cannot be read as patterns. + await createAssetsIgnoreFunction(assetDirectory).catch((error: unknown) => { if (error instanceof InvalidAssetsIgnoreError) { throw new MiniflareCoreError("ERR_VALIDATION", error.message); } throw error; - } - ); - let counter = 0; - await Promise.all( - files.map(async (file) => { - if (assetsIgnoreFunction(file)) { - return; - } - /** absolute file path */ - const filepath = path.join(dir, file); - const relativeFilepath = path.relative(dir, filepath); - const filestat = await fs.stat(filepath); + }); + const read = (file: string) => maybeGetFile(join(assetDirectory, file)); + // The config of workerd's assets service. + const assetConfig = { + html_handling: assets.htmlHandling, + not_found_handling: assets.notFoundHandling, + base_path: assets.basePath, + run_worker_first: assets.runWorkerFirst, + headers: read(HEADERS_FILENAME), + redirects: read(REDIRECTS_FILENAME), + ignore: read(CF_ASSETS_IGNORE_FILENAME), + debug: true, + }; - // TODO: decide whether to follow symbolic links - if (filestat.isSymbolicLink() || filestat.isDirectory()) { - return; - } else { - // TODO: Warn about _worker.js - - if (filestat.size > MAX_ASSET_SIZE) { - throw new Error( - `Asset too large.\n` + - `Cloudflare Workers supports assets with sizes of up to ${prettyBytes( - MAX_ASSET_SIZE, - { - binary: true, - } - )}. We found a file ${filepath} with a size of ${prettyBytes( - filestat.size, - { - binary: true, - } - )}.\n` + - `Ensure all assets in your assets directory "${dir}" conform with the Workers maximum size requirement.` - ); - } - - /* - * Each manifest entry is a series of bytes that store data in a [header, - * pathHash, contentHash] format, where: - * - `header` is a fixed 20 bytes reserved but currently unused - * - `pathHash` is the hashed file path - * - `contentHash` is the hashed file content in prod - * - * The `contentHash` of a file is determined by reading the contents of - * the file, and applying a hash function on the read data. In local - * development, performing this operation for each asset file would - * become very expensive very quickly, as it would have to be performed - * every time a dev server reload is trigerred. In watch mode, depending - * on the user's setup, this could potentially be on every file change. - * - * To avoid this from becoming a performance bottleneck, we're doing - * things a bit differently for dev, and implementing the `contentHash` - * as a hash function of the file path and the modified timestamp. - * (`hash(filePath + modifiedTime)`). - * This way a file's corresponding 'contentHash' will always update - * if the file changes, and `wrangler dev` will serve the updated asset - * files instead of incorrectly returning 304s. - */ - - const [pathHash, contentHash] = await Promise.all([ - hashPath(normalizeFilePath(relativeFilepath)), - // used absolute filepath here so that changes to the enclosing asset folder will be registered - hashPath(filepath + filestat.mtimeMs.toString()), - ]); - manifest.push({ - pathHash, - contentHash, - }); - assetsReverseMap[bytesToHex(contentHash)] = { - filePath: relativeFilepath.split(path.sep).join("/"), - contentType: getContentType(filepath), - }; - counter++; - } - }) - ); - if (counter > MAX_ASSET_COUNT) { - throw new Error( - `Maximum number of assets exceeded.\n` + - `Cloudflare Workers supports up to ${MAX_ASSET_COUNT.toLocaleString()} assets in a version. We found ${counter.toLocaleString()} files in the specified assets directory "${dir}".\n` + - `Ensure your assets directory contains a maximum of ${MAX_ASSET_COUNT.toLocaleString()} files, and that you have specified your assets directory correctly.` - ); - } - return { manifest, assetsReverseMap }; -}; - -// sorts ascending by path hash -const sortManifest = (manifest: ManifestEntry[]) => { - return manifest.sort(comparisonFn); -}; - -const comparisonFn = (a: ManifestEntry, b: ManifestEntry) => { - if (a.pathHash.length < b.pathHash.length) { - return -1; - } - if (a.pathHash.length > b.pathHash.length) { - return 1; - } - for (const [i, v] of a.pathHash.entries()) { - if (v < b.pathHash[i]) { - return -1; - } - if (v > b.pathHash[i]) { - return 1; - } - } - return 1; -}; - -const encodeManifest = (manifest: ManifestEntry[]) => { - const assetManifestBytes = new Uint8Array( - HEADER_SIZE + manifest.length * ENTRY_SIZE - ); - - for (const [i, entry] of manifest.entries()) { - const entryOffset = HEADER_SIZE + i * ENTRY_SIZE; - assetManifestBytes.set(entry.pathHash, entryOffset + PATH_HASH_OFFSET); - // content hash here in dev is hash(path + last modified time of file) - assetManifestBytes.set( - entry.contentHash, - entryOffset + CONTENT_HASH_OFFSET - ); - } - return assetManifestBytes; -}; - -const bytesToHex = (buffer: Uint8Array) => { - return [...new Uint8Array(buffer)] - .map((b) => b.toString(16).padStart(2, "0")) - .join(""); -}; - -const hashPath = async (path: string) => { - const encoder = new TextEncoder(); - const data = encoder.encode(path); - const hashBuffer = await crypto.subtle.digest( - "SHA-256", - data.buffer as ArrayBuffer - ); - return new Uint8Array(hashBuffer, 0, PATH_HASH_SIZE); + const id = options.config.name; + // workerd's assets service routes each request to the User Worker or to a + // file, and sends the User Worker every other event (RPC, scheduled, tail). + // `has_user_worker` comes from the explicit `assets.hasUserWorker` flag: + // wrangler injects a placeholder script for assets-only workers. + const routerService: Service = { + name: `${ROUTER_SERVICE_NAME}:${id}`, + assets: { + path: assetDirectory, + config: JSON.stringify(assetConfig), + ...(assets.hasUserWorker + ? { userWorker: { name: getUserServiceName(id) } } + : { + compatibilityDate: + options.config.compatibilityDate ?? FALLBACK_COMPATIBILITY_DATE, + compatibilityFlags: options.config.compatibilityFlags, + }), + }, + }; + return [routerService]; + }, }; diff --git a/packages/miniflare/src/plugins/core/index.ts b/packages/miniflare/src/plugins/core/index.ts index d73b9812b0..3b711a342f 100644 --- a/packages/miniflare/src/plugins/core/index.ts +++ b/packages/miniflare/src/plugins/core/index.ts @@ -268,7 +268,7 @@ function maybeGetCustomServiceService( } } -const FALLBACK_COMPATIBILITY_DATE = "2000-01-01"; +export const FALLBACK_COMPATIBILITY_DATE = "2000-01-01"; function validateCompatibilityDate(compatibilityDate: string) { if (compatibilityDate > getTodaysCompatDate()) { diff --git a/packages/miniflare/src/runtime/config/generated/workerd.ts b/packages/miniflare/src/runtime/config/generated/workerd.ts index 5c38570aeb..9c19efebff 100644 --- a/packages/miniflare/src/runtime/config/generated/workerd.ts +++ b/packages/miniflare/src/runtime/config/generated/workerd.ts @@ -821,6 +821,11 @@ export const Service_Which = { * */ DISK: 4, + /** + * A directory of static assets, served as Cloudflare serves a Worker's `assets`. + * + */ + ASSETS: 6, } as const; export type Service_Which = (typeof Service_Which)[keyof typeof Service_Which]; /** @@ -834,6 +839,7 @@ export class Service extends $.Struct { static readonly NETWORK = Service_Which.NETWORK; static readonly EXTERNAL = Service_Which.EXTERNAL; static readonly DISK = Service_Which.DISK; + static readonly ASSETS = Service_Which.ASSETS; static readonly _capnp = { displayName: "Service", id: "e5c88e8bb7bcb6b9", @@ -977,6 +983,35 @@ export class Service extends $.Struct { $.utils.setUint16(0, 4, this); $.utils.copyFrom(value, $.utils.getPointer(1, this)); } + _adoptAssets(value: $.Orphan): void { + $.utils.setUint16(0, 6, this); + $.utils.adopt(value, $.utils.getPointer(1, this)); + } + _disownAssets(): $.Orphan { + return $.utils.disown(this.assets); + } + /** + * A directory of static assets, served as Cloudflare serves a Worker's `assets`. + * + */ + get assets(): AssetsService { + $.utils.testWhich("assets", $.utils.getUint16(0, this), 6, this); + return $.utils.getStruct(1, AssetsService, this); + } + _hasAssets(): boolean { + return !$.utils.isNull($.utils.getPointer(1, this)); + } + _initAssets(): AssetsService { + $.utils.setUint16(0, 6, this); + return $.utils.initStructAt(1, AssetsService, this); + } + get _isAssets(): boolean { + return $.utils.getUint16(0, this) === 6; + } + set assets(value: AssetsService) { + $.utils.setUint16(0, 6, this); + $.utils.copyFrom(value, $.utils.getPointer(1, this)); + } toString(): string { return "Service_" + super.toString(); } @@ -984,6 +1019,104 @@ export class Service extends $.Struct { return $.utils.getUint16(0, this) as Service_Which; } } +/** + * Static assets, as Cloudflare serves a Worker's `assets` (wrangler's `assets` config): each + * request goes to the user Worker or to a file of the directory, as `run_worker_first` and the + * files ask. Files are looked up as each request arrives, so a change to the directory is seen + * at once. A directory is never an asset, nor is a file `.assetsignore` ignores, which by + * default are `_headers`, `_redirects` and `.assetsignore` themselves. Experimental: requires + * `--experimental`. + * + * A designator with no entrypoint names the router; one with the entrypoint `AssetWorker` + * serves files only, and is what a Worker's assets binding names. + * + */ +export class AssetsService extends $.Struct { + static readonly _capnp = { + displayName: "AssetsService", + id: "d1d743b2d711ca07", + size: new $.ObjectSize(0, 5), + }; + /** + * The directory. Relative paths are interpreted relative to the current directory, as a + * `disk` service's are. + * + */ + get path(): string { + return $.utils.getText(0, this); + } + set path(value: string) { + $.utils.setText(0, value, this); + } + /** + * JSON, every field optional: `html_handling`, `not_found_handling`, `base_path` and + * `run_worker_first`, as in wrangler's `assets`; `headers`, `redirects` and `ignore`, the text + * of the directory's `_headers`, `_redirects` and `.assetsignore`; `debug`, for Miniflare, which + * logs and removes the header it adds to a navigation that `not_found_handling` answered. + * + */ + get config(): string { + return $.utils.getText(1, this); + } + set config(value: string) { + $.utils.setText(1, value, this); + } + _adoptUserWorker(value: $.Orphan): void { + $.utils.adopt(value, $.utils.getPointer(2, this)); + } + _disownUserWorker(): $.Orphan { + return $.utils.disown(this.userWorker); + } + /** + * The user Worker, if any. Events other than HTTP requests go to it, and its compatibility flags + * select assets behavior, as they do on Cloudflare. + * + */ + get userWorker(): ServiceDesignator { + return $.utils.getStruct(2, ServiceDesignator, this); + } + _hasUserWorker(): boolean { + return !$.utils.isNull($.utils.getPointer(2, this)); + } + _initUserWorker(): ServiceDesignator { + return $.utils.initStructAt(2, ServiceDesignator, this); + } + set userWorker(value: ServiceDesignator) { + $.utils.copyFrom(value, $.utils.getPointer(2, this)); + } + get compatibilityDate(): string { + return $.utils.getText(3, this); + } + set compatibilityDate(value: string) { + $.utils.setText(3, value, this); + } + _adoptCompatibilityFlags(value: $.Orphan<$.List>): void { + $.utils.adopt(value, $.utils.getPointer(4, this)); + } + _disownCompatibilityFlags(): $.Orphan<$.List> { + return $.utils.disown(this.compatibilityFlags); + } + /** + * Without a user Worker, the compatibility date and flags that select assets behavior, compiled + * as a Worker's are. + * + */ + get compatibilityFlags(): $.List { + return $.utils.getList(4, $.TextList, this); + } + _hasCompatibilityFlags(): boolean { + return !$.utils.isNull($.utils.getPointer(4, this)); + } + _initCompatibilityFlags(length: number): $.List { + return $.utils.initList(4, $.TextList, length, this); + } + set compatibilityFlags(value: $.List) { + $.utils.copyFrom(value, $.utils.getPointer(4, this)); + } + toString(): string { + return "AssetsService_" + super.toString(); + } +} /** * Value to provide in `ctx.props` in the target worker. * diff --git a/packages/miniflare/src/runtime/config/workerd.ts b/packages/miniflare/src/runtime/config/workerd.ts index 402dc1f740..05b49cda2a 100644 --- a/packages/miniflare/src/runtime/config/workerd.ts +++ b/packages/miniflare/src/runtime/config/workerd.ts @@ -56,6 +56,7 @@ export type Service = { | { network?: Network } | { external?: ExternalServer } | { disk?: DiskDirectory } + | { assets?: AssetsService } ); export interface ServiceDesignator { @@ -273,6 +274,14 @@ export interface Network { tlsOptions?: TlsOptions; } +export interface AssetsService { + path?: string; + config?: string; + userWorker?: ServiceDesignator; + compatibilityDate?: string; + compatibilityFlags?: string[]; +} + export interface DiskDirectory { path?: string; writable?: boolean; diff --git a/packages/miniflare/src/shared/DEV_REGISTRY.md b/packages/miniflare/src/shared/DEV_REGISTRY.md index 5bc479819d..5defe73d83 100644 --- a/packages/miniflare/src/shared/DEV_REGISTRY.md +++ b/packages/miniflare/src/shared/DEV_REGISTRY.md @@ -175,7 +175,7 @@ The `defaultEntrypointService` in `WorkerDefinition` varies by worker type: | Worker Type | defaultEntrypointService | Why | | --------------- | -------------------------- | ---------------------------- | | Plain worker | `core:user:` | Direct access | -| Worker + Assets | `assets:rpc-proxy:` | Routes through asset handler | +| Worker + Assets | `assets:router:` | Routes through asset handler | | Vite worker | `core:user:` | `unsafeOverrideFetchWorker` | Named entrypoints and DO access always use `userWorkerService` (`core:user:`) to bypass any asset/vite proxy layer. diff --git a/packages/miniflare/src/workers/assets/assets-kv.worker.ts b/packages/miniflare/src/workers/assets/assets-kv.worker.ts deleted file mode 100644 index 0cc2c80edd..0000000000 --- a/packages/miniflare/src/workers/assets/assets-kv.worker.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { SharedBindings } from "miniflare:shared"; - -interface Env { - [SharedBindings.MAYBE_SERVICE_BLOBS]: Fetcher; - ASSETS_REVERSE_MAP: AssetReverseMap; -} - -type AssetReverseMap = { - [pathHash: string]: { filePath: string; contentType: string }; -}; //map to actual filepath - -export default >{ - async fetch(request, env) { - // Only permit reads - if (request.method !== "GET") { - const message = `Cannot ${request.method.toLowerCase()}() with Workers Assets namespace`; - return new Response(message, { status: 405, statusText: message }); - } - - const pathHash = new URL(request.url).pathname.substring(1); - const entry = env.ASSETS_REVERSE_MAP[pathHash]; - if (entry === undefined) { - return new Response("Not Found", { status: 404 }); - } - - const { filePath, contentType } = entry; - const blobsService = env[SharedBindings.MAYBE_SERVICE_BLOBS]; - const response = await blobsService.fetch( - new URL( - // somewhere in blobservice I think this is being decoded again - filePath - .split("/") - .map((x) => encodeURIComponent(x)) - .join("/"), - "http://placeholder" - ) - ); - const newResponse = new Response(response.body, response); - // ensure the runtime will return the metadata we need - if (contentType !== null) { - newResponse.headers.append( - "cf-kv-metadata", - `{"contentType": "${contentType}"}` - ); - } - return newResponse; - }, -}; diff --git a/packages/miniflare/src/workers/assets/assets.worker.ts b/packages/miniflare/src/workers/assets/assets.worker.ts deleted file mode 100644 index f2ade4c324..0000000000 --- a/packages/miniflare/src/workers/assets/assets.worker.ts +++ /dev/null @@ -1,7 +0,0 @@ -// Re-export the asset-worker entrypoints so ctx.exports loopback bindings are -// preserved in the bundled worker template. -export { - default, - AssetWorkerInner, - AssetWorkerOuter, -} from "@cloudflare/workers-shared/asset-worker"; diff --git a/packages/miniflare/src/workers/assets/router.worker.ts b/packages/miniflare/src/workers/assets/router.worker.ts deleted file mode 100644 index 3fab4b54c4..0000000000 --- a/packages/miniflare/src/workers/assets/router.worker.ts +++ /dev/null @@ -1,7 +0,0 @@ -// Re-export the router-worker outer and inner entrypoints so ctx.exports loopback -// bindings are preserved in the bundled worker template. -export { - default, - RouterInnerEntrypoint, - RouterOuterEntrypoint, -} from "@cloudflare/workers-shared/router-worker"; diff --git a/packages/miniflare/src/workers/assets/rpc-proxy.worker.ts b/packages/miniflare/src/workers/assets/rpc-proxy.worker.ts deleted file mode 100644 index e2d36a7744..0000000000 --- a/packages/miniflare/src/workers/assets/rpc-proxy.worker.ts +++ /dev/null @@ -1,87 +0,0 @@ -import { WorkerEntrypoint } from "cloudflare:workers"; -import { - tailEventsReplacer, - tailEventsReviver, -} from "../core/dev-registry-proxy-shared.worker"; -import type RouterWorker from "@cloudflare/workers-shared/asset-worker"; - -interface Env { - ROUTER_WORKER: Service; - USER_WORKER: Fetcher | Service; -} - -/* - * The RPCProxyWorker is a Miniflare abstraction of the filter stage used in - * production Workers + Assets pipelines. It lives in front of the Router Worker, - * and is meant to redirect incoming `fetch` requests to the Router Worker, and - * all other RPC requests to the User Worker. It does so via a JSProxy mechanism. - * The RPCProxyWorker is specifically relevant in the context of - * Worker <> Worker+Assets service bindings. - * - * Please note that this conceptual pipeline architecture is applied only for - * Workers + Assets and default entrypoints, class or non-class based. - */ -export default class RPCProxyWorker extends WorkerEntrypoint { - async fetch(request: Request) { - return this.env.ROUTER_WORKER.fetch(request); - } - - // Forward scheduled events to the User Worker. The proxy itself doesn't run - // any scheduled logic; it just dispatches a real scheduled event to the user - // worker via the Fetcher built-in, then propagates the user worker's noRetry - // decision back onto this controller so the outcome surfaces correctly to - // the caller (e.g. the entry worker's `/cdn-cgi/local/scheduled` handler). - async scheduled(controller: ScheduledController) { - const result = await this.env.USER_WORKER.scheduled?.({ - cron: controller.cron, - scheduledTime: new Date(controller.scheduledTime), - }); - if (result?.noRetry) { - controller.noRetry(); - } - if (result?.outcome !== "ok") { - // Re-throw so workerd surfaces `outcome: "exception"` to the caller - // rather than swallowing the user worker's failure. - throw new Error( - `User Worker scheduled handler failed with outcome: ${result?.outcome}` - ); - } - } - - tail(events: TraceItem[]) { - // Temporary workaround: the tail events is not serializable over capnproto yet - // But they are effectively JSON, so we are serializing them to JSON and parsing it back to make it transferable. - // @ts-expect-error FIXME when https://github.com/cloudflare/workerd/pull/4595 lands - return this.env.USER_WORKER.tail( - JSON.parse(JSON.stringify(events, tailEventsReplacer), tailEventsReviver) - ); - } - - constructor(ctx: ExecutionContext, env: Env) { - super(ctx, env); - /* - * Create a proxy of the RPCProxyWorker instance - * - * see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Proxy - * see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Reflect - */ - return new Proxy(this, { - get(target, prop) { - /* - * If `prop` is defined on the RPCProxyWorker, then we are - * intentionally meaning to intercept `env.USER_WORKER[prop]` - * calls and override their return value . We should therefore - * return whatever `RPCProxyWorker[prop]` returns - */ - if (Reflect.has(target, prop)) { - return Reflect.get(target, prop); - } - - /* - * Otherwise, forward to the USER_WORKER and return its response - */ - return Reflect.get(target.env.USER_WORKER, prop); - }, - }); - } -} diff --git a/packages/miniflare/src/workers/assets/sentry-noop.ts b/packages/miniflare/src/workers/assets/sentry-noop.ts deleted file mode 100644 index 278c2baf3d..0000000000 --- a/packages/miniflare/src/workers/assets/sentry-noop.ts +++ /dev/null @@ -1,9 +0,0 @@ -/** - * Miniflare's asset services do not bind Sentry credentials, so this preserves - * the production setup's credential-free behaviour without bundling Toucan. - * - * @returns `undefined`, matching production when credentials are absent. - */ -export function setupSentry(): undefined { - return undefined; -} diff --git a/packages/miniflare/test/plugins/assets/index.spec.ts b/packages/miniflare/test/plugins/assets/index.spec.ts index 7b98853f9e..cab0f803aa 100644 --- a/packages/miniflare/test/plugins/assets/index.spec.ts +++ b/packages/miniflare/test/plugins/assets/index.spec.ts @@ -41,27 +41,6 @@ function makeOptions( }; } -test("omits Sentry from the embedded asset service Workers", async ({ - expect, -}) => { - for (const worker of ["assets", "router"]) { - for (const extension of ["js", "js.map"]) { - const artifact = await fs.readFile( - path.join( - "dist", - "src", - "workers", - "assets", - `${worker}.worker.${extension}` - ), - "utf8" - ); - expect(artifact).not.toContain("node_modules/toucan-js"); - expect(artifact).not.toContain("node_modules/@sentry"); - } - } -}); - test("starts without error when assets directory does not exist", async ({ expect, }) => { diff --git a/packages/miniflare/test/plugins/assets/read-only.spec.ts b/packages/miniflare/test/plugins/assets/read-only.spec.ts index cad2e073af..4231138083 100644 --- a/packages/miniflare/test/plugins/assets/read-only.spec.ts +++ b/packages/miniflare/test/plugins/assets/read-only.spec.ts @@ -6,9 +6,10 @@ import { test } from "vitest"; import { singleModuleManifest, useDispose, useTmp } from "../../test-shared"; import type { MiniflareOptions, WorkerDefinition } from "miniflare"; -test.for(["assets", "Workers Sites"] as const)( +// Static assets have no disk service: workerd's assets service only reads. +test.for(["Workers Sites"] as const)( "%s disk service rejects writes through debug RPC", - async (type, { expect }) => { + async (_type, { expect }) => { const directory = await useTmp(); const filePath = path.join(directory, "test.txt"); await fs.writeFile(filePath, "original"); @@ -27,11 +28,9 @@ test.for(["assets", "Workers Sites"] as const)( return new Response(await env.__STATIC_CONTENT.get(key)); } }`), - assets: type === "assets" ? { directory } : undefined, env: { KV: { type: "kv", id: "writable-kv" } }, }, - legacy: - type === "Workers Sites" ? { sitePath: directory } : undefined, + legacy: { sitePath: directory }, }, ], }; @@ -43,8 +42,7 @@ test.for(["assets", "Workers Sites"] as const)( // the namespace Worker's method guard. Omitting instanceId forces TCP RPC. const definition = getWorkerRegistry(registryPath)["static-worker"]; assert(definition); - const serviceName = - type === "assets" ? "assets:storage:static-worker" : "kv:site:storage"; + const serviceName = "kv:site:storage"; const clientRegistryPath = await useTmp(); await fs.writeFile( path.join(clientRegistryPath, "disk"),