feat(cloudflare): support pass through bundling of custom file extens…

…ions

.changeset/proud-jeans-wave.md

@@ -0,0 +1,5 @@
+---
+'@astrojs/cloudflare': minor
+---
+
+Add support for bundling .bin and .txt files to cloudflare pages functions

packages/cloudflare/src/index.ts

@@ -1,7 +1,6 @@
 import type { AstroConfig, AstroIntegration, RouteData } from 'astro';
 import type { OutputChunk, ProgramNode } from 'rollup';
 import type { PluginOption } from 'vite';
-import type { CloudflareModulePluginExtra } from './utils/wasm-module-loader.js';
 
 import { createReadStream } from 'node:fs';
 import { appendFile, rename, stat, unlink } from 'node:fs/promises';
@@ -16,11 +15,14 @@ import { AstroError } from 'astro/errors';
 import { walk } from 'estree-walker';
 import MagicString from 'magic-string';
 import { getPlatformProxy } from 'wrangler';
+import {
+ type CloudflareModulePluginExtra,
+ cloudflareModuleLoader,
+} from './utils/cloudflare-module-loader.js';
 import { createRoutesFile, getParts } from './utils/generate-routes-json.js';
 import { setImageConfig } from './utils/image-config.js';
 import { mutateDynamicPageImportsInPlace, mutatePageMapInPlace } from './utils/index.js';
 import { NonServerChunkDetector } from './utils/non-server-chunk-detector.js';
-import { cloudflareModuleLoader } from './utils/wasm-module-loader.js';
 
 export type { Runtime } from './entrypoints/server.js';
 
@@ -64,18 +66,25 @@ export type Options = {
  /** Configuration persistence settings. Default '.wrangler/state/v3' */
  persist?: boolean | { path: string };
  };
+
  /**
- * Allow bundling cloudflare worker specific file types
- * https://developers.cloudflare.com/workers/wrangler/bundling/
+ * Allow bundling cloudflare worker specific file types as importable modules. Defaults to true.
+ * When enabled, allows imports of '.wasm', '.bin', and '.txt' file types
+ *
+ * See https://developers.cloudflare.com/pages/functions/module-support/
+ * for reference on how these file types are exported
  */
+ cloudflareModules?: boolean;
+
+ /** @deprecated - use `cloudflareModules`, which defaults to true. You can set `cloudflareModuleLoading: false` to disable */
  wasmModuleImports?: boolean;
 };
 
 export default function createIntegration(args?: Options): AstroIntegration {
  let _config: AstroConfig;
 
  const cloudflareModulePlugin: PluginOption & CloudflareModulePluginExtra = cloudflareModuleLoader(
- args?.wasmModuleImports ?? false
+ args?.cloudflareModules ?? args?.wasmModuleImports ?? true
  );
 
  // Initialize the unused chunk analyzer as a shared state between hooks.

packages/cloudflare/src/utils/wasm-module-loader.ts → packages/cloudflare/src/utils/cloudflare-module-loader.ts

@@ -8,48 +8,66 @@ import type { PluginOption } from 'vite';
 export interface CloudflareModulePluginExtra {
  afterBuildCompleted(config: AstroConfig): Promise<void>;
 }
+
+export type ModuleType = 'CompiledWasm' | 'Text' | 'Data';
+
 /**
- * Enables support for wasm modules within cloudflare pages functions
+ * Enables support for various non-standard extensions in module imports that cloudflare workers supports.
+ *
+ * See https://developers.cloudflare.com/pages/functions/module-support/ for reference
  *
- * Loads '*.wasm?module' and `*.wasm` imports as WebAssembly modules, which is the only way to load WASM in cloudflare workers.
- * Current proposal for WASM modules: https://github.com/WebAssembly/esm-integration/tree/main/proposals/esm-integration
- * Cloudflare worker WASM from javascript support: https://developers.cloudflare.com/workers/runtime-apis/webassembly/javascript/
- * @param enabled - if true, load '.wasm' imports as Uint8Arrays, otherwise will throw errors when encountered to clarify that it must be enabled
- * @returns Vite plugin to load WASM tagged with '?module' as a WASM modules
+ * This adds supports for imports in the following formats:
+ * - .wasm
+ * - .wasm?module
+ * - .bin
+ * - .txt
+ *
+ * @param enabled - if true, will load all cloudflare pages supported types
+ * @returns Vite plugin with additional extension method to hook into astro build
  */
 export function cloudflareModuleLoader(
  enabled: boolean
 ): PluginOption & CloudflareModulePluginExtra {
- const enabledAdapters = cloudflareImportAdapters.filter((x) => enabled);
+ /**
+ * It's likely that eventually cloudflare will add support for custom extensions, like they do in vanilla cloudflare workers,
+ * by adding rules to your wrangler.tome
+ * https://developers.cloudflare.com/workers/wrangler/bundling/
+ */
+ const adaptersByExtension: Record<string, ModuleType> = enabled ? { ...defaultAdapters } : {};
+
+ const extensions = Object.keys(adaptersByExtension);
+
  let isDev = false;
  const MAGIC_STRING = '__CLOUDFLARE_ASSET__';
  const replacements: Replacement[] = [];
 
  return {
- name: 'vite:wasm-module-loader',
+ name: 'vite:cf-module-loader',
  enforce: 'pre',
  configResolved(config) {
  isDev = config.command === 'serve';
  },
  config(_, __) {
  // let vite know that file format and the magic import string is intentional, and will be handled in this plugin
  return {
- assetsInclude: enabledAdapters.map((x) => `**/*.${x.qualifiedExtension}`),
+ assetsInclude: extensions.map((x) => `**/*${x}`),
  build: {
  rollupOptions: {
  // mark the wasm files as external so that they are not bundled and instead are loaded from the files
- external: enabledAdapters.map(
- (x) => new RegExp(`^${MAGIC_STRING}.+\\.${x.extension}.mjs$`, 'i')
+ external: extensions.map(
+ (x) => new RegExp(`^${MAGIC_STRING}.+${escapeRegExp(x)}.mjs$`, 'i')
  ),
  },
  },
  };
  },
 
  async load(id, _) {
- const suffix = id.split('.').at(-1);
- const importAdapter = cloudflareImportAdapters.find((x) => x.qualifiedExtension === suffix);
- if (!importAdapter) {
+ const name = id.split('/').at(-1);
+ let suffix = name?.split('.').slice(1).join('.');
+ suffix = suffix ? `.${suffix}` : '';
+ const moduleType: ModuleType | undefined = adaptersByExtension[suffix];
+ if (!moduleType) {
  return;
  }
  if (!enabled) {
@@ -58,12 +76,15 @@ export function cloudflareModuleLoader(
  );
  }
 
- const filePath = id.replace(/\?module$/, '');
+ const moduleLoader = renderers[moduleType];
+
+ const filePath = id.replace(/\?\w+$/, '');
+ const extension = suffix.replace(/\?\w+$/, '');
 
  const data = await fs.readFile(filePath);
  const base64 = data.toString('base64');
 
- const inlineModule = importAdapter.asNodeModule(data);
+ const inlineModule = moduleLoader(data);
 
  if (isDev) {
  // no need to wire up the assets in dev mode, just rewrite
@@ -73,9 +94,7 @@ export function cloudflareModuleLoader(
  const hash = hashString(base64);
  // emit the wasm binary as an asset file, to be picked up later by the esbuild bundle for the worker.
  // give it a shared deterministic name to make things easy for esbuild to switch on later
- const assetName = `${path.basename(filePath).split('.')[0]}.${hash}.${
- importAdapter.extension
- }`;
+ const assetName = `${path.basename(filePath).split('.')[0]}.${hash}${extension}`;
  this.emitFile({
  type: 'asset',
  // emit the data explicitly as an esset with `fileName` rather than `name` so that
@@ -92,7 +111,7 @@ export function cloudflareModuleLoader(
  code: inlineModule,
  });
 
- return `import module from "${MAGIC_STRING}${chunkId}.${importAdapter.extension}.mjs";export default module;`;
+ return `import module from "${MAGIC_STRING}${chunkId}${extension}.mjs";export default module;`;
  },
 
  // output original wasm file relative to the chunk now that chunking has been achieved
@@ -104,9 +123,10 @@ export function cloudflareModuleLoader(
  // SSR will need the .mjs suffix removed from the import before this works in cloudflare, but this is done as a final step
  // so as to support prerendering from nodejs runtime
  let replaced = code;
- for (const loader of enabledAdapters) {
+ for (const ext of extensions) {
+ const extension = ext.replace(/\?\w+$/, '');
  replaced = replaced.replaceAll(
- new RegExp(`${MAGIC_STRING}([A-Za-z\\d]+)\\.${loader.extension}\\.mjs`, 'g'),
+ new RegExp(`${MAGIC_STRING}([A-Za-z\\$\\d]+)${escapeRegExp(extension)}\\.mjs`, 'g'),
  (s, assetId) => {
  const fileName = this.getFileName(assetId);
  const relativePath = path
@@ -123,9 +143,6 @@ export function cloudflareModuleLoader(
  }
  );
  }
- if (replaced.includes(MAGIC_STRING)) {
- console.error('failed to replace', replaced);
- }
 
  return { code: replaced };
  },
@@ -176,8 +193,6 @@ export function cloudflareModuleLoader(
  };
 }
 
-export type ImportType = 'wasm';
-
 interface Replacement {
  fileName?: string;
  chunkName: string;
@@ -187,22 +202,30 @@ interface Replacement {
  nodejsImport: string;
 }
 
-interface ModuleImportAdapter {
- extension: ImportType;
- qualifiedExtension: string;
- asNodeModule(fileContents: Buffer): string;
-}
-
-const wasmImportAdapter: ModuleImportAdapter = {
- extension: 'wasm',
- qualifiedExtension: 'wasm?module',
- asNodeModule(fileContents: Buffer) {
+const renderers: Record<ModuleType, (fileContents: Buffer) => string> = {
+ CompiledWasm(fileContents: Buffer) {
  const base64 = fileContents.toString('base64');
  return `const wasmModule = new WebAssembly.Module(Uint8Array.from(atob("${base64}"), c => c.charCodeAt(0)));export default wasmModule;`;
  },
+ Data(fileContents: Buffer) {
+ const base64 = fileContents.toString('base64');
+ return `const binModule = Uint8Array.from(atob("${base64}"), c => c.charCodeAt(0)).buffer;export default binModule;`;
+ },
+ Text(fileContents: Buffer) {
+ const escaped = JSON.stringify(fileContents.toString('utf-8'));
+ return `const stringModule = ${escaped};export default stringModule;`;
+ },
 };
 
-const cloudflareImportAdapters = [wasmImportAdapter];
+const defaultAdapters: Record<string, ModuleType> = {
+ // Loads '*.wasm?module' imports as WebAssembly modules, which is the only way to load WASM in cloudflare workers.
+ // Current proposal for WASM modules: https://github.com/WebAssembly/esm-integration/tree/main/proposals/esm-integration
+ '.wasm?module': 'CompiledWasm',
+ // treats the module as a WASM module
+ '.wasm': 'CompiledWasm',
+ '.bin': 'Data',
+ '.txt': 'Text',
+};
 
 /**
  * Returns a deterministic 32 bit hash code from a string
@@ -216,3 +239,7 @@ function hashString(str: string): string {
  }
  return new Uint32Array([hash])[0].toString(36);
 }
+
+function escapeRegExp(string: string) {
+ return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); // $& means the whole matched string
+}

packages/cloudflare/test/fixtures/wasm/astro.config.mjs → packages/cloudflare/test/fixtures/module-loader/astro.config.mjs

@@ -2,8 +2,6 @@ import cloudflare from '@astrojs/cloudflare';
 import { defineConfig } from 'astro/config';
 
 export default defineConfig({
- adapter: cloudflare({
- wasmModuleImports: true
- }),
+ adapter: cloudflare({}),
  output: 'hybrid'
 });

packages/cloudflare/test/fixtures/module-loader/src/pages/bin.ts

@@ -0,0 +1,16 @@
+import { type APIContext } from 'astro';
+
+import data from '../util/file.bin';
+
+export const prerender = false;
+
+export async function GET(
+ context: APIContext
+): Promise<Response> {
+ return new Response(data, {
+ status: 200,
+ headers: {
+ 'Content-Type': 'binary/octet-stream',
+ },
+ });
+}

packages/cloudflare/test/fixtures/wasm/src/pages/hybrid.ts → packages/cloudflare/test/fixtures/module-loader/src/pages/hybrid.ts

@@ -1,6 +1,6 @@
 import { type APIContext } from 'astro';
 // @ts-ignore
-import mod from '../util/add.wasm?module';
+import mod from '../util/add.wasm';
 
 const addModule: any = new WebAssembly.Instance(mod);
 

packages/cloudflare/test/fixtures/module-loader/src/pages/text.ts

@@ -0,0 +1,16 @@
+import { type APIContext } from 'astro';
+
+import text from '../util/file.txt';
+
+export const prerender = false;
+
+export async function GET(
+ context: APIContext
+): Promise<Response> {
+ return new Response(text, {
+ status: 200,
+ headers: {
+ 'Content-Type': 'text/plain; charset=utf-8',
+ },
+ });
+}

packages/cloudflare/test/fixtures/wasm/src/util/add.ts → packages/cloudflare/test/fixtures/module-loader/src/util/add.ts

@@ -1,4 +1,4 @@
-import mod from './add.wasm?module';
+import mod from './add.wasm';
 
 
 const addModule: any = new WebAssembly.Instance(mod);

packages/cloudflare/test/fixtures/module-loader/src/util/file.txt

@@ -0,0 +1 @@
+Hello

packages/cloudflare/test/wasm.test.js → packages/cloudflare/test/module-loader.test.js

@@ -1,11 +1,12 @@
 import * as assert from 'node:assert/strict';
 import { after, before, describe, it } from 'node:test';
 import { fileURLToPath } from 'node:url';
+import zlib from 'node:zlib';
 import { astroCli, wranglerCli } from './_test-utils.js';
 
-const root = new URL('./fixtures/wasm/', import.meta.url);
+const root = new URL('./fixtures/module-loader/', import.meta.url);
 
-describe('WasmImport', () => {
+describe('CloudflareModuleLoading', () => {
  let wrangler;
  before(async () => {
  await astroCli(fileURLToPath(root), 'build');
@@ -26,31 +27,40 @@ describe('WasmImport', () => {
  wrangler.kill();
  });
 
- it('can render', async () => {
+ it('can render server side', async () => {
  const res = await fetch('http://127.0.0.1:8788/add/40/2');
  assert.equal(res.status, 200);
  const json = await res.json();
  assert.deepEqual(json, { answer: 42 });
  });
-
  it('can render static', async () => {
  const res = await fetch('http://127.0.0.1:8788/hybrid');
  assert.equal(res.status, 200);
  const json = await res.json();
  assert.deepEqual(json, { answer: 21 });
  });
-
  it('can render shared', async () => {
  const res = await fetch('http://127.0.0.1:8788/shared/40/2');
  assert.equal(res.status, 200);
  const json = await res.json();
  assert.deepEqual(json, { answer: 42 });
  });
-
  it('can render static shared', async () => {
  const res = await fetch('http://127.0.0.1:8788/hybridshared');
  assert.equal(res.status, 200);
  const json = await res.json();
  assert.deepEqual(json, { answer: 21 });
  });
+ it('can render txt', async () => {
+ const res = await fetch('http://127.0.0.1:8788/text');
+ assert.equal(res.status, 200);
+ const text = await res.text();
+ assert.equal(text, 'Hello\n');
+ });
+ it('can render binary', async () => {
+ const res = await fetch('http://127.0.0.1:8788/bin');
+ assert.equal(res.status, 200);
+ const text = zlib.gunzipSync(await res.arrayBuffer()).toString('utf-8');
+ assert.equal(text, 'Hello\n');
+ });
 });