import text file types into bundles (#107)

* import text file types into bundles

This is the first of what I think will be a few PRs for supporting non-js modules in workers. When esbuild scans the module graph, we collect references to some well known file types, mark them as external, and instead add them directly to the worker form upload. This prevents the need to configure this in wrangler.toml or whatever.

In the next PR, I'll do wasm support in a similar vein.

Once we have that in place, I'll probably work on a `--loader` flag to define custom types.

* Create pretty-monkeys-invent.md

.changeset/pretty-monkeys-invent.md

@@ -0,0 +1,5 @@
+---
+"wrangler": patch
+---
+
+import text file types into workers

packages/wrangler/src/dev.tsx

@@ -22,6 +22,7 @@ import commandExists from "command-exists";
 import assert from "assert";
 import { getAPIToken } from "./user";
 import fetch from "node-fetch";
+import makeModuleCollector from "./module-collection";
 
 type CfScriptFormat = void | "modules" | "service-worker";
 
@@ -169,7 +170,7 @@ function Remote(props: {
     name: props.name,
     bundle: props.bundle,
     format: props.format,
-    modules: [],
+    modules: props.bundle ? props.bundle.modules : [],
     accountId: props.accountId,
     apiToken: props.apiToken,
     variables: props.variables,
@@ -343,6 +344,7 @@ type EsbuildBundle = {
   entry: string;
   type: "esm" | "commonjs";
   exports: string[];
+  modules: CfModule[];
 };
 
 function useEsbuild(props: {
@@ -358,6 +360,7 @@ function useEsbuild(props: {
     let result: esbuild.BuildResult;
     async function build() {
       if (!destination) return;
+      const moduleCollector = makeModuleCollector();
       result = await esbuild.build({
         entryPoints: [entry],
         bundle: true,
@@ -371,6 +374,7 @@ function useEsbuild(props: {
         ...(jsxFactory && { jsxFactory }),
         ...(jsxFragment && { jsxFragment }),
         external: ["__STATIC_CONTENT_MANIFEST"],
+        plugins: [moduleCollector.plugin],
         // TODO: import.meta.url
         watch: {
           async onRebuild(error) {
@@ -394,6 +398,7 @@ function useEsbuild(props: {
         path: chunks[0],
         type: chunks[1].exports.length > 0 ? "esm" : "commonjs",
         exports: chunks[1].exports,
+        modules: moduleCollector.modules,
       });
     }
     build();

packages/wrangler/src/module-collection.ts

@@ -0,0 +1,63 @@
+import { CfModule } from "./api/worker";
+import type esbuild from "esbuild";
+import path from "node:path";
+import { readFile } from "node:fs/promises";
+import crypto from "node:crypto";
+
+// This is a combination of an esbuild plugin and a mutable array
+// that we use to collect module references from source code.
+// There will be modules that _shouldn't_ be inlined directly into
+// the bundle. (eg. wasm modules, some text files, etc). We can include
+// those files as modules in the multi part forker form upload. This
+// plugin+array is used to collect references to these modules, reference
+// them correctly in the bundle, and add them to the form upload.
+
+export default function makeModuleCollector(): {
+  modules: CfModule[];
+  plugin: esbuild.Plugin;
+} {
+  const modules: CfModule[] = [];
+  return {
+    modules,
+    plugin: {
+      name: "wrangler-module-collector",
+      setup(build) {
+        build.onStart(() => {
+          // reset the moduels collection
+          modules.splice(0);
+        });
+
+        build.onResolve(
+          // filter on "known" file types,
+          // we can expand this list later
+          { filter: /.*\.(pem|txt|html)$/ },
+          async (args: esbuild.OnResolveArgs) => {
+            // take the file and massage it to a
+            // transportable/manageable format
+            const filePath = path.join(args.resolveDir, args.path);
+            const fileContent = await readFile(filePath);
+            const fileHash = crypto
+              .createHash("sha1")
+              .update(fileContent)
+              .digest("hex");
+            const fileName = `${fileHash}-${path.basename(args.path)}`;
+
+            // add the module to the array
+            modules.push({
+              name: fileName,
+              content: fileContent,
+              type: "text",
+            });
+
+            return {
+              path: fileName, // change the reference to the changed module
+              external: true, // mark it as external in the bundle
+              namespace: "wrangler-module-collector-ns", // just a tag, this isn't strictly necessary
+              watchFiles: [filePath], // we also add the file to esbuild's watch list
+            };
+          }
+        );
+      },
+    },
+  };
+}

packages/wrangler/src/publish.ts

@@ -8,6 +8,7 @@ import { readFile } from "fs/promises";
 import cfetch from "./cfetch";
 import assert from "node:assert";
 import { syncAssets } from "./sites";
+import makeModuleCollector from "./module-collection";
 
 type CfScriptFormat = void | "modules" | "service-worker";
 
@@ -75,6 +76,7 @@ export default async function publish(props: Props): Promise<void> {
 
   const destination = await tmp.dir({ unsafeCleanup: true });
 
+  const moduleCollector = makeModuleCollector();
   const result = await esbuild.build({
     ...(props.public
       ? {
@@ -100,6 +102,7 @@ export default async function publish(props: Props): Promise<void> {
     loader: {
       ".js": "jsx",
     },
+    plugins: [moduleCollector.plugin],
     ...(jsxFactory && { jsxFactory }),
     ...(jsxFragment && { jsxFragment }),
   });
@@ -216,12 +219,12 @@ export default async function publish(props: Props): Promise<void> {
     },
     ...(migrations && { migrations }),
     modules: assets.manifest
-      ? [].concat({
+      ? moduleCollector.modules.concat({
           name: "__STATIC_CONTENT_MANIFEST",
           content: JSON.stringify(assets.manifest),
           type: "text",
         })
-      : [],
+      : moduleCollector.modules,
     compatibility_date: config.compatibility_date,
     compatibility_flags: config.compatibility_flags,
     usage_model: config.usage_model,