Make buffer.toString("base64") 4x faster (#3486)

* Add libbase64

* Add bench

* Update licensing.md

---------

Co-authored-by: Jarred Sumner <709451+Jarred-Sumner@users.noreply.github.com>

.gitmodules

@@ -68,4 +68,10 @@ fetchRecurseSubmodules = false
 [submodule "src/deps/zstd"]
 	path = src/deps/zstd
 	url = https://github.com/facebook/zstd.git
-	ignore = dirty
+	ignore = dirty
+[submodule "src/deps/base64"]
+	path = src/deps/base64
+	url = https://github.com/aklomp/base64.git
+	ignore = dirty
+	depth = 1
+	shallow = true

Dockerfile

@@ -295,6 +295,27 @@ WORKDIR $BUN_DIR
 RUN cd $BUN_DIR && \
     make uws && rm -rf src/deps/uws Makefile
 
+FROM bun-base as base64
+
+ARG DEBIAN_FRONTEND
+ARG GITHUB_WORKSPACE
+ARG ZIG_PATH
+# Directory extracts to "bun-webkit"
+ARG WEBKIT_DIR
+ARG BUN_RELEASE_DIR
+ARG BUN_DEPS_OUT_DIR
+ARG BUN_DIR
+ARG CPU_TARGET
+ENV CPU_TARGET=${CPU_TARGET}
+
+COPY Makefile ${BUN_DIR}/Makefile
+COPY src/deps/base64 ${BUN_DIR}/src/deps/base64
+
+WORKDIR $BUN_DIR
+
+RUN cd $BUN_DIR && \
+    make base64 && rm -rf src/deps/base64 Makefile
+
 FROM bun-base as picohttp
 
 ARG DEBIAN_FRONTEND
@@ -556,6 +577,7 @@ ENV JSC_BASE_DIR=${WEBKIT_DIR}
 ENV LIB_ICU_PATH=${WEBKIT_DIR}/lib
 
 COPY --from=zlib ${BUN_DEPS_OUT_DIR}/*.a ${BUN_DEPS_OUT_DIR}/
+COPY --from=base64 ${BUN_DEPS_OUT_DIR}/*.a ${BUN_DEPS_OUT_DIR}/
 COPY --from=libarchive ${BUN_DEPS_OUT_DIR}/*.a ${BUN_DEPS_OUT_DIR}/
 COPY --from=boringssl ${BUN_DEPS_OUT_DIR}/*.a ${BUN_DEPS_OUT_DIR}/
 COPY --from=lolhtml ${BUN_DEPS_OUT_DIR}/*.a ${BUN_DEPS_OUT_DIR}/

Makefile

@@ -453,7 +453,8 @@ MINIMUM_ARCHIVE_FILES = -L$(BUN_DEPS_OUT_DIR) \
 	-ldecrepit \
 	-lssl \
 	-lcrypto \
-	-llolhtml
+	-llolhtml \
+	-lbase64
 
 ARCHIVE_FILES_WITHOUT_LIBCRYPTO = $(MINIMUM_ARCHIVE_FILES) \
 		-larchive \
@@ -1850,6 +1851,10 @@ copy-to-bun-release-dir-bin:
 
 PACKAGE_MAP = --pkg-begin async_io $(BUN_DIR)/src/io/io_darwin.zig --pkg-begin bun $(BUN_DIR)/src/bun_redirect.zig --pkg-end --pkg-end --pkg-begin javascript_core $(BUN_DIR)/src/jsc.zig --pkg-begin bun $(BUN_DIR)/src/bun_redirect.zig --pkg-end --pkg-end --pkg-begin bun $(BUN_DIR)/src/bun_redirect.zig --pkg-end
 
+.PHONY: base64
+base64:
+	cd $(BUN_DEPS_DIR)/base64 && make clean && cmake $(CMAKE_FLAGS) . && make
+	cp $(BUN_DEPS_DIR)/base64/libbase64.a $(BUN_DEPS_OUT_DIR)/libbase64.a
 
 .PHONY: cold-jsc-start
 cold-jsc-start:
@@ -1868,7 +1873,8 @@ cold-jsc-start:
 		misctools/cold-jsc-start.cpp -o cold-jsc-start
 
 .PHONY: vendor-without-npm
-vendor-without-npm: node-fallbacks runtime_js fallback_decoder bun_error mimalloc picohttp zlib boringssl libarchive lolhtml sqlite usockets uws tinycc c-ares zstd
+vendor-without-npm: node-fallbacks runtime_js fallback_decoder bun_error mimalloc picohttp zlib boringssl libarchive lolhtml sqlite usockets uws tinycc c-ares zstd base64
+
 
 .PHONY: vendor-without-check
 vendor-without-check: npm-install vendor-without-npm

bench/snippets/base64-buffer-to-string.mjs

@@ -0,0 +1,14 @@
+import { bench, run } from "./runner.mjs";
+import { Buffer } from "node:buffer";
+
+const bigBuffer = Buffer.from("hello world".repeat(10000));
+const converted = bigBuffer.toString("base64");
+bench("Buffer.toString('base64')", () => {
+  return bigBuffer.toString("base64");
+});
+
+// bench("Buffer.from(str, 'base64')", () => {
+//   return Buffer.from(converted, "base64");
+// });
+
+await run();

docs/project/licensing.md

@@ -85,6 +85,11 @@ Bun statically links these libraries:
 
 ---
 
+- [`libbase64`](https://github.com/aklomp/base64/blob/master/LICENSE)
+- BSD 2-Clause
+
+---
+
 - A fork of [`uWebsockets`](https://github.com/jarred-sumner/uwebsockets)
 - Apache 2.0 licensed
 

src/base64/base64.zig

@@ -5,6 +5,22 @@ pub const DecodeResult = struct {
     fail: bool = false,
 };
 
+pub const LibBase64 = struct {
+    pub const State = extern struct {
+        eof: c_int,
+        bytes: c_int,
+        flags: c_int,
+        carry: u8,
+    };
+    pub extern fn base64_encode(src: [*]const u8, srclen: usize, out: [*]u8, outlen: *usize, flags: c_int) void;
+    pub extern fn base64_stream_encode_init(state: *State, flags: c_int) void;
+    pub extern fn base64_stream_encode(state: *State, src: [*]const u8, srclen: usize, out: [*]u8, outlen: *usize) void;
+    pub extern fn base64_stream_encode_final(state: *State, out: [*]u8, outlen: *usize) void;
+    pub extern fn base64_decode(src: [*]const u8, srclen: usize, out: [*]u8, outlen: *usize, flags: c_int) c_int;
+    pub extern fn base64_stream_decode_init(state: *State, flags: c_int) void;
+    pub extern fn base64_stream_decode(state: *State, src: [*]const u8, srclen: usize, out: [*]u8, outlen: *usize) c_int;
+};
+
 const mixed_decoder = brk: {
     var decoder = zig_base64.standard.decoderWithIgnore("\xff \t\r\n" ++ [_]u8{
         std.ascii.control_code.vt,
@@ -30,7 +46,9 @@ pub fn decode(destination: []u8, source: []const u8) DecodeResult {
 }
 
 pub fn encode(destination: []u8, source: []const u8) usize {
-    return zig_base64.standard.Encoder.encode(destination, source).len;
+    var outlen: usize = destination.len;
+    LibBase64.base64_encode(source.ptr, source.len, destination.ptr, &outlen, 0);
+    return outlen;
 }
 
 pub fn decodeLenUpperBound(len: usize) usize {

src/bun.js/node/types.zig

@@ -524,8 +524,8 @@ pub const Encoding = enum(u8) {
         switch (encoding) {
             .base64 => {
                 var base64: [std.base64.standard.Encoder.calcSize(size)]u8 = undefined;
-                const result = JSC.ZigString.init(std.base64.standard.Encoder.encode(&base64, input)).toValueGC(globalThis);
-                return result;
+                const len = bun.base64.encode(&base64, input);
+                return JSC.ZigString.init(base64[0..len]).toValueGC(globalThis);
             },
             .base64url => {
                 var buf: [std.base64.url_safe.Encoder.calcSize(size) + "data:;base64,".len]u8 = undefined;