Support decoding base64 to byte[]

jackson-databind writes byte[] as base64 and supports reading from base64 or array. Serde 2 writes byte[] as array, but does not support reading from base64, so prior to this change we are only compatible with jackson in one direction (serde encoder -> jackson decoder).

This patch adds decode support for base64, and for the bson binary type. For oracle jdbc json, a decodeBinary method already exists.

On the encode side, there is now a config flag to use format-specific byte array encoding, which is base64 for json. The default remains to write an array of numbers.

Fixes #520

serde-api/src/main/java/io/micronaut/serde/Decoder.java

@@ -19,6 +19,7 @@
 import io.micronaut.core.annotation.Nullable;
 import io.micronaut.core.type.Argument;
 import io.micronaut.json.tree.JsonNode;
+import io.micronaut.serde.util.BinaryCodecUtil;
 
 import java.io.IOException;
 import java.math.BigDecimal;
@@ -283,6 +284,38 @@ default BigDecimal decodeBigDecimalNullable() throws IOException {
         return decodeNull() ? null : decodeBigDecimal();
     }
 
+    /**
+     * Decode binary data from this stream. Binary data can be serialized in multiple different
+     * ways that differ by format.
+     *
+     * <ul>
+     *     <li>An array of numbers must be supported by all implementations, for compatibility.
+     *     This is also the default implementation.</li>
+     *     <li>A base64 string. This is convenient for text-based formats like json, and is
+     *     supported by jackson.</li>
+     *     <li>A format-specific type, for binary formats such as bson.</li>
+     *     <li>Other format specific behavior. Oracle JDBC Json will parse strings as hex, for
+     *     example.</li>
+     * </ul>
+     *
+     * Implementations <b>must</b> support the array shape, but the other shapes are optional.
+     *
+     * @return The decoded byte array
+     */
+    default byte @NonNull [] decodeBinary() throws IOException {
+        return BinaryCodecUtil.decodeFromArray(this);
+    }
+
+    /**
+     * Equivalent to {@code decodeNull() ? null : decodeBinary()}.
+     *
+     * @return The value
+     * @throws IOException If an unrecoverable error occurs
+     */
+    default byte @Nullable [] decodeBinaryNullable() throws IOException {
+        return decodeNull() ? null : decodeBinary();
+    }
+
     /**
      * Attempt to decode a null value. Returns {@code false} if this value is not null, and another method should be
      * used for decoding. Returns {@code true} if this value was null, and the cursor has been advanced to the next

serde-api/src/main/java/io/micronaut/serde/Encoder.java

@@ -17,6 +17,7 @@
 
 import io.micronaut.core.annotation.NonNull;
 import io.micronaut.core.type.Argument;
+import io.micronaut.serde.util.BinaryCodecUtil;
 
 import java.io.IOException;
 import java.math.BigDecimal;
@@ -143,6 +144,19 @@ default void close() throws IOException {
      */
     void encodeBigDecimal(@NonNull BigDecimal value) throws IOException;
 
+    /**
+     * Encode the given binary data. The shape of the data in the output is unspecified, the only
+     * requirement is that the equivalent {@link Decoder#decodeBinary()} must be able to parse to
+     * the same data.
+     *
+     * @param data The input data
+     * @implNote For symmetry with {@link Decoder#decodeBinary()}, the default implementation
+     * writes to an array, but most implementations should write base64 instead.
+     */
+    default void encodeBinary(byte @NonNull [] data) throws IOException {
+        BinaryCodecUtil.encodeToArray(this, data);
+    }
+
     /**
      * Encode {@code null}.
      * @throws IOException If an error occurs

serde-api/src/main/java/io/micronaut/serde/config/SerdeConfiguration.java

@@ -59,6 +59,17 @@ public interface SerdeConfiguration {
     @Bindable(defaultValue = "SECONDS")
     NumericTimeUnit getNumericTimeUnit();
 
+    /**
+     * Control whether to use legacy behavior for writing byte arrays. When set to {@code true} (the
+     * default in serde 2.x), byte arrays will always be written as arrays of numbers. When set to
+     * {@code false}, the encoding may be format-specific instead, and will be a base64 string for
+     * JSON.
+     *
+     * @return Whether to use legacy byte array writing behavior
+     */
+    @Bindable(defaultValue = "true")
+    boolean isWriteLegacyByteArrays();
+
     /**
      * @return The default locale to use.
      */

serde-api/src/main/java/io/micronaut/serde/util/BinaryCodecUtil.java

@@ -0,0 +1,68 @@
+/*
+ * Copyright 2017-2023 original authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.micronaut.serde.util;
+
+import io.micronaut.core.annotation.Internal;
+import io.micronaut.core.type.Argument;
+import io.micronaut.serde.Decoder;
+import io.micronaut.serde.Encoder;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.Base64;
+
+/**
+ * Common implementations for reading/writing byte arrays.
+ */
+@Internal
+public final class BinaryCodecUtil {
+    private static final Argument<byte[]> BYTE_ARRAY = Argument.of(byte[].class);
+
+    private BinaryCodecUtil() {
+    }
+
+    public static byte[] decodeFromArray(Decoder base) throws IOException {
+        ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+        try (Decoder arrayDecoder = base.decodeArray(BYTE_ARRAY)) {
+            while (arrayDecoder.hasNextArrayValue()) {
+                Byte b = arrayDecoder.decodeByteNullable();
+                buffer.write(b == null ? 0 : b);
+            }
+        }
+        return buffer.toByteArray();
+    }
+
+    public static byte[] decodeFromString(Decoder base) throws IOException {
+        String s = base.decodeString();
+        try {
+            return Base64.getDecoder().decode(s);
+        } catch (IllegalArgumentException e) {
+            throw base.createDeserializationException("Illegal base64 input: " + e.getMessage(), null);
+        }
+    }
+
+    public static void encodeToArray(Encoder encoder, byte[] data) throws IOException {
+        try (Encoder arrayEncoder = encoder.encodeArray(BYTE_ARRAY)) {
+            for (byte i : data) {
+                arrayEncoder.encodeByte(i);
+            }
+        }
+    }
+
+    public static void encodeToString(Encoder encoder, byte[] data) throws IOException {
+        encoder.encodeString(Base64.getEncoder().encodeToString(data));
+    }
+}

serde-bson/src/main/java/io/micronaut/serde/bson/BsonReaderDecoder.java

@@ -316,6 +316,15 @@ protected Number getBestNumber() {
         }
     }
 
+    @Override
+    public byte @NonNull [] decodeBinary() throws IOException {
+        if (currentBsonType == BsonType.BINARY) {
+            return decodeCustom(parser -> ((BsonReaderDecoder) parser).bsonReader.readBinaryData().getData());
+        } else {
+            return super.decodeBinary();
+        }
+    }
+
     @Override
     protected void skipChildren() {
         bsonReader.skipValue();

serde-bson/src/main/java/io/micronaut/serde/bson/BsonWriterEncoder.java

@@ -21,10 +21,12 @@
 import io.micronaut.serde.Encoder;
 import io.micronaut.serde.LimitingStream;
 import io.micronaut.serde.exceptions.SerdeException;
+import org.bson.BsonBinary;
 import org.bson.BsonWriter;
 import org.bson.types.Decimal128;
 import org.bson.types.ObjectId;
 
+import java.io.IOException;
 import java.math.BigDecimal;
 import java.math.BigInteger;
 
@@ -157,6 +159,12 @@ public void encodeBigDecimal(BigDecimal value) {
         postEncodeValue();
     }
 
+    @Override
+    public void encodeBinary(byte @NonNull [] data) throws IOException {
+        bsonWriter.writeBinaryData(new BsonBinary(data));
+        postEncodeValue();
+    }
+
     @Override
     public void encodeNull() {
         bsonWriter.writeNull();

serde-bson/src/test/groovy/io/micronaut/serde/bson/BsonSpec.groovy

@@ -2,7 +2,16 @@ package io.micronaut.serde.bson
 
 import io.micronaut.test.extensions.spock.annotation.MicronautTest
 import jakarta.inject.Inject
-import org.bson.*
+import org.bson.BsonBinary
+import org.bson.BsonBinarySubType
+import org.bson.BsonDateTime
+import org.bson.BsonDbPointer
+import org.bson.BsonDecimal128
+import org.bson.BsonDocument
+import org.bson.BsonNull
+import org.bson.BsonObjectId
+import org.bson.BsonRegularExpression
+import org.bson.BsonString
 import org.bson.types.Decimal128
 import org.bson.types.ObjectId
 import spock.lang.Specification
@@ -118,4 +127,20 @@ class BsonSpec extends Specification implements BsonJsonSpec, BsonBinarySpec {
             value.objectId == null
     }
 
+    def "decode binary types"() {
+        given:
+        def document = new BsonDocument()
+        def uuid = new BsonBinary(UUID.randomUUID())
+        def normal = new BsonBinary([1, 2, 3] as byte[])
+        def userDefined = new BsonBinary(BsonBinarySubType.USER_DEFINED, [1, 2, 3] as byte[])
+        document.put("uuid", uuid)
+        document.put("normal", normal)
+        document.put("userDefined", userDefined)
+        when:
+        def value = encodeAsBinaryDecodeAsObject(document, BinaryTypes)
+        then:
+        value.uuid() == uuid.data
+        value.normal() == normal.data
+        value.userDefined() == userDefined.data
+    }
 }

serde-bson/src/test/java/io/micronaut/serde/bson/BinaryTypes.java

@@ -0,0 +1,11 @@
+package io.micronaut.serde.bson;
+
+import io.micronaut.serde.annotation.Serdeable;
+
+@Serdeable
+public record BinaryTypes(
+    byte[] uuid,
+    byte[] normal,
+    byte[] userDefined
+) {
+}

serde-jackson/src/main/java/io/micronaut/serde/jackson/JacksonDecoder.java

@@ -27,6 +27,7 @@
 import io.micronaut.serde.exceptions.InvalidFormatException;
 import io.micronaut.serde.exceptions.SerdeException;
 import io.micronaut.serde.support.util.JsonNodeDecoder;
+import io.micronaut.serde.util.BinaryCodecUtil;
 
 import java.io.EOFException;
 import java.io.IOException;
@@ -777,6 +778,15 @@ private Number decodeNumber() throws IOException {
         return parser.getNumberValue();
     }
 
+    @Override
+    public byte @NonNull [] decodeBinary() throws IOException {
+        return switch (peekToken()) {
+            case VALUE_STRING -> BinaryCodecUtil.decodeFromString(this);
+            case START_ARRAY -> BinaryCodecUtil.decodeFromArray(this);
+            default -> throw unexpectedToken(JsonToken.START_ARRAY, nextToken());
+        };
+    }
+
     @Override
     public boolean decodeNull() throws IOException {
         if (peekToken() == JsonToken.VALUE_NULL) {

serde-jackson/src/main/java/io/micronaut/serde/jackson/JacksonEncoder.java

@@ -194,6 +194,11 @@ public final void encodeBigDecimal(@NonNull BigDecimal value) throws IOException
         generator.writeNumber(value);
     }
 
+    @Override
+    public void encodeBinary(byte @NonNull [] data) throws IOException {
+        generator.writeBinary(data);
+    }
+
     @Override
     public final void encodeNull() throws IOException {
         generator.writeNull();

serde-jsonp/src/main/java/io/micronaut/serde/json/stream/JsonStreamEncoder.java

@@ -19,6 +19,7 @@
 import io.micronaut.core.type.Argument;
 import io.micronaut.serde.Encoder;
 import io.micronaut.serde.LimitingStream;
+import io.micronaut.serde.util.BinaryCodecUtil;
 import jakarta.json.stream.JsonGenerator;
 
 import java.io.IOException;
@@ -140,6 +141,12 @@ public void encodeBigDecimal(BigDecimal value) throws IOException {
         postEncodeValue();
     }
 
+    @Override
+    public void encodeBinary(byte @NonNull [] data) throws IOException {
+        // we're allowed to encode to string because our decoder can handle it
+        BinaryCodecUtil.encodeToString(this, data);
+    }
+
     @Override
     public void encodeNull() throws IOException {
         jsonGenerator.writeNull();

serde-oracle-jdbc-json/src/main/java/io/micronaut/serde/oracle/jdbc/json/OracleJdbcJsonGeneratorEncoder.java

@@ -23,6 +23,7 @@
 import io.micronaut.serde.exceptions.SerdeException;
 import oracle.sql.json.OracleJsonGenerator;
 
+import java.io.IOException;
 import java.math.BigDecimal;
 import java.math.BigInteger;
 import java.time.LocalDateTime;
@@ -150,6 +151,13 @@ public void encodeBigDecimal(BigDecimal value) {
         postEncodeValue();
     }
 
+    @Override
+    public void encodeBinary(byte @NonNull [] data) throws IOException {
+        // custom oson type, can be read by our decoder
+        jsonGenerator.write(data);
+        postEncodeValue();
+    }
+
     @Override
     public void encodeNull() {
         jsonGenerator.writeNull();

serde-oracle-jdbc-json/src/main/java/io/micronaut/serde/oracle/jdbc/json/OracleJdbcJsonParserDecoder.java

@@ -174,6 +174,7 @@ public IOException createDeserializationException(@NonNull String message, @Null
      *
      * @return the byte array for Oracle JSON binary
      */
+    @Override
     public byte[] decodeBinary() {
         if (currentEvent == OracleJsonParser.Event.VALUE_BINARY) {
             byte[] bytes = jsonParser.getBytes();

serde-oracle-jdbc-json/src/main/java/io/micronaut/serde/oracle/jdbc/json/serde/OracleJsonBinarySerde.java

@@ -35,11 +35,20 @@
 @Singleton
 @Order(-100)
 public class OracleJsonBinarySerde extends AbstractOracleJsonSerde<byte[]> {
+    private final DefaultSerdeRegistry.ByteArraySerde byteArraySerde;
+
+    public OracleJsonBinarySerde(DefaultSerdeRegistry.ByteArraySerde byteArraySerde) {
+        this.byteArraySerde = byteArraySerde;
+    }
+
+    @Deprecated
+    public OracleJsonBinarySerde() {
+        this(DefaultSerdeRegistry.BYTE_ARRAY_SERDE);
+    }
 
     @Override
-    @NonNull
-    protected byte[] doDeserializeNonNull(@NonNull OracleJdbcJsonParserDecoder decoder, @NonNull DecoderContext decoderContext,
-                                          @NonNull Argument<? super byte[]> type) {
+    protected byte @NonNull [] doDeserializeNonNull(@NonNull OracleJdbcJsonParserDecoder decoder, @NonNull DecoderContext decoderContext,
+                                                    @NonNull Argument<? super byte[]> type) {
         return decoder.decodeBinary();
     }
 
@@ -51,7 +60,7 @@ protected void doSerializeNonNull(@NonNull OracleJdbcJsonGeneratorEncoder encode
 
     @Override
     protected Serde<byte[]> getDefault() {
-        return DefaultSerdeRegistry.BYTE_ARRAY_SERDE;
+        return byteArraySerde;
     }
 
 }