From bb68eb22dde9fbefc0413b7c3f836b03d062df4c Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Tue, 14 May 2024 12:36:44 -0700 Subject: [PATCH] Implement new Debug API with redaction. Implement emittingSingleLine TextFormat printer option. PiperOrigin-RevId: 633672722 --- java/core/BUILD.bazel | 1 + .../java/com/google/protobuf/DebugFormat.java | 79 +++++++++ .../java/com/google/protobuf/TextFormat.java | 146 +++++++++++++--- .../com/google/protobuf/DebugFormatTest.java | 158 ++++++++++++++++++ src/google/protobuf/unittest.proto | 6 + 5 files changed, 371 insertions(+), 19 deletions(-) create mode 100644 java/core/src/main/java/com/google/protobuf/DebugFormat.java create mode 100644 java/core/src/test/java/com/google/protobuf/DebugFormatTest.java diff --git a/java/core/BUILD.bazel b/java/core/BUILD.bazel index 41802133edf1..bc2b592589f0 100644 --- a/java/core/BUILD.bazel +++ b/java/core/BUILD.bazel @@ -523,6 +523,7 @@ LITE_TEST_EXCLUSIONS = [ "src/test/java/com/google/protobuf/AnyTest.java", "src/test/java/com/google/protobuf/CodedInputStreamTest.java", "src/test/java/com/google/protobuf/DeprecatedFieldTest.java", + "src/test/java/com/google/protobuf/DebugFormatTest.java", "src/test/java/com/google/protobuf/DescriptorsTest.java", "src/test/java/com/google/protobuf/DiscardUnknownFieldsTest.java", "src/test/java/com/google/protobuf/DynamicMessageTest.java", diff --git a/java/core/src/main/java/com/google/protobuf/DebugFormat.java b/java/core/src/main/java/com/google/protobuf/DebugFormat.java new file mode 100644 index 000000000000..2d63ef17196b --- /dev/null +++ b/java/core/src/main/java/com/google/protobuf/DebugFormat.java @@ -0,0 +1,79 @@ +package com.google.protobuf; + +import com.google.protobuf.Descriptors.FieldDescriptor; + +/** + * Provides an explicit API for unstable, redacting debug output suitable for debug logging. This + * implementation is based on TextFormat, but should not be parsed. + */ +public final class DebugFormat { + + private final boolean isSingleLine; + + private DebugFormat(boolean singleLine) { + isSingleLine = singleLine; + } + + public static DebugFormat singleLine() { + return new DebugFormat(true); + } + + public static DebugFormat multiline() { + return new DebugFormat(false); + } + + public String toString(MessageOrBuilder message) { + return TextFormat.printer() + .emittingSingleLine(this.isSingleLine) + .enablingSafeDebugFormat(true) + .printToString(message); + } + + public String toString(FieldDescriptor field, Object value) { + return TextFormat.printer() + .emittingSingleLine(this.isSingleLine) + .enablingSafeDebugFormat(true) + .printFieldToString(field, value); + } + + public String toString(UnknownFieldSet fields) { + return TextFormat.printer() + .emittingSingleLine(this.isSingleLine) + .enablingSafeDebugFormat(true) + .printToString(fields); + } + + public Object lazyToString(MessageOrBuilder message) { + return new LazyDebugOutput(message, this); + } + + public Object lazyToString(UnknownFieldSet fields) { + return new LazyDebugOutput(fields, this); + } + + private static class LazyDebugOutput { + private final MessageOrBuilder message; + private final UnknownFieldSet fields; + private final DebugFormat format; + + LazyDebugOutput(MessageOrBuilder message, DebugFormat format) { + this.message = message; + this.fields = null; + this.format = format; + } + + LazyDebugOutput(UnknownFieldSet fields, DebugFormat format) { + this.message = null; + this.fields = fields; + this.format = format; + } + + @Override + public String toString() { + if (message != null) { + return format.toString(message); + } + return format.toString(fields); + } + } +} diff --git a/java/core/src/main/java/com/google/protobuf/TextFormat.java b/java/core/src/main/java/com/google/protobuf/TextFormat.java index 71b2b33f674c..8e5f2a0d87c2 100644 --- a/java/core/src/main/java/com/google/protobuf/TextFormat.java +++ b/java/core/src/main/java/com/google/protobuf/TextFormat.java @@ -38,6 +38,8 @@ private TextFormat() {} private static final String DEBUG_STRING_SILENT_MARKER = "\t "; + private static final String REDACTED_MARKER = "[REDACTED]"; + /** * Generates a human readable form of this message, useful for debugging and other purposes, with * no newline characters. This is just a trivial wrapper around {@link @@ -58,7 +60,7 @@ public static String shortDebugString(final MessageOrBuilder message) { */ public static void printUnknownFieldValue( final int tag, final Object value, final Appendable output) throws IOException { - printUnknownFieldValue(tag, value, multiLineOutput(output)); + printUnknownFieldValue(tag, value, setSingleLineOutput(output, false)); } private static void printUnknownFieldValue( @@ -109,7 +111,11 @@ public static final class Printer { // Printer instance which escapes non-ASCII characters. private static final Printer DEFAULT = new Printer( - true, TypeRegistry.getEmptyTypeRegistry(), ExtensionRegistryLite.getEmptyRegistry()); + true, + TypeRegistry.getEmptyTypeRegistry(), + ExtensionRegistryLite.getEmptyRegistry(), + false, + false); /** Whether to escape non ASCII characters with backslash and octal. */ private final boolean escapeNonAscii; @@ -117,13 +123,25 @@ public static final class Printer { private final TypeRegistry typeRegistry; private final ExtensionRegistryLite extensionRegistry; + /** + * Whether to enable redaction of sensitive fields and introduce randomization. Note that when + * this is enabled, the output will no longer be deserializable. + */ + private final boolean enablingSafeDebugFormat; + + private final boolean singleLine; + private Printer( boolean escapeNonAscii, TypeRegistry typeRegistry, - ExtensionRegistryLite extensionRegistry) { + ExtensionRegistryLite extensionRegistry, + boolean enablingSafeDebugFormat, + boolean singleLine) { this.escapeNonAscii = escapeNonAscii; this.typeRegistry = typeRegistry; this.extensionRegistry = extensionRegistry; + this.enablingSafeDebugFormat = enablingSafeDebugFormat; + this.singleLine = singleLine; } /** @@ -136,7 +154,8 @@ private Printer( * with the escape mode set to the given parameter. */ public Printer escapingNonAscii(boolean escapeNonAscii) { - return new Printer(escapeNonAscii, typeRegistry, extensionRegistry); + return new Printer( + escapeNonAscii, typeRegistry, extensionRegistry, enablingSafeDebugFormat, singleLine); } /** @@ -149,7 +168,8 @@ public Printer usingTypeRegistry(TypeRegistry typeRegistry) { if (this.typeRegistry != TypeRegistry.getEmptyTypeRegistry()) { throw new IllegalArgumentException("Only one typeRegistry is allowed."); } - return new Printer(escapeNonAscii, typeRegistry, extensionRegistry); + return new Printer( + escapeNonAscii, typeRegistry, extensionRegistry, enablingSafeDebugFormat, singleLine); } /** @@ -162,7 +182,34 @@ public Printer usingExtensionRegistry(ExtensionRegistryLite extensionRegistry) { if (this.extensionRegistry != ExtensionRegistryLite.getEmptyRegistry()) { throw new IllegalArgumentException("Only one extensionRegistry is allowed."); } - return new Printer(escapeNonAscii, typeRegistry, extensionRegistry); + return new Printer( + escapeNonAscii, typeRegistry, extensionRegistry, enablingSafeDebugFormat, singleLine); + } + + /** + * Return a new Printer instance that outputs a redacted and unstable format suitable for + * debugging. + * + * @param enablingSafeDebugFormat If true, the new Printer will redact all proto fields that are + * marked by a debug_redact=true option, and apply an unstable prefix to the output. + * @return a new Printer that clones all other configurations from the current {@link Printer}, + * with the enablingSafeDebugFormat mode set to the given parameter. + */ + Printer enablingSafeDebugFormat(boolean enablingSafeDebugFormat) { + return new Printer( + escapeNonAscii, typeRegistry, extensionRegistry, enablingSafeDebugFormat, singleLine); + } + + /** + * Return a new Printer instance with the specified line formatting status. + * + * @param singleLine If true, the new Printer will output no newline characters. + * @return a new Printer that clones all other configurations from the current {@link Printer}, + * with the singleLine mode set to the given parameter. + */ + public Printer emittingSingleLine(boolean singleLine) { + return new Printer( + escapeNonAscii, typeRegistry, extensionRegistry, enablingSafeDebugFormat, singleLine); } /** @@ -171,12 +218,12 @@ public Printer usingExtensionRegistry(ExtensionRegistryLite extensionRegistry) { * original Protocol Buffer system) */ public void print(final MessageOrBuilder message, final Appendable output) throws IOException { - print(message, multiLineOutput(output)); + print(message, setSingleLineOutput(output, this.singleLine)); } /** Outputs a textual representation of {@code fields} to {@code output}. */ public void print(final UnknownFieldSet fields, final Appendable output) throws IOException { - printUnknownFields(fields, multiLineOutput(output)); + printUnknownFields(fields, setSingleLineOutput(output, this.singleLine)); } private void print(final MessageOrBuilder message, final TextGenerator generator) @@ -188,6 +235,14 @@ && printAny(message, generator)) { printMessage(message, generator); } + private void applyUnstablePrefix(final Appendable output) { + try { + output.append(""); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + /** * Attempt to print the 'google.protobuf.Any' message in a human-friendly format. Returns false * if the message isn't a valid 'google.protobuf.Any' message (in which case the message should @@ -244,6 +299,9 @@ private boolean printAny(final MessageOrBuilder message, final TextGenerator gen public String printFieldToString(final FieldDescriptor field, final Object value) { try { final StringBuilder text = new StringBuilder(); + if (enablingSafeDebugFormat) { + applyUnstablePrefix(text); + } printField(field, value, text); return text.toString(); } catch (IOException e) { @@ -253,7 +311,7 @@ public String printFieldToString(final FieldDescriptor field, final Object value public void printField(final FieldDescriptor field, final Object value, final Appendable output) throws IOException { - printField(field, value, multiLineOutput(output)); + printField(field, value, setSingleLineOutput(output, this.singleLine)); } private void printField( @@ -358,12 +416,19 @@ public int compareTo(MapEntryAdapter b) { public void printFieldValue( final FieldDescriptor field, final Object value, final Appendable output) throws IOException { - printFieldValue(field, value, multiLineOutput(output)); + printFieldValue(field, value, setSingleLineOutput(output, this.singleLine)); } private void printFieldValue( final FieldDescriptor field, final Object value, final TextGenerator generator) throws IOException { + if (shouldRedact(field)) { + generator.print(REDACTED_MARKER); + if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE) { + generator.eol(); + } + return; + } switch (field.getType()) { case INT32: case SINT32: @@ -429,10 +494,54 @@ private void printFieldValue( } } + private boolean shouldRedactOptionValue(EnumValueDescriptor optionValue) { + if (optionValue.getOptions().hasDebugRedact()) { + return optionValue.getOptions().getDebugRedact(); + } + return false; + } + + // The criteria for redacting a field is as follows: 1) The enablingSafeDebugFormat printer + // option + // must be on. 2) The field must be marked by a debug_redact=true option, or is marked by an + // option with an enum value that is marked by a debug_redact=true option. + private boolean shouldRedact(final FieldDescriptor field) { + if (!this.enablingSafeDebugFormat) { + return false; + } + if (field.getOptions().hasDebugRedact()) { + return field.getOptions().getDebugRedact(); + } + // Iterate through every option; if it's an enum, we check each enum value for debug_redact. + for (Map.Entry entry : + field.getOptions().getAllFields().entrySet()) { + Descriptors.FieldDescriptor option = entry.getKey(); + if (option.getType() != Descriptors.FieldDescriptor.Type.ENUM) { + continue; + } + if (option.isRepeated()) { + for (EnumValueDescriptor value : (List) entry.getValue()) { + if (shouldRedactOptionValue(value)) { + return true; + } + } + } else { + EnumValueDescriptor optionValue = (EnumValueDescriptor) entry.getValue(); + if (shouldRedactOptionValue(optionValue)) { + return true; + } + } + } + return false; + } + /** Like {@code print()}, but writes directly to a {@code String} and returns it. */ public String printToString(final MessageOrBuilder message) { try { final StringBuilder text = new StringBuilder(); + if (enablingSafeDebugFormat) { + applyUnstablePrefix(text); + } print(message, text); return text.toString(); } catch (IOException e) { @@ -443,6 +552,9 @@ public String printToString(final MessageOrBuilder message) { public String printToString(final UnknownFieldSet fields) { try { final StringBuilder text = new StringBuilder(); + if (enablingSafeDebugFormat) { + applyUnstablePrefix(text); + } print(fields, text); return text.toString(); } catch (IOException e) { @@ -457,7 +569,7 @@ public String printToString(final UnknownFieldSet fields) { public String shortDebugString(final MessageOrBuilder message) { try { final StringBuilder text = new StringBuilder(); - print(message, singleLineOutput(text)); + print(message, setSingleLineOutput(text, true)); return text.toString(); } catch (IOException e) { throw new IllegalStateException(e); @@ -471,7 +583,7 @@ public String shortDebugString(final MessageOrBuilder message) { public String shortDebugString(final FieldDescriptor field, final Object value) { try { final StringBuilder text = new StringBuilder(); - printField(field, value, singleLineOutput(text)); + printField(field, value, setSingleLineOutput(text, true)); return text.toString(); } catch (IOException e) { throw new IllegalStateException(e); @@ -485,7 +597,7 @@ public String shortDebugString(final FieldDescriptor field, final Object value) public String shortDebugString(final UnknownFieldSet fields) { try { final StringBuilder text = new StringBuilder(); - printUnknownFields(fields, singleLineOutput(text)); + printUnknownFields(fields, setSingleLineOutput(text, true)); return text.toString(); } catch (IOException e) { throw new IllegalStateException(e); @@ -640,12 +752,8 @@ public static String unsignedToString(final long value) { } } - private static TextGenerator multiLineOutput(Appendable output) { - return new TextGenerator(output, false); - } - - private static TextGenerator singleLineOutput(Appendable output) { - return new TextGenerator(output, true); + private static TextGenerator setSingleLineOutput(Appendable output, boolean singleLine) { + return new TextGenerator(output, singleLine); } /** An inner class for writing text to the output stream. */ diff --git a/java/core/src/test/java/com/google/protobuf/DebugFormatTest.java b/java/core/src/test/java/com/google/protobuf/DebugFormatTest.java new file mode 100644 index 000000000000..3f20ecf6f009 --- /dev/null +++ b/java/core/src/test/java/com/google/protobuf/DebugFormatTest.java @@ -0,0 +1,158 @@ +package com.google.protobuf; + +import static com.google.common.truth.Truth.assertThat; +import static protobuf_unittest.UnittestProto.redactedExtension; + +import com.google.protobuf.Descriptors.FieldDescriptor; +import protobuf_unittest.UnittestProto.RedactedFields; +import protobuf_unittest.UnittestProto.TestNestedMessageRedaction; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public final class DebugFormatTest { + + private static final String REDACTED_REGEX = "\\[REDACTED\\]"; + private static final String UNSTABLE_PREFIX_SINGLE_LINE = getUnstablePrefix(true); + private static final String UNSTABLE_PREFIX_MULTILINE = getUnstablePrefix(false); + + private static String getUnstablePrefix(boolean singleLine) { + return ""; + } + + @Test + public void multilineMessageFormat_returnsMultiline() { + RedactedFields message = RedactedFields.newBuilder().setOptionalUnredactedString("foo").build(); + + String result = DebugFormat.multiline().toString(message); + assertThat(result) + .matches( + String.format("%soptional_unredacted_string: \"foo\"\n", UNSTABLE_PREFIX_MULTILINE)); + } + + @Test + public void singleLineMessageFormat_returnsSingleLine() { + RedactedFields message = RedactedFields.newBuilder().setOptionalUnredactedString("foo").build(); + + String result = DebugFormat.singleLine().toString(message); + assertThat(result) + .matches( + String.format("%soptional_unredacted_string: \"foo\"", UNSTABLE_PREFIX_SINGLE_LINE)); + } + + @Test + public void messageFormat_debugRedactFieldIsRedacted() { + RedactedFields message = RedactedFields.newBuilder().setOptionalRedactedString("foo").build(); + + String result = DebugFormat.multiline().toString(message); + + assertThat(result) + .matches( + String.format( + "%soptional_redacted_string: %s\n", UNSTABLE_PREFIX_MULTILINE, REDACTED_REGEX)); + } + + @Test + public void messageFormat_debugRedactMessageIsRedacted() { + RedactedFields message = + RedactedFields.newBuilder() + .setOptionalRedactedMessage( + TestNestedMessageRedaction.newBuilder().setOptionalUnredactedNestedString("foo")) + .build(); + + String result = DebugFormat.multiline().toString(message); + + assertThat(result) + .matches( + String.format( + "%soptional_redacted_message \\{\n %s\n\\}\n", + UNSTABLE_PREFIX_MULTILINE, REDACTED_REGEX)); + } + + @Test + public void messageFormat_debugRedactMapIsRedacted() { + RedactedFields message = RedactedFields.newBuilder().putMapRedactedString("foo", "bar").build(); + + String result = DebugFormat.multiline().toString(message); + + assertThat(result) + .matches( + String.format( + "%smap_redacted_string \\{\\n %s\n\\}\n", + UNSTABLE_PREFIX_MULTILINE, REDACTED_REGEX)); + } + + @Test + public void messageFormat_debugRedactExtensionIsRedacted() { + RedactedFields message = + RedactedFields.newBuilder().setExtension(redactedExtension, "foo").build(); + + String result = DebugFormat.multiline().toString(message); + + assertThat(result) + .matches( + String.format( + "%s\\[protobuf_unittest\\.redacted_extension\\]: %s\n", + UNSTABLE_PREFIX_MULTILINE, REDACTED_REGEX)); + } + + @Test + public void messageFormat_redactFalseIsNotRedacted() { + RedactedFields message = + RedactedFields.newBuilder().setOptionalRedactedFalseString("foo").build(); + + String result = DebugFormat.multiline().toString(message); + assertThat(result) + .matches( + String.format( + "%soptional_redacted_false_string: \"foo\"\n", UNSTABLE_PREFIX_MULTILINE)); + } + + @Test + public void messageFormat_nonSensitiveFieldIsNotRedacted() { + RedactedFields message = RedactedFields.newBuilder().setOptionalUnredactedString("foo").build(); + + String result = DebugFormat.multiline().toString(message); + + assertThat(result) + .matches( + String.format("%soptional_unredacted_string: \"foo\"\n", UNSTABLE_PREFIX_MULTILINE)); + } + + @Test + public void descriptorDebugFormat_returnsExpectedFormat() { + FieldDescriptor field = + RedactedFields.getDescriptor().findFieldByName("optional_redacted_string"); + String result = DebugFormat.multiline().toString(field, "foo"); + assertThat(result) + .matches( + String.format( + "%soptional_redacted_string: %s\n", UNSTABLE_PREFIX_MULTILINE, REDACTED_REGEX)); + } + + @Test + public void unstableFormat_isStablePerProcess() { + RedactedFields message1 = + RedactedFields.newBuilder().setOptionalUnredactedString("foo").build(); + RedactedFields message2 = + RedactedFields.newBuilder().setOptionalUnredactedString("foo").build(); + for (int i = 0; i < 5; i++) { + String result1 = DebugFormat.multiline().toString(message1); + String result2 = DebugFormat.multiline().toString(message2); + assertThat(result1).isEqualTo(result2); + } + } + + @Test + public void lazyDebugFormatMessage_supportsImplicitFormatting() { + RedactedFields message = RedactedFields.newBuilder().setOptionalUnredactedString("foo").build(); + + Object lazyDebug = DebugFormat.singleLine().lazyToString(message); + + assertThat(String.format("%s", lazyDebug)) + .matches( + String.format("%soptional_unredacted_string: \"foo\"", UNSTABLE_PREFIX_SINGLE_LINE)); + } + +} diff --git a/src/google/protobuf/unittest.proto b/src/google/protobuf/unittest.proto index 7c65fb594f87..cb46956d31f1 100644 --- a/src/google/protobuf/unittest.proto +++ b/src/google/protobuf/unittest.proto @@ -1738,6 +1738,12 @@ message RedactedFields { repeated TestNestedMessageRedaction repeated_unredacted_message = 8; map map_redacted_string = 9 [debug_redact = true]; map map_unredacted_string = 10; + optional string optional_redacted_false_string = 11 [debug_redact = false]; + extensions 20 to 30; +} + +extend RedactedFields { + optional string redacted_extension = 20 [debug_redact = true]; } message TestCord{