Skip to content

Commit

Permalink
Cont. - Add Proto Message Extraction http filter (#35689)
Browse files Browse the repository at this point in the history 
The Proto Message Extraction Filter supports extracting gRPC
requests/responses (proto messages) as `google.protobuf.Struct` and
storing results in the dynamic metadata
`envoy.filters.http.proto_message_scrubbing` for later access.

---------

Signed-off-by: dchakarwarti@google.com <dchakarwarti@google.com>

Mirrored from https://github.com/envoyproxy/envoy @ 16759b97e02131cc9c8ca64d13cf663c75dae047
  • Loading branch information
@update-envoy
update-envoy[bot] committed Aug 27, 2024
1 parent 536d5aa commit 396df43
Show file tree
Hide file tree
Showing 4 changed files with 83 additions and 69 deletions.
2 changes: 1 addition & 1 deletion BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,7 @@ proto_library(
"//envoy/extensions/filters/http/oauth2/v3:pkg",
"//envoy/extensions/filters/http/on_demand/v3:pkg",
"//envoy/extensions/filters/http/original_src/v3:pkg",
"//envoy/extensions/filters/http/proto_message_logging/v3:pkg",
"//envoy/extensions/filters/http/proto_message_extraction/v3:pkg",
"//envoy/extensions/filters/http/rate_limit_quota/v3:pkg",
"//envoy/extensions/filters/http/ratelimit/v3:pkg",
"//envoy/extensions/filters/http/rbac/v3:pkg",
Expand Down
0 ...lters/http/proto_message_logging/v3/BUILD → ...rs/http/proto_message_extraction/v3/BUILD
View file
File renamed without changes.
148 changes: 81 additions & 67 deletions ...ttp/proto_message_logging/v3/config.proto → .../proto_message_extraction/v3/config.proto
View file
Original file line number Diff line number Diff line change
@@ -1,40 +1,46 @@
syntax = "proto3";

package envoy.extensions.filters.http.proto_message_logging.v3;
package envoy.extensions.filters.http.proto_message_extraction.v3;

import "envoy/config/core/v3/base.proto";

import "xds/annotations/v3/status.proto";

import "udpa/annotations/status.proto";

option java_package = "io.envoyproxy.envoy.extensions.filters.http.proto_message_logging.v3";
option java_package = "io.envoyproxy.envoy.extensions.filters.http.proto_message_extraction.v3";
option java_outer_classname = "ConfigProto";
option java_multiple_files = true;
option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/proto_message_logging/v3;proto_message_loggingv3";
option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/proto_message_extraction/v3;proto_message_extractionv3";
option (udpa.annotations.file_status).package_version_status = ACTIVE;
option (xds.annotations.v3.file_status).work_in_progress = true;

// [#not-implemented-hide:]
// [#protodoc-title: Proto Message Logging]
// Proto Message Logging :ref:`configuration overview
// <config_http_filters_proto_message_logging>`.
// [#extension: envoy.filters.http.proto_message_logging]
// [#protodoc-title: Proto Message Extraction]
//
// ProtoMessageLogging filter supports logging scrubbed gRPC requests/responses(proto messages)
// as google.protobuf.Struct and storing results
// in the dynamic metadata `envoy.filters.http.proto_message_logging` for later access.
// [#extension: envoy.filters.http.proto_message_extraction]
//
// Overview
// --------
//
// ProtoMessageExtraction filter supports extracting gRPC
// requests/responses(proto messages) into google.protobuf.Struct and storing
// results in the dynamic metadata `envoy.filters.http.proto_message_extraction`
// for later access.
//
// Assumptions
// -----------
//
// # Assumptions
// This filter assumes it is only applicable for gRPC with Protobuf as payload.
//
// # Process Flow
// Process Flow
// ------------
//
// On the request path, it will check
//
// 1. if the incoming gRPC request is configured, the filter tries to:
//
// a. buffer the incoming data to complete protobuf messages
// b. log individual protobuf messages according to directives
// b. extract individual protobuf messages according to directives
// c. write the result into the dynamic metadata.
// d. pass through the request data
//
Expand All @@ -45,50 +51,57 @@ option (xds.annotations.v3.file_status).work_in_progress = true;
// 1. if the incoming gRPC request is configured, the filter tries to:
//
// a. buffer the incoming data to complete protobuf messages
// b. log individual protobuf messages according to directives
// b. extract individual protobuf messages according to directives
// c. write the result into the dynamic metadata.
// d. pass through the response data
//
// 2. otherwise, pass through the response.
//
// # Config Requirements
// Config Requirements
// -------------------

// Here are config requirements
//
// 1. the log target field should be among the following primitive types: `string`, `uint32`, `uint64`, `int32`, `int64`, `sint32`, `sint64`, `fixed32`, `fixed64`, `sfixed32`, `sfixed64`, `float`, `double`.
//

// 1. the extracted target field should be among the following primitive types:
// `string`, `uint32`, `uint64`, `int32`, `int64`, `sint32`, `sint64`,
// `fixed32`, `fixed64`, `sfixed32`, `sfixed64`, `float`, `double`.

// 2. the target field could be repeated.
//

// 3. the intermediate type could also be repeated.
//
// # Output Format
// The logged requests and responses will be will be added in the dynamic ``metadata<google.protobuf.Struct>`` with the same layout of the message.
//

// Output Format
// -------------

// The extracted requests and responses will be will be added in the dynamic
// ``metadata<google.protobuf.Struct>`` with the same layout of the message.

// For the default `FIRST_AND_LAST` mode, it will be like:

// .. code-block:: json

// {
// "requests":{
// "first":{
// "foo": "val_foo1",
// ...
// }
// "last":{
// "foo": "val_foo3",
// ...
// }
// },
// "responses":{
// "first":{
// "baz": "val_baz1",
// ...
// }
// "last":{
// "baz": "val_foo3",
// ...
// }
// }
// }

// Example for `FIRST_AND_LAST` mode
// ---------------------------------
//
//
// # Example for `FIRST_AND_LAST` mode
// Let's say we have the following definition for the bi-streaming request
// `pkg.svc.Method`.
//
Expand All @@ -111,7 +124,7 @@ option (xds.annotations.v3.file_status).work_in_progress = true;
//
// message Msg {
// string bar = 1;
// string not_logged = 2;
// string not_extracted = 2;
// }
//
// This is the filter config in JSON.
Expand All @@ -121,15 +134,15 @@ option (xds.annotations.v3.file_status).work_in_progress = true;
// {
// "descriptor_set":{},
// "mode": "FIRST_AND_LAST",
// "logging_by_method":{
// "extraction_by_method":{
// "pkg.svc.Method":{
// "request_logging_by_field":{
// "foo":"LOG",
// "nested.doubled_nested.bar":"LOG",
// "redacted":"LOG_REDACT"
// "request_extraction_by_field":{
// "foo":"EXTRACT",
// "nested.doubled_nested.bar":"EXTRACT",
// "redacted":"EXTRACT_REDACT"
// },
// "response_logging_by_field":{
// "bar":"LOG",
// "response_extraction_by_field":{
// "bar":"EXTRACT",
// }
// }
// }
Expand All @@ -142,18 +155,18 @@ option (xds.annotations.v3.file_status).work_in_progress = true;
//
// {
// "foo": "val_foo1",
// "nested": { "double_nested": {"bar": "val_bar1", "not_logged": "val_not_logged1"},
// "redacted": { "bar": "val_redacted_bar1"}
// "nested": { "double_nested": {"bar": "val_bar1", "not_extracted":
// "val_not_extracted1"}, "redacted": { "bar": "val_redacted_bar1"}
// }
// {
// "foo": "val_foo2",
// "nested": { "double_nested": {"bar": "val_bar2", "not_logged": "val_not_logged2"},
// "redacted": { "bar": "val_redacted_bar2"}
// "nested": { "double_nested": {"bar": "val_bar2", "not_extracted":
// "val_not_extracted2"}, "redacted": { "bar": "val_redacted_bar2"}
// }
// {
// "foo": "val_foo3",
// "nested": { "double_nested": {"bar": "val_bar3", "not_logged": "val_not_logged3"},
// "redacted": { "bar": "val_redacted_bar3"}
// "nested": { "double_nested": {"bar": "val_bar3", "not_extracted":
// "val_not_extracted3"}, "redacted": { "bar": "val_redacted_bar3"}
// }
//
// the filter receives the following `MethodResponse` message in
Expand All @@ -172,7 +185,7 @@ option (xds.annotations.v3.file_status).work_in_progress = true;
// }
//
// The filter will write the following dynamic
// metadata(`envoy.filters.http.proto_message_logging`) in JSON.
// metadata(`envoy.filters.http.proto_message_extraction`) in JSON.
//
// .. code-block:: json
//
Expand All @@ -199,11 +212,11 @@ option (xds.annotations.v3.file_status).work_in_progress = true;
// }
// }

message ProtoMessageLoggingConfig {
enum LogMode {
LogMode_UNSPECIFIED = 0;
message ProtoMessageExtractionConfig {
enum ExtractMode {
ExtractMode_UNSPECIFIED = 0;

// The filter will log the first and the last message for
// The filter will extract the first and the last message for
// for streaming cases, containing
// client-side streaming, server-side streaming or bi-directional streaming.
FIRST_AND_LAST = 1;
Expand All @@ -216,40 +229,41 @@ message ProtoMessageLoggingConfig {
config.core.v3.DataSource data_source = 1;

// Unimplemented, the key of proto descriptor TypedMetadata.
// Among filters depending on the proto descriptor, we can have a TypedMetadata
// for proto descriptors, so that these filters can share one copy of proto
// descriptor in memory.
// Among filters depending on the proto descriptor, we can have a
// TypedMetadata for proto descriptors, so that these filters can share one
// copy of proto descriptor in memory.
string proto_descriptor_typed_metadata = 2;
}

LogMode mode = 3;
ExtractMode mode = 3;

// Specify the message logging info.
// Specify the message extraction info.
// The key is the fully qualified gRPC method name.
// ``${package}.${Service}.${Method}``, like
// ``endpoints.examples.bookstore.BookStore.GetShelf``
//
// The value is the message logging information for individual gRPC methods.
map<string, MethodLogging> logging_by_method = 4;
// The value is the message extraction information for individual gRPC
// methods.
map<string, MethodExtraction> extraction_by_method = 4;
}

// This message can be used to support per route config approach later even
// though the Istio doesn't support that so far.
message MethodLogging {
enum LogDirective {
LogDirective_UNSPECIFIED = 0;
message MethodExtraction {
enum ExtractDirective {
ExtractDirective_UNSPECIFIED = 0;

// The value of this field will be logged.
LOG = 1;
// The value of this field will be extracted.
EXTRACT = 1;

// It should be only annotated on Message type fields so if the field isn't
// empty, an empty Struct will be logged.
LOG_REDACT = 2;
// empty, an empty Struct will be extracted.
EXTRACT_REDACT = 2;
}

// The mapping of field path to its LogDirective for request messages
map<string, LogDirective> request_logging_by_field = 2;
// The mapping of field path to its ExtractDirective for request messages
map<string, ExtractDirective> request_extraction_by_field = 2;

// The mapping of field path to its LogDirective for response messages
map<string, LogDirective> response_logging_by_field = 3;
// The mapping of field path to its ExtractDirective for response messages
map<string, ExtractDirective> response_extraction_by_field = 3;
}
2 changes: 1 addition & 1 deletion versioning/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ proto_library(
"//envoy/extensions/filters/http/oauth2/v3:pkg",
"//envoy/extensions/filters/http/on_demand/v3:pkg",
"//envoy/extensions/filters/http/original_src/v3:pkg",
"//envoy/extensions/filters/http/proto_message_logging/v3:pkg",
"//envoy/extensions/filters/http/proto_message_extraction/v3:pkg",
"//envoy/extensions/filters/http/rate_limit_quota/v3:pkg",
"//envoy/extensions/filters/http/ratelimit/v3:pkg",
"//envoy/extensions/filters/http/rbac/v3:pkg",
Expand Down

0 comments on commit 396df43

Please sign in to comment.