diff --git a/api/bazel/api_build_system.bzl b/api/bazel/api_build_system.bzl index e9307f5a67d2..497d82c5ccc0 100644 --- a/api/bazel/api_build_system.bzl +++ b/api/bazel/api_build_system.bzl @@ -31,6 +31,7 @@ def api_py_proto_library(name, srcs = [], deps = [], has_services = 0): protoc = "@com_google_protobuf//:protoc", deps = [_LibrarySuffix(d, _PY_SUFFIX) for d in deps] + [ "@com_lyft_protoc_gen_validate//validate:validate_py", + "@googleapis//:api_httpbody_protos_py", "@googleapis//:http_api_protos_py", "@googleapis//:rpc_status_protos_py", "@com_github_gogo_protobuf//:gogo_proto_py", @@ -106,6 +107,7 @@ def api_proto_library(name, visibility = ["//visibility:private"], srcs = [], de "@com_google_protobuf//:struct_proto", "@com_google_protobuf//:timestamp_proto", "@com_google_protobuf//:wrappers_proto", + "@googleapis//:api_httpbody_protos_proto", "@googleapis//:http_api_protos_proto", "@googleapis//:rpc_status_protos_lib", "@com_github_gogo_protobuf//:gogo_proto", diff --git a/api/bazel/repositories.bzl b/api/bazel/repositories.bzl index d46771a7642b..ba56da977af7 100644 --- a/api/bazel/repositories.bzl +++ b/api/bazel/repositories.bzl @@ -17,10 +17,59 @@ def api_dependencies(): name = "googleapis", strip_prefix = "googleapis-" + GOOGLEAPIS_SHA, url = "https://github.com/googleapis/googleapis/archive/" + GOOGLEAPIS_SHA + ".tar.gz", + # TODO(dio): Consider writing a Skylark macro for importing Google API proto. build_file_content = """ load("@com_google_protobuf//:protobuf.bzl", "cc_proto_library", "py_proto_library") load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") +filegroup( + name = "api_httpbody_protos_src", + srcs = [ + "google/api/httpbody.proto", + ], + visibility = ["//visibility:public"], +) + +proto_library( + name = "api_httpbody_protos_proto", + srcs = [":api_httpbody_protos_src"], + deps = ["@com_google_protobuf//:descriptor_proto"], + visibility = ["//visibility:public"], +) + +cc_proto_library( + name = "api_httpbody_protos", + srcs = [ + "google/api/httpbody.proto", + ], + default_runtime = "@com_google_protobuf//:protobuf", + protoc = "@com_google_protobuf//:protoc", + deps = ["@com_google_protobuf//:cc_wkt_protos"], + visibility = ["//visibility:public"], +) + +py_proto_library( + name = "api_httpbody_protos_py", + srcs = [ + "google/api/httpbody.proto", + ], + include = ".", + default_runtime = "@com_google_protobuf//:protobuf_python", + protoc = "@com_google_protobuf//:protoc", + visibility = ["//visibility:public"], + deps = ["@com_google_protobuf//:protobuf_python"], +) + +go_proto_library( + name = "api_httpbody_go_proto", + importpath = "google.golang.org/genproto/googleapis/api/httpbody", + proto = ":api_httpbody_protos_proto", + visibility = ["//visibility:public"], + deps = [ + ":descriptor_go_proto", + ], +) + filegroup( name = "http_api_protos_src", srcs = [ @@ -28,7 +77,7 @@ filegroup( "google/api/http.proto", ], visibility = ["//visibility:public"], - ) +) go_proto_library( name = "descriptor_go_proto", @@ -93,6 +142,7 @@ proto_library( deps = ["@com_google_protobuf//:any_proto"], visibility = ["//visibility:public"], ) + cc_proto_library( name = "rpc_status_protos", srcs = ["google/rpc/status.proto"], diff --git a/bazel/envoy_build_system.bzl b/bazel/envoy_build_system.bzl index cad0fec13b23..374ecb4612d3 100644 --- a/bazel/envoy_build_system.bzl +++ b/bazel/envoy_build_system.bzl @@ -383,6 +383,10 @@ def envoy_proto_library( cc_proto_deps = [] py_proto_deps = ["@com_google_protobuf//:protobuf_python"] + if "api_httpbody_protos" in external_deps: + cc_proto_deps.append("@googleapis//:api_httpbody_protos") + py_proto_deps.append("@googleapis//:api_httpbody_protos_py") + if "http_api_protos" in external_deps: cc_proto_deps.append("@googleapis//:http_api_protos") py_proto_deps.append("@googleapis//:http_api_protos_py") @@ -420,6 +424,10 @@ def envoy_proto_descriptor(name, out, srcs = [], external_deps = []): input_files = ["$(location " + src + ")" for src in srcs] include_paths = [".", PACKAGE_NAME] + if "api_httpbody_protos" in external_deps: + srcs.append("@googleapis//:api_httpbody_protos_src") + include_paths.append("external/googleapis") + if "http_api_protos" in external_deps: srcs.append("@googleapis//:http_api_protos_src") include_paths.append("external/googleapis") diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index 21d04ff9ba90..c85aaf358c8f 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -192,6 +192,10 @@ def _envoy_api_deps(): if "envoy_api" not in native.existing_rules().keys(): _default_envoy_api(name = "envoy_api") + native.bind( + name = "api_httpbody_protos", + actual = "@googleapis//:api_httpbody_protos", + ) native.bind( name = "http_api_protos", actual = "@googleapis//:http_api_protos", diff --git a/docs/root/configuration/http_filters/grpc_json_transcoder_filter.rst b/docs/root/configuration/http_filters/grpc_json_transcoder_filter.rst index 0c77708dbac5..471297286a12 100644 --- a/docs/root/configuration/http_filters/grpc_json_transcoder_filter.rst +++ b/docs/root/configuration/http_filters/grpc_json_transcoder_filter.rst @@ -62,3 +62,16 @@ match the incoming request path, set `match_incoming_request_route` to true. }; } } + +Sending arbitrary content +------------------------- + +By default, when transcoding occurs, gRPC-JSON encodes the message output of a gRPC service method into +JSON and sets the HTTP response `Content-Type` header to `application/json`. To send abritrary content, +a gRPC service method can use +`google.api.HttpBody `_ +as its output message type. The implementation needs to set +`content_type `_ +(which sets the value of the HTTP response `Content-Type` header) and +`data `_ +(which sets the HTTP response body) accordingly. \ No newline at end of file diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 77813440b8ac..2f74c99e65f7 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -7,6 +7,9 @@ Version history to filter based on the presence of Envoy response flags. * admin: added :http:get:`/hystrix_event_stream` as an endpoint for monitoring envoy's statistics through `Hystrix dashboard `_. +* grpc-json: added support for building HTTP response from + `google.api.HttpBody `_. +* config: v1 disabled by default. v1 support remains available until October via flipping --v2-config-only=false. * config: v1 disabled by default. v1 support remains available until October via setting :option:`--allow-deprecated-v1-api`. * health check: added support for :ref:`custom health check `. * health check: added support for :ref:`specifying jitter as a percentage `. diff --git a/source/extensions/filters/http/grpc_json_transcoder/BUILD b/source/extensions/filters/http/grpc_json_transcoder/BUILD index 81b357aa6d71..3128edcf4c52 100644 --- a/source/extensions/filters/http/grpc_json_transcoder/BUILD +++ b/source/extensions/filters/http/grpc_json_transcoder/BUILD @@ -20,11 +20,11 @@ envoy_cc_library( "path_matcher", "grpc_transcoding", "http_api_protos", + "api_httpbody_protos", ], deps = [ ":transcoder_input_stream_lib", "//include/envoy/http:filter_interface", - "//source/common/common:base64_lib", "//source/common/grpc:codec_lib", "//source/common/grpc:common_lib", "//source/common/http:headers_lib", diff --git a/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.cc b/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.cc index 4328fb2ab760..0d02ea3553dd 100644 --- a/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.cc +++ b/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.cc @@ -14,6 +14,7 @@ #include "google/api/annotations.pb.h" #include "google/api/http.pb.h" +#include "google/api/httpbody.pb.h" #include "grpc_transcoding/json_request_translator.h" #include "grpc_transcoding/path_matcher_utility.h" #include "grpc_transcoding/response_to_json_translator.h" @@ -222,6 +223,7 @@ Http::FilterHeadersStatus JsonTranscoderFilter::decodeHeaders(Http::HeaderMap& h // just pass-through the request to upstream. return Http::FilterHeadersStatus::Continue; } + has_http_body_output_ = !method_->server_streaming() && hasHttpBodyAsOutputType(); headers.removeContentLength(); headers.insertContentType().value().setReference(Http::Headers::get().ContentTypeValues.Grpc); @@ -340,6 +342,12 @@ Http::FilterDataStatus JsonTranscoderFilter::encodeData(Buffer::Instance& data, return Http::FilterDataStatus::Continue; } + // TODO(dio): Add support for streaming case. + if (has_http_body_output_) { + buildResponseFromHttpBodyOutput(*response_headers_, data); + return Http::FilterDataStatus::StopIterationAndBuffer; + } + response_in_.move(data); if (end_stream) { @@ -415,6 +423,32 @@ bool JsonTranscoderFilter::readToBuffer(Protobuf::io::ZeroCopyInputStream& strea return false; } +void JsonTranscoderFilter::buildResponseFromHttpBodyOutput(Http::HeaderMap& response_headers, + Buffer::Instance& data) { + std::vector frames; + decoder_.decode(data, frames); + if (frames.empty()) { + return; + } + + google::api::HttpBody http_body; + for (auto& frame : frames) { + if (frame.length_ > 0) { + Buffer::ZeroCopyInputStreamImpl stream(std::move(frame.data_)); + http_body.ParseFromZeroCopyStream(&stream); + const auto& body = http_body.data(); + data.add(body); + response_headers.insertContentType().value(http_body.content_type()); + response_headers.insertContentLength().value(body.length()); + return; + } + } +} + +bool JsonTranscoderFilter::hasHttpBodyAsOutputType() { + return method_->output_type()->full_name() == google::api::HttpBody::descriptor()->full_name(); +} + } // namespace GrpcJsonTranscoder } // namespace HttpFilters } // namespace Extensions diff --git a/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.h b/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.h index e8544ea2600f..5849bcc96b2f 100644 --- a/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.h +++ b/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.h @@ -7,6 +7,7 @@ #include "envoy/json/json_object.h" #include "common/common/logger.h" +#include "common/grpc/codec.h" #include "common/protobuf/protobuf.h" #include "extensions/filters/http/grpc_json_transcoder/transcoder_input_stream_impl.h" @@ -117,6 +118,8 @@ class JsonTranscoderFilter : public Http::StreamFilter, public Logger::Loggable< private: bool readToBuffer(Protobuf::io::ZeroCopyInputStream& stream, Buffer::Instance& data); + void buildResponseFromHttpBodyOutput(Http::HeaderMap& response_headers, Buffer::Instance& data); + bool hasHttpBodyAsOutputType(); JsonTranscoderConfig& config_; std::unique_ptr transcoder_; @@ -126,8 +129,10 @@ class JsonTranscoderFilter : public Http::StreamFilter, public Logger::Loggable< Http::StreamEncoderFilterCallbacks* encoder_callbacks_{nullptr}; const Protobuf::MethodDescriptor* method_{nullptr}; Http::HeaderMap* response_headers_{nullptr}; + Grpc::Decoder decoder_; bool error_{false}; + bool has_http_body_output_{false}; }; } // namespace GrpcJsonTranscoder diff --git a/test/extensions/filters/http/grpc_json_transcoder/grpc_json_transcoder_integration_test.cc b/test/extensions/filters/http/grpc_json_transcoder/grpc_json_transcoder_integration_test.cc index a63ac5120f11..2417a31794de 100644 --- a/test/extensions/filters/http/grpc_json_transcoder/grpc_json_transcoder_integration_test.cc +++ b/test/extensions/filters/http/grpc_json_transcoder/grpc_json_transcoder_integration_test.cc @@ -172,6 +172,17 @@ TEST_P(GrpcJsonTranscoderIntegrationTest, UnaryGet) { R"({"shelves":[{"id":"20","theme":"Children"},{"id":"1","theme":"Foo"}]})"); } +TEST_P(GrpcJsonTranscoderIntegrationTest, UnaryGetHttpBody) { + testTranscoding( + Http::TestHeaderMapImpl{{":method", "GET"}, {":path", "/index"}, {":authority", "host"}}, "", + {""}, {R"(content_type: "text/html" data: "

Hello!

" )"}, Status(), + Http::TestHeaderMapImpl{{":status", "200"}, + {"content-type", "text/html"}, + {"content-length", "15"}, + {"grpc-status", "0"}}, + R"(

Hello!

)"); +} + TEST_P(GrpcJsonTranscoderIntegrationTest, UnaryGetError) { testTranscoding( Http::TestHeaderMapImpl{ diff --git a/test/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter_test.cc b/test/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter_test.cc index 06a5ede72416..f388588a085e 100644 --- a/test/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter_test.cc +++ b/test/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter_test.cc @@ -550,6 +550,84 @@ TEST_F(GrpcJsonTranscoderFilterTest, TranscodingUnaryNotGrpcResponse) { EXPECT_EQ(Http::FilterDataStatus::Continue, filter_.encodeData(request_data, true)); } +TEST_F(GrpcJsonTranscoderFilterTest, TranscodingUnaryWithHttpBodyAsOutput) { + Http::TestHeaderMapImpl request_headers{{":method", "GET"}, {":path", "/index"}}; + + EXPECT_CALL(decoder_callbacks_, clearRouteCache()); + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(request_headers, false)); + EXPECT_EQ("application/grpc", request_headers.get_("content-type")); + EXPECT_EQ("/index", request_headers.get_("x-envoy-original-path")); + EXPECT_EQ("/bookstore.Bookstore/GetIndex", request_headers.get_(":path")); + EXPECT_EQ("trailers", request_headers.get_("te")); + + Http::TestHeaderMapImpl response_headers{{"content-type", "application/grpc"}, + {":status", "200"}}; + + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, + filter_.encodeHeaders(response_headers, false)); + EXPECT_EQ("application/json", response_headers.get_("content-type")); + + google::api::HttpBody response; + response.set_content_type("text/html"); + response.set_data("

Hello, world!

"); + + auto response_data = Grpc::Common::serializeBody(response); + + EXPECT_EQ(Http::FilterDataStatus::StopIterationAndBuffer, + filter_.encodeData(*response_data, false)); + + EXPECT_EQ(response.content_type(), response_headers.get_("content-type")); + EXPECT_EQ(response.data(), response_data->toString()); + + Http::TestHeaderMapImpl response_trailers{{"grpc-status", "0"}, {"grpc-message", ""}}; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_.decodeTrailers(response_trailers)); +} + +TEST_F(GrpcJsonTranscoderFilterTest, TranscodingUnaryWithHttpBodyAsOutputAndSplitTwoEncodeData) { + Http::TestHeaderMapImpl request_headers{{":method", "GET"}, {":path", "/index"}}; + + EXPECT_CALL(decoder_callbacks_, clearRouteCache()); + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(request_headers, false)); + EXPECT_EQ("application/grpc", request_headers.get_("content-type")); + EXPECT_EQ("/index", request_headers.get_("x-envoy-original-path")); + EXPECT_EQ("/bookstore.Bookstore/GetIndex", request_headers.get_(":path")); + EXPECT_EQ("trailers", request_headers.get_("te")); + + Http::TestHeaderMapImpl response_headers{{"content-type", "application/grpc"}, + {":status", "200"}}; + + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, + filter_.encodeHeaders(response_headers, false)); + EXPECT_EQ("application/json", response_headers.get_("content-type")); + + google::api::HttpBody response; + response.set_content_type("text/html"); + response.set_data("

Hello, world!

"); + + auto response_data = Grpc::Common::serializeBody(response); + + // Firstly, the response data buffer is splitted into two parts. + Buffer::OwnedImpl response_data_first_part; + response_data_first_part.move(*response_data, response_data->length() / 2); + + // Secondly, we send the first part of response data to the data encoding step. + EXPECT_EQ(Http::FilterDataStatus::StopIterationAndBuffer, + filter_.encodeData(response_data_first_part, false)); + + // Finaly, since half of the response data buffer is moved already, here we can send the rest + // of it to the next data encoding step. + EXPECT_EQ(Http::FilterDataStatus::StopIterationAndBuffer, + filter_.encodeData(*response_data, false)); + + EXPECT_EQ(response.content_type(), response_headers.get_("content-type")); + EXPECT_EQ(response.data(), response_data->toString()); + + Http::TestHeaderMapImpl response_trailers{{"grpc-status", "0"}, {"grpc-message", ""}}; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_.decodeTrailers(response_trailers)); +} + struct GrpcJsonTranscoderFilterPrintTestParam { std::string config_json_; std::string expected_response_; diff --git a/test/proto/BUILD b/test/proto/BUILD index 3b21fb2d2ea2..fa1674251719 100644 --- a/test/proto/BUILD +++ b/test/proto/BUILD @@ -20,8 +20,9 @@ envoy_proto_library( name = "bookstore_proto", srcs = [":bookstore.proto"], external_deps = [ - "well_known_protos", + "api_httpbody_protos", "http_api_protos", + "well_known_protos", ], ) @@ -32,6 +33,7 @@ envoy_proto_descriptor( ], out = "bookstore.descriptor", external_deps = [ + "api_httpbody_protos", "http_api_protos", "well_known_protos", ], diff --git a/test/proto/bookstore.proto b/test/proto/bookstore.proto index 45ca15bf54c6..6fb65b42d119 100644 --- a/test/proto/bookstore.proto +++ b/test/proto/bookstore.proto @@ -3,6 +3,7 @@ syntax = "proto3"; package bookstore; import "google/api/annotations.proto"; +import "google/api/httpbody.proto"; import "google/protobuf/empty.proto"; // A simple Bookstore API. @@ -84,6 +85,11 @@ service Bookstore { get: "/authors/{author}" }; } + rpc GetIndex(google.protobuf.Empty) returns (google.api.HttpBody) { + option (google.api.http) = { + get: "/index" + }; + } } // A shelf resource.