From 5bd6eea04e16a25a94211ce07ca6a437f75e93de Mon Sep 17 00:00:00 2001 From: styblope Date: Tue, 6 Dec 2022 00:24:12 +0100 Subject: [PATCH] shippingservice: add http client request span (#610) * shippingservice: Add http client request span The commit adds new http client span for outgoing calls to `/getquote` endpoint of quoteservice via the `reqwests` client middleware instrumentation. This provides a bit of the server/client tasks separation visibility as well as enables span pairing with the downstream quoteservice server span. The instrumentation uses the available `reqwests-tracing` and `tracing-opentelementry` crates. * update shippingservice docs * update changelog * update doc section position * apply syntax change suggestion Co-authored-by: Juliano Costa --- CHANGELOG.md | 2 + docs/services/shippingservice.md | 17 +- src/shippingservice/Cargo.lock | 217 +++++++++++++++--- src/shippingservice/Cargo.toml | 7 +- src/shippingservice/src/main.rs | 11 +- .../src/shipping_service/quote.rs | 6 +- 6 files changed, 228 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bf5b05b289..2396f8471d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -144,3 +144,5 @@ significant modifications will be credited to OpenTelemetry Authors. ([#587](https://github.com/open-telemetry/opentelemetry-demo/pull/587)) * Pass product's `categories` as an input for the Ad service ([#600](https://github.com/open-telemetry/opentelemetry-demo/pull/600)) +* Add HTTP client instrumentation to shippingservice +([#610](https://github.com/open-telemetry/opentelemetry-demo/pull/610)) diff --git a/docs/services/shippingservice.md b/docs/services/shippingservice.md index 2b6e538e60..d64a3b75ab 100644 --- a/docs/services/shippingservice.md +++ b/docs/services/shippingservice.md @@ -57,7 +57,7 @@ Be mindful of async runtime, [context guards](https://docs.rs/opentelemetry/latest/opentelemetry/struct.ContextGuard.html), and inability to move and clone `spans` when replicating from these samples. -### Adding gRPC/HTTP instrumentation +### Adding gRPC instrumentation This service receives gRPC requests, which are instrumented in the middleware. @@ -107,6 +107,21 @@ You may also notice the `attributes` set on the span in this example, and context) the [OpenTelemetry API](https://docs.rs/opentelemetry/0.17.0/opentelemetry/trace/struct.SpanRef.html) will work. +### Adding HTTP instrumentation + +A child *client* span is also produced for the outoing HTTP call to +`quoteservice` via the `reqwest` client. This span pairs up with the +corresponding `quoteservice` *server* span. The tracing instrumentation is +implemented in the client middleware making use of the available +`reqwest-middleware`, `reqwest-tracing` and `tracing-opentelementry` libraries: + +```rust + let reqwest_client = reqwest::Client::new(); + let client = ClientBuilder::new(reqwest_client) + .with(TracingMiddleware::::new()) + .build(); +``` + ### Add span attributes Provided you are on the same thread, or in a context passed from a diff --git a/src/shippingservice/Cargo.lock b/src/shippingservice/Cargo.lock index 5cb67153a9..5ccc08cc8d 100644 --- a/src/shippingservice/Cargo.lock +++ b/src/shippingservice/Cargo.lock @@ -111,9 +111,9 @@ checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba" [[package]] name = "bytes" -version = "1.2.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec8a7b6a70fde80372154c65702f00a0f56f3e1c36abbc6c440484be248856db" +checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" [[package]] name = "cfg-if" @@ -133,9 +133,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.12" +version = "0.8.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edbafec5fa1f196ca66527c1b12c2ec4745ca14b50f1ad8f9f6f720b55d11fac" +checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f" dependencies = [ "cfg-if", ] @@ -379,9 +379,9 @@ checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" [[package]] name = "hyper" -version = "0.14.22" +version = "0.14.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abfba89e19b959ca163c7752ba59d737c1ceea53a5d31a149c805446fc958064" +checksum = "034711faac9d2166cb1baf1a2fb0b60b1f277f8492fd72176c17f3515e1abd3c" dependencies = [ "bytes", "futures-channel", @@ -425,9 +425,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.9.1" +version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e" +checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" dependencies = [ "autocfg", "hashbrown", @@ -444,9 +444,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.5.0" +version = "2.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b" +checksum = "f88c5561171189e69df9d98bcf18fd5f9558300f7ea7b801eb8a0fd748bd8745" [[package]] name = "itertools" @@ -521,6 +521,16 @@ version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" +[[package]] +name = "mime_guess" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4192263c238a5f0d0c6bfd21f336a313a4ce1c450542449ca191bb657b4642ef" +dependencies = [ + "mime", + "unicase", +] + [[package]] name = "mio" version = "0.8.5" @@ -539,6 +549,16 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + [[package]] name = "num_cpus" version = "1.14.0" @@ -665,6 +685,12 @@ dependencies = [ "tokio-stream", ] +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + [[package]] name = "parking_lot_core" version = "0.9.4" @@ -856,18 +882,18 @@ dependencies = [ [[package]] name = "regex" -version = "1.6.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b" +checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a" dependencies = [ "regex-syntax", ] [[package]] name = "regex-syntax" -version = "0.6.27" +version = "0.6.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244" +checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" [[package]] name = "remove_dir_all" @@ -880,9 +906,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.12" +version = "0.11.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "431949c384f4e2ae07605ccaa56d1d9d2ecdb5cadd4f9577ccfab29f2e5149fc" +checksum = "68cc60575865c7831548863cc02356512e3f1dc2f3f82cb837d7fc4cc8f3c97c" dependencies = [ "base64", "bytes", @@ -897,6 +923,7 @@ dependencies = [ "js-sys", "log", "mime", + "mime_guess", "once_cell", "percent-encoding", "pin-project-lite", @@ -912,6 +939,36 @@ dependencies = [ "winreg", ] +[[package]] +name = "reqwest-middleware" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a1c03e9011a8c59716ad13115550469e081e2e9892656b0ba6a47c907921894" +dependencies = [ + "anyhow", + "async-trait", + "http", + "reqwest", + "serde", + "task-local-extensions", + "thiserror", +] + +[[package]] +name = "reqwest-tracing" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b739d87a6b2cf4743968ad2b4cef648fbe0204c19999509824425babb2097bce" +dependencies = [ + "async-trait", + "opentelemetry", + "reqwest", + "reqwest-middleware", + "task-local-extensions", + "tracing", + "tracing-opentelemetry", +] + [[package]] name = "ryu" version = "1.0.11" @@ -932,9 +989,9 @@ checksum = "d193d69bae983fc11a79df82342761dfbf28a99fc8d203dca4c3c1b590948965" [[package]] name = "serde_json" -version = "1.0.87" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce777b7b150d76b9cf60d28b55f5847135a003f7d7350c6be7a773508ce7d45" +checksum = "020ff22c755c2ed3f8cf162dbb41a7268d934702f3ed3631656ea597e08fc3db" dependencies = [ "itoa", "ryu", @@ -953,6 +1010,15 @@ dependencies = [ "serde", ] +[[package]] +name = "sharded-slab" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" +dependencies = [ + "lazy_static", +] + [[package]] name = "shippingservice" version = "0.1.0" @@ -966,11 +1032,16 @@ dependencies = [ "prost", "prost-types", "reqwest", + "reqwest-middleware", + "reqwest-tracing", "simplelog", "tokio", "tonic", "tonic-build", "tonic-health", + "tracing", + "tracing-opentelemetry", + "tracing-subscriber", "uuid", ] @@ -1027,6 +1098,15 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20518fe4a4c9acf048008599e464deb21beeae3d3578418951a189c235a7a9a8" +[[package]] +name = "task-local-extensions" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4167afbec18ae012de40f8cf1b9bf48420abb390678c34821caa07d924941cc4" +dependencies = [ + "tokio", +] + [[package]] name = "tempfile" version = "3.3.0" @@ -1070,23 +1150,43 @@ dependencies = [ "syn", ] +[[package]] +name = "thread_local" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180" +dependencies = [ + "once_cell", +] + [[package]] name = "time" -version = "0.3.15" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d634a985c4d4238ec39cacaed2e7ae552fbd3c476b552c1deac3021b7d7eaf0c" +checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376" dependencies = [ "itoa", "libc", "num_threads", + "serde", + "time-core", "time-macros", ] +[[package]] +name = "time-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" + [[package]] name = "time-macros" -version = "0.2.4" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42657b1a6f4d817cda8e7a0ace261fe0cc946cf3a80314390b22cc61ae080792" +checksum = "d967f99f534ca7e495c575c62638eebc2898a8c84c119b89e250477bc4ba16b2" +dependencies = [ + "time-core", +] [[package]] name = "tinyvec" @@ -1105,9 +1205,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.21.2" +version = "1.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9e03c497dc955702ba729190dc4aac6f2a0ce97f913e5b1b5912fc5039d9099" +checksum = "d76ce4a75fb488c605c54bf610f221cea8b0dafb53333c1a67e8ee199dcd2ae3" dependencies = [ "autocfg", "bytes", @@ -1308,6 +1408,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" dependencies = [ "once_cell", + "valuable", ] [[package]] @@ -1320,12 +1421,60 @@ dependencies = [ "tracing", ] +[[package]] +name = "tracing-log" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922" +dependencies = [ + "lazy_static", + "log", + "tracing-core", +] + +[[package]] +name = "tracing-opentelemetry" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21ebb87a95ea13271332df069020513ab70bdb5637ca42d6e492dc3bbbad48de" +dependencies = [ + "once_cell", + "opentelemetry", + "tracing", + "tracing-core", + "tracing-log", + "tracing-subscriber", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70" +dependencies = [ + "nu-ansi-term", + "sharded-slab", + "smallvec", + "thread_local", + "tracing-core", + "tracing-log", +] + [[package]] name = "try-lock" version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" +[[package]] +name = "unicase" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +dependencies = [ + "version_check", +] + [[package]] name = "unicode-bidi" version = "0.3.8" @@ -1360,9 +1509,9 @@ dependencies = [ [[package]] name = "uuid" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "feb41e78f93363bb2df8b0e86a2ca30eed7806ea16ea0c790d757cf93f79be83" +checksum = "422ee0de9031b5b948b97a8fc04e3aa35230001a722ddd27943e0be31564ce4c" dependencies = [ "getrandom", "rand", @@ -1371,15 +1520,27 @@ dependencies = [ [[package]] name = "uuid-macro-internal" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44cc305b0c375fa2f078537bbe564516cb1946c1defa4041aa0e3f9cd5e07c4a" +checksum = "73bc89f2894593e665241e0052c3791999e6787b7c4831daa0a5c2e637e276d8" dependencies = [ "proc-macro2", "quote", "syn", ] +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + [[package]] name = "want" version = "0.3.0" diff --git a/src/shippingservice/Cargo.toml b/src/shippingservice/Cargo.toml index f1d59601fd..b30808937a 100644 --- a/src/shippingservice/Cargo.toml +++ b/src/shippingservice/Cargo.toml @@ -24,6 +24,11 @@ opentelemetry-http = "0.7.0" opentelemetry-otlp = "0.11.0" log = "0.4.17" simplelog = "0.12" +reqwest-middleware = "0.2.0" +reqwest-tracing = { version = "0.4.0", features = ["opentelemetry_0_18"] } +tracing = "0.1" +tracing-opentelemetry = "0.18.0" +tracing-subscriber = "0.3" [dependencies.uuid] version = "1.1.2" @@ -34,4 +39,4 @@ features = [ ] [build-dependencies] -tonic-build = "0.8" \ No newline at end of file +tonic-build = "0.8" diff --git a/src/shippingservice/src/main.rs b/src/shippingservice/src/main.rs index f4c0087008..8dd87c5117 100644 --- a/src/shippingservice/src/main.rs +++ b/src/shippingservice/src/main.rs @@ -29,6 +29,9 @@ use opentelemetry_otlp::{self, WithExportConfig}; use tonic::transport::Server; +use tracing_subscriber::Registry; +use tracing_subscriber::layer::SubscriberExt; + use log::*; use simplelog::*; @@ -74,6 +77,12 @@ fn init_tracer() -> Result { .install_batch(opentelemetry::runtime::Tokio) } +fn init_reqwest_tracing(tracer: sdktrace::Tracer) -> Result<(), tracing::subscriber::SetGlobalDefaultError> { + let telemetry = tracing_opentelemetry::layer().with_tracer(tracer); + let subscriber = Registry::default().with(telemetry); + tracing::subscriber::set_global_default(subscriber) +} + #[tokio::main] async fn main() -> Result<(), Box> { let (mut health_reporter, health_service) = tonic_health::server::health_reporter(); @@ -82,7 +91,7 @@ async fn main() -> Result<(), Box> { .await; init_logger()?; - init_tracer()?; + init_reqwest_tracing(init_tracer()?)?; info!("OTel pipeline created"); let port = env::var("SHIPPING_SERVICE_PORT").expect("$SHIPPING_SERVICE_PORT is not set"); let addr = format!("0.0.0.0:{}", port).parse()?; diff --git a/src/shippingservice/src/shipping_service/quote.rs b/src/shippingservice/src/shipping_service/quote.rs index 833b7c002e..26b8756f57 100644 --- a/src/shippingservice/src/shipping_service/quote.rs +++ b/src/shippingservice/src/shipping_service/quote.rs @@ -6,6 +6,8 @@ use opentelemetry::global; use opentelemetry::{trace::get_active_span, Context, KeyValue}; use opentelemetry_http::HeaderInjector; use reqwest::header::HeaderMap; +use reqwest_middleware::ClientBuilder; +use reqwest_tracing::{TracingMiddleware, SpanBackendWithUrl}; use reqwest::Method; @@ -48,7 +50,9 @@ async fn request_quote(count: u32) -> Result> { let mut reqbody = HashMap::new(); reqbody.insert("numberOfItems", count); - let client = reqwest::Client::new(); + let client = ClientBuilder::new(reqwest::Client::new()) + .with(TracingMiddleware::::new()) + .build(); let req = client.request(Method::POST, quote_service_addr);