[hotrod] Handle both OT and OTEL baggage (#4572)

## Which problem is this PR solving?
- Part of #3380

## Short description of the changes
- Move explicit baggage access into a util func that checks both OT and
OTEL baggage. Later the OT part can be retired completely.
- Update usage sites to use the new function instead of OT
`Span.BaggageItem`

A couple of less related changes in `tracing/mutex.go`:
- Use OTEL Span API instead of OT
- Replace direct-to-span logging with the regular logger that logs both
to stdout and to the span

Signed-off-by: Yuri Shkuro <github@ysh.us>

examples/hotrod/pkg/tracing/baggage.go

@@ -0,0 +1,44 @@
+// Copyright (c) 2023 The Jaeger Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tracing
+
+import (
+	"context"
+
+	"github.com/opentracing/opentracing-go"
+	"go.opentelemetry.io/otel/baggage"
+)
+
+func BaggageItem(ctx context.Context, key string) string {
+	val := opentracingBaggageItem(ctx, key)
+	if val != "" {
+		return val
+	}
+	return otelBaggageItem(ctx, key)
+}
+
+func opentracingBaggageItem(ctx context.Context, key string) string {
+	span := opentracing.SpanFromContext(ctx)
+	if span == nil {
+		return ""
+	}
+	return span.BaggageItem(key)
+}
+
+func otelBaggageItem(ctx context.Context, key string) string {
+	b := baggage.FromContext(ctx)
+	m := b.Member(key)
+	return m.Value()
+}

examples/hotrod/pkg/tracing/mutex.go

@@ -20,14 +20,18 @@ import (
 	"fmt"
 	"sync"
 
-	"github.com/opentracing/opentracing-go"
-	"github.com/opentracing/opentracing-go/log"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
+	"go.uber.org/zap"
+
+	"github.com/jaegertracing/jaeger/examples/hotrod/pkg/log"
 )
 
 // Mutex is just like the standard sync.Mutex, except that it is aware of the Context
 // and logs some diagnostic information into the current span.
 type Mutex struct {
 	SessionBaggageKey string
+	LogFactory        log.Factory
 
 	realLock sync.Mutex
 	holder   string
@@ -38,19 +42,17 @@ type Mutex struct {
 
 // Lock acquires an exclusive lock.
 func (sm *Mutex) Lock(ctx context.Context) {
-	var session string
-	activeSpan := opentracing.SpanFromContext(ctx)
-	if activeSpan != nil {
-		session = activeSpan.BaggageItem(sm.SessionBaggageKey)
-		activeSpan.SetTag(sm.SessionBaggageKey, session)
-	}
+	logger := sm.LogFactory.For(ctx)
+	session := BaggageItem(ctx, sm.SessionBaggageKey)
+	activeSpan := trace.SpanFromContext(ctx)
+	activeSpan.SetAttributes(attribute.String(sm.SessionBaggageKey, session))
 
 	sm.waitersLock.Lock()
 	if waiting := len(sm.waiters); waiting > 0 && activeSpan != nil {
-		activeSpan.LogFields(
-			log.String("event", fmt.Sprintf("Waiting for lock behind %d transactions", waiting)),
-			log.String("blockers", fmt.Sprintf("%v", sm.waiters))) // avoid deferred slice.String()
-		fmt.Printf("%s Waiting for lock behind %d transactions: %v\n", session, waiting, sm.waiters)
+		logger.Info(
+			fmt.Sprintf("Waiting for lock behind %d transactions", waiting),
+			zap.String("blockers", fmt.Sprintf("%v", sm.waiters)),
+		)
 	}
 	sm.waiters = append(sm.waiters, session)
 	sm.waitersLock.Unlock()
@@ -60,12 +62,13 @@ func (sm *Mutex) Lock(ctx context.Context) {
 
 	sm.waitersLock.Lock()
 	behindLen := len(sm.waiters) - 1
+	behindIDs := fmt.Sprintf("%v", sm.waiters[1:]) // skip self
 	sm.waitersLock.Unlock()
 
-	if activeSpan != nil {
-		activeSpan.LogFields(log.String("event",
-			fmt.Sprintf("Acquired lock with %d transactions waiting behind", behindLen)))
-	}
+	logger.Info(
+		fmt.Sprintf("Acquired lock; %d transactions waiting behind", behindLen),
+		zap.String("waiters", behindIDs),
+	)
 }
 
 // Unlock releases the lock.

examples/hotrod/services/customer/database.go

@@ -44,6 +44,7 @@ func newDatabase(tracer trace.Tracer, logger log.Factory) *database {
 		logger: logger,
 		lock: &tracing.Mutex{
 			SessionBaggageKey: "request",
+			LogFactory:        logger,
 		},
 		customers: map[string]*Customer{
 			"123": {
@@ -76,7 +77,10 @@ func (d *database) Get(ctx context.Context, customerID string) (*Customer, error
 	// simulate opentracing instrumentation of an SQL query
 	ctx, span := d.tracer.Start(ctx, "SQL SELECT", trace.WithSpanKind(trace.SpanKindClient))
 	// #nosec
-	span.SetAttributes(semconv.PeerServiceKey.String("mysql"), attribute.Key("sql.query").String("SELECT * FROM customer WHERE customer_id=" + customerID))
+	span.SetAttributes(
+		semconv.PeerServiceKey.String("mysql"),
+		attribute.Key("sql.query").String("SELECT * FROM customer WHERE customer_id="+customerID),
+	)
 	defer span.End()
 
 	if !config.MySQLMutexDisabled {

examples/hotrod/services/route/stats.go

@@ -20,28 +20,30 @@ import (
 	"expvar"
 	"time"
 
-	"github.com/opentracing/opentracing-go"
+	"github.com/jaegertracing/jaeger/examples/hotrod/pkg/tracing"
 )
 
 var routeCalcByCustomer = expvar.NewMap("route.calc.by.customer.sec")
 var routeCalcBySession = expvar.NewMap("route.calc.by.session.sec")
 
 var stats = []struct {
-	expvar  *expvar.Map
-	baggage string
+	expvar     *expvar.Map
+	baggageKey string
 }{
-	{routeCalcByCustomer, "customer"},
-	{routeCalcBySession, "session"},
+	{
+		expvar:     routeCalcByCustomer,
+		baggageKey: "customer",
+	},
+	{
+		expvar:     routeCalcBySession,
+		baggageKey: "session",
+	},
 }
 
 func updateCalcStats(ctx context.Context, delay time.Duration) {
-	span := opentracing.SpanFromContext(ctx)
-	if span == nil {
-		return
-	}
 	delaySec := float64(delay/time.Millisecond) / 1000.0
 	for _, s := range stats {
-		key := span.BaggageItem(s.baggage)
+		key := tracing.BaggageItem(ctx, s.baggageKey)
 		if key != "" {
 			s.expvar.AddFloat(key, delaySec)
 		}