Merge branch 'main' into main

.github/workflows/operator-integration-test.yml

@@ -68,24 +68,45 @@ jobs:
             echo "Pod is created test case: PASS"
           fi
 
-      - name: Test for default instrumentation resources
+      - name: Test for default instrumentation resources for Java
         run: |
-          cd integration-tests/manifests
-          kubectl apply -f ./sample-deployment.yaml
-          sleep 60
+          kubectl apply -f integration-tests/java/sample-deployment-java.yaml
+          sleep 5
           kubectl get pods -A 
           kubectl describe pods -n default
-          cd ./cmd
-          go run validate_instrumentation_vars.go default default_instrumentation_env_variables.json
+          go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/java/default_instrumentation_java_env_variables.json
 
-      - name: Test for defined instrumentation resources
+      - name: Test for defined instrumentation resources for Java
         run: |
-          cd integration-tests/manifests
-          kubectl apply -f ./sample-instrumentation.yaml
-          sleep 60
+          kubectl apply -f integration-tests/manifests/sample-instrumentation.yaml
+          sleep 5
           kubectl rollout restart deployment nginx
-          sleep 60
+          sleep 5
+          kubectl get pods -A
+          kubectl describe pods -n default
+          cd integration-tests/manifests/cmd
+          go run validate_instrumentation_vars.go default ns_instrumentation_env_variables.json     
+          kubectl delete instrumentation sample-instrumentation
+
+      - name: Test for default instrumentation resources for python
+        run: |
+          kubectl apply -f integration-tests/python/sample-deployment-python.yaml
+          sleep 5
+          kubectl rollout restart deployment nginx
+          sleep 5
+          kubectl get pods -A 
+          kubectl describe pods -n default
+          go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/python/default_instrumentation_python_env_variables.json
+
+
+      - name: Test for defined instrumentation resources for python
+        run: |
+          kubectl apply -f integration-tests/manifests/sample-instrumentation.yaml
+          sleep 5
+          kubectl rollout restart deployment nginx
+          sleep 5
           kubectl get pods -A
           kubectl describe pods -n default
-          cd ./cmd
-          go run validate_instrumentation_vars.go default ns_instrumentation_env_variables.json
+          cd integration-tests/manifests/cmd
+          go run validate_instrumentation_vars.go default ns_instrumentation_env_variables.json
+          kubectl delete instrumentation sample-instrumentation

helm/templates/operator-deployment.yaml

@@ -29,6 +29,7 @@ spec:
         - {{ printf "--auto-annotation-config=%s" (.Values.manager.autoAnnotateAutoInstrumentation | toJson) | quote }}
         - "--auto-instrumentation-java-image={{ .Values.manager.autoInstrumentationImage.java.repository }}:{{ .Values.manager.autoInstrumentationImage.java.tag }}"
         - "--auto-instrumentation-python-image={{ .Values.manager.autoInstrumentationImage.python.repository }}:{{ .Values.manager.autoInstrumentationImage.python.tag }}"
+        - "--feature-gates=operator.autoinstrumentation.multi-instrumentation,operator.autoinstrumentation.multi-instrumentation.skip-container-validation"
         command:
         - /manager
         name: manager

helm/values.yaml

@@ -147,7 +147,7 @@ agent:
       us-gov-west-1: 743662458514.dkr.ecr.us-gov-west-1.amazonaws.com
   enabled: true
   serviceAccount:
-    name: # override agent service account name 
+    name: # override agent service account name
   config: # optional config that can be provided to override the defaultConfig
   defaultConfig:
     {

integration-tests/python/default_instrumentation_python_env_variables.json

@@ -0,0 +1,14 @@
+
+{
+  "PYTHONPATH": "/otel-auto-instrumentation-python/opentelemetry/instrumentation/auto_instrumentation:/otel-auto-instrumentation-python",
+  "OTEL_AWS_APP_SIGNALS_ENABLED": "true",
+  "OTEL_TRACES_SAMPLER_ARG": "endpoint=http://cloudwatch-agent.amazon-cloudwatch:2000",
+  "OTEL_TRACES_SAMPLER": "xray",
+  "OTEL_EXPORTER_OTLP_PROTOCOL": "http/protobuf",
+  "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": "http://cloudwatch-agent.amazon-cloudwatch:4316/v1/traces",
+  "OTEL_AWS_APP_SIGNALS_EXPORTER_ENDPOINT": "http://cloudwatch-agent.amazon-cloudwatch:4315",
+  "OTEL_METRICS_EXPORTER": "none",
+  "OTEL_PYTHON_DISTRO": "aws_distro",
+  "OTEL_PYTHON_CONFIGURATOR": "aws_configurator"
+
+}

integration-tests/python/sample-deployment-python.yaml

@@ -0,0 +1,20 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx
+spec:
+  selector:
+    matchLabels:
+      app: nginx
+  template:
+    metadata:
+      labels:
+        app: nginx
+      annotations:
+        instrumentation.opentelemetry.io/inject-python: "true"
+    spec:
+      containers:
+        - name: nginx
+          image: nginx:1.14.2
+      restartPolicy: Always
+status: {}

main.go

@@ -3,13 +3,15 @@
 package main
 
 import (
+	"context"
 	"crypto/tls"
 	"encoding/json"
 	"flag"
 	"fmt"
 	"os"
 	"runtime"
 	"strings"
+	"time"
 
 	routev1 "github.com/openshift/api/route/v1"
 	monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
@@ -211,8 +213,15 @@ func main() {
 			mgr.GetWebhookServer().Register("/mutate-v1-namespace", &webhook.Admission{
 				Handler: namespacemutation.NewWebhookHandler(decoder, autoAnnotationMutators),
 			})
-			setupLog.Info("Starting auto-annotation")
-			go autoAnnotationMutators.MutateAndPatchAll(ctx)
+			setupLog.Info("Auto-annotation is enabled")
+			go waitForWebhookServerStart(
+				ctx,
+				mgr.GetWebhookServer().StartedChecker(),
+				func(ctx context.Context) {
+					setupLog.Info("Applying auto-annotation")
+					autoAnnotationMutators.MutateAndPatchAll(ctx)
+				},
+			)
 		}
 	}
 
@@ -253,6 +262,23 @@ func main() {
 	}
 }
 
+func waitForWebhookServerStart(ctx context.Context, checker healthz.Checker, callback func(context.Context)) {
+	ticker := time.NewTicker(time.Second)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ticker.C:
+			if err := checker(nil); err == nil {
+				setupLog.Info("Webhook server has started")
+				callback(ctx)
+				return
+			}
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
 // This function get the option from command argument (tlsConfig), check the validity through k8sapiflag
 // and set the config for webhook server.
 // refer to https://pkg.go.dev/k8s.io/component-base/cli/flag

pkg/featuregate/featuregate.go

@@ -79,6 +79,15 @@ var (
 		featuregate.WithRegisterDescription("enables features associated to the Prometheus Operator"),
 		featuregate.WithRegisterFromVersion("v0.82.0"),
 	)
+
+	// SkipMultiInstrumentationContainerValidation is the feature gate that controls whether the operator will skip
+	// container name validation during pod mutation for multi-instrumentation. Enabling this feature allows multiple
+	// instrumentations for pods without specified container name annotations. Does not prevent specification
+	// annotations from being used.
+	SkipMultiInstrumentationContainerValidation = featuregate.GlobalRegistry().MustRegister(
+		"operator.autoinstrumentation.multi-instrumentation.skip-container-validation",
+		featuregate.StageAlpha,
+		featuregate.WithRegisterDescription("controls whether the operator validates the container annotations when multi-instrumentation is enabled"))
 )
 
 // Flags creates a new FlagSet that represents the available featuregate flags using the supplied featuregate registry.

pkg/instrumentation/annotationmutator.go

@@ -95,10 +95,11 @@ func (m *AnnotationMutator) Mutate(obj metav1.Object) bool {
 	if annotations == nil {
 		annotations = make(map[string]string)
 	}
-	var mutated bool
+	var anyMutated bool
 	for _, mutation := range m.mutations {
-		mutated = mutated || mutation.Mutate(annotations)
+		mutated := mutation.Mutate(annotations)
+		anyMutated = anyMutated || mutated
 	}
 	obj.SetAnnotations(annotations)
-	return mutated
+	return anyMutated
 }

pkg/instrumentation/annotationmutator_test.go

@@ -48,6 +48,22 @@ func TestMutateAnnotations(t *testing.T) {
 			},
 			wantMutated: true,
 		},
+		"TestInsert/Multiple": {
+			annotations: nil,
+			mutations: []AnnotationMutation{
+				NewInsertAnnotationMutation(map[string]string{
+					"keyA": "3",
+				}),
+				NewInsertAnnotationMutation(map[string]string{
+					"keyC": "4",
+				}),
+			},
+			wantAnnotations: map[string]string{
+				"keyA": "3",
+				"keyC": "4",
+			},
+			wantMutated: true,
+		},
 		"TestRemove/Conflicts": {
 			annotations: map[string]string{
 				"keyA": "1",
@@ -79,6 +95,41 @@ func TestMutateAnnotations(t *testing.T) {
 			wantAnnotations: map[string]string{},
 			wantMutated:     true,
 		},
+		"TestRemove/Multiple": {
+			annotations: map[string]string{
+				"keyA": "1",
+				"keyB": "2",
+			},
+			mutations: []AnnotationMutation{
+				NewRemoveAnnotationMutation([]string{
+					"keyA",
+				}),
+				NewRemoveAnnotationMutation([]string{
+					"keyB",
+				}),
+			},
+			wantAnnotations: map[string]string{},
+			wantMutated:     true,
+		},
+		"TestBoth": {
+			annotations: map[string]string{
+				"keyA": "1",
+				"keyB": "2",
+			},
+			mutations: []AnnotationMutation{
+				NewRemoveAnnotationMutation([]string{
+					"keyA",
+				}),
+				NewInsertAnnotationMutation(map[string]string{
+					"keyA": "3",
+				}),
+			},
+			wantAnnotations: map[string]string{
+				"keyA": "3",
+				"keyB": "2",
+			},
+			wantMutated: true,
+		},
 	}
 	for name, testCase := range testCases {
 		t.Run(name, func(t *testing.T) {

pkg/instrumentation/podmutator.go

@@ -87,6 +87,10 @@ func (langInsts languageInstrumentations) areContainerNamesConfiguredForMultiple
 	var instrWithContainers int
 	var allContainers []string
 
+	if featuregate.SkipMultiInstrumentationContainerValidation.IsEnabled() {
+		return true, nil
+	}
+
 	// Check for instrumentations with and without containers.
 	if langInsts.Java.Instrumentation != nil {
 		instrWithContainers += isInstrWithContainers(langInsts.Java)