Skip to content

Commit

Permalink
Add map and array attribute value type support (#1656)
Browse files Browse the repository at this point in the history 
Map and array attribute value types which were recently added to the specification had not been implemented in the collector yet. This PR implements across all tracing receivers and exporters.

**Link to tracking Issue:** #1590

**Testing:** Added map and array attribute values to golden dataset. All unit and correctness tests continue to pass.
  • Loading branch information
@kbrockhoff
kbrockhoff committed Sep 16, 2020
1 parent e00fdab commit 471c4a6
Show file tree
Hide file tree
Showing 21 changed files with 764 additions and 169 deletions.
10 changes: 10 additions & 0 deletions internal/goldendataset/generator_commons.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,16 @@ func constructAttributeKeyValue(key string, value interface{}) *otlpcommon.KeyVa
Key: key,
Value: &otlpcommon.AnyValue{Value: &otlpcommon.AnyValue_BoolValue{BoolValue: cast.ToBool(val)}},
}
case *otlpcommon.ArrayValue:
attr = otlpcommon.KeyValue{
Key: key,
Value: &otlpcommon.AnyValue{Value: &otlpcommon.AnyValue_ArrayValue{ArrayValue: val}},
}
case *otlpcommon.KeyValueList:
attr = otlpcommon.KeyValue{
Key: key,
Value: &otlpcommon.AnyValue{Value: &otlpcommon.AnyValue_KvlistValue{KvlistValue: val}},
}
default:
attr = otlpcommon.KeyValue{
Key: key,
Expand Down
30 changes: 23 additions & 7 deletions internal/goldendataset/resource_generator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
package goldendataset

import (
otlpcommon "go.opentelemetry.io/collector/internal/data/opentelemetry-proto-gen/common/v1"
otlpresource "go.opentelemetry.io/collector/internal/data/opentelemetry-proto-gen/resource/v1"
"go.opentelemetry.io/collector/translator/conventions"
)
Expand Down Expand Up @@ -69,10 +70,14 @@ func generateOnpremVMAttributes() map[string]interface{} {
attrMap[conventions.AttributeServiceName] = "customers"
attrMap[conventions.AttributeServiceNamespace] = "production"
attrMap[conventions.AttributeServiceVersion] = "semver:0.7.3"
attrMap[conventions.AttributeHostHostname] = "tc-prod9.internal.example.com"
attrMap[conventions.AttributeHostName] = "172.18.36.18"
subMap := make(map[string]interface{})
subMap["public"] = "tc-prod9.internal.example.com"
subMap["internal"] = "172.18.36.18"
attrMap[conventions.AttributeHostName] = &otlpcommon.KeyValueList{
Values: convertMapToAttributeKeyValues(subMap),
}
attrMap[conventions.AttributeHostImageID] = "661ADFA6-E293-4870-9EFA-1AA052C49F18"
attrMap[conventions.AttributeTelemetrySDKLanguage] = "java"
attrMap[conventions.AttributeTelemetrySDKLanguage] = conventions.AttributeSDKLangValueJava
attrMap[conventions.AttributeTelemetrySDKName] = "opentelemetry"
attrMap[conventions.AttributeTelemetrySDKVersion] = "0.3.0"
return attrMap
Expand All @@ -84,7 +89,7 @@ func generateCloudVMAttributes() map[string]interface{} {
attrMap[conventions.AttributeServiceName] = "customers"
attrMap[conventions.AttributeServiceNamespace] = "production"
attrMap[conventions.AttributeServiceVersion] = "semver:0.7.3"
attrMap[conventions.AttributeTelemetrySDKLanguage] = "java"
attrMap[conventions.AttributeTelemetrySDKLanguage] = conventions.AttributeSDKLangValueJava
attrMap[conventions.AttributeTelemetrySDKName] = "opentelemetry"
attrMap[conventions.AttributeTelemetrySDKVersion] = "0.3.0"
attrMap[conventions.AttributeHostHostname] = "env-check"
Expand Down Expand Up @@ -119,8 +124,11 @@ func generateCloudK8sAttributes() map[string]interface{} {
attrMap[conventions.AttributeK8sCluster] = "erp-dev"
attrMap[conventions.AttributeK8sNamespace] = "monitoring"
attrMap[conventions.AttributeK8sDeployment] = "otel-collector"
attrMap[conventions.AttributeK8sDeploymentUID] = "4D614B27-EDAF-409B-B631-6963D8F6FCD4"
attrMap[conventions.AttributeK8sReplicaSet] = "otel-collector-2983fd34"
attrMap[conventions.AttributeK8sReplicaSetUID] = "EC7D59EF-D5B6-48B7-881E-DA6B7DD539B6"
attrMap[conventions.AttributeK8sPod] = "otel-collector-6484db5844-c6f9m"
attrMap[conventions.AttributeHostHostname] = "ip-10-99-118-157.ec2.internal"
attrMap[conventions.AttributeK8sPodUID] = "FDFD941E-2A7A-4945-B601-88DD486161A4"
attrMap[conventions.AttributeHostID] = "ec2e3fdaffa294348bdf355156b94cda"
attrMap[conventions.AttributeHostName] = "10.99.118.157"
attrMap[conventions.AttributeHostImageID] = "ami-011c865bf7da41a9d"
Expand All @@ -147,10 +155,18 @@ func generateFassAttributes() map[string]interface{} {
func generateExecAttributes() map[string]interface{} {
attrMap := make(map[string]interface{})
attrMap[conventions.AttributeProcessExecutableName] = "otelcol"
attrMap[conventions.AttributeProcessCommandLine] =
"--config=/etc/otel-collector-config.yaml --mem-ballast-size-mib=683"
parts := make([]*otlpcommon.AnyValue, 3)
parts[0] = &otlpcommon.AnyValue{Value: &otlpcommon.AnyValue_StringValue{StringValue: "otelcol"}}
parts[1] = &otlpcommon.AnyValue{Value: &otlpcommon.AnyValue_StringValue{StringValue: "--config=/etc/otel-collector-config.yaml"}}
parts[2] = &otlpcommon.AnyValue{Value: &otlpcommon.AnyValue_StringValue{StringValue: "--mem-ballast-size-mib=683"}}
attrMap[conventions.AttributeProcessCommandLine] = &otlpcommon.ArrayValue{
Values: parts,
}
attrMap[conventions.AttributeProcessExecutablePath] = "/usr/local/bin/otelcol"
attrMap[conventions.AttributeProcessID] = 2020
attrMap[conventions.AttributeProcessOwner] = "otel"
attrMap[conventions.AttributeOSType] = "LINUX"
attrMap[conventions.AttributeOSDescription] =
"Linux ubuntu 5.4.0-42-generic #46-Ubuntu SMP Fri Jul 10 00:24:02 UTC 2020 x86_64 x86_64 x86_64 GNU/Linux"
return attrMap
}
16 changes: 14 additions & 2 deletions internal/goldendataset/span_generator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -435,8 +435,20 @@ func generateMaxCountAttributes(includeStatus bool) map[string]interface{} {
attrMap["ai-sampler.absolute"] = false
attrMap["ai-sampler.maxhops"] = int64(6)
attrMap["application.create.location"] = "https://api.opentelemetry.io/blog/posts/806673B9-4F4D-4284-9635-3A3E3E3805BE"
attrMap["application.svcmap"] = "Blogosphere"
attrMap["application.abflags"] = "UIx=false,UI4=true,flow-alt3=false"
stages := make([]*otlpcommon.AnyValue, 3)
stages[0] = &otlpcommon.AnyValue{Value: &otlpcommon.AnyValue_StringValue{StringValue: "Launch"}}
stages[1] = &otlpcommon.AnyValue{Value: &otlpcommon.AnyValue_StringValue{StringValue: "Injestion"}}
stages[2] = &otlpcommon.AnyValue{Value: &otlpcommon.AnyValue_StringValue{StringValue: "Validation"}}
attrMap["application.stages"] = &otlpcommon.ArrayValue{
Values: stages,
}
subMap := make(map[string]interface{})
subMap["UIx"] = false
subMap["UI4"] = true
subMap["flow-alt3"] = false
attrMap["application.abflags"] = &otlpcommon.KeyValueList{
Values: convertMapToAttributeKeyValues(subMap),
}
attrMap["application.thread"] = "proc-pool-14"
attrMap["application.session"] = ""
attrMap["application.persist.size"] = int64(1172184)
Expand Down
2 changes: 2 additions & 0 deletions testbed/correctness/.gitignore
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
results

8 changes: 4 additions & 4 deletions testbed/correctness/traces/correctness_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import (
"testing"
"time"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"go.opentelemetry.io/collector/service/defaultcomponents"
"go.opentelemetry.io/collector/testbed/correctness"
Expand All @@ -34,7 +34,7 @@ func TestMain(m *testing.M) {

func TestTracingGoldenData(t *testing.T) {
tests, err := correctness.LoadPictOutputPipelineDefs("testdata/generated_pict_pairs_traces_pipeline.txt")
assert.NoError(t, err)
require.NoError(t, err)
processors := map[string]string{
"batch": `
batch:
Expand Down Expand Up @@ -64,12 +64,12 @@ func testWithTracingGoldenDataset(
"",
161803)
factories, err := defaultcomponents.Components()
assert.NoError(t, err)
require.NoError(t, err, "default components resulted in: %v", err)
runner := testbed.NewInProcessCollector(factories, sender.GetCollectorPort())
validator := testbed.NewCorrectTestValidator(dataProvider)
config := correctness.CreateConfigYaml(sender, receiver, processors, "traces")
configCleanup, cfgErr := runner.PrepareConfig(config)
assert.NoError(t, cfgErr)
require.NoError(t, cfgErr, "collector configuration resulted in: %v", cfgErr)
defer configCleanup()
tc := testbed.NewTestCase(
t,
Expand Down
137 changes: 124 additions & 13 deletions testbed/testbed/validator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ package testbed

import (
"encoding/hex"
"encoding/json"
"fmt"
"log"
"reflect"
Expand Down Expand Up @@ -420,20 +421,15 @@ func (v *CorrectnessTestValidator) diffAttributesSlice(spanName string, recdAttr
if ok {
sentVal := retrieveAttributeValue(sentAttr)
recdVal := retrieveAttributeValue(recdAttr)
if !reflect.DeepEqual(sentVal, recdVal) {
sentStr := fmt.Sprintf("%v", sentVal)
recdStr := fmt.Sprintf("%v", recdVal)
if sentStr != recdStr {
af := &TraceAssertionFailure{
typeName: "Span",
dataComboName: spanName,
fieldPath: fmt.Sprintf(fmtStr, sentAttr.Key),
expectedValue: sentVal,
actualValue: recdVal,
}
v.assertionFailures = append(v.assertionFailures, af)
}
switch val := sentVal.(type) {
case *otlpcommon.KeyValueList:
v.compareKeyValueList(spanName, val, recdVal, fmtStr, sentAttr.Key)
case *otlpcommon.ArrayValue:
v.compareArrayList(spanName, val, recdVal, fmtStr, sentAttr.Key)
default:
v.compareSimpleValues(spanName, sentVal, recdVal, fmtStr, sentAttr.Key)
}

} else {
af := &TraceAssertionFailure{
typeName: "Span",
Expand All @@ -447,6 +443,64 @@ func (v *CorrectnessTestValidator) diffAttributesSlice(spanName string, recdAttr
}
}

func (v *CorrectnessTestValidator) compareSimpleValues(spanName string, sentVal interface{}, recdVal interface{},
fmtStr string, attrKey string) {
if !reflect.DeepEqual(sentVal, recdVal) {
sentStr := fmt.Sprintf("%v", sentVal)
recdStr := fmt.Sprintf("%v", recdVal)
if !strings.EqualFold(sentStr, recdStr) {
af := &TraceAssertionFailure{
typeName: "Span",
dataComboName: spanName,
fieldPath: fmt.Sprintf(fmtStr, attrKey),
expectedValue: sentVal,
actualValue: recdVal,
}
v.assertionFailures = append(v.assertionFailures, af)
}
}
}

func (v *CorrectnessTestValidator) compareKeyValueList(spanName string, sentKVList *otlpcommon.KeyValueList,
recdVal interface{}, fmtStr string, attrKey string) {
switch val := recdVal.(type) {
case *otlpcommon.KeyValueList:
v.diffAttributesSlice(spanName, val.Values, sentKVList.Values, fmtStr)
case string:
jsonStr := convertKVListToJSONString(sentKVList.Values)
v.compareSimpleValues(spanName, jsonStr, val, fmtStr, attrKey)
default:
af := &TraceAssertionFailure{
typeName: "Span",
dataComboName: spanName,
fieldPath: fmt.Sprintf(fmtStr, attrKey),
expectedValue: sentKVList,
actualValue: recdVal,
}
v.assertionFailures = append(v.assertionFailures, af)
}
}

func (v *CorrectnessTestValidator) compareArrayList(spanName string, sentArray *otlpcommon.ArrayValue,
recdVal interface{}, fmtStr string, attrKey string) {
switch val := recdVal.(type) {
case *otlpcommon.ArrayValue:
v.compareSimpleValues(spanName, sentArray.Values, val.Values, fmtStr, attrKey)
case string:
jsonStr := convertArrayValuesToJSONString(sentArray.Values)
v.compareSimpleValues(spanName, jsonStr, val, fmtStr, attrKey)
default:
af := &TraceAssertionFailure{
typeName: "Span",
dataComboName: spanName,
fieldPath: fmt.Sprintf(fmtStr, attrKey),
expectedValue: sentArray,
actualValue: recdVal,
}
v.assertionFailures = append(v.assertionFailures, af)
}
}

func convertAttributesSliceToMap(attributes []*otlpcommon.KeyValue) map[string]*otlpcommon.KeyValue {
attrMap := make(map[string]*otlpcommon.KeyValue)
for _, attr := range attributes {
Expand Down Expand Up @@ -517,3 +571,60 @@ func notWithinOneMillisecond(sentNs uint64, recdNs uint64) bool {
}
return diff > uint64(1100000)
}

func convertKVListToJSONString(values []*otlpcommon.KeyValue) string {
jsonStr, err := json.Marshal(convertKVListToRawMap(values))
if err == nil {
return string(jsonStr)
}
return ""
}

func convertArrayValuesToJSONString(values []*otlpcommon.AnyValue) string {
jsonStr, err := json.Marshal(convertArrayValuesToRawSlice(values))
if err == nil {
return string(jsonStr)
}
return ""
}

func convertKVListToRawMap(values []*otlpcommon.KeyValue) map[string]interface{} {
rawMap := make(map[string]interface{})
for _, kv := range values {
var value *otlpcommon.AnyValue = kv.GetValue()
switch val := value.GetValue().(type) {
case *otlpcommon.AnyValue_StringValue:
rawMap[kv.Key] = val.StringValue
case *otlpcommon.AnyValue_IntValue:
rawMap[kv.Key] = val.IntValue
case *otlpcommon.AnyValue_DoubleValue:
rawMap[kv.Key] = val.DoubleValue
case *otlpcommon.AnyValue_BoolValue:
rawMap[kv.Key] = val.BoolValue
case *otlpcommon.AnyValue_KvlistValue:
rawMap[kv.Key] = convertKVListToRawMap(val.KvlistValue.Values)
case *otlpcommon.AnyValue_ArrayValue:
rawMap[kv.Key] = convertArrayValuesToRawSlice(val.ArrayValue.Values)
default:
rawMap[kv.Key] = val
}
}
return rawMap
}

func convertArrayValuesToRawSlice(values []*otlpcommon.AnyValue) []interface{} {
rawSlice := make([]interface{}, 0, len(values))
for _, v := range values {
switch val := v.GetValue().(type) {
case *otlpcommon.AnyValue_StringValue:
rawSlice = append(rawSlice, val.StringValue)
case *otlpcommon.AnyValue_IntValue:
rawSlice = append(rawSlice, val.IntValue)
case *otlpcommon.AnyValue_DoubleValue:
rawSlice = append(rawSlice, val.DoubleValue)
case *otlpcommon.AnyValue_BoolValue:
rawSlice = append(rawSlice, val.BoolValue)
}
}
return rawSlice
}
36 changes: 35 additions & 1 deletion translator/conventions/opentelemetry.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
package conventions

// OpenTelemetry Semantic Convention values for Resource attribute names.
// See: https://github.com/open-telemetry/opentelemetry-specification/blob/master/specification/resource/semantic_conventions/README.md
// See: https://github.com/open-telemetry/opentelemetry-specification/tree/master/specification/resource/semantic_conventions/README.md
const (
AttributeCloudAccount = "cloud.account.id"
AttributeCloudProvider = "cloud.provider"
Expand Down Expand Up @@ -54,6 +54,8 @@ const (
AttributeK8sReplicaSetUID = "k8s.replicaset.uid"
AttributeK8sStatefulSet = "k8s.statefulset.name"
AttributeK8sStatefulSetUID = "k8s.statefulset.uid"
AttributeOSType = "os.type"
AttributeOSDescription = "os.description"
AttributeProcessCommand = "process.command"
AttributeProcessCommandLine = "process.command_line"
AttributeProcessExecutableName = "process.executable.name"
Expand All @@ -70,13 +72,29 @@ const (
AttributeTelemetrySDKVersion = "telemetry.sdk.version"
)

// OpenTelemetry Semantic Convention values for Resource attribute "telemetry.sdk.language" values.
// See: https://github.com/open-telemetry/opentelemetry-specification/tree/master/specification/resource/semantic_conventions/README.md
const (
AttributeSDKLangValueCPP = "cpp"
AttributeSDKLangValueDotNET = "dotnet"
AttributeSDKLangValueErlang = "erlang"
AttributeSDKLangValueGo = "go"
AttributeSDKLangValueJava = "java"
AttributeSDKLangValueNodeJS = "nodejs"
AttributeSDKLangValuePHP = "php"
AttributeSDKLangValuePython = "python"
AttributeSDKLangValueRuby = "ruby"
AttributeSDKLangValueWebJS = "webjs"
)

// GetResourceSemanticConventionAttributeNames a slice with all the Resource Semantic Conventions attribute names.
func GetResourceSemanticConventionAttributeNames() []string {
return []string{
AttributeCloudAccount,
AttributeCloudProvider,
AttributeCloudRegion,
AttributeCloudZone,
AttributeContainerID,
AttributeContainerImage,
AttributeContainerName,
AttributeContainerTag,
Expand All @@ -93,9 +111,24 @@ func GetResourceSemanticConventionAttributeNames() []string {
AttributeHostName,
AttributeHostType,
AttributeK8sCluster,
AttributeK8sContainer,
AttributeK8sCronJob,
AttributeK8sCronJobUID,
AttributeK8sDaemonSet,
AttributeK8sDaemonSetUID,
AttributeK8sDeployment,
AttributeK8sDeploymentUID,
AttributeK8sJob,
AttributeK8sJobUID,
AttributeK8sNamespace,
AttributeK8sPod,
AttributeK8sPodUID,
AttributeK8sReplicaSet,
AttributeK8sReplicaSetUID,
AttributeK8sStatefulSet,
AttributeK8sStatefulSetUID,
AttributeOSType,
AttributeOSDescription,
AttributeProcessCommand,
AttributeProcessCommandLine,
AttributeProcessExecutableName,
Expand All @@ -106,6 +139,7 @@ func GetResourceSemanticConventionAttributeNames() []string {
AttributeServiceName,
AttributeServiceNamespace,
AttributeServiceVersion,
AttributeTelemetryAutoVersion,
AttributeTelemetrySDKLanguage,
AttributeTelemetrySDKName,
AttributeTelemetrySDKVersion,
Expand Down
2 changes: 1 addition & 1 deletion translator/internaldata/metrics_to_oc_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ func TestMetricsToOC(t *testing.T) {
attrs.Upsert(conventions.AttributeHostHostname, pdata.NewAttributeValueString("host1"))
attrs.Upsert(conventions.OCAttributeProcessID, pdata.NewAttributeValueInt(123))
attrs.Upsert(conventions.OCAttributeProcessStartTime, pdata.NewAttributeValueString("2020-02-11T20:26:00Z"))
attrs.Upsert(conventions.AttributeTelemetrySDKLanguage, pdata.NewAttributeValueString("CPP"))
attrs.Upsert(conventions.AttributeTelemetrySDKLanguage, pdata.NewAttributeValueString("cpp"))
attrs.Upsert(conventions.AttributeTelemetrySDKVersion, pdata.NewAttributeValueString("v2.0.1"))
attrs.Upsert(conventions.OCAttributeExporterVersion, pdata.NewAttributeValueString("v1.2.0"))

Expand Down
Loading

0 comments on commit 471c4a6

Please sign in to comment.