intake: Add rows_affected to span db information (#3095)

Allow to include number of rows affected by a sql database statement in the db information of a span. closes #2802
elastic · Jan 6, 2020 · 2a032a1 · 2a032a1
1 parent 8d8d9b2
commit 2a032a1
Show file tree

Hide file tree

Showing 12 changed files with 80 additions and 31 deletions.
diff --git a/beater/test_approved_es_documents/TestPublishIntegrationSpans.approved.json b/beater/test_approved_es_documents/TestPublishIntegrationSpans.approved.json
@@ -283,6 +283,7 @@
                 "db": {
                     "instance": "customers",
                     "link": "other.db.com",
+                    "rows_affected": 2,
                     "statement": "SELECT * FROM product_types WHERE user_id=?",
                     "type": "sql",
                     "user": {

diff --git a/changelogs/head.asciidoc b/changelogs/head.asciidoc
@@ -7,6 +7,10 @@ https://github.com/elastic/apm-server/compare/7.5\...master[View commits]
 ==== Breaking Changes
 - Respect `apm-server.ilm.setup.overwrite` flag when running `setup --index-management` {pull}2984[2984].
 
+[float]
+==== Intake API Changes
+- Add support for `span.context.db.rows_affected` {pull}3095[3095].
+
 [float]
 ==== Added
 - Adds support for global labels in spans {pull}2806[2806].
@@ -18,3 +22,4 @@ https://github.com/elastic/apm-server/compare/7.5\...master[View commits]
 - Build UBI based images also {pull}2994[2994].
 - Adds support for API Key authorization for incoming requests {pull}3004[3004]
 - Upgrade Go to 1.13.5 {pull}3069[3069].
+
diff --git a/docs/data/elasticsearch/generated/spans.json b/docs/data/elasticsearch/generated/spans.json
@@ -198,6 +198,7 @@
                 "db": {
                     "instance": "customers",
                     "link": "other.db.com",
+                    "rows_affected": 2,
                     "statement": "SELECT * FROM product_types WHERE user_id=?",
                     "type": "sql",
                     "user": {

diff --git a/docs/fields.asciidoc b/docs/fields.asciidoc
@@ -1442,6 +1442,16 @@ type: keyword
 
 --
 
+*`span.db.rows_affected`*::
++
+--
+Number of rows affected by the database statement.
+
+
+type: long
+
+--
+
 
 [float]
 === service

diff --git a/docs/spec/spans/span.json b/docs/spec/spans/span.json
@@ -74,7 +74,7 @@
                                             "maxLength": 1024
                                         }
                                     },
-				    "required": ["type", "name", "resource"]
+                                    "required": ["type", "name", "resource"]
                                 }
                             }
                         },
@@ -102,6 +102,10 @@
                                 "user": {
                                     "type": ["string", "null"],
                                     "description": "Username for accessing database"
+                                },
+                                "rows_affected": {
+                                    "type": ["integer", "null"],
+                                    "description": "Number of rows affected by the SQL statement (if applicable)"
                                 }
                             }
                         },

diff --git a/include/fields.go b/include/fields.go
diff --git a/model/span/_meta/fields.yml b/model/span/_meta/fields.yml
@@ -62,6 +62,11 @@
               description: >
                 Database link.
 
+            - name: rows_affected
+              type: long
+              description: >
+                Number of rows affected by the database statement.
+
         - name: destination
           type: group
           dynamic: false

diff --git a/model/span/event.go b/model/span/event.go
@@ -89,11 +89,12 @@ type Event struct {
 }
 
 type db struct {
-	Instance  *string
-	Statement *string
-	Type      *string
-	UserName  *string
-	Link      *string
+	Instance     *string
+	Statement    *string
+	Type         *string
+	UserName     *string
+	Link         *string
+	RowsAffected *int
 }
 
 func decodeDB(input interface{}, err error) (*db, error) {
@@ -115,6 +116,7 @@ func decodeDB(input interface{}, err error) (*db, error) {
 		decoder.StringPtr(dbInput, "type"),
 		decoder.StringPtr(dbInput, "user"),
 		decoder.StringPtr(dbInput, "link"),
+		decoder.IntPtr(dbInput, "rows_affected"),
 	}
 	return &db, decoder.Err
 }
@@ -127,6 +129,7 @@ func (db *db) fields() common.MapStr {
 	utility.Set(fields, "instance", db.Instance)
 	utility.Set(fields, "statement", db.Statement)
 	utility.Set(fields, "type", db.Type)
+	utility.Set(fields, "rows_affected", db.RowsAffected)
 	if db.UserName != nil {
 		utility.Set(fields, "user", common.MapStr{"name": db.UserName})
 	}

diff --git a/model/span/event_test.go b/model/span/event_test.go
@@ -44,14 +44,16 @@ func TestDecodeSpan(t *testing.T) {
 	name, spType := "foo", "db"
 	start, duration := 1.2, 3.4
 	method, statusCode, url := "get", 200, "http://localhost"
-	instance, statement, dbType, user, link := "db01", "select *", "sql", "joe", "other.db.com"
+	instance, statement, dbType, user, link, rowsAffected := "db01", "select *", "sql", "joe", "other.db.com", 34
 	address, port := "localhost", 8080
 	destServiceType, destServiceName, destServiceResource := "db", "elasticsearch", "elasticsearch"
 	context := map[string]interface{}{
 		"a":    "b",
 		"tags": map[string]interface{}{"a": "tag", "tag.key": 17},
 		"http": map[string]interface{}{"method": "GET", "status_code": json.Number("200"), "url": url},
-		"db":   map[string]interface{}{"instance": instance, "statement": statement, "type": dbType, "user": user, "link": link},
+		"db": map[string]interface{}{
+			"instance": instance, "statement": statement, "type": dbType,
+			"user": user, "link": link, "rows_affected": json.Number("34")},
 		"destination": map[string]interface{}{
 			"address": address,
 			"port":    float64(port),
@@ -220,8 +222,15 @@ func TestDecodeSpan(t *testing.T) {
 				ParentId:      parentId,
 				TransactionId: &transactionId,
 				HTTP:          &http{Method: &method, StatusCode: &statusCode, Url: &url},
-				DB:            &db{Instance: &instance, Statement: &statement, Type: &dbType, UserName: &user, Link: &link},
-				Destination:   &destination{Address: &address, Port: &port},
+				DB: &db{
+					Instance:     &instance,
+					Statement:    &statement,
+					Type:         &dbType,
+					UserName:     &user,
+					Link:         &link,
+					RowsAffected: &rowsAffected,
+				},
+				Destination: &destination{Address: &address, Port: &port},
 				DestinationService: &destinationService{
 					Type:     &destServiceType,
 					Name:     &destServiceName,
@@ -255,7 +264,7 @@ func TestSpanTransform(t *testing.T) {
 		time.FixedZone("+0100", 3600))
 	timestampUs := timestamp.UnixNano() / 1000
 	method, statusCode, url := "get", 200, "http://localhost"
-	instance, statement, dbType, user := "db01", "select *", "sql", "jane"
+	instance, statement, dbType, user, rowsAffected := "db01", "select *", "sql", "jane", 5
 	metadataLabels := common.MapStr{"label.a": "a", "label.b": "b", "c": 1}
 	address, port := "127.0.0.1", 8080
 	destServiceType, destServiceName, destServiceResource := "db", "elasticsearch", "elasticsearch"
@@ -282,19 +291,24 @@ func TestSpanTransform(t *testing.T) {
 		},
 		{
 			Event: Event{
-				Id:          hexId,
-				TraceId:     traceId,
-				ParentId:    parentId,
-				Name:        "myspan",
-				Type:        "myspantype",
-				Subtype:     &subtype,
-				Action:      &action,
-				Start:       &start,
-				Duration:    1.20,
-				Stacktrace:  m.Stacktrace{{AbsPath: &path}},
-				Labels:      common.MapStr{"label.a": 12},
-				HTTP:        &http{Method: &method, StatusCode: &statusCode, Url: &url},
-				DB:          &db{Instance: &instance, Statement: &statement, Type: &dbType, UserName: &user},
+				Id:         hexId,
+				TraceId:    traceId,
+				ParentId:   parentId,
+				Name:       "myspan",
+				Type:       "myspantype",
+				Subtype:    &subtype,
+				Action:     &action,
+				Start:      &start,
+				Duration:   1.20,
+				Stacktrace: m.Stacktrace{{AbsPath: &path}},
+				Labels:     common.MapStr{"label.a": 12},
+				HTTP:       &http{Method: &method, StatusCode: &statusCode, Url: &url},
+				DB: &db{
+					Instance:     &instance,
+					Statement:    &statement,
+					Type:         &dbType,
+					UserName:     &user,
+					RowsAffected: &rowsAffected},
 				Destination: &destination{Address: &address, Port: &port},
 				DestinationService: &destinationService{
 					Type:     &destServiceType,
@@ -320,10 +334,11 @@ func TestSpanTransform(t *testing.T) {
 							"updated": false,
 						}}},
 					"db": common.MapStr{
-						"instance":  instance,
-						"statement": statement,
-						"type":      dbType,
-						"user":      common.MapStr{"name": user},
+						"instance":      instance,
+						"statement":     statement,
+						"type":          dbType,
+						"user":          common.MapStr{"name": user},
+						"rows_affected": rowsAffected,
 					},
 					"http": common.MapStr{
 						"url":      common.MapStr{"original": url},

diff --git a/model/span/generated/schema/span.go b/model/span/generated/schema/span.go
@@ -120,7 +120,7 @@ const ModelSchema = `{
                                             "maxLength": 1024
                                         }
                                     },
-				    "required": ["type", "name", "resource"]
+                                    "required": ["type", "name", "resource"]
                                 }
                             }
                         },
@@ -148,6 +148,10 @@ const ModelSchema = `{
                                 "user": {
                                     "type": ["string", "null"],
                                     "description": "Username for accessing database"
+                                },
+                                "rows_affected": {
+                                    "type": ["integer", "null"],
+                                    "description": "Number of rows affected by the SQL statement (if applicable)"
                                 }
                             }
                         },

diff --git a/processor/stream/test_approved_es_documents/testIntakeIntegrationSpans.approved.json b/processor/stream/test_approved_es_documents/testIntakeIntegrationSpans.approved.json
@@ -198,6 +198,7 @@
                 "db": {
                     "instance": "customers",
                     "link": "other.db.com",
+                    "rows_affected": 2,
                     "statement": "SELECT * FROM product_types WHERE user_id=?",
                     "type": "sql",
                     "user": {

diff --git a/testdata/intake-v2/spans.ndjson b/testdata/intake-v2/spans.ndjson
@@ -3,4 +3,4 @@
 {"span": {"trace_id": "abcdef0123456789abcdef9876543210", "parent_id": "0000000011111111", "id": "1234abcdef567895", "transaction_id": "ab45781d265894fe", "name": "GET /api/types", "type": "request", "start": 22, "duration": 32.592981, "timestamp": 1532976822281000,"context":{"service":{"environment":"prod","agent":{}}}}}
 {"span": {"trace_id": "abcdef0123456789abcdef9876543210", "parent_id": "abcdefabcdef7890", "id": "0123456a89012345", "transaction_id": "ab23456a89012345", "name": "GET /api/types", "type": "request.http", "start": 1.845, "duration": 3.5642981, "stacktrace": [], "context":{"tags": {"tag1": "value1", "tag2": 123, "tag3": 12.34, "tag4": true, "tag5": null},"service":{}}}}}
 {"span": {"trace_id": "abcdef0123456789abcdef9876543210", "parent_id": "ababcdcdefefabde", "id": "abcde56a89012345", "transaction_id": null, "name": "get /api/types",  "sync": false, "type": "request", "subtype": "http", "action": "call", "start": 0, "duration": 13.9802981, "stacktrace": null, "context": null }}
-{"span": {"trace_id": "abcdef0123456789abcdef9876543210", "parent_id": "abcdef0123456789", "id": "1234567890aaaade", "sync": true, "name": "SELECT FROM product_types", "type": "db.postgresql.query", "start": 2.83092, "duration": 3.781912, "stacktrace": [{ "filename": "net.js", "lineno": 547},{"filename": "file2.js", "lineno": 12, "post_context": [ "    ins.currentTransaction = prev", "}"]}, { "function": "onread", "abs_path": "net.js", "filename": "net.js", "lineno": 547, "library_frame": true, "vars": { "key": "value" }, "module": "some module", "colno": 4, "context_line": "line3", "pre_context": [ "  var trans = this.currentTransaction", "" ], "post_context": [ "    ins.currentTransaction = prev", "    return result"] }], "context": { "db": { "instance": "customers", "statement": "SELECT * FROM product_types WHERE user_id=?", "type": "sql", "user": "readonly_user", "link": "other.db.com" }, "http": { "url": "http://localhost:8000", "status_code": 200, "method": "GET" }, "destination": {"address": "0:0::0:1", "port": 5432, "service": {"type": "db", "name": "postgresql", "resource": "postgresql"}}, "service":{"name":"service1","agent":{"version":"2.2","name":"elastic-ruby", "ephemeral_id": "justanid"}}}}}
+{"span": {"trace_id": "abcdef0123456789abcdef9876543210", "parent_id": "abcdef0123456789", "id": "1234567890aaaade", "sync": true, "name": "SELECT FROM product_types", "type": "db.postgresql.query", "start": 2.83092, "duration": 3.781912, "stacktrace": [{ "filename": "net.js", "lineno": 547},{"filename": "file2.js", "lineno": 12, "post_context": [ "    ins.currentTransaction = prev", "}"]}, { "function": "onread", "abs_path": "net.js", "filename": "net.js", "lineno": 547, "library_frame": true, "vars": { "key": "value" }, "module": "some module", "colno": 4, "context_line": "line3", "pre_context": [ "  var trans = this.currentTransaction", "" ], "post_context": [ "    ins.currentTransaction = prev", "    return result"] }], "context": { "db": { "instance": "customers", "statement": "SELECT * FROM product_types WHERE user_id=?", "type": "sql", "user": "readonly_user", "link": "other.db.com", "rows_affected": 2 }, "http": { "url": "http://localhost:8000", "status_code": 200, "method": "GET" }, "destination": {"address": "0:0::0:1", "port": 5432, "service": {"type": "db", "name": "postgresql", "resource": "postgresql"}}, "service":{"name":"service1","agent":{"version":"2.2","name":"elastic-ruby", "ephemeral_id": "justanid"}}}}}