[AWS] [CloudWatch] Add dimensions metadata (#6827)

* cloudwatch: add dimension fields

Signed-off-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co>

* remove aws.s3.bucket.name field

Signed-off-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co>

---------

Signed-off-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co>

packages/aws/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.46.8"
+  changes:
+    - description: Add dimensions metadata to the cloudwatch data_stream
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/6827
 - version: "1.46.7"
   changes:
     - description: Enable time series data streams for the API Gateway and EMR data streams. This improves storage usage and query performance. For more details, see https://www.elastic.co/guide/en/elasticsearch/reference/current/tsds.html.

packages/aws/data_stream/cloudwatch_metrics/elasticsearch/ingest_pipeline/default.yml

@@ -0,0 +1,11 @@
+---
+description: "Pipeline for AWS CloudWatch metrics"
+processors:
+  - fingerprint:
+      fields: ["aws.dimensions"]
+      target_field: 'aws.dimensions.fingerprint'
+      ignore_missing: true
+on_failure:
+  - set:
+      field: error.message
+      value: "{{ _ingest.on_failure_message }}"

packages/aws/data_stream/cloudwatch_metrics/fields/ecs.yml

@@ -2,6 +2,7 @@
   name: cloud
 - external: ecs
   name: cloud.account.id
+  dimension: true
 - external: ecs
   name: cloud.account.name
 - external: ecs
@@ -14,6 +15,7 @@
   name: cloud.provider
 - external: ecs
   name: cloud.region
+  dimension: true
 - external: ecs
   name: ecs.version
 - external: ecs
@@ -60,3 +62,6 @@
   name: container.labels
 - external: ecs
   name: container.name
+- name: agent.id
+  external: ecs
+  dimension: true

packages/aws/data_stream/cloudwatch_metrics/fields/package-fields.yml

@@ -5,14 +5,15 @@
       type: object
       description: |
         Tag key value pairs from aws resources.
-    - name: s3.bucket.name
-      type: keyword
-      description: |
-        Name of a S3 bucket.
     - name: dimensions.*
       type: object
       description: |
         Metric dimensions.
+    - name: dimensions.fingerprint
+      type: keyword
+      description: |
+        Autogenerated ID representing the fingerprint of the aws.dimensions object
+      dimension: true
     - name: '*.metrics.*.*'
       type: object
       description: |

packages/aws/docs/cloudwatch.md

@@ -204,10 +204,11 @@ An example event for `cloudwatch` looks as following:
 | Field | Description | Type |
 |---|---|---|
 | @timestamp | Event timestamp. | date |
+| agent.id | Unique identifier of this agent (if one exists). Example: For Beats this would be beat.id. | keyword |
 | aws.\*.metrics.\*.\* | Metrics that returned from Cloudwatch API query. | object |
 | aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword |
 | aws.dimensions.\* | Metric dimensions. | object |
-| aws.s3.bucket.name | Name of a S3 bucket. | keyword |
+| aws.dimensions.fingerprint | Autogenerated ID representing the fingerprint of the aws.dimensions object | keyword |
 | aws.tags.\* | Tag key value pairs from aws resources. | object |
 | cloud | Fields related to the cloud or infrastructure the events are coming from. | group |
 | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword |

packages/aws/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 1.0.0
 name: aws
 title: AWS
-version: 1.46.7
+version: 1.46.8
 license: basic
 description: Collect logs and metrics from Amazon Web Services (AWS) with Elastic Agent.
 type: integration