From 3ce066e4b81169ad068909313e4cc6336c261dd8 Mon Sep 17 00:00:00 2001 From: Tetiana Kravchenko Date: Tue, 18 Jul 2023 11:06:19 +0200 Subject: [PATCH] [AWS] [CloudWatch] Add dimensions metadata (#6827) * cloudwatch: add dimension fields Signed-off-by: Tetiana Kravchenko * remove aws.s3.bucket.name field Signed-off-by: Tetiana Kravchenko --------- Signed-off-by: Tetiana Kravchenko --- packages/aws/changelog.yml | 5 +++++ .../elasticsearch/ingest_pipeline/default.yml | 11 +++++++++++ .../aws/data_stream/cloudwatch_metrics/fields/ecs.yml | 5 +++++ .../cloudwatch_metrics/fields/package-fields.yml | 9 +++++---- packages/aws/docs/cloudwatch.md | 3 ++- packages/aws/manifest.yml | 2 +- 6 files changed, 29 insertions(+), 6 deletions(-) create mode 100644 packages/aws/data_stream/cloudwatch_metrics/elasticsearch/ingest_pipeline/default.yml diff --git a/packages/aws/changelog.yml b/packages/aws/changelog.yml index d29d6fdaa11d..7b9517601f4e 100644 --- a/packages/aws/changelog.yml +++ b/packages/aws/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.46.8" + changes: + - description: Add dimensions metadata to the cloudwatch data_stream + type: enhancement + link: https://github.com/elastic/integrations/pull/6827 - version: "1.46.7" changes: - description: Enable time series data streams for the API Gateway and EMR data streams. This improves storage usage and query performance. For more details, see https://www.elastic.co/guide/en/elasticsearch/reference/current/tsds.html. diff --git a/packages/aws/data_stream/cloudwatch_metrics/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/cloudwatch_metrics/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 000000000000..8da84149a463 --- /dev/null +++ b/packages/aws/data_stream/cloudwatch_metrics/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,11 @@ +--- +description: "Pipeline for AWS CloudWatch metrics" +processors: + - fingerprint: + fields: ["aws.dimensions"] + target_field: 'aws.dimensions.fingerprint' + ignore_missing: true +on_failure: + - set: + field: error.message + value: "{{ _ingest.on_failure_message }}" diff --git a/packages/aws/data_stream/cloudwatch_metrics/fields/ecs.yml b/packages/aws/data_stream/cloudwatch_metrics/fields/ecs.yml index 20f154e37bdb..8e6c369edfc8 100644 --- a/packages/aws/data_stream/cloudwatch_metrics/fields/ecs.yml +++ b/packages/aws/data_stream/cloudwatch_metrics/fields/ecs.yml @@ -2,6 +2,7 @@ name: cloud - external: ecs name: cloud.account.id + dimension: true - external: ecs name: cloud.account.name - external: ecs @@ -14,6 +15,7 @@ name: cloud.provider - external: ecs name: cloud.region + dimension: true - external: ecs name: ecs.version - external: ecs @@ -60,3 +62,6 @@ name: container.labels - external: ecs name: container.name +- name: agent.id + external: ecs + dimension: true diff --git a/packages/aws/data_stream/cloudwatch_metrics/fields/package-fields.yml b/packages/aws/data_stream/cloudwatch_metrics/fields/package-fields.yml index a8a7ee8dcce3..a641c8ff08e4 100644 --- a/packages/aws/data_stream/cloudwatch_metrics/fields/package-fields.yml +++ b/packages/aws/data_stream/cloudwatch_metrics/fields/package-fields.yml @@ -5,14 +5,15 @@ type: object description: | Tag key value pairs from aws resources. - - name: s3.bucket.name - type: keyword - description: | - Name of a S3 bucket. - name: dimensions.* type: object description: | Metric dimensions. + - name: dimensions.fingerprint + type: keyword + description: | + Autogenerated ID representing the fingerprint of the aws.dimensions object + dimension: true - name: '*.metrics.*.*' type: object description: | diff --git a/packages/aws/docs/cloudwatch.md b/packages/aws/docs/cloudwatch.md index b9bc24ba49f0..9309aca34204 100644 --- a/packages/aws/docs/cloudwatch.md +++ b/packages/aws/docs/cloudwatch.md @@ -204,10 +204,11 @@ An example event for `cloudwatch` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | +| agent.id | Unique identifier of this agent (if one exists). Example: For Beats this would be beat.id. | keyword | | aws.\*.metrics.\*.\* | Metrics that returned from Cloudwatch API query. | object | | aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword | | aws.dimensions.\* | Metric dimensions. | object | -| aws.s3.bucket.name | Name of a S3 bucket. | keyword | +| aws.dimensions.fingerprint | Autogenerated ID representing the fingerprint of the aws.dimensions object | keyword | | aws.tags.\* | Tag key value pairs from aws resources. | object | | cloud | Fields related to the cloud or infrastructure the events are coming from. | group | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | diff --git a/packages/aws/manifest.yml b/packages/aws/manifest.yml index cb4553266d4b..3b5ac8efefaf 100644 --- a/packages/aws/manifest.yml +++ b/packages/aws/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: aws title: AWS -version: 1.46.7 +version: 1.46.8 license: basic description: Collect logs and metrics from Amazon Web Services (AWS) with Elastic Agent. type: integration