diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index c6908806c54..f4871d41df6 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -312,6 +312,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add Consul Metricbeat module with Agent Metricset {pull}8631[8631] - Add filters and pie chart for AWS EC2 dashboard. {pull}10596[10596] - Add AWS SQS metricset. {pull}10684[10684] {issue}10053[10053] +- Add AWS s3_request metricset. {pull}10949[10949] {issue}10055[10055] *Packetbeat* diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 00080797d30..4df0fba5db7 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -1083,6 +1083,183 @@ type: integer The state of the instance (pending | running | shutting-down | terminated | stopping | stopped). +-- + +[float] +== s3_request fields + +`s3_request` contains request metrics that were scraped from AWS CloudWatch which contains monitoring metrics sent by AWS S3. + + + +*`aws.s3_request.bucket.name`*:: ++ +-- +type: keyword + +Name of a S3 bucket. + + +-- + +*`aws.s3_request.requests.total`*:: ++ +-- +type: long + +The total number of HTTP requests made to an Amazon S3 bucket, regardless of type. + + +-- + +*`aws.s3_request.requests.get`*:: ++ +-- +type: long + +The number of HTTP GET requests made for objects in an Amazon S3 bucket. + + +-- + +*`aws.s3_request.requests.put`*:: ++ +-- +type: long + +The number of HTTP PUT requests made for objects in an Amazon S3 bucket. + + +-- + +*`aws.s3_request.requests.delete`*:: ++ +-- +type: long + +The number of HTTP DELETE requests made for objects in an Amazon S3 bucket. + + +-- + +*`aws.s3_request.requests.head`*:: ++ +-- +type: long + +The number of HTTP HEAD requests made to an Amazon S3 bucket. + + +-- + +*`aws.s3_request.requests.post`*:: ++ +-- +type: long + +The number of HTTP POST requests made to an Amazon S3 bucket. + + +-- + +*`aws.s3_request.requests.select`*:: ++ +-- +type: long + +The number of Amazon S3 SELECT Object Content requests made for objects in an Amazon S3 bucket. + + +-- + +*`aws.s3_request.requests.select.scanned.bytes`*:: ++ +-- +type: scaled_float + +The number of bytes of data scanned with Amazon S3 SELECT Object Content requests in an Amazon S3 bucket. + + +-- + +*`aws.s3_request.requests.select.returned.bytes`*:: ++ +-- +type: scaled_float + +The number of bytes of data returned with Amazon S3 SELECT Object Content requests in an Amazon S3 bucket. + + +-- + +*`aws.s3_request.requests.list`*:: ++ +-- +type: long + +The number of HTTP requests that list the contents of a bucket. + + +-- + +*`aws.s3_request.downloaded.bytes`*:: ++ +-- +type: scaled_float + +The number bytes downloaded for requests made to an Amazon S3 bucket, where the response includes a body. + + +-- + +*`aws.s3_request.uploaded.bytes`*:: ++ +-- +type: scaled_float + +The number bytes uploaded that contain a request body, made to an Amazon S3 bucket. + + +-- + +*`aws.s3_request.errors.4xx`*:: ++ +-- +type: long + +The number of HTTP 4xx client error status code requests made to an Amazon S3 bucket with a value of either 0 or 1. + + +-- + +*`aws.s3_request.errors.5xx`*:: ++ +-- +type: long + +The number of HTTP 5xx server error status code requests made to an Amazon S3 bucket with a value of either 0 or 1. + + +-- + +*`aws.s3_request.latency.first_byte.ms`*:: ++ +-- +type: long + +The per-request time from the complete request being received by an Amazon S3 bucket to when the response starts to be returned. + + +-- + +*`aws.s3_request.latency.total_request.ms`*:: ++ +-- +type: long + +The elapsed per-request time from the first byte received to the last byte sent to an Amazon S3 bucket. + + -- [float] diff --git a/metricbeat/docs/modules/aws.asciidoc b/metricbeat/docs/modules/aws.asciidoc index 8eae60fd964..542c4f03792 100644 --- a/metricbeat/docs/modules/aws.asciidoc +++ b/metricbeat/docs/modules/aws.asciidoc @@ -9,7 +9,7 @@ This module periodically fetches monitoring metrics from AWS Cloudwatch using https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_GetMetricData.html[GetMetricData API] for running EC2 instances. Note: extra AWS charges on GetMetricData API requests will be generated by this module. -The default metricsets are `ec2` and `sqs`. +The default metricsets are `ec2`, `sqs` and `s3_request`. [float] === Module-specific configuration notes @@ -51,6 +51,7 @@ metricbeat.modules: default_region: '${AWS_REGION:us-west-1}' ---- +=== ec2 metricset By default, Amazon EC2 sends metric data to CloudWatch every 5 minutes. With this basic monitoring, `period` in aws module configuration should be larger or equal than `300s`. If `period` is set to be less than `300s`, the same cloudwatch metrics will be collected more than once which will cause extra fees without getting more granular metrics. For example, in `US East (N. Virginia)` region, it costs @@ -63,13 +64,23 @@ larger than `60s`. Since AWS sends metric data to CloudWatch in 1-minute periods than `60s` will cause extra API requests which means extra charges on AWS. To avoid unnecessary charges, `period` is preferred to be set to `60s` or multiples of `60s`, such as `120s` and `180s`. -Since cloudWatch metrics for Amazon SQS queues are automatically collected and pushed to CloudWatch every 5 minutes, -`period` for `sqs` is recommended to set to `300s` or multiples of `300s`. - -The AWS module comes with a predefined dashboard. For example: +The ec2 metricset comes with a predefined dashboard. For example: image::./images/metricbeat-aws-ec2-overview.png[] +=== sqs metricset +Cloudwatch metrics for Amazon SQS queues are automatically collected and pushed to CloudWatch every 5 minutes, +the `period` for `sqs` metricset is recommended to be `300s` or multiples of `300s`. + +=== s3_request metricset +Request metrics are available at 1-minute intervals with additional charges. The s3_request metricset will give more +granular data to track S3 bucket usage. The `period` for `s3_request` metricset can be set to `60s` or multiples of `60s`. +But because of the extra charges for querying these metrics, the `period` is recommended to set to `86400s`. The user can +always adjust this to the granularity they want. Request metrics are not enabled by default for S3 buckets. Please see +https://docs.aws.amazon.com/AmazonS3/latest/user-guide/configure-metrics.html[How to +Configure Request Metrics for S3] for instructions on how to enable request metrics for +each S3 bucket. + [float] === Example configuration @@ -89,6 +100,14 @@ metricbeat.modules: secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}' default_region: '${AWS_REGION:us-west-1}' +- module: aws + period: 86400s + metricsets: + - "s3_request" + access_key_id: '${AWS_ACCESS_KEY_ID:""}' + secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' + session_token: '${AWS_SESSION_TOKEN:""}' + default_region: '${AWS_REGION:us-west-1}' ---- [float] @@ -98,9 +117,13 @@ The following metricsets are available: * <> +* <> + * <> include::aws/ec2.asciidoc[] +include::aws/s3_request.asciidoc[] + include::aws/sqs.asciidoc[] diff --git a/metricbeat/docs/modules/aws/s3_request.asciidoc b/metricbeat/docs/modules/aws/s3_request.asciidoc new file mode 100644 index 00000000000..03c070c1bf8 --- /dev/null +++ b/metricbeat/docs/modules/aws/s3_request.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-metricset-aws-s3_request]] +=== aws s3_request metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/aws/s3_request/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/aws/s3_request/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 7aef20c7339..485cab8fb1f 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -10,7 +10,8 @@ This file is generated! See scripts/docs_collector.py |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | .1+| .1+| |<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | -.2+| .2+| |<> +.3+| .3+| |<> +|<> beta[] |<> beta[] |<> |image:./images/icon-no.png[No prebuilt dashboards] | .7+| .7+| |<> diff --git a/x-pack/metricbeat/include/list.go b/x-pack/metricbeat/include/list.go index 4e75b505f43..c20490fb53b 100644 --- a/x-pack/metricbeat/include/list.go +++ b/x-pack/metricbeat/include/list.go @@ -10,6 +10,7 @@ import ( // Import packages that need to register themselves. _ "github.com/elastic/beats/x-pack/metricbeat/module/aws" _ "github.com/elastic/beats/x-pack/metricbeat/module/aws/ec2" + _ "github.com/elastic/beats/x-pack/metricbeat/module/aws/s3_request" _ "github.com/elastic/beats/x-pack/metricbeat/module/aws/sqs" _ "github.com/elastic/beats/x-pack/metricbeat/module/mssql" _ "github.com/elastic/beats/x-pack/metricbeat/module/mssql/performance" diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index a588f13d2d9..0e7bcc0a0cb 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -157,6 +157,14 @@ metricbeat.modules: secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}' default_region: '${AWS_REGION:us-west-1}' +- module: aws + period: 86400s + metricsets: + - "s3_request" + access_key_id: '${AWS_ACCESS_KEY_ID:""}' + secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' + session_token: '${AWS_SESSION_TOKEN:""}' + default_region: '${AWS_REGION:us-west-1}' #--------------------------------- Ceph Module --------------------------------- - module: ceph diff --git a/x-pack/metricbeat/module/aws/_meta/config.yml b/x-pack/metricbeat/module/aws/_meta/config.yml index d831e5e094d..3561b58b11f 100644 --- a/x-pack/metricbeat/module/aws/_meta/config.yml +++ b/x-pack/metricbeat/module/aws/_meta/config.yml @@ -7,3 +7,11 @@ secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}' default_region: '${AWS_REGION:us-west-1}' +- module: aws + period: 86400s + metricsets: + - "s3_request" + access_key_id: '${AWS_ACCESS_KEY_ID:""}' + secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' + session_token: '${AWS_SESSION_TOKEN:""}' + default_region: '${AWS_REGION:us-west-1}' diff --git a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc index 6706525d8d0..5b5ae11018f 100644 --- a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc @@ -2,7 +2,7 @@ This module periodically fetches monitoring metrics from AWS Cloudwatch using https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_GetMetricData.html[GetMetricData API] for running EC2 instances. Note: extra AWS charges on GetMetricData API requests will be generated by this module. -The default metricsets are `ec2` and `sqs`. +The default metricsets are `ec2`, `sqs` and `s3_request`. [float] === Module-specific configuration notes @@ -44,6 +44,7 @@ metricbeat.modules: default_region: '${AWS_REGION:us-west-1}' ---- +=== ec2 metricset By default, Amazon EC2 sends metric data to CloudWatch every 5 minutes. With this basic monitoring, `period` in aws module configuration should be larger or equal than `300s`. If `period` is set to be less than `300s`, the same cloudwatch metrics will be collected more than once which will cause extra fees without getting more granular metrics. For example, in `US East (N. Virginia)` region, it costs @@ -56,9 +57,19 @@ larger than `60s`. Since AWS sends metric data to CloudWatch in 1-minute periods than `60s` will cause extra API requests which means extra charges on AWS. To avoid unnecessary charges, `period` is preferred to be set to `60s` or multiples of `60s`, such as `120s` and `180s`. -Since cloudWatch metrics for Amazon SQS queues are automatically collected and pushed to CloudWatch every 5 minutes, -`period` for `sqs` is recommended to set to `300s` or multiples of `300s`. - -The AWS module comes with a predefined dashboard. For example: +The ec2 metricset comes with a predefined dashboard. For example: image::./images/metricbeat-aws-ec2-overview.png[] + +=== sqs metricset +Cloudwatch metrics for Amazon SQS queues are automatically collected and pushed to CloudWatch every 5 minutes, +the `period` for `sqs` metricset is recommended to be `300s` or multiples of `300s`. + +=== s3_request metricset +Request metrics are available at 1-minute intervals with additional charges. The s3_request metricset will give more +granular data to track S3 bucket usage. The `period` for `s3_request` metricset can be set to `60s` or multiples of `60s`. +But because of the extra charges for querying these metrics, the `period` is recommended to set to `86400s`. The user can +always adjust this to the granularity they want. Request metrics are not enabled by default for S3 buckets. Please see +https://docs.aws.amazon.com/AmazonS3/latest/user-guide/configure-metrics.html[How to +Configure Request Metrics for S3] for instructions on how to enable request metrics for +each S3 bucket. diff --git a/x-pack/metricbeat/module/aws/aws.go b/x-pack/metricbeat/module/aws/aws.go index d120d8d0654..2a709aba79a 100644 --- a/x-pack/metricbeat/module/aws/aws.go +++ b/x-pack/metricbeat/module/aws/aws.go @@ -139,3 +139,13 @@ func convertPeriodToDuration(period string) (string, int, error) { return duration, numberPeriod, err } } + +// StringInSlice checks if a string is already exists in list +func StringInSlice(str string, list []string) bool { + for _, v := range list { + if v == str { + return true + } + } + return false +} diff --git a/x-pack/metricbeat/module/aws/ec2/ec2_integration_test.go b/x-pack/metricbeat/module/aws/ec2/ec2_integration_test.go index 6525e3f3b7c..d327c539af9 100644 --- a/x-pack/metricbeat/module/aws/ec2/ec2_integration_test.go +++ b/x-pack/metricbeat/module/aws/ec2/ec2_integration_test.go @@ -2,6 +2,8 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +// +build integration + package ec2 import ( @@ -14,7 +16,7 @@ import ( ) func TestFetch(t *testing.T) { - config, info := mtest.GetConfigForTest("ec2") + config, info := mtest.GetConfigForTest("ec2", "300s") if info != "" { t.Skip("Skipping TestFetch: " + info) } @@ -63,7 +65,7 @@ func TestFetch(t *testing.T) { } func TestData(t *testing.T) { - config, info := mtest.GetConfigForTest("ec2") + config, info := mtest.GetConfigForTest("ec2", "300s") if info != "" { t.Skip("Skipping TestData: " + info) } diff --git a/x-pack/metricbeat/module/aws/fields.go b/x-pack/metricbeat/module/aws/fields.go index 15e67526b2a..f65009a75f8 100644 --- a/x-pack/metricbeat/module/aws/fields.go +++ b/x-pack/metricbeat/module/aws/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAws returns asset data. // This is the base64 encoded gzipped contents of module/aws. func AssetAws() string { - return "eJzEWM9v27gSvuevGPTUAo2AVzy8Qw4PaNMecthFtmnRozMmx9YgFKlyhvY66B+/ICU7sSMnTmpldQgCUZ75vo+cXzyFG1qdAS7lBEBZHZ3BG1zKmxMAS2Iit8rBn8H/TwAArnEp19AEmxyBCc6RUYGPP66gCZ41RPZzaEgjG4FZDE1ZO3ch2SWqqasTgEiOUOgM5ngCMGNyVs6K9VPw2NAaTX501eYPY0ht/2YA1LaR+4bIfNi8GzK212D3XJP5cA0meEX2AlrThpvWqLCkSCAmYkt2h+2PzBaWNZv6zsCARkJeYboqP/xy/qG6539bp/WzS/U+XdOmSoOiq1qjW1+syYtBR3YycwF3P3hEh/x8qwlaioa84pwgzACdCwaVbAYOJjRtUoLkWXt5MBKYFCN5dStgD0kIgi86shdFb6jaS8REsqyTJDinEbj41EwpZh7nl9+hcyYgbb8f9zHCLMTyVVJ2fIvZ7JO4p+jyb0dFThg92S0CnfD+DnuNAmhMTGRBOL9hhSUKOEze1GQhRBDFqGT3k5IUW5dk8orkepfbzGpcEEyJ/N1OoYfkHTecT+KG9rImD/ln55ffz4uFTx1mWKBLBCxwSzEcylgmpsY4Jzsu5cJpkHiOJR8UWmQLNix9pv5w/98DetunHa2TAHuTYtYIreWMAh10VIape9JliDcV+6pFc0MqozLufUAkQ7zIh9HnvLKGAeyV4gwNyW5QPg4/JH1V/CWNh6THws++mq6UxgVfPIwi/WthP5bsluWGQxUJ7SjYP/VKY98mZKybVCUaIsEiuNSQAC6QHU4dgYbDkS8jK40IPdtX8hnT0bEX1UN7bODnoWkd5aJQdA8txVK55eVbkHsYzFna8IzJ5n6Ig83HUbk5ZIPGZFk83Kf5wr16CUlR1CSVqcncTGbIbk+hdMHPn8fvK7UhquR6rjXFbaS5t2lRhCxMg9bbix0mKJhKVcyrshKlZnuNu47UoSg07JMeTnLS2XtlrmMQWfv5F6gM79ihZDYpxoSY/yQ/PPnkkjCn+NuDQogkZSB4Or9tVrnBOVU8HBM3tFqGuLt2ALCLzyUoM4xsP09XNgdz19g/B9/dXFrlPRg+BC/CeeEt5xnx7iRY0nLi7g/DLEA+56I9E8gGaBt5gUqV9TLJS8cVtLcOn/+8Ko7X8j7oKg5Eye3wSdx9/QxoF5eL/+ZWPpIIoEgwXCbwJffp79lY09SxGUvQYvyBngeeyh7aEVVcC9fj+JKTCxu4uNysvM0Cv4NpSN6uC+NzJS0hVJlgh9V8cSIqdnc1fA95vof//O90ygrJC899mYOLkyeQap1bI5m0FCc5sb0CXnjbkrc56H9BTN53/0mdVNnPT8tk+wuUYsO+nOxfuW9p2/67/C/Zd9WDmz75Kb910yc/5RVv+q7+uhq86ZuSHnrXF5wl0UlDIjinSS4wQuZI5bsES9vG8Dc3eRP7G78sS+cXfPCnlrrOs8ewLtk/E6U9MdJ/WVlyuDpaj/htu0z3TmQLz90tSu+7tIQ+6L1GOFf1HBClJDUN2Zxb3epxKj7oZMHCU3es3mkPmw0B9jBzPK/1cWCvAmpXPI1MC3RdiBx+GPIxGhfo+qw+B9j6ZmRcZJE0lWu86QoMOifruvO1c/9HH11o9t87rxHnHDPyhlvbdZn4mILUtLqa9PoNj7y/i2hHnY+XF2v1cpxY7oK7Exdwo9DwQET+LpMK345zk1JqIt/SE3r+EwAA//8lh/10" + return "eJzEWk1vIzcSvftXFHJKgHFjk5nswYcFJh5jYyCbeGMPctSUyJKaazbZwyIly5gfvyj2hyy5Zcu2WtHBMJot1nuPVcUqUqdwS6szwCWfAEQTLZ3Bd7jk704ANLEKpo7GuzP41wkAwBdc8heovE6WQHlrSUWGj39dQ+WdiT4YN4eKYjCKYRZ8lcfOrU96iVGVxQlAIEvIdAZzPAGYGbKaz/Lsp+Cwog6NfOKqlheDT3X7ZADU5iQPJyL1U/9saLKdEzafL6R++gLKu4jGMcSSem6xxAhLCgSsAtakt9j+JWxhWRpVricY0IjJRZiu8hcvzn8qHtjf1Kn7bFN9SFfVqYg+oi1qFTfe6MizQkt6MrMet194Qgf53JQENQVFLuKcwM8ArfUKI2kBDspXdYoEyZnYyoOBQKUQyEW7AuMgMYF3WUfjOKJTVOwkogJpEyeJcU4jcHGpmlIQHudXn6ExxsB1ux4PMcLMh/xWisaae5Rpn8U9RSvfHRU5YXCkNwg0wrs19hIZUKmQSAMbeWIiLJHBYnKqJA0+AEcMkfRuUpxCbRNPjkiuNbnJrMQFwZTIrVcKHSRnTWXEE3vay5IcyNfOrz6f5xl+aTDDAm0iMAz3FPy+jHmiSgxz0uNSzpwGiUssOR+hRqNB+6UT6o/X/x2g023aiWViME6lIBqh1kZQoIWGyjB1R3Hpw21hXFGjuqXIozJubUAgRWYhzugkr3QwwLhIYYaKeDson4bvUzwq/pzGfYqHwm9cMV1FGhd8tjCK9MfCfijZteFb44tAqEfB/kurNLZlgmDtUxVHHwgW3qaKGHCBxuLUEkS/P/JlMJFGhC7zR3KC6eDYs+q+PjTwc1/VlmRTyLr7mkLeufn1SyA1DEqWVmZmSEs9ZLwWd4ym2meBxmSZLTyk+cq1eg1JjhgTF6okdTuZobE7Nkrr3fxl/P6k2ofIsp/HksImUqltamQmDVMfy83BBhNkTHlXlFFecaRqc8w0FalFjlAZl+L+JCfNfEfmOgaRzs7fQGV4xfYl06cY5YP8SW6485EtYU7hzY2CD8S5IXg+v/WjpsI5FWY4Jm5ptfRhe2wPYJefclAKDJlfuistwdwU9i/Bt+5LC1mDYSd4Fc5Lp430iGtP0BSzxz1shg0DOclFOzqQHmgdzAIjFdrxRIYOK2g7O3z6/Tob7uR9VFXsidLUw564/fgF0C6vFh+klA/EDMjslckd+NK06e/FWNPUGjWWoHnyR3ru6ZUttAOq2AnX4riQ5GIUXF71I9+LwD/A1Cenu43xpZLmECqU18NqvjoR5Xm3NXwH0t/Dj/88nZoIybGZu9wHZyPPII2llEY8qSlMJLEdAS98X5PTEvTfICTnmv+4TDEaNz/Nne03iBQq47Jnf5O6pa7b9+Rf0j8Uj076+P0k0NdEHN904Lee5sG5X/tk1LO/6/eDR39Tivse/k2TtKLFYaP49zZsEa7fdxYGrbcacXP+eKAqQrwoz/dgE/715uaqtwYV6lzAooOPFd57t8b5DgLNMWjbRfyq3hG5PfY5DdcOr0O+hfnfFzdbuKWK8NP/5SN0Kbofc3gGb51GxHv1+eB4NUmrMh7kTxe/XdxcHBp1SXiobmYA868XHz/t5c/P+YLnMZ3hj+ttb3gVSiZLO64m3opzjeT64reL8xv4Iy86nHsXJdEe2CsaJgUrdI7GOTUaOvHyM9AYEVq7TeG3N/W3MA0UU/g7qHaGj8HVmjGjqMeWKwixlWsj1UDnZqN9CqeUR9ajHn8VmiVY28sBs9+2uyylNBJigbj2jqX6UzZpkkp16vVqmFyqj0mts9asRVumAfbFnuB89/JMRyH4wMWHu7vx3OjD3R0oa8Tbs7n+4MRr2muNmkjC9irMz4BM7s3/AT7Aj08S+3lMYj/f3QFTWFA4IjGLkZxaFTMTOE7EOYpq2Ptex7GmcNo5VTQVNc1CE/fN2e3a50jag/5CprncfMQx+uZ2cyPC8g1uPvOdUp8xnyacK+uu3TksZ7JYc3N2vIN7VjuH4ppv22/nI788khuk56Kv7wC/8ttav698xN96XP/3+q0Nn7eaOE4qYsY5TXBOBZM64CpiXQd/Zypp49vffIgsjV1w3p02Bb2GFkN3aPs1UdrRa7VvSiuAq4PdEtxsJpPWCG/gWd+jt7bzpYDz8cFVSLPJYT6fMFVF2mAku2O/6qg4HycLw2Zqx2ltejY9AeNgZs283LEHdcCOAmpbvBgMLdCuI30/ZxA3Ghdo56svAdalpnGR9QXudAUKreUuE/7ZmP9PG12odv/yqEMsOWbkBde6ydX4lIJU1XE1afU75NayRrSlzsery049iRNtmuBuxAXsFRqEK7L1mZTN/Th36flU1NzTM3r+PwAA//+efQFO" } diff --git a/x-pack/metricbeat/module/aws/mtest/integration.go b/x-pack/metricbeat/module/aws/mtest/integration.go index 117ec5e5d42..afacfdcbb0d 100644 --- a/x-pack/metricbeat/module/aws/mtest/integration.go +++ b/x-pack/metricbeat/module/aws/mtest/integration.go @@ -5,6 +5,7 @@ package mtest import ( + "errors" "os" "testing" @@ -14,7 +15,7 @@ import ( ) // GetConfigForTest function gets aws credentials for integration tests. -func GetConfigForTest(metricSetName string) (map[string]interface{}, string) { +func GetConfigForTest(metricSetName string, period string) (map[string]interface{}, string) { accessKeyID, okAccessKeyID := os.LookupEnv("AWS_ACCESS_KEY_ID") secretAccessKey, okSecretAccessKey := os.LookupEnv("AWS_SECRET_ACCESS_KEY") sessionToken, okSessionToken := os.LookupEnv("AWS_SESSION_TOKEN") @@ -32,7 +33,7 @@ func GetConfigForTest(metricSetName string) (map[string]interface{}, string) { } else { config = map[string]interface{}{ "module": "aws", - "period": "300s", + "period": period, "metricsets": []string{metricSetName}, "access_key_id": accessKeyID, "secret_access_key": secretAccessKey, @@ -48,36 +49,43 @@ func GetConfigForTest(metricSetName string) (map[string]interface{}, string) { // CheckEventField function checks a given field type and compares it with the expected type for integration tests. func CheckEventField(metricName string, expectedType string, event mb.Event, t *testing.T) { - if ok, err := event.MetricSetFields.HasKey(metricName); ok { - assert.NoError(t, err) - metricValue, err := event.MetricSetFields.GetValue(metricName) - assert.NoError(t, err) - compareType(metricValue, expectedType, t) - } else if ok, err := event.RootFields.HasKey(metricName); ok { - assert.NoError(t, err) - rootValue, err := event.RootFields.GetValue(metricName) - assert.NoError(t, err) - compareType(rootValue, expectedType, t) + ok1, err1 := event.MetricSetFields.HasKey(metricName) + ok2, err2 := event.RootFields.HasKey(metricName) + if ok1 || ok2 { + if ok1 { + assert.NoError(t, err1) + metricValue, err := event.MetricSetFields.GetValue(metricName) + assert.NoError(t, err) + err = compareType(metricValue, expectedType, metricName) + assert.NoError(t, err) + t.Log("Succeed: Field " + metricName + " matches type " + expectedType) + } else if ok2 { + assert.NoError(t, err2) + rootValue, err := event.RootFields.GetValue(metricName) + assert.NoError(t, err) + err = compareType(rootValue, expectedType, metricName) + assert.NoError(t, err) + t.Log("Succeed: Field " + metricName + " matches type " + expectedType) + } + } else { + t.Log("Field " + metricName + " does not exist in metric set fields") } } -func compareType(metricValue interface{}, expectedType string, t *testing.T) { +func compareType(metricValue interface{}, expectedType string, metricName string) (err error) { switch metricValue.(type) { case float64: if expectedType != "float" { - t.Log("Failed: Field is not in type " + expectedType) - t.Fail() + err = errors.New("Failed: Field " + metricName + "is not in type " + expectedType) } case string: if expectedType != "string" { - t.Log("Failed: Field is not in type " + expectedType) - t.Fail() + err = errors.New("Failed: Field " + metricName + "is not in type " + expectedType) } case int64: if expectedType != "int" { - t.Log("Failed: Field is not in type " + expectedType) - t.Fail() + err = errors.New("Failed: Field " + metricName + "is not in type " + expectedType) } } - t.Log("Succeed: Field matches type " + expectedType) + return } diff --git a/x-pack/metricbeat/module/aws/s3_request/_meta/data.json b/x-pack/metricbeat/module/aws/s3_request/_meta/data.json new file mode 100644 index 00000000000..4814b96c7db --- /dev/null +++ b/x-pack/metricbeat/module/aws/s3_request/_meta/data.json @@ -0,0 +1,46 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "agent": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "aws": { + "s3_request": { + "bucket": { + "name": "test-s3-ks" + }, + "downloaded": { + "bytes": 1303 + }, + "errors": { + "4xx": 0, + "5xx": 0 + }, + "latency": { + "first_byte.ms": 544, + "total_request.ms": 545 + }, + "requests": { + "head": 1, + "list": 1, + "total": 2 + }, + "uploaded": {} + } + }, + "cloud": { + "region": "ap-southeast-1" + }, + "event": { + "dataset": "aws.s3_request", + "duration": 115000, + "module": "aws" + }, + "metricset": { + "name": "s3_request" + }, + "service": { + "name": "s3_request", + "type": "s3_request" + } +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/aws/s3_request/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/s3_request/_meta/docs.asciidoc new file mode 100644 index 00000000000..1c0810be196 --- /dev/null +++ b/x-pack/metricbeat/module/aws/s3_request/_meta/docs.asciidoc @@ -0,0 +1,10 @@ +The s3_request metricset of aws module allows you to monitor your AWS S3 buckets. `s3_request` metricset +fetches Cloudwatch daily storage metrics for each S3 bucket from +https://docs.aws.amazon.com/AmazonS3/latest/dev/cloudwatch-monitoring.html[S3 CloudWatch Request Metrics for Buckets]. + + === AWS Permissions +Some specific AWS permissions are required for IAM user to collect AWS s3_request metrics. +---- +ec2:DescribeRegions +cloudwatch:GetMetricData +---- diff --git a/x-pack/metricbeat/module/aws/s3_request/_meta/fields.yml b/x-pack/metricbeat/module/aws/s3_request/_meta/fields.yml new file mode 100644 index 00000000000..a5469353f96 --- /dev/null +++ b/x-pack/metricbeat/module/aws/s3_request/_meta/fields.yml @@ -0,0 +1,74 @@ +- name: s3_request + type: group + description: > + `s3_request` contains request metrics that were scraped from AWS CloudWatch which contains monitoring metrics sent by AWS S3. + release: beta + fields: + - name: bucket.name + type: keyword + description: > + Name of a S3 bucket. + - name: requests.total + type: long + description: > + The total number of HTTP requests made to an Amazon S3 bucket, regardless of type. + - name: requests.get + type: long + description: > + The number of HTTP GET requests made for objects in an Amazon S3 bucket. + - name: requests.put + type: long + description: > + The number of HTTP PUT requests made for objects in an Amazon S3 bucket. + - name: requests.delete + type: long + description: > + The number of HTTP DELETE requests made for objects in an Amazon S3 bucket. + - name: requests.head + type: long + description: > + The number of HTTP HEAD requests made to an Amazon S3 bucket. + - name: requests.post + type: long + description: > + The number of HTTP POST requests made to an Amazon S3 bucket. + - name: requests.select + type: long + description: > + The number of Amazon S3 SELECT Object Content requests made for objects in an Amazon S3 bucket. + - name: requests.select.scanned.bytes + type: scaled_float + description: > + The number of bytes of data scanned with Amazon S3 SELECT Object Content requests in an Amazon S3 bucket. + - name: requests.select.returned.bytes + type: scaled_float + description: > + The number of bytes of data returned with Amazon S3 SELECT Object Content requests in an Amazon S3 bucket. + - name: requests.list + type: long + description: > + The number of HTTP requests that list the contents of a bucket. + - name: downloaded.bytes + type: scaled_float + description: > + The number bytes downloaded for requests made to an Amazon S3 bucket, where the response includes a body. + - name: uploaded.bytes + type: scaled_float + description: > + The number bytes uploaded that contain a request body, made to an Amazon S3 bucket. + - name: errors.4xx + type: long + description: > + The number of HTTP 4xx client error status code requests made to an Amazon S3 bucket with a value of either 0 or 1. + - name: errors.5xx + type: long + description: > + The number of HTTP 5xx server error status code requests made to an Amazon S3 bucket with a value of either 0 or 1. + - name: latency.first_byte.ms + type: long + description: > + The per-request time from the complete request being received by an Amazon S3 bucket to when the response starts to be returned. + - name: latency.total_request.ms + type: long + description: > + The elapsed per-request time from the first byte received to the last byte sent to an Amazon S3 bucket. diff --git a/x-pack/metricbeat/module/aws/s3_request/data.go b/x-pack/metricbeat/module/aws/s3_request/data.go new file mode 100644 index 00000000000..06e6f4cb290 --- /dev/null +++ b/x-pack/metricbeat/module/aws/s3_request/data.go @@ -0,0 +1,41 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package s3_request + +import ( + s "github.com/elastic/beats/libbeat/common/schema" + c "github.com/elastic/beats/libbeat/common/schema/mapstrstr" +) + +var ( + schemaMetricSetFields = s.Schema{ + "requests": s.Object{ + "total": c.Int("AllRequests"), + "get": c.Int("GetRequests"), + "put": c.Int("PutRequests"), + "delete": c.Int("DeleteRequests"), + "head": c.Int("HeadRequests"), + "post": c.Int("PostRequests"), + "select": c.Int("SelectRequests"), + "select.scanned.bytes": c.Float("SelectScannedBytes"), + "select.returned.bytes": c.Float("SelectReturnedBytes"), + "list": c.Int("ListRequests"), + }, + "downloaded": s.Object{ + "bytes": c.Float("BytesDownloaded"), + }, + "uploaded": s.Object{ + "bytes": c.Float("BytesUploaded"), + }, + "errors": s.Object{ + "4xx": c.Int("4xxErrors"), + "5xx": c.Int("5xxErrors"), + }, + "latency": s.Object{ + "first_byte.ms": c.Float("FirstByteLatency"), + "total_request.ms": c.Float("TotalRequestLatency"), + }, + } +) diff --git a/x-pack/metricbeat/module/aws/s3_request/s3_request.go b/x-pack/metricbeat/module/aws/s3_request/s3_request.go new file mode 100644 index 00000000000..eb81ba06b34 --- /dev/null +++ b/x-pack/metricbeat/module/aws/s3_request/s3_request.go @@ -0,0 +1,228 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package s3_request + +import ( + "fmt" + "strconv" + "strings" + + "github.com/aws/aws-sdk-go-v2/service/cloudwatch" + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/common/cfgwarn" + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/x-pack/metricbeat/module/aws" +) + +var metricsetName = "s3_request" + +// init registers the MetricSet with the central registry as soon as the program +// starts. The New function will be called later to instantiate an instance of +// the MetricSet for each host defined in the module's configuration. After the +// MetricSet has been created then Fetch will begin to be called periodically. +func init() { + mb.Registry.MustAddMetricSet(aws.ModuleName, metricsetName, New, + mb.DefaultMetricSet(), + ) +} + +// MetricSet holds any configuration or state information. It must implement +// the mb.MetricSet interface. And this is best achieved by embedding +// mb.BaseMetricSet because it implements all of the required mb.MetricSet +// interface methods except for Fetch. +type MetricSet struct { + *aws.MetricSet + logger *logp.Logger +} + +// New creates a new instance of the MetricSet. New is responsible for unpacking +// any MetricSet specific configuration options if there are any. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + cfgwarn.Beta("The aws s3_request metricset is beta.") + s3Logger := logp.NewLogger(aws.ModuleName) + + moduleConfig := aws.Config{} + if err := base.Module().UnpackConfig(&moduleConfig); err != nil { + return nil, err + } + + if moduleConfig.Period == "" { + err := errors.New("period is not set in AWS module config") + s3Logger.Error(err) + } + + metricSet, err := aws.NewMetricSet(base) + if err != nil { + return nil, errors.Wrap(err, "error creating aws metricset") + } + + // Check if period is set to be multiple of 60s + remainder := metricSet.PeriodInSec % 60 + if remainder != 0 { + err := errors.New("period needs to be set to 60s (or a multiple of 60s). " + + "To avoid data missing or extra costs, please make sure period is set correctly " + + "in config.yml") + s3Logger.Info(err) + } + + return &MetricSet{ + MetricSet: metricSet, + logger: s3Logger, + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(report mb.ReporterV2) { + namespace := "AWS/S3" + // Get startTime and endTime + startTime, endTime, err := aws.GetStartTimeEndTime(m.DurationString) + if err != nil { + logp.Error(errors.Wrap(err, "Error ParseDuration")) + m.logger.Error(err.Error()) + report.Error(err) + return + } + + // GetMetricData for AWS S3 from Cloudwatch + for _, regionName := range m.MetricSet.RegionsList { + m.MetricSet.AwsConfig.Region = regionName + svcCloudwatch := cloudwatch.New(*m.MetricSet.AwsConfig) + listMetricsOutputs, err := aws.GetListMetricsOutput(namespace, regionName, svcCloudwatch) + if err != nil { + m.logger.Error(err.Error()) + report.Error(err) + continue + } + + if listMetricsOutputs == nil || len(listMetricsOutputs) == 0 { + continue + } + + metricDataQueries := constructMetricQueries(listMetricsOutputs, m.PeriodInSec) + // This happens when S3 cloudwatch request metrics are not enabled. + if len(metricDataQueries) == 0 { + continue + } + // Use metricDataQueries to make GetMetricData API calls + metricDataOutputs, err := aws.GetMetricDataResults(metricDataQueries, svcCloudwatch, startTime, endTime) + if err != nil { + err = errors.Wrap(err, "GetMetricDataResults failed, skipping region "+regionName) + m.logger.Error(err.Error()) + report.Error(err) + continue + } + + // Create Cloudwatch Events for s3_request + bucketNames := getBucketNames(listMetricsOutputs) + for _, bucketName := range bucketNames { + event, err := createS3RequestEvents(metricDataOutputs, regionName, bucketName) + if err != nil { + m.logger.Error(err.Error()) + event.Error = err + report.Event(event) + continue + } + report.Event(event) + } + } +} + +func getBucketNames(listMetricsOutputs []cloudwatch.Metric) (bucketNames []string) { + for _, output := range listMetricsOutputs { + for _, dim := range output.Dimensions { + if *dim.Name == "BucketName" { + if aws.StringInSlice(*dim.Value, bucketNames) { + continue + } + bucketNames = append(bucketNames, *dim.Value) + } + } + } + return +} + +func createMetricDataQuery(metric cloudwatch.Metric, periodInSec int, index int) (metricDataQuery cloudwatch.MetricDataQuery) { + statistic := "Sum" + period := int64(periodInSec) + id := "s3r" + strconv.Itoa(index) + metricDims := metric.Dimensions + bucketName := "" + filterID := "" + for _, dim := range metricDims { + if *dim.Name == "BucketName" { + bucketName = *dim.Value + } else if *dim.Name == "FilterId" { + filterID = *dim.Value + } + } + metricName := *metric.MetricName + label := bucketName + " " + filterID + " " + metricName + metricDataQuery = cloudwatch.MetricDataQuery{ + Id: &id, + MetricStat: &cloudwatch.MetricStat{ + Period: &period, + Stat: &statistic, + Metric: &metric, + }, + Label: &label, + } + return +} + +func constructMetricQueries(listMetricsOutputs []cloudwatch.Metric, periodInSec int) []cloudwatch.MetricDataQuery { + metricDataQueries := []cloudwatch.MetricDataQuery{} + metricDataQueryEmpty := cloudwatch.MetricDataQuery{} + dailyMetricNames := []string{"NumberOfObjects", "BucketSizeBytes"} + for i, listMetric := range listMetricsOutputs { + if aws.StringInSlice(*listMetric.MetricName, dailyMetricNames) { + continue + } + + metricDataQuery := createMetricDataQuery(listMetric, periodInSec, i) + if metricDataQuery == metricDataQueryEmpty { + continue + } + metricDataQueries = append(metricDataQueries, metricDataQuery) + } + return metricDataQueries +} + +// CreateS3Events creates s3_request and s3_daily_storage events from Cloudwatch metric data. +func createS3RequestEvents(outputs []cloudwatch.MetricDataResult, regionName string, bucketName string) (event mb.Event, err error) { + event.Service = metricsetName + event.RootFields = common.MapStr{} + // Cloud fields in ECS + event.RootFields.Put("service.name", metricsetName) + event.RootFields.Put("cloud.region", regionName) + + // AWS s3_request metrics + mapOfMetricSetFieldResults := make(map[string]interface{}) + // Find a timestamp for all metrics in output + if len(outputs) > 0 && len(outputs[0].Timestamps) > 0 { + timestamp := outputs[0].Timestamps[0] + for _, output := range outputs { + labels := strings.Split(*output.Label, " ") + // check timestamp to make sure metrics come from the same timestamp + if len(labels) == 3 && labels[0] == bucketName && len(output.Values) > 0 && output.Timestamps[0] == timestamp { + mapOfMetricSetFieldResults[labels[2]] = fmt.Sprint(output.Values[0]) + } + } + } + + resultMetricSetFields, err := aws.EventMapping(mapOfMetricSetFieldResults, schemaMetricSetFields) + if err != nil { + err = errors.Wrap(err, "Error trying to apply schema schemaMetricSetFields in AWS s3_request metricbeat module.") + return + } + + resultMetricSetFields.Put("bucket.name", bucketName) + event.MetricSetFields = resultMetricSetFields + return +} diff --git a/x-pack/metricbeat/module/aws/s3_request/s3_request_integration_test.go b/x-pack/metricbeat/module/aws/s3_request/s3_request_integration_test.go new file mode 100644 index 00000000000..8a3d369216e --- /dev/null +++ b/x-pack/metricbeat/module/aws/s3_request/s3_request_integration_test.go @@ -0,0 +1,74 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build integration + +package s3_request + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + "github.com/elastic/beats/x-pack/metricbeat/module/aws/mtest" +) + +func TestFetch(t *testing.T) { + config, info := mtest.GetConfigForTest("s3_request", "86400s") + if info != "" { + t.Skip("Skipping TestFetch: " + info) + } + + s3DailyMetricSet := mbtest.NewReportingMetricSetV2(t, config) + events, err := mbtest.ReportingFetchV2(s3DailyMetricSet) + if err != nil { + t.Skip("Skipping TestFetch: failed to make api calls. Please check $AWS_ACCESS_KEY_ID, " + + "$AWS_SECRET_ACCESS_KEY and $AWS_SESSION_TOKEN in config.yml") + } + + assert.Empty(t, err) + if !assert.NotEmpty(t, events) { + t.FailNow() + } + t.Logf("Module: %s Metricset: %s", s3DailyMetricSet.Module().Name(), s3DailyMetricSet.Name()) + + for _, event := range events { + // RootField + mtest.CheckEventField("service.name", "string", event, t) + mtest.CheckEventField("cloud.region", "string", event, t) + + // MetricSetField + mtest.CheckEventField("bucket.name", "string", event, t) + mtest.CheckEventField("requests.total", "int", event, t) + mtest.CheckEventField("requests.get", "int", event, t) + mtest.CheckEventField("requests.put", "int", event, t) + mtest.CheckEventField("requests.delete", "int", event, t) + mtest.CheckEventField("requests.head", "int", event, t) + mtest.CheckEventField("requests.post", "int", event, t) + mtest.CheckEventField("select.requests", "int", event, t) + mtest.CheckEventField("select.scanned.bytes", "float", event, t) + mtest.CheckEventField("select.returned.bytes", "float", event, t) + mtest.CheckEventField("requests.list", "int", event, t) + mtest.CheckEventField("downloaded.bytes", "float", event, t) + mtest.CheckEventField("uploaded.bytes", "float", event, t) + mtest.CheckEventField("errors.4xx", "int", event, t) + mtest.CheckEventField("errors.5xx", "int", event, t) + mtest.CheckEventField("latency.first_byte.ms", "float", event, t) + mtest.CheckEventField("latency.total_request.ms", "float", event, t) + } +} + +func TestData(t *testing.T) { + config, info := mtest.GetConfigForTest("s3_request", "86400s") + if info != "" { + t.Skip("Skipping TestData: " + info) + } + + ec2MetricSet := mbtest.NewReportingMetricSetV2(t, config) + errs := mbtest.WriteEventsReporterV2(ec2MetricSet, t, "/") + if errs != nil { + t.Fatal("write", errs) + } +} diff --git a/x-pack/metricbeat/module/aws/sqs/sqs_integration_test.go b/x-pack/metricbeat/module/aws/sqs/sqs_integration_test.go index 704242f5211..65497b7fc18 100644 --- a/x-pack/metricbeat/module/aws/sqs/sqs_integration_test.go +++ b/x-pack/metricbeat/module/aws/sqs/sqs_integration_test.go @@ -17,7 +17,7 @@ import ( ) func TestFetch(t *testing.T) { - config, info := mtest.GetConfigForTest("sqs") + config, info := mtest.GetConfigForTest("sqs", "300s") if info != "" { t.Skip("Skipping TestFetch: " + info) } @@ -52,7 +52,7 @@ func TestFetch(t *testing.T) { } func TestData(t *testing.T) { - config, info := mtest.GetConfigForTest("sqs") + config, info := mtest.GetConfigForTest("sqs", "300s") if info != "" { t.Skip("Skipping TestData: " + info) } diff --git a/x-pack/metricbeat/modules.d/aws.yml.disabled b/x-pack/metricbeat/modules.d/aws.yml.disabled index d831e5e094d..3561b58b11f 100644 --- a/x-pack/metricbeat/modules.d/aws.yml.disabled +++ b/x-pack/metricbeat/modules.d/aws.yml.disabled @@ -7,3 +7,11 @@ secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}' default_region: '${AWS_REGION:us-west-1}' +- module: aws + period: 86400s + metricsets: + - "s3_request" + access_key_id: '${AWS_ACCESS_KEY_ID:""}' + secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' + session_token: '${AWS_SESSION_TOKEN:""}' + default_region: '${AWS_REGION:us-west-1}'