[Metrics UI] Add AWS Metricsets to Inventory Models (#49983)

* Adding initial code for EC2

* Removing obsolute files; Adding EC2;

* Removing currentTimerange and replacing it with currentTime; Timerange will now be calcuated on the server

* Fixing AWS.s3 with Metrics Explorer

* Auto calculating timerange and interval based on metricset.period

* Adding S3 metricset

* Inital addition of RDS metrics

* Adding SQS and fixing a few things

* Fixing typescript error

* Adding RDS; Adjusting fields for S3; adding new formatter

* Return 60 seconds by detault

* Fixing types

* Removing i18n

* Fixing tests

* Fixing translations

* Fixes from merge

* Removing IDX from code not covered by #52354

* fixing tests

* Adding controls for crossliking; consolidating display name

* remove obsolete import

* Adding drop_last_bucket_support to TSVB models

* Changing type

* Fixing value per type

* remvoing obsolete translation

* Removing duplicate lines

* Removing icons from switcher

* Reducing boilerplate in Toolbar Items

* Changing file name

x-pack/legacy/plugins/infra/common/ecs_allowed_list.ts

@@ -45,6 +45,8 @@ export const DOCKER_ALLOWED_LIST = [
   'docker.container.labels',
 ];
 
+export const AWS_S3_ALLOWED_LIST = ['aws.s3'];
+
 export const getAllowedListForPrefix = (prefix: string) => {
   const firstPart = first(prefix.split(/\./));
   const defaultAllowedList = prefix ? [...ECS_ALLOWED_LIST, prefix] : ECS_ALLOWED_LIST;
@@ -55,6 +57,10 @@ export const getAllowedListForPrefix = (prefix: string) => {
       return [...defaultAllowedList, ...PROMETHEUS_ALLOWED_LIST];
     case 'kubernetes':
       return [...defaultAllowedList, ...K8S_ALLOWED_LIST];
+    case 'aws':
+      if (prefix === 'aws.s3_daily_storage') {
+        return [...defaultAllowedList, ...AWS_S3_ALLOWED_LIST];
+      }
     default:
       return defaultAllowedList;
   }

x-pack/legacy/plugins/infra/common/graphql/shared/schema.gql.ts

@@ -30,5 +30,9 @@ export const sharedSchema = gql`
     pod
     container
     host
+    awsEC2
+    awsS3
+    awsRDS
+    awsSQS
   }
 `;

x-pack/legacy/plugins/infra/common/graphql/types.ts

@@ -552,6 +552,10 @@ export enum InfraNodeType {
   pod = 'pod',
   container = 'container',
   host = 'host',
+  awsEC2 = 'awsEC2',
+  awsS3 = 'awsS3',
+  awsRDS = 'awsRDS',
+  awsSQS = 'awsSQS',
 }
 
 export enum InfraSnapshotMetricType {
@@ -562,6 +566,22 @@ export enum InfraSnapshotMetricType {
   tx = 'tx',
   rx = 'rx',
   logRate = 'logRate',
+  diskIOReadBytes = 'diskIOReadBytes',
+  diskIOWriteBytes = 'diskIOWriteBytes',
+  s3TotalRequests = 's3TotalRequests',
+  s3NumberOfObjects = 's3NumberOfObjects',
+  s3BucketSize = 's3BucketSize',
+  s3DownloadBytes = 's3DownloadBytes',
+  s3UploadBytes = 's3UploadBytes',
+  rdsConnections = 'rdsConnections',
+  rdsQueriesExecuted = 'rdsQueriesExecuted',
+  rdsActiveTransactions = 'rdsActiveTransactions',
+  rdsLatency = 'rdsLatency',
+  sqsMessagesVisible = 'sqsOldestMessage',
+  sqsMessagesDelayed = 'sqsMessagesDelayed',
+  sqsMessagesSent = 'sqsMessagesSent',
+  sqsMessagesEmpty = 'sqsMessagesEmpty',
+  sqsOldestMessage = 'sqsOldestMessage',
 }
 
 export enum InfraMetric {
@@ -602,6 +622,24 @@ export enum InfraMetric {
   awsNetworkPackets = 'awsNetworkPackets',
   awsDiskioBytes = 'awsDiskioBytes',
   awsDiskioOps = 'awsDiskioOps',
+  awsEC2CpuUtilization = 'awsEC2CpuUtilization',
+  awsEC2DiskIOBytes = 'awsEC2DiskIOBytes',
+  awsEC2NetworkTraffic = 'awsEC2NetworkTraffic',
+  awsS3TotalRequests = 'awsS3TotalRequests',
+  awsS3NumberOfObjects = 'awsS3NumberOfObjects',
+  awsS3BucketSize = 'awsS3BucketSize',
+  awsS3DownloadBytes = 'awsS3DownloadBytes',
+  awsS3UploadBytes = 'awsS3UploadBytes',
+  awsRDSCpuTotal = 'awsRDSCpuTotal',
+  awsRDSConnections = 'awsRDSConnections',
+  awsRDSQueriesExecuted = 'awsRDSQueriesExecuted',
+  awsRDSActiveTransactions = 'awsRDSActiveTransactions',
+  awsRDSLatency = 'awsRDSLatency',
+  awsSQSMessagesVisible = 'awsSQSMessagesVisible',
+  awsSQSMessagesDelayed = 'awsSQSMessagesDelayed',
+  awsSQSMessagesSent = 'awsSQSMessagesSent',
+  awsSQSMessagesEmpty = 'awsSQSMessagesEmpty',
+  awsSQSOldestMessage = 'awsSQSOldestMessage',
   custom = 'custom',
 }
 

x-pack/legacy/plugins/infra/common/http_api/metadata_api.ts

@@ -5,16 +5,11 @@
  */
 
 import * as rt from 'io-ts';
-
-export const InfraMetadataNodeTypeRT = rt.keyof({
-  host: null,
-  pod: null,
-  container: null,
-});
+import { ItemTypeRT } from '../../common/inventory_models/types';
 
 export const InfraMetadataRequestRT = rt.type({
   nodeId: rt.string,
-  nodeType: InfraMetadataNodeTypeRT,
+  nodeType: ItemTypeRT,
   sourceId: rt.string,
 });
 
@@ -96,5 +91,3 @@ export type InfraMetadataMachine = rt.TypeOf<typeof InfraMetadataMachineRT>;
 export type InfraMetadataHost = rt.TypeOf<typeof InfraMetadataHostRT>;
 
 export type InfraMetadataOS = rt.TypeOf<typeof InfraMetadataOSRT>;
-
-export type InfraMetadataNodeType = rt.TypeOf<typeof InfraMetadataNodeTypeRT>;

x-pack/legacy/plugins/infra/common/inventory_models/aws_ec2/index.ts

@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { i18n } from '@kbn/i18n';
+import { metrics } from './metrics';
+import { InventoryModel } from '../types';
+
+export const awsEC2: InventoryModel = {
+  id: 'awsEC2',
+  displayName: i18n.translate('xpack.infra.inventoryModels.awsEC2.displayName', {
+    defaultMessage: 'EC2 Instances',
+  }),
+  requiredModules: ['aws'],
+  crosslinkSupport: {
+    details: true,
+    logs: true,
+    apm: true,
+    uptime: true,
+  },
+  metrics,
+  fields: {
+    id: 'cloud.instance.id',
+    name: 'cloud.instance.name',
+    ip: 'aws.ec2.instance.public.ip',
+  },
+  requiredMetrics: ['awsEC2CpuUtilization', 'awsEC2NetworkTraffic', 'awsEC2DiskIOBytes'],
+};

x-pack/legacy/plugins/infra/common/inventory_models/aws_ec2/layout.tsx

@@ -0,0 +1,116 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import React from 'react';
+import { i18n } from '@kbn/i18n';
+import { LayoutPropsWithTheme } from '../../../public/pages/metrics/types';
+import { Section } from '../../../public/pages/metrics/components/section';
+import { SubSection } from '../../../public/pages/metrics/components/sub_section';
+import { ChartSectionVis } from '../../../public/pages/metrics/components/chart_section_vis';
+import { withTheme } from '../../../../../common/eui_styled_components';
+
+export const Layout = withTheme(({ metrics, theme }: LayoutPropsWithTheme) => (
+  <React.Fragment>
+    <Section
+      navLabel="AWS EC2"
+      sectionLabel={i18n.translate(
+        'xpack.infra.metricDetailPage.ec2MetricsLayout.overviewSection.sectionLabel',
+        {
+          defaultMessage: 'Aws EC2 Overview',
+        }
+      )}
+      metrics={metrics}
+    >
+      <SubSection
+        id="awsEC2CpuUtilization"
+        label={i18n.translate(
+          'xpack.infra.metricDetailPage.ec2MetricsLayout.cpuUsageSection.sectionLabel',
+          {
+            defaultMessage: 'CPU Usage',
+          }
+        )}
+      >
+        <ChartSectionVis
+          stacked={true}
+          type="area"
+          formatter="percent"
+          seriesOverrides={{
+            total: { color: theme.eui.euiColorVis1 },
+          }}
+        />
+      </SubSection>
+      <SubSection
+        id="awsEC2NetworkTraffic"
+        label={i18n.translate(
+          'xpack.infra.metricDetailPage.ec2MetricsLayout.networkTrafficSection.sectionLabel',
+          {
+            defaultMessage: 'Network Traffic',
+          }
+        )}
+      >
+        <ChartSectionVis
+          formatter="bits"
+          formatterTemplate="{{value}}/s"
+          type="area"
+          seriesOverrides={{
+            rx: {
+              color: theme.eui.euiColorVis1,
+              name: i18n.translate(
+                'xpack.infra.metricDetailPage.hostMetricsLayout.networkTrafficSection.networkRxRateSeriesLabel',
+                {
+                  defaultMessage: 'in',
+                }
+              ),
+            },
+            tx: {
+              color: theme.eui.euiColorVis2,
+              name: i18n.translate(
+                'xpack.infra.metricDetailPage.hostMetricsLayout.networkTrafficSection.networkTxRateSeriesLabel',
+                {
+                  defaultMessage: 'out',
+                }
+              ),
+            },
+          }}
+        />
+      </SubSection>
+      <SubSection
+        id="awsEC2DiskIOBytes"
+        label={i18n.translate(
+          'xpack.infra.metricDetailPage.ec2MetricsLayout.diskIOBytesSection.sectionLabel',
+          {
+            defaultMessage: 'Disk IO (Bytes)',
+          }
+        )}
+      >
+        <ChartSectionVis
+          formatter="bytes"
+          formatterTemplate="{{value}}/s"
+          type="area"
+          seriesOverrides={{
+            write: {
+              color: theme.eui.euiColorVis2,
+              name: i18n.translate(
+                'xpack.infra.metricDetailPage.ec2MetricsLayout.diskIOBytesSection.writeLabel',
+                {
+                  defaultMessage: 'writes',
+                }
+              ),
+            },
+            read: {
+              color: theme.eui.euiColorVis1,
+              name: i18n.translate(
+                'xpack.infra.metricDetailPage.ec2MetricsLayout.diskIOBytesSection.readLabel',
+                {
+                  defaultMessage: 'reads',
+                }
+              ),
+            },
+          }}
+        />
+      </SubSection>
+    </Section>
+  </React.Fragment>
+));

x-pack/legacy/plugins/infra/common/inventory_models/aws_ec2/metrics/index.ts

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { cpu } from './snapshot/cpu';
+import { rx } from './snapshot/rx';
+import { tx } from './snapshot/tx';
+import { diskIOReadBytes } from './snapshot/disk_io_read_bytes';
+import { diskIOWriteBytes } from './snapshot/disk_io_write_bytes';
+
+import { awsEC2CpuUtilization } from './tsvb/aws_ec2_cpu_utilization';
+import { awsEC2NetworkTraffic } from './tsvb/aws_ec2_network_traffic';
+import { awsEC2DiskIOBytes } from './tsvb/aws_ec2_diskio_bytes';
+
+import { InventoryMetrics } from '../../types';
+
+export const metrics: InventoryMetrics = {
+  tsvb: {
+    awsEC2CpuUtilization,
+    awsEC2NetworkTraffic,
+    awsEC2DiskIOBytes,
+  },
+  snapshot: { cpu, rx, tx, diskIOReadBytes, diskIOWriteBytes },
+  defaultSnapshot: 'cpu',
+  defaultTimeRangeInSeconds: 14400, // 4 hours
+};

x-pack/legacy/plugins/infra/common/inventory_models/aws_ec2/metrics/snapshot/cpu.ts

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { SnapshotModel } from '../../../types';
+
+export const cpu: SnapshotModel = {
+  cpu_avg: {
+    avg: {
+      field: 'aws.ec2.cpu.total.pct',
+    },
+  },
+  cpu: {
+    bucket_script: {
+      buckets_path: {
+        cpu: 'cpu_avg',
+      },
+      script: {
+        source: 'params.cpu / 100',
+        lang: 'painless',
+      },
+      gap_policy: 'skip',
+    },
+  },
+};

x-pack/legacy/plugins/infra/common/inventory_models/aws_ec2/metrics/snapshot/disk_io_read_bytes.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { SnapshotModel } from '../../../types';
+
+export const diskIOReadBytes: SnapshotModel = {
+  diskIOReadBytes: {
+    avg: {
+      field: 'aws.ec2.diskio.read.bytes_per_sec',
+    },
+  },
+};

x-pack/legacy/plugins/infra/common/inventory_models/aws_ec2/metrics/snapshot/disk_io_write_bytes.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { SnapshotModel } from '../../../types';
+
+export const diskIOWriteBytes: SnapshotModel = {
+  diskIOWriteBytes: {
+    avg: {
+      field: 'aws.ec2.diskio.write.bytes_per_sec',
+    },
+  },
+};

x-pack/legacy/plugins/infra/common/inventory_models/aws_ec2/metrics/snapshot/rx.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { SnapshotModel } from '../../../types';
+
+export const rx: SnapshotModel = {
+  rx: {
+    avg: {
+      field: 'aws.ec2.network.in.bytes_per_sec',
+    },
+  },
+};

x-pack/legacy/plugins/infra/common/inventory_models/aws_ec2/metrics/snapshot/tx.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { SnapshotModel } from '../../../types';
+
+export const tx: SnapshotModel = {
+  tx: {
+    avg: {
+      field: 'aws.ec2.network.in.bytes_per_sec',
+    },
+  },
+};