[Infra UI] Limit Metric Explorer fields (#43322) (#46051)

* [Infra UI] Limit Metric Explorer fields

- Closes #41090
- Closes #39613
- Adds allowed list for ECS, Promehteus, Kubernetes, and Docker fields
- Filters "graph pre" fields by selected metrics
- Only displays allowed fields for metric selection, graph per, and
Kquery bar

* Fixing test

* Changing all caps to camel case

* Fixing logic to be more clear and handle null use cases

* Changing to singular

x-pack/legacy/plugins/infra/common/ecs_allowed_list.test.ts

@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import {
+  getAllowedListForPrefix,
+  ECS_ALLOWED_LIST,
+  K8S_ALLOWED_LIST,
+  PROMETHEUS_ALLOWED_LIST,
+  DOCKER_ALLOWED_LIST,
+} from './ecs_allowed_list';
+describe('getAllowedListForPrefix()', () => {
+  test('kubernetes', () => {
+    expect(getAllowedListForPrefix('kubernetes.pod')).toEqual([
+      ...ECS_ALLOWED_LIST,
+      'kubernetes.pod',
+      ...K8S_ALLOWED_LIST,
+    ]);
+  });
+  test('docker', () => {
+    expect(getAllowedListForPrefix('docker.container')).toEqual([
+      ...ECS_ALLOWED_LIST,
+      'docker.container',
+      ...DOCKER_ALLOWED_LIST,
+    ]);
+  });
+  test('prometheus', () => {
+    expect(getAllowedListForPrefix('prometheus.metrics')).toEqual([
+      ...ECS_ALLOWED_LIST,
+      'prometheus.metrics',
+      ...PROMETHEUS_ALLOWED_LIST,
+    ]);
+  });
+  test('anything.else', () => {
+    expect(getAllowedListForPrefix('anything.else')).toEqual([
+      ...ECS_ALLOWED_LIST,
+      'anything.else',
+    ]);
+  });
+});

x-pack/legacy/plugins/infra/common/ecs_allowed_list.ts

@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import { first } from 'lodash';
+
+export const ECS_ALLOWED_LIST = [
+  'host',
+  'cloud',
+  'event',
+  'agent',
+  'fields',
+  'service',
+  'ecs',
+  'metricset',
+  'tags',
+  'message',
+  'labels',
+  '@timestamp',
+  'source',
+  'container',
+];
+
+export const K8S_ALLOWED_LIST = [
+  'kubernetes.pod.name',
+  'kubernetes.pod.uid',
+  'kubernetes.namespace',
+  'kubernetes.node.name',
+  'kubernetes.labels',
+  'kubernetes.annotations',
+  'kubernetes.replicaset.name',
+  'kubernetes.deployment.name',
+  'kubernetes.statefulset.name',
+  'kubernetes.container.name',
+  'kubernetes.container.image',
+];
+
+export const PROMETHEUS_ALLOWED_LIST = ['prometheus.labels', 'prometheus.metrics'];
+
+export const DOCKER_ALLOWED_LIST = [
+  'docker.container.id',
+  'docker.container.image',
+  'docker.container.name',
+  'docker.container.labels',
+];
+
+export const getAllowedListForPrefix = (prefix: string) => {
+  const firstPart = first(prefix.split(/\./));
+  const defaultAllowedList = prefix ? [...ECS_ALLOWED_LIST, prefix] : ECS_ALLOWED_LIST;
+  switch (firstPart) {
+    case 'docker':
+      return [...defaultAllowedList, ...DOCKER_ALLOWED_LIST];
+    case 'prometheus':
+      return [...defaultAllowedList, ...PROMETHEUS_ALLOWED_LIST];
+    case 'kubernetes':
+      return [...defaultAllowedList, ...K8S_ALLOWED_LIST];
+    default:
+      return defaultAllowedList;
+  }
+};

x-pack/legacy/plugins/infra/common/graphql/types.ts

@@ -131,6 +131,8 @@ export interface InfraIndexField {
   searchable: boolean;
   /** Whether the field's values can be aggregated */
   aggregatable: boolean;
+  /** Whether the field should be displayed based on event.module and a ECS allowed list */
+  displayable: boolean;
 }
 /** A consecutive sequence of log entries */
 export interface InfraLogEntryInterval {
@@ -1138,6 +1140,8 @@ export namespace SourceStatusFields {
     searchable: boolean;
 
     aggregatable: boolean;
+
+    displayable: boolean;
   };
 }
 

x-pack/legacy/plugins/infra/public/components/metrics_explorer/group_by.tsx

@@ -9,6 +9,7 @@ import { InjectedIntl, injectI18n } from '@kbn/i18n/react';
 import React, { useCallback } from 'react';
 import { FieldType } from 'ui/index_patterns';
 import { MetricsExplorerOptions } from '../../containers/metrics_explorer/use_metrics_explorer_options';
+import { isDisplayable } from '../../utils/is_displayable';
 
 interface Props {
   intl: InjectedIntl;
@@ -25,6 +26,19 @@ export const MetricsExplorerGroupBy = injectI18n(({ intl, options, onChange, fie
     },
     [onChange]
   );
+
+  const metricPrefixes = options.metrics
+    .map(
+      metric =>
+        (metric.field &&
+          metric.field
+            .split(/\./)
+            .slice(0, 2)
+            .join('.')) ||
+        null
+    )
+    .filter(metric => metric) as string[];
+
   return (
     <EuiComboBox
       placeholder={intl.formatMessage({
@@ -35,7 +49,7 @@ export const MetricsExplorerGroupBy = injectI18n(({ intl, options, onChange, fie
       singleSelection={true}
       selectedOptions={(options.groupBy && [{ label: options.groupBy }]) || []}
       options={fields
-        .filter(f => f.aggregatable && f.type === 'string')
+        .filter(f => isDisplayable(f, metricPrefixes) && f.aggregatable && f.type === 'string')
         .map(f => ({ label: f.name }))}
       onChange={handleChange}
       isClearable={true}

x-pack/legacy/plugins/infra/public/components/metrics_explorer/kuery_bar.tsx

@@ -10,6 +10,7 @@ import React, { useEffect, useState } from 'react';
 import { StaticIndexPattern } from 'ui/index_patterns';
 import { WithKueryAutocompletion } from '../../containers/with_kuery_autocompletion';
 import { AutocompleteField } from '../autocomplete_field';
+import { isDisplayable } from '../../utils/is_displayable';
 
 interface Props {
   intl: InjectedIntl;
@@ -44,8 +45,13 @@ export const MetricsExplorerKueryBar = injectI18n(
       setDraftQuery(query);
     };
 
+    const filteredDerivedIndexPattern = {
+      ...derivedIndexPattern,
+      fields: derivedIndexPattern.fields.filter(field => isDisplayable(field)),
+    };
+
     return (
-      <WithKueryAutocompletion indexPattern={derivedIndexPattern}>
+      <WithKueryAutocompletion indexPattern={filteredDerivedIndexPattern}>
         {({ isLoadingSuggestions, loadSuggestions, suggestions }) => (
           <AutocompleteField
             isLoadingSuggestions={isLoadingSuggestions}

x-pack/legacy/plugins/infra/public/components/metrics_explorer/metrics.tsx

@@ -14,6 +14,7 @@ import {
   MetricsExplorerAggregation,
 } from '../../../server/routes/metrics_explorer/types';
 import { MetricsExplorerOptions } from '../../containers/metrics_explorer/use_metrics_explorer_options';
+import { isDisplayable } from '../../utils/is_displayable';
 
 interface Props {
   intl: InjectedIntl;
@@ -62,7 +63,9 @@ export const MetricsExplorerMetrics = injectI18n(
       [options, onChange]
     );
 
-    const comboOptions = fields.map(field => ({ label: field.name, value: field.name }));
+    const comboOptions = fields
+      .filter(field => isDisplayable(field))
+      .map(field => ({ label: field.name, value: field.name }));
     const selectedOptions = options.metrics
       .filter(m => m.aggregation !== MetricsExplorerAggregation.count)
       .map(metric => ({

x-pack/legacy/plugins/infra/public/containers/source/source_fields_fragment.gql_query.ts

@@ -48,6 +48,7 @@ export const sourceStatusFieldsFragment = gql`
       type
       searchable
       aggregatable
+      displayable
     }
     logIndicesExist
     metricIndicesExist

x-pack/legacy/plugins/infra/public/graphql/introspection.json

@@ -1125,6 +1125,18 @@
             },
             "isDeprecated": false,
             "deprecationReason": null
+          },
+          {
+            "name": "displayable",
+            "description": "Whether the field should be displayed based on event.module and a ECS allowed list",
+            "args": [],
+            "type": {
+              "kind": "NON_NULL",
+              "name": null,
+              "ofType": { "kind": "SCALAR", "name": "Boolean", "ofType": null }
+            },
+            "isDeprecated": false,
+            "deprecationReason": null
           }
         ],
         "inputFields": null,

x-pack/legacy/plugins/infra/public/graphql/types.ts

@@ -131,6 +131,8 @@ export interface InfraIndexField {
   searchable: boolean;
   /** Whether the field's values can be aggregated */
   aggregatable: boolean;
+  /** Whether the field should be displayed based on event.module and a ECS allowed list */
+  displayable: boolean;
 }
 /** A consecutive sequence of log entries */
 export interface InfraLogEntryInterval {
@@ -1138,6 +1140,8 @@ export namespace SourceStatusFields {
     searchable: boolean;
 
     aggregatable: boolean;
+
+    displayable: boolean;
   };
 }
 

x-pack/legacy/plugins/infra/public/utils/is_displayable.test.ts

@@ -0,0 +1,65 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import { isDisplayable } from './is_displayable';
+
+describe('isDisplayable()', () => {
+  test('field that is not displayable', () => {
+    const field = {
+      name: 'some.field',
+      type: 'number',
+      displayable: false,
+    };
+    expect(isDisplayable(field)).toBe(false);
+  });
+  test('field that is displayable', () => {
+    const field = {
+      name: 'some.field',
+      type: 'number',
+      displayable: true,
+    };
+    expect(isDisplayable(field)).toBe(true);
+  });
+  test('field that an ecs field', () => {
+    const field = {
+      name: '@timestamp',
+      type: 'date',
+      displayable: true,
+    };
+    expect(isDisplayable(field)).toBe(true);
+  });
+  test('field that matches same prefix', () => {
+    const field = {
+      name: 'system.network.name',
+      type: 'string',
+      displayable: true,
+    };
+    expect(isDisplayable(field, ['system.network'])).toBe(true);
+  });
+  test('field that does not matches same prefix', () => {
+    const field = {
+      name: 'system.load.1',
+      type: 'number',
+      displayable: true,
+    };
+    expect(isDisplayable(field, ['system.network'])).toBe(false);
+  });
+  test('field that is an K8s allowed field but does not match prefix', () => {
+    const field = {
+      name: 'kubernetes.namespace',
+      type: 'string',
+      displayable: true,
+    };
+    expect(isDisplayable(field, ['kubernetes.pod'])).toBe(true);
+  });
+  test('field that is a Prometheus allowed field but does not match prefix', () => {
+    const field = {
+      name: 'prometheus.labels.foo.bar',
+      type: 'string',
+      displayable: true,
+    };
+    expect(isDisplayable(field, ['prometheus.metrics'])).toBe(true);
+  });
+});

x-pack/legacy/plugins/infra/public/utils/is_displayable.ts

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { FieldType } from 'ui/index_patterns';
+import { startsWith, uniq } from 'lodash';
+import { getAllowedListForPrefix } from '../../common/ecs_allowed_list';
+
+interface DisplayableFieldType extends FieldType {
+  displayable?: boolean;
+}
+
+const fieldStartsWith = (field: DisplayableFieldType) => (name: string) =>
+  startsWith(field.name, name);
+
+export const isDisplayable = (field: DisplayableFieldType, additionalPrefixes: string[] = []) => {
+  // We need to start with at least one prefix, even if it's empty
+  const prefixes = additionalPrefixes && additionalPrefixes.length ? additionalPrefixes : [''];
+  // Create a set of allowed list based on the prefixes
+  const allowedList = prefixes.reduce(
+    (acc, prefix) => {
+      return uniq([...acc, ...getAllowedListForPrefix(prefix)]);
+    },
+    [] as string[]
+  );
+  // If the field is displayable and part of the allowed list or covered by the prefix
+  return (
+    (field.displayable && prefixes.some(fieldStartsWith(field))) ||
+    allowedList.some(fieldStartsWith(field))
+  );
+};

x-pack/legacy/plugins/infra/server/graphql/source_status/schema.gql.ts

@@ -17,6 +17,8 @@ export const sourceStatusSchema = gql`
     searchable: Boolean!
     "Whether the field's values can be aggregated"
     aggregatable: Boolean!
+    "Whether the field should be displayed based on event.module and a ECS allowed list"
+    displayable: Boolean!
   }
 
   extend type InfraSourceStatus {

x-pack/legacy/plugins/infra/server/graphql/types.ts

@@ -159,6 +159,8 @@ export interface InfraIndexField {
   searchable: boolean;
   /** Whether the field's values can be aggregated */
   aggregatable: boolean;
+  /** Whether the field should be displayed based on event.module and a ECS allowed list */
+  displayable: boolean;
 }
 /** A consecutive sequence of log entries */
 export interface InfraLogEntryInterval {
@@ -1123,6 +1125,8 @@ export namespace InfraIndexFieldResolvers {
     searchable?: SearchableResolver<boolean, TypeParent, Context>;
     /** Whether the field's values can be aggregated */
     aggregatable?: AggregatableResolver<boolean, TypeParent, Context>;
+    /** Whether the field should be displayed based on event.module and a ECS allowed list */
+    displayable?: DisplayableResolver<boolean, TypeParent, Context>;
   }
 
   export type NameResolver<R = string, Parent = InfraIndexField, Context = InfraContext> = Resolver<
@@ -1145,6 +1149,11 @@ export namespace InfraIndexFieldResolvers {
     Parent = InfraIndexField,
     Context = InfraContext
   > = Resolver<R, Parent, Context>;
+  export type DisplayableResolver<
+    R = boolean,
+    Parent = InfraIndexField,
+    Context = InfraContext
+  > = Resolver<R, Parent, Context>;
 }
 /** A consecutive sequence of log entries */
 export namespace InfraLogEntryIntervalResolvers {

x-pack/legacy/plugins/infra/server/lib/adapters/fields/adapter_types.ts

@@ -15,4 +15,5 @@ export interface IndexFieldDescriptor {
   type: string;
   searchable: boolean;
   aggregatable: boolean;
+  displayable: boolean;
 }