[TSVB] Multi-field group by (#126015)

* fieldSelect

* activate multifield support for table

* update table>pivot request_processor

* fix some tests

* apply some changes

* fix JEST

* push initial logic for series request_processor

* fix some broken cases for Table tab

* update convert_series_to_datatable / convert_series_to_vars

* add some logic

* fix table/terms

* do some logic

* fix some issues

* push some logic

* navigation to Lens

* fix CI

* add excludedFieldFormatsIds param into excludedFieldFormatsIds

* fix ci

* fix translations

* fix some comments

* fix series_agg label

* update labels in lens

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

docs/user/dashboard/tsvb.asciidoc

@@ -276,20 +276,4 @@ For other types of month over month calculations, use <<timelion, *Timelion*>> o
 
 Calculating the duration between the start and end of an event is unsupported in *TSVB* because *TSVB* requires correlation between different time periods. 
 *TSVB* requires that the duration is pre-calculated.
-====
-
-[discrete]
-[group-on-multiple-fields]
-.*How do I group on multiple fields?*
-[%collapsible]
-====
-
-To group with multiple fields, create runtime fields in the {data-source} you are visualizing.
-
-. Create a runtime field. Refer to <<managing-data-views, Manage data views>> for more information.
-+
-[role="screenshot"]
-image::images/tsvb_group_by_multiple_fields.png[Group by multiple fields]
-
-. Create a *TSVB* visualization and group by this field.
 ====

src/plugins/vis_types/timeseries/common/calculate_label.test.ts

@@ -9,6 +9,7 @@
 import { calculateLabel } from './calculate_label';
 import type { Metric } from './types';
 import { SanitizedFieldType } from './types';
+import { KBN_FIELD_TYPES } from '../../../data/common';
 
 describe('calculateLabel(metric, metrics)', () => {
   test('returns the metric.alias if set', () => {
@@ -90,7 +91,7 @@ describe('calculateLabel(metric, metrics)', () => {
       { id: 1, type: 'max', field: 'network.out.bytes', alias: 'Outbound Traffic' },
       metric,
     ] as unknown as Metric[];
-    const fields: SanitizedFieldType[] = [{ name: '2', label: '2', type: 'field' }];
+    const fields: SanitizedFieldType[] = [{ name: '2', label: '2', type: KBN_FIELD_TYPES.DATE }];
 
     expect(() => calculateLabel(metric, metrics, fields)).toThrowError('Field "3" not found');
   });
@@ -101,7 +102,7 @@ describe('calculateLabel(metric, metrics)', () => {
       { id: 1, type: 'max', field: 'network.out.bytes', alias: 'Outbound Traffic' },
       metric,
     ] as unknown as Metric[];
-    const fields: SanitizedFieldType[] = [{ name: '2', label: '2', type: 'field' }];
+    const fields: SanitizedFieldType[] = [{ name: '2', label: '2', type: KBN_FIELD_TYPES.DATE }];
 
     expect(calculateLabel(metric, metrics, fields, false)).toBe('Max of 3');
   });

src/plugins/vis_types/timeseries/common/fields_utils.test.ts

@@ -6,8 +6,16 @@
  * Side Public License, v 1.
  */
 
-import { toSanitizedFieldType } from './fields_utils';
-import type { FieldSpec } from '../../../data/common';
+import {
+  getFieldsForTerms,
+  toSanitizedFieldType,
+  getMultiFieldLabel,
+  createCachedFieldValueFormatter,
+} from './fields_utils';
+import { FieldSpec, KBN_FIELD_TYPES } from '../../../data/common';
+import { DataView } from '../../../data_views/common';
+import { stubLogstashDataView } from '../../../data/common/stubs';
+import { FieldFormatsRegistry, StringFormat } from '../../../field_formats/common';
 
 describe('fields_utils', () => {
   describe('toSanitizedFieldType', () => {
@@ -59,4 +67,92 @@ describe('fields_utils', () => {
       expect(toSanitizedFieldType(fields)).toMatchInlineSnapshot(`Array []`);
     });
   });
+
+  describe('getFieldsForTerms', () => {
+    test('should return fields as array', () => {
+      expect(getFieldsForTerms('field')).toEqual(['field']);
+      expect(getFieldsForTerms(['field', 'field1'])).toEqual(['field', 'field1']);
+    });
+
+    test('should exclude empty values', () => {
+      expect(getFieldsForTerms([null, ''])).toEqual([]);
+    });
+
+    test('should return empty array in case of undefined field', () => {
+      expect(getFieldsForTerms(undefined)).toEqual([]);
+    });
+  });
+
+  describe('getMultiFieldLabel', () => {
+    test('should return label for single field', () => {
+      expect(
+        getMultiFieldLabel(
+          ['field'],
+          [{ name: 'field', label: 'Label', type: KBN_FIELD_TYPES.DATE }]
+        )
+      ).toBe('Label');
+    });
+
+    test('should return label for multi fields', () => {
+      expect(
+        getMultiFieldLabel(
+          ['field', 'field1'],
+          [
+            { name: 'field', label: 'Label', type: KBN_FIELD_TYPES.DATE },
+            { name: 'field2', label: 'Label1', type: KBN_FIELD_TYPES.DATE },
+          ]
+        )
+      ).toBe('Label + 1 other');
+    });
+
+    test('should return label for multi fields (2 others)', () => {
+      expect(
+        getMultiFieldLabel(
+          ['field', 'field1', 'field2'],
+          [
+            { name: 'field', label: 'Label', type: KBN_FIELD_TYPES.DATE },
+            { name: 'field1', label: 'Label1', type: KBN_FIELD_TYPES.DATE },
+            { name: 'field3', label: 'Label2', type: KBN_FIELD_TYPES.DATE },
+          ]
+        )
+      ).toBe('Label + 2 others');
+    });
+  });
+
+  describe('createCachedFieldValueFormatter', () => {
+    let dataView: DataView;
+
+    beforeEach(() => {
+      dataView = stubLogstashDataView;
+    });
+
+    test('should use data view formatters', () => {
+      const getFormatterForFieldSpy = jest.spyOn(dataView, 'getFormatterForField');
+
+      const cache = createCachedFieldValueFormatter(dataView);
+
+      cache('bytes', '10001');
+      cache('bytes', '20002');
+
+      expect(getFormatterForFieldSpy).toHaveBeenCalledTimes(1);
+    });
+
+    test('should use default formatters in case of Data view not defined', () => {
+      const fieldFormatServiceMock = {
+        getDefaultInstance: jest.fn().mockReturnValue(new StringFormat()),
+      } as unknown as FieldFormatsRegistry;
+
+      const cache = createCachedFieldValueFormatter(
+        null,
+        [{ name: 'field', label: 'Label', type: KBN_FIELD_TYPES.STRING }],
+        fieldFormatServiceMock
+      );
+
+      cache('field', '10001');
+      cache('field', '20002');
+
+      expect(fieldFormatServiceMock.getDefaultInstance).toHaveBeenCalledTimes(1);
+      expect(fieldFormatServiceMock.getDefaultInstance).toHaveBeenCalledWith('string');
+    });
+  });
 });

src/plugins/vis_types/timeseries/common/fields_utils.ts

@@ -5,11 +5,12 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
+import { i18n } from '@kbn/i18n';
 
-import { FieldSpec } from '../../../data/common';
-import { isNestedField } from '../../../data/common';
-import { FetchedIndexPattern, SanitizedFieldType } from './types';
+import { isNestedField, FieldSpec, DataView } from '../../../data/common';
 import { FieldNotFoundError } from './errors';
+import type { FetchedIndexPattern, SanitizedFieldType } from './types';
+import { FieldFormat, FieldFormatsRegistry, FIELD_FORMAT_IDS } from '../../../field_formats/common';
 
 export const extractFieldLabel = (
   fields: SanitizedFieldType[],
@@ -49,3 +50,63 @@ export const toSanitizedFieldType = (fields: FieldSpec[]) =>
           type: field.type,
         } as SanitizedFieldType)
     );
+
+export const getFieldsForTerms = (fields: string | Array<string | null> | undefined): string[] => {
+  return fields ? ([fields].flat().filter(Boolean) as string[]) : [];
+};
+
+export const getMultiFieldLabel = (fieldForTerms: string[], fields?: SanitizedFieldType[]) => {
+  const firstFieldLabel = fields ? extractFieldLabel(fields, fieldForTerms[0]) : fieldForTerms[0];
+
+  if (fieldForTerms.length > 1) {
+    return i18n.translate('visTypeTimeseries.fieldUtils.multiFieldLabel', {
+      defaultMessage: '{firstFieldLabel} + {count} {count, plural, one {other} other {others}}',
+      values: {
+        firstFieldLabel,
+        count: fieldForTerms.length - 1,
+      },
+    });
+  }
+  return firstFieldLabel ?? '';
+};
+
+export const createCachedFieldValueFormatter = (
+  dataView?: DataView | null,
+  fields?: SanitizedFieldType[],
+  fieldFormatService?: FieldFormatsRegistry,
+  excludedFieldFormatsIds: FIELD_FORMAT_IDS[] = []
+) => {
+  const cache = new Map<string, FieldFormat>();
+
+  return (fieldName: string, value: string, contentType: 'text' | 'html' = 'text') => {
+    const cachedFormatter = cache.get(fieldName);
+    if (cachedFormatter) {
+      return cachedFormatter.convert(value, contentType);
+    }
+
+    if (dataView && !excludedFieldFormatsIds.includes(dataView.fieldFormatMap?.[fieldName]?.id)) {
+      const field = dataView.fields.getByName(fieldName);
+      if (field) {
+        const formatter = dataView.getFormatterForField(field);
+
+        if (formatter) {
+          cache.set(fieldName, formatter);
+          return formatter.convert(value, contentType);
+        }
+      }
+    } else if (fieldFormatService && fields) {
+      const f = fields.find((item) => item.name === fieldName);
+
+      if (f) {
+        const formatter = fieldFormatService.getDefaultInstance(f.type);
+
+        if (formatter) {
+          cache.set(fieldName, formatter);
+          return formatter.convert(value, contentType);
+        }
+      }
+    }
+  };
+};
+
+export const MULTI_FIELD_VALUES_SEPARATOR = ' › ';

src/plugins/vis_types/timeseries/common/types/index.ts

@@ -7,7 +7,7 @@
  */
 
 import { Filter } from '@kbn/es-query';
-import { IndexPattern, Query } from '../../../../data/common';
+import { IndexPattern, KBN_FIELD_TYPES, Query } from '../../../../data/common';
 import { Panel } from './panel_model';
 
 export type { Metric, Series, Panel, MetricType } from './panel_model';
@@ -28,7 +28,7 @@ export interface FetchedIndexPattern {
 
 export interface SanitizedFieldType {
   name: string;
-  type: string;
+  type: KBN_FIELD_TYPES;
   label?: string;
 }
 

src/plugins/vis_types/timeseries/common/types/panel_model.ts

@@ -6,10 +6,15 @@
  * Side Public License, v 1.
  */
 
-import { METRIC_TYPES, Query } from '../../../../data/common';
+import { Query, METRIC_TYPES, KBN_FIELD_TYPES } from '../../../../data/common';
 import { PANEL_TYPES, TOOLTIP_MODES, TSVB_METRIC_TYPES } from '../enums';
-import { IndexPatternValue, Annotation } from './index';
-import { ColorRules, BackgroundColorRules, BarColorRules, GaugeColorRules } from './color_rules';
+import type { IndexPatternValue, Annotation } from './index';
+import type {
+  ColorRules,
+  BackgroundColorRules,
+  BarColorRules,
+  GaugeColorRules,
+} from './color_rules';
 
 interface MetricVariable {
   field?: string;
@@ -109,7 +114,7 @@ export interface Series {
   steps: number;
   terms_direction?: string;
   terms_exclude?: string;
-  terms_field?: string;
+  terms_field?: string | Array<string | null>;
   terms_include?: string;
   terms_order_by?: string;
   terms_size?: string;
@@ -155,10 +160,10 @@ export interface Panel {
   markdown_scrollbars: number;
   markdown_vertical_align?: string;
   max_bars: number;
-  pivot_id?: string;
+  pivot_id?: string | Array<string | null>;
   pivot_label?: string;
   pivot_rows?: string;
-  pivot_type?: string;
+  pivot_type?: KBN_FIELD_TYPES | Array<KBN_FIELD_TYPES | null>;
   series: Series[];
   show_grid: number;
   show_legend: number;

src/plugins/vis_types/timeseries/common/types/vis_data.ts

@@ -46,7 +46,7 @@ export interface PanelSeries {
 
 export interface PanelData {
   id: string;
-  label: string;
+  label: string | string[];
   labelFormatted?: string;
   data: PanelDataArray[];
   seriesId: string;