[ES|QL][UnifiedFieldList] Support field stats for ES|QL query (#178433)

- Closes #174984

## Summary

This PR adds field stats for ES|QL mode in Discover. It will show "Top
values" for `keyword`, `ip`, `boolean`, `number`, `version` fields and
"Examples" for `text` and geo fields.

Also this PR extends text based column's meta with `esType` to make it
easier to differentiate between `keyword` and `text` columns when kibana
type for both is `string`. This change also has a UI improvement: `k`
token will be shown next to keyword fields in data grid column header
and in doc viewer instead of `t` token. And it will be possible to
filter by Keyword/Text types in the fields sidebar for ES|QL mode.

<img width="400" alt="Screenshot 2024-03-13 at 18 29 47"
src="https://github.com/elastic/kibana/assets/1415710/d88e85c3-329f-44fc-af54-3493dd9896d0">
<img width="400" alt="Screenshot 2024-03-13 at 18 30 01"
src="https://github.com/elastic/kibana/assets/1415710/b0808860-28db-472b-a0e3-37fc857bb40c">
<img width="400" alt="Screenshot 2024-03-13 at 18 30 19"
src="https://github.com/elastic/kibana/assets/1415710/b571199c-5956-472c-a0f7-c1b3d9bfe473">
<img width="400" alt="Screenshot 2024-03-13 at 18 30 13"
src="https://github.com/elastic/kibana/assets/1415710/a8e46b97-da38-4cbb-8980-fff214e09818">
<img width="400" alt="Screenshot 2024-03-13 at 18 30 06"
src="https://github.com/elastic/kibana/assets/1415710/8d1495ea-36d5-4203-9e86-90e7ea22cfe9">
<img width="400" alt="Screenshot 2024-03-13 at 18 31 50"
src="https://github.com/elastic/kibana/assets/1415710/b0660c57-aea8-4570-a72b-22df4c15dcc8">
<img width="400" alt="Screenshot 2024-03-13 at 18 30 50"
src="https://github.com/elastic/kibana/assets/1415710/9f81f7a8-7cba-44db-bbef-f1f1cb8e1093">

25x flaky test
https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/5470

### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [x] This was checked for [cross-browser
compatibility](https://www.elastic.co/support/matrix#matrix_browsers)

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

api_docs/kbn_unified_data_table.devdocs.json

@@ -2581,4 +2581,4 @@
       }
     ]
   }
-}
+}

packages/kbn-esql-utils/index.ts

@@ -14,5 +14,6 @@ export {
   removeDropCommandsFromESQLQuery,
   getIndexForESQLQuery,
   getInitialESQLQuery,
+  getESQLWithSafeLimit,
   TextBasedLanguages,
 } from './src';

packages/kbn-esql-utils/src/index.ts

@@ -9,6 +9,7 @@
 export { TextBasedLanguages } from './types';
 export { getESQLAdHocDataview, getIndexForESQLQuery } from './utils/get_esql_adhoc_dataview';
 export { getInitialESQLQuery } from './utils/get_initial_esql_query';
+export { getESQLWithSafeLimit } from './utils/get_esql_with_safe_limit';
 export {
   getIndexPatternFromSQLQuery,
   getIndexPatternFromESQLQuery,

packages/kbn-esql-utils/src/utils/get_esql_with_safe_limit.test.ts

@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { getESQLWithSafeLimit } from './get_esql_with_safe_limit';
+
+const LIMIT = 10000;
+
+describe('getESQLWithSafeLimit()', () => {
+  it('should not add the limit', () => {
+    expect(getESQLWithSafeLimit('show info', LIMIT)).toBe('show info');
+    expect(getESQLWithSafeLimit('row t = 5', LIMIT)).toBe('row t = 5');
+  });
+
+  it('should add the limit', () => {
+    expect(getESQLWithSafeLimit(' from logs', LIMIT)).toBe('from logs | LIMIT 10000');
+    expect(getESQLWithSafeLimit('FROM logs* | LIMIT 5', LIMIT)).toBe(
+      'FROM logs* | LIMIT 10000| LIMIT 5'
+    );
+    expect(getESQLWithSafeLimit('FROM logs* | SORT @timestamp | LIMIT 5', LIMIT)).toBe(
+      'FROM logs* |SORT @timestamp | LIMIT 10000| LIMIT 5'
+    );
+    expect(getESQLWithSafeLimit('from logs* | STATS MIN(a) BY b', LIMIT)).toBe(
+      'from logs* | LIMIT 10000| STATS MIN(a) BY b'
+    );
+  });
+});

packages/kbn-esql-utils/src/utils/get_esql_with_safe_limit.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export function getESQLWithSafeLimit(esql: string, limit: number): string {
+  if (!esql.trim().toLowerCase().startsWith('from')) {
+    return esql;
+  }
+  const parts = esql.split('|');
+
+  if (!parts.length) {
+    return esql;
+  }
+
+  const fromCommandIndex = 0;
+  const sortCommandIndex = 1;
+  const index =
+    parts.length > 1 && parts[1].trim().toLowerCase().startsWith('sort')
+      ? sortCommandIndex
+      : fromCommandIndex;
+
+  return parts
+    .map((part, i) => {
+      if (i === index) {
+        return `${part.trim()} | LIMIT ${limit}`;
+      }
+      return part;
+    })
+    .join('|');
+}

packages/kbn-field-utils/index.ts

@@ -14,6 +14,7 @@ export {
   KNOWN_FIELD_TYPE_LIST,
 } from './src/utils/field_types';
 
+export { getTextBasedColumnIconType } from './src/utils/get_text_based_column_icon_type';
 export { getFieldIconType } from './src/utils/get_field_icon_type';
 export { getFieldType } from './src/utils/get_field_type';
 export { getFieldTypeDescription } from './src/utils/get_field_type_description';

packages/kbn-field-utils/src/utils/get_text_based_column_icon_type.ts

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { DatatableColumnMeta } from '@kbn/expressions-plugin/common';
+import { getFieldIconType } from './get_field_icon_type';
+
+export function getTextBasedColumnIconType(
+  columnMeta:
+    | {
+        type: DatatableColumnMeta['type'];
+        esType?: DatatableColumnMeta['esType'];
+      }
+    | undefined
+    | null
+): string | null {
+  return columnMeta && columnMeta.type
+    ? getFieldIconType({
+        name: '',
+        type: columnMeta.type,
+        esTypes: columnMeta.esType ? [columnMeta.esType] : [],
+      })
+    : null;
+}

packages/kbn-field-utils/tsconfig.json

@@ -9,6 +9,7 @@
     "@kbn/data-views-plugin",
     "@kbn/react-field",
     "@kbn/field-types",
+    "@kbn/expressions-plugin",
   ],
   "exclude": ["target/**/*"]
 }

packages/kbn-unified-data-table/index.ts

@@ -17,7 +17,7 @@ export {
   type RowHeightSettingsProps,
 } from './src/components/row_height_settings';
 export { getDisplayedColumns } from './src/utils/columns';
-export { getTextBasedColumnTypes } from './src/utils/get_column_types';
+export { getTextBasedColumnsMeta } from './src/utils/get_columns_meta';
 export { ROWS_HEIGHT_OPTIONS } from './src/constants';
 
 export { JSONCodeEditorCommonMemoized } from './src/components/json_code_editor/json_code_editor_common';

packages/kbn-unified-data-table/src/components/data_table.test.tsx

@@ -28,6 +28,7 @@ import {
   testLeadingControlColumn,
   testTrailingControlColumns,
 } from '../../__mocks__/external_control_columns';
+import { DatatableColumnType } from '@kbn/expressions-plugin/common';
 
 const mockUseDataGridColumnsCellActions = jest.fn((prop: unknown) => []);
 jest.mock('@kbn/cell-actions', () => ({
@@ -531,7 +532,7 @@ describe('UnifiedDataTable', () => {
       },
       flattened: { test: jest.fn() },
     };
-    const columnTypesOverride = { testField: 'number ' };
+    const columnsMetaOverride = { testField: { type: 'number' as DatatableColumnType } };
     const renderDocumentViewMock = jest.fn((hit: DataTableRecord) => (
       <div data-test-subj="test-document-view">{hit.id}</div>
     ));
@@ -540,7 +541,7 @@ describe('UnifiedDataTable', () => {
       ...getProps(),
       expandedDoc,
       setExpandedDoc: jest.fn(),
-      columnTypes: columnTypesOverride,
+      columnsMeta: columnsMetaOverride,
       renderDocumentView: renderDocumentViewMock,
       externalControlColumns: [testLeadingControlColumn],
     });
@@ -551,7 +552,7 @@ describe('UnifiedDataTable', () => {
       expandedDoc,
       getProps().rows,
       ['_source'],
-      columnTypesOverride
+      columnsMetaOverride
     );
   });
 

packages/kbn-unified-data-table/src/components/data_table.tsx

@@ -49,7 +49,7 @@ import type { DocViewFilterFn } from '@kbn/unified-doc-viewer/types';
 import {
   UnifiedDataTableSettings,
   ValueToStringConverter,
-  DataTableColumnTypes,
+  DataTableColumnsMeta,
   CustomCellRenderer,
   CustomGridColumnsConfiguration,
   CustomControlColumnConfiguration,
@@ -124,10 +124,10 @@ export interface UnifiedDataTableProps {
   columns: string[];
   /**
    * If not provided, types will be derived by default from the dataView field types.
-   * For displaying text-based search results, pass column types (which are available separately in the fetch request) down here.
-   * Check available utils in `utils/get_column_types.ts`
+   * For displaying text-based search results, pass columns meta (which are available separately in the fetch request) down here.
+   * Check available utils in `utils/get_columns_meta.ts`
    */
-  columnTypes?: DataTableColumnTypes;
+  columnsMeta?: DataTableColumnsMeta;
   /**
    * Field tokens could be rendered in column header next to the field name.
    */
@@ -283,7 +283,7 @@ export interface UnifiedDataTableProps {
     hit: DataTableRecord,
     displayedRows: DataTableRecord[],
     displayedColumns: string[],
-    columnTypes?: DataTableColumnTypes
+    columnsMeta?: DataTableColumnsMeta
   ) => JSX.Element | undefined;
   /**
    * Optional value for providing configuration setting for enabling to display the complex fields in the table. Default is true.
@@ -376,7 +376,7 @@ const CONTROL_COLUMN_IDS_DEFAULT = ['openDetails', 'select'];
 export const UnifiedDataTable = ({
   ariaLabelledBy,
   columns,
-  columnTypes,
+  columnsMeta,
   showColumnTokens,
   configHeaderRowHeight,
   headerRowHeightState,
@@ -637,11 +637,11 @@ export const UnifiedDataTable = ({
       canPrependTimeFieldColumn(
         activeColumns,
         timeFieldName,
-        columnTypes,
+        columnsMeta,
         showTimeCol,
         isPlainRecord
       ),
-    [timeFieldName, isPlainRecord, showTimeCol, columnTypes]
+    [timeFieldName, isPlainRecord, showTimeCol, columnsMeta]
   );
 
   const visibleColumns = useMemo(
@@ -724,13 +724,13 @@ export const UnifiedDataTable = ({
         onFilter,
         editField,
         visibleCellActions,
-        columnTypes,
+        columnsMeta,
         showColumnTokens,
         headerRowHeightLines,
         customGridColumnsConfiguration,
       }),
     [
-      columnTypes,
+      columnsMeta,
       columnsCellActions,
       customGridColumnsConfiguration,
       dataView,
@@ -1044,7 +1044,7 @@ export const UnifiedDataTable = ({
         )}
         {canSetExpandedDoc &&
           expandedDoc &&
-          renderDocumentView!(expandedDoc, displayedRows, displayedColumns, columnTypes)}
+          renderDocumentView!(expandedDoc, displayedRows, displayedColumns, columnsMeta)}
       </span>
     </UnifiedDataTableContext.Provider>
   );

packages/kbn-unified-data-table/src/components/data_table_column_header.test.tsx

@@ -78,8 +78,11 @@ describe('DataTableColumnHeader', function () {
       <DataTableColumnHeader
         columnName="bytes"
         columnDisplayName="bytesDisplayName"
-        columnTypes={{
-          bytes: 'keyword',
+        columnsMeta={{
+          bytes: {
+            type: 'string',
+            esType: 'keyword',
+          },
         }}
         dataView={stubLogstashDataView}
         showColumnTokens

packages/kbn-unified-data-table/src/components/data_table_column_header.tsx

@@ -10,18 +10,18 @@ import React, { useMemo } from 'react';
 import { css, CSSObject } from '@emotion/react';
 import { EuiIcon, EuiToolTip } from '@elastic/eui';
 import type { DataView, DataViewField } from '@kbn/data-views-plugin/common';
-import { FieldIcon, getFieldIconProps } from '@kbn/field-utils';
+import { FieldIcon, getFieldIconProps, getTextBasedColumnIconType } from '@kbn/field-utils';
 import { isNestedFieldParent } from '@kbn/discover-utils';
 import { i18n } from '@kbn/i18n';
 import { euiThemeVars } from '@kbn/ui-theme';
-import type { DataTableColumnTypes } from '../types';
+import type { DataTableColumnsMeta } from '../types';
 import ColumnHeaderTruncateContainer from './column_header_truncate_container';
 
 interface DataTableColumnHeaderProps {
   dataView: DataView;
   columnName: string | null;
   columnDisplayName: string;
-  columnTypes?: DataTableColumnTypes;
+  columnsMeta?: DataTableColumnsMeta;
   headerRowHeight?: number;
   showColumnTokens?: boolean;
 }
@@ -30,7 +30,7 @@ export const DataTableColumnHeader: React.FC<DataTableColumnHeaderProps> = ({
   columnDisplayName,
   showColumnTokens,
   columnName,
-  columnTypes,
+  columnsMeta,
   dataView,
   headerRowHeight,
 }) => {
@@ -39,7 +39,7 @@ export const DataTableColumnHeader: React.FC<DataTableColumnHeaderProps> = ({
       {showColumnTokens && (
         <DataTableColumnToken
           columnName={columnName}
-          columnTypes={columnTypes}
+          columnsMeta={columnsMeta}
           dataView={dataView}
         />
       )}
@@ -49,12 +49,12 @@ export const DataTableColumnHeader: React.FC<DataTableColumnHeaderProps> = ({
 };
 
 const DataTableColumnToken: React.FC<
-  Pick<DataTableColumnHeaderProps, 'columnName' | 'columnTypes' | 'dataView'>
+  Pick<DataTableColumnHeaderProps, 'columnName' | 'columnsMeta' | 'dataView'>
 > = (props) => {
-  const { columnName, columnTypes, dataView } = props;
+  const { columnName, columnsMeta, dataView } = props;
   const columnToken = useMemo(
-    () => getRenderedToken({ columnName, columnTypes, dataView }),
-    [columnName, columnTypes, dataView]
+    () => getRenderedToken({ columnName, columnsMeta, dataView }),
+    [columnName, columnsMeta, dataView]
   );
 
   return columnToken ? (
@@ -73,16 +73,18 @@ const fieldIconCss: CSSObject = { verticalAlign: 'bottom' };
 function getRenderedToken({
   dataView,
   columnName,
-  columnTypes,
-}: Pick<DataTableColumnHeaderProps, 'dataView' | 'columnName' | 'columnTypes'>) {
+  columnsMeta,
+}: Pick<DataTableColumnHeaderProps, 'dataView' | 'columnName' | 'columnsMeta'>) {
   if (!columnName || columnName === '_source') {
     return null;
   }
 
   // for text-based searches
-  if (columnTypes) {
-    return columnTypes[columnName] && columnTypes[columnName] !== 'unknown' ? ( // renders an icon or nothing
-      <FieldIcon type={columnTypes[columnName]} css={fieldIconCss} />
+  if (columnsMeta) {
+    const columnMeta = columnsMeta[columnName];
+    const columnIconType = getTextBasedColumnIconType(columnMeta);
+    return columnIconType && columnIconType !== 'unknown' ? ( // renders an icon or nothing
+      <FieldIcon type={columnIconType} css={fieldIconCss} />
     ) : null;
   }