From e2da5af35887529c56c000479b639a0f1fe9582a Mon Sep 17 00:00:00 2001 From: Julia Rechkunova Date: Fri, 22 Mar 2024 13:30:04 +0100 Subject: [PATCH] [ES|QL][UnifiedFieldList] Support field stats for ES|QL query (#178433) - Closes https://github.com/elastic/kibana/issues/174984 ## Summary This PR adds field stats for ES|QL mode in Discover. It will show "Top values" for `keyword`, `ip`, `boolean`, `number`, `version` fields and "Examples" for `text` and geo fields. Also this PR extends text based column's meta with `esType` to make it easier to differentiate between `keyword` and `text` columns when kibana type for both is `string`. This change also has a UI improvement: `k` token will be shown next to keyword fields in data grid column header and in doc viewer instead of `t` token. And it will be possible to filter by Keyword/Text types in the fields sidebar for ES|QL mode. Screenshot 2024-03-13 at 18 29 47 Screenshot 2024-03-13 at 18 30 01 Screenshot 2024-03-13 at 18 30 19 Screenshot 2024-03-13 at 18 30 13 Screenshot 2024-03-13 at 18 30 06 Screenshot 2024-03-13 at 18 31 50 Screenshot 2024-03-13 at 18 30 50 25x flaky test https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/5470 ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [x] This was checked for [cross-browser compatibility](https://www.elastic.co/support/matrix#matrix_browsers) --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- api_docs/kbn_unified_data_table.devdocs.json | 2 +- packages/kbn-esql-utils/index.ts | 1 + packages/kbn-esql-utils/src/index.ts | 1 + .../utils/get_esql_with_safe_limit.test.ts | 31 ++ .../src/utils/get_esql_with_safe_limit.ts | 34 ++ packages/kbn-field-utils/index.ts | 1 + .../utils/get_text_based_column_icon_type.ts | 28 ++ packages/kbn-field-utils/tsconfig.json | 1 + packages/kbn-unified-data-table/index.ts | 2 +- .../data_table_columns.test.tsx.snap | 24 +- .../src/components/data_table.test.tsx | 7 +- .../src/components/data_table.tsx | 22 +- .../data_table_column_header.test.tsx | 7 +- .../components/data_table_column_header.tsx | 30 +- .../components/data_table_columns.test.tsx | 34 +- .../src/components/data_table_columns.tsx | 20 +- packages/kbn-unified-data-table/src/types.ts | 9 +- ...types.test.ts => get_columns_meta.test.ts} | 20 +- ...et_column_types.ts => get_columns_meta.ts} | 10 +- .../src/services/types.ts | 9 +- packages/kbn-unified-doc-viewer/tsconfig.json | 1 + .../field_popover/field_popover_header.tsx | 2 +- .../field_stats/field_stats.test.tsx | 3 +- .../components/field_stats/field_stats.tsx | 184 +++++++---- .../field_stats/field_top_values.tsx | 8 +- .../field_stats/field_top_values_bucket.tsx | 1 + .../kbn-unified-field-list/src/constants.ts | 13 + .../field_list_item.tsx | 12 +- .../field_examples_calculator.test.ts | 76 ++++- .../field_examples_calculator.ts | 43 +-- .../field_examples_calculator/index.ts | 9 + .../field_stats/field_stats_utils.test.ts | 1 + .../services/field_stats/field_stats_utils.ts | 53 ++- .../field_stats_utils_text_based.test.ts | 129 ++++++++ .../field_stats_utils_text_based.ts | 151 +++++++++ .../services/field_stats_text_based/index.ts | 9 + .../load_field_stats_text_based.ts | 104 ++++++ packages/kbn-unified-field-list/src/types.ts | 1 + .../src/utils/can_provide_stats.test.ts | 164 ++++++++++ .../src/utils/can_provide_stats.ts | 84 +++++ packages/kbn-unified-field-list/tsconfig.json | 1 + .../data/common/search/expressions/esql.ts | 2 +- .../components/layout/discover_documents.tsx | 14 +- .../components/sidebar/lib/get_field_list.ts | 1 + .../sidebar/lib/sidebar_reducer.test.ts | 7 +- .../discover_grid_flyout.tsx | 9 +- .../embeddable/saved_search_embeddable.tsx | 6 +- .../public/embeddable/saved_search_grid.tsx | 6 +- .../expression_types/specs/datatable.ts | 7 + .../components/doc_viewer_table/table.tsx | 16 +- .../group2/_data_grid_field_tokens.ts | 14 +- .../apps/discover/group3/_sidebar.ts | 11 +- .../discover/group3/_sidebar_field_stats.ts | 305 ++++++++++++++++++ test/functional/apps/discover/group3/index.ts | 1 + test/functional/page_objects/discover_page.ts | 5 +- .../page_objects/unified_field_list.ts | 7 + 56 files changed, 1485 insertions(+), 268 deletions(-) create mode 100644 packages/kbn-esql-utils/src/utils/get_esql_with_safe_limit.test.ts create mode 100644 packages/kbn-esql-utils/src/utils/get_esql_with_safe_limit.ts create mode 100644 packages/kbn-field-utils/src/utils/get_text_based_column_icon_type.ts rename packages/kbn-unified-data-table/src/utils/{get_column_types.test.ts => get_columns_meta.test.ts} (68%) rename packages/kbn-unified-data-table/src/utils/{get_column_types.ts => get_columns_meta.ts} (70%) create mode 100644 packages/kbn-unified-field-list/src/constants.ts rename packages/kbn-unified-field-list/src/services/{field_stats => field_examples_calculator}/field_examples_calculator.test.ts (74%) rename packages/kbn-unified-field-list/src/services/{field_stats => field_examples_calculator}/field_examples_calculator.ts (79%) create mode 100644 packages/kbn-unified-field-list/src/services/field_examples_calculator/index.ts create mode 100644 packages/kbn-unified-field-list/src/services/field_stats_text_based/field_stats_utils_text_based.test.ts create mode 100644 packages/kbn-unified-field-list/src/services/field_stats_text_based/field_stats_utils_text_based.ts create mode 100644 packages/kbn-unified-field-list/src/services/field_stats_text_based/index.ts create mode 100644 packages/kbn-unified-field-list/src/services/field_stats_text_based/load_field_stats_text_based.ts create mode 100644 packages/kbn-unified-field-list/src/utils/can_provide_stats.test.ts create mode 100644 packages/kbn-unified-field-list/src/utils/can_provide_stats.ts create mode 100644 test/functional/apps/discover/group3/_sidebar_field_stats.ts diff --git a/api_docs/kbn_unified_data_table.devdocs.json b/api_docs/kbn_unified_data_table.devdocs.json index c5ceadd6327683..b8299aa351488b 100644 --- a/api_docs/kbn_unified_data_table.devdocs.json +++ b/api_docs/kbn_unified_data_table.devdocs.json @@ -2581,4 +2581,4 @@ } ] } -} \ No newline at end of file +} diff --git a/packages/kbn-esql-utils/index.ts b/packages/kbn-esql-utils/index.ts index b4fc1f6548c69d..1592777d136f30 100644 --- a/packages/kbn-esql-utils/index.ts +++ b/packages/kbn-esql-utils/index.ts @@ -14,5 +14,6 @@ export { removeDropCommandsFromESQLQuery, getIndexForESQLQuery, getInitialESQLQuery, + getESQLWithSafeLimit, TextBasedLanguages, } from './src'; diff --git a/packages/kbn-esql-utils/src/index.ts b/packages/kbn-esql-utils/src/index.ts index f2e537a240f779..58e241c1ebc6eb 100644 --- a/packages/kbn-esql-utils/src/index.ts +++ b/packages/kbn-esql-utils/src/index.ts @@ -9,6 +9,7 @@ export { TextBasedLanguages } from './types'; export { getESQLAdHocDataview, getIndexForESQLQuery } from './utils/get_esql_adhoc_dataview'; export { getInitialESQLQuery } from './utils/get_initial_esql_query'; +export { getESQLWithSafeLimit } from './utils/get_esql_with_safe_limit'; export { getIndexPatternFromSQLQuery, getIndexPatternFromESQLQuery, diff --git a/packages/kbn-esql-utils/src/utils/get_esql_with_safe_limit.test.ts b/packages/kbn-esql-utils/src/utils/get_esql_with_safe_limit.test.ts new file mode 100644 index 00000000000000..a78dd9f9c06475 --- /dev/null +++ b/packages/kbn-esql-utils/src/utils/get_esql_with_safe_limit.test.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { getESQLWithSafeLimit } from './get_esql_with_safe_limit'; + +const LIMIT = 10000; + +describe('getESQLWithSafeLimit()', () => { + it('should not add the limit', () => { + expect(getESQLWithSafeLimit('show info', LIMIT)).toBe('show info'); + expect(getESQLWithSafeLimit('row t = 5', LIMIT)).toBe('row t = 5'); + }); + + it('should add the limit', () => { + expect(getESQLWithSafeLimit(' from logs', LIMIT)).toBe('from logs | LIMIT 10000'); + expect(getESQLWithSafeLimit('FROM logs* | LIMIT 5', LIMIT)).toBe( + 'FROM logs* | LIMIT 10000| LIMIT 5' + ); + expect(getESQLWithSafeLimit('FROM logs* | SORT @timestamp | LIMIT 5', LIMIT)).toBe( + 'FROM logs* |SORT @timestamp | LIMIT 10000| LIMIT 5' + ); + expect(getESQLWithSafeLimit('from logs* | STATS MIN(a) BY b', LIMIT)).toBe( + 'from logs* | LIMIT 10000| STATS MIN(a) BY b' + ); + }); +}); diff --git a/packages/kbn-esql-utils/src/utils/get_esql_with_safe_limit.ts b/packages/kbn-esql-utils/src/utils/get_esql_with_safe_limit.ts new file mode 100644 index 00000000000000..e8b63b21dd1d47 --- /dev/null +++ b/packages/kbn-esql-utils/src/utils/get_esql_with_safe_limit.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export function getESQLWithSafeLimit(esql: string, limit: number): string { + if (!esql.trim().toLowerCase().startsWith('from')) { + return esql; + } + const parts = esql.split('|'); + + if (!parts.length) { + return esql; + } + + const fromCommandIndex = 0; + const sortCommandIndex = 1; + const index = + parts.length > 1 && parts[1].trim().toLowerCase().startsWith('sort') + ? sortCommandIndex + : fromCommandIndex; + + return parts + .map((part, i) => { + if (i === index) { + return `${part.trim()} | LIMIT ${limit}`; + } + return part; + }) + .join('|'); +} diff --git a/packages/kbn-field-utils/index.ts b/packages/kbn-field-utils/index.ts index 26dd359a359a16..643e22ce6450a9 100644 --- a/packages/kbn-field-utils/index.ts +++ b/packages/kbn-field-utils/index.ts @@ -14,6 +14,7 @@ export { KNOWN_FIELD_TYPE_LIST, } from './src/utils/field_types'; +export { getTextBasedColumnIconType } from './src/utils/get_text_based_column_icon_type'; export { getFieldIconType } from './src/utils/get_field_icon_type'; export { getFieldType } from './src/utils/get_field_type'; export { getFieldTypeDescription } from './src/utils/get_field_type_description'; diff --git a/packages/kbn-field-utils/src/utils/get_text_based_column_icon_type.ts b/packages/kbn-field-utils/src/utils/get_text_based_column_icon_type.ts new file mode 100644 index 00000000000000..0ccad94674208d --- /dev/null +++ b/packages/kbn-field-utils/src/utils/get_text_based_column_icon_type.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { DatatableColumnMeta } from '@kbn/expressions-plugin/common'; +import { getFieldIconType } from './get_field_icon_type'; + +export function getTextBasedColumnIconType( + columnMeta: + | { + type: DatatableColumnMeta['type']; + esType?: DatatableColumnMeta['esType']; + } + | undefined + | null +): string | null { + return columnMeta && columnMeta.type + ? getFieldIconType({ + name: '', + type: columnMeta.type, + esTypes: columnMeta.esType ? [columnMeta.esType] : [], + }) + : null; +} diff --git a/packages/kbn-field-utils/tsconfig.json b/packages/kbn-field-utils/tsconfig.json index 9aaae0d119e684..4b75159b5f7feb 100644 --- a/packages/kbn-field-utils/tsconfig.json +++ b/packages/kbn-field-utils/tsconfig.json @@ -9,6 +9,7 @@ "@kbn/data-views-plugin", "@kbn/react-field", "@kbn/field-types", + "@kbn/expressions-plugin", ], "exclude": ["target/**/*"] } diff --git a/packages/kbn-unified-data-table/index.ts b/packages/kbn-unified-data-table/index.ts index 9cb8b67dc4b2e7..f59b98cda99d92 100644 --- a/packages/kbn-unified-data-table/index.ts +++ b/packages/kbn-unified-data-table/index.ts @@ -17,7 +17,7 @@ export { type RowHeightSettingsProps, } from './src/components/row_height_settings'; export { getDisplayedColumns } from './src/utils/columns'; -export { getTextBasedColumnTypes } from './src/utils/get_column_types'; +export { getTextBasedColumnsMeta } from './src/utils/get_columns_meta'; export { ROWS_HEIGHT_OPTIONS } from './src/constants'; export { JSONCodeEditorCommonMemoized } from './src/components/json_code_editor/json_code_editor_common'; diff --git a/packages/kbn-unified-data-table/src/components/__snapshots__/data_table_columns.test.tsx.snap b/packages/kbn-unified-data-table/src/components/__snapshots__/data_table_columns.test.tsx.snap index 3c00afb1c1dd8c..28d43dc69b6cd3 100644 --- a/packages/kbn-unified-data-table/src/components/__snapshots__/data_table_columns.test.tsx.snap +++ b/packages/kbn-unified-data-table/src/components/__snapshots__/data_table_columns.test.tsx.snap @@ -1393,10 +1393,15 @@ Array [ "display": []); jest.mock('@kbn/cell-actions', () => ({ @@ -531,7 +532,7 @@ describe('UnifiedDataTable', () => { }, flattened: { test: jest.fn() }, }; - const columnTypesOverride = { testField: 'number ' }; + const columnsMetaOverride = { testField: { type: 'number' as DatatableColumnType } }; const renderDocumentViewMock = jest.fn((hit: DataTableRecord) => (
{hit.id}
)); @@ -540,7 +541,7 @@ describe('UnifiedDataTable', () => { ...getProps(), expandedDoc, setExpandedDoc: jest.fn(), - columnTypes: columnTypesOverride, + columnsMeta: columnsMetaOverride, renderDocumentView: renderDocumentViewMock, externalControlColumns: [testLeadingControlColumn], }); @@ -551,7 +552,7 @@ describe('UnifiedDataTable', () => { expandedDoc, getProps().rows, ['_source'], - columnTypesOverride + columnsMetaOverride ); }); diff --git a/packages/kbn-unified-data-table/src/components/data_table.tsx b/packages/kbn-unified-data-table/src/components/data_table.tsx index 6b488b16805bee..f935f0ff7daace 100644 --- a/packages/kbn-unified-data-table/src/components/data_table.tsx +++ b/packages/kbn-unified-data-table/src/components/data_table.tsx @@ -49,7 +49,7 @@ import type { DocViewFilterFn } from '@kbn/unified-doc-viewer/types'; import { UnifiedDataTableSettings, ValueToStringConverter, - DataTableColumnTypes, + DataTableColumnsMeta, CustomCellRenderer, CustomGridColumnsConfiguration, CustomControlColumnConfiguration, @@ -124,10 +124,10 @@ export interface UnifiedDataTableProps { columns: string[]; /** * If not provided, types will be derived by default from the dataView field types. - * For displaying text-based search results, pass column types (which are available separately in the fetch request) down here. - * Check available utils in `utils/get_column_types.ts` + * For displaying text-based search results, pass columns meta (which are available separately in the fetch request) down here. + * Check available utils in `utils/get_columns_meta.ts` */ - columnTypes?: DataTableColumnTypes; + columnsMeta?: DataTableColumnsMeta; /** * Field tokens could be rendered in column header next to the field name. */ @@ -283,7 +283,7 @@ export interface UnifiedDataTableProps { hit: DataTableRecord, displayedRows: DataTableRecord[], displayedColumns: string[], - columnTypes?: DataTableColumnTypes + columnsMeta?: DataTableColumnsMeta ) => JSX.Element | undefined; /** * Optional value for providing configuration setting for enabling to display the complex fields in the table. Default is true. @@ -376,7 +376,7 @@ const CONTROL_COLUMN_IDS_DEFAULT = ['openDetails', 'select']; export const UnifiedDataTable = ({ ariaLabelledBy, columns, - columnTypes, + columnsMeta, showColumnTokens, configHeaderRowHeight, headerRowHeightState, @@ -637,11 +637,11 @@ export const UnifiedDataTable = ({ canPrependTimeFieldColumn( activeColumns, timeFieldName, - columnTypes, + columnsMeta, showTimeCol, isPlainRecord ), - [timeFieldName, isPlainRecord, showTimeCol, columnTypes] + [timeFieldName, isPlainRecord, showTimeCol, columnsMeta] ); const visibleColumns = useMemo( @@ -724,13 +724,13 @@ export const UnifiedDataTable = ({ onFilter, editField, visibleCellActions, - columnTypes, + columnsMeta, showColumnTokens, headerRowHeightLines, customGridColumnsConfiguration, }), [ - columnTypes, + columnsMeta, columnsCellActions, customGridColumnsConfiguration, dataView, @@ -1044,7 +1044,7 @@ export const UnifiedDataTable = ({ )} {canSetExpandedDoc && expandedDoc && - renderDocumentView!(expandedDoc, displayedRows, displayedColumns, columnTypes)} + renderDocumentView!(expandedDoc, displayedRows, displayedColumns, columnsMeta)} ); diff --git a/packages/kbn-unified-data-table/src/components/data_table_column_header.test.tsx b/packages/kbn-unified-data-table/src/components/data_table_column_header.test.tsx index 2420b1508cbf58..a9035b9fdafbc8 100644 --- a/packages/kbn-unified-data-table/src/components/data_table_column_header.test.tsx +++ b/packages/kbn-unified-data-table/src/components/data_table_column_header.test.tsx @@ -78,8 +78,11 @@ describe('DataTableColumnHeader', function () { = ({ columnDisplayName, showColumnTokens, columnName, - columnTypes, + columnsMeta, dataView, headerRowHeight, }) => { @@ -39,7 +39,7 @@ export const DataTableColumnHeader: React.FC = ({ {showColumnTokens && ( )} @@ -49,12 +49,12 @@ export const DataTableColumnHeader: React.FC = ({ }; const DataTableColumnToken: React.FC< - Pick + Pick > = (props) => { - const { columnName, columnTypes, dataView } = props; + const { columnName, columnsMeta, dataView } = props; const columnToken = useMemo( - () => getRenderedToken({ columnName, columnTypes, dataView }), - [columnName, columnTypes, dataView] + () => getRenderedToken({ columnName, columnsMeta, dataView }), + [columnName, columnsMeta, dataView] ); return columnToken ? ( @@ -73,16 +73,18 @@ const fieldIconCss: CSSObject = { verticalAlign: 'bottom' }; function getRenderedToken({ dataView, columnName, - columnTypes, -}: Pick) { + columnsMeta, +}: Pick) { if (!columnName || columnName === '_source') { return null; } // for text-based searches - if (columnTypes) { - return columnTypes[columnName] && columnTypes[columnName] !== 'unknown' ? ( // renders an icon or nothing - + if (columnsMeta) { + const columnMeta = columnsMeta[columnName]; + const columnIconType = getTextBasedColumnIconType(columnMeta); + return columnIconType && columnIconType !== 'unknown' ? ( // renders an icon or nothing + ) : null; } diff --git a/packages/kbn-unified-data-table/src/components/data_table_columns.test.tsx b/packages/kbn-unified-data-table/src/components/data_table_columns.test.tsx index b98f514690aee5..e50a0073d31a2d 100644 --- a/packages/kbn-unified-data-table/src/components/data_table_columns.test.tsx +++ b/packages/kbn-unified-data-table/src/components/data_table_columns.test.tsx @@ -6,9 +6,10 @@ * Side Public License, v 1. */ +import React from 'react'; import { dataViewMock } from '@kbn/discover-utils/src/__mocks__'; import type { DataView } from '@kbn/data-views-plugin/public'; -import React from 'react'; +import type { DatatableColumnType } from '@kbn/expressions-plugin/common'; import { deserializeHeaderRowHeight, getEuiGridColumns, @@ -124,11 +125,14 @@ describe('Data table columns', function () { describe('canPrependTimeFieldColumn', () => { function buildColumnTypes(dataView: DataView) { - const columnTypes: Record = {}; + const columnsMeta: Record< + string, + { type: DatatableColumnType; esType?: string | undefined } + > = {}; for (const field of dataView.fields) { - columnTypes[field.name] = field.type; + columnsMeta[field.name] = { type: field.type as DatatableColumnType }; } - return columnTypes; + return columnsMeta; } describe('dataView with timeField', () => { @@ -190,16 +194,16 @@ describe('Data table columns', function () { it('should return false if _source column is passed but time field is not returned, text-based datasource', () => { // ... | DROP @timestamp test case - const columnTypes = buildColumnTypes(dataViewWithTimefieldMock); + const columnsMeta = buildColumnTypes(dataViewWithTimefieldMock); if (dataViewWithTimefieldMock.timeFieldName) { - delete columnTypes[dataViewWithTimefieldMock.timeFieldName]; + delete columnsMeta[dataViewWithTimefieldMock.timeFieldName]; } for (const showTimeCol of [true, false]) { expect( canPrependTimeFieldColumn( ['_source'], dataViewWithTimefieldMock.timeFieldName, - columnTypes, + columnsMeta, showTimeCol, true ) @@ -294,9 +298,9 @@ describe('Data table columns', function () { it('returns eui grid columns with tokens for custom column types', async () => { const actual = getEuiGridColumns({ showColumnTokens: true, - columnTypes: { - extension: 'number', - message: 'keyword', + columnsMeta: { + extension: { type: 'number' }, + message: { type: 'string', esType: 'keyword' }, }, columns, settings: {}, @@ -343,14 +347,14 @@ describe('Data table columns', function () { hasEditDataViewPermission: () => servicesMock.dataViewFieldEditor.userPermissions.editIndexPattern(), onFilter: () => {}, - columnTypes: { - var_test: 'number', + columnsMeta: { + var_test: { type: 'number' }, }, }); expect(gridColumns[1].schema).toBe('string'); }); - it('returns eui grid with in memory sorting for text based languages and columns not on the columnTypes', async () => { + it('returns eui grid with in memory sorting for text based languages and columns not on the columnsMeta', async () => { const columnsNotInDataview = getVisibleColumns( ['var_test'], dataViewWithTimefieldMock, @@ -373,8 +377,8 @@ describe('Data table columns', function () { hasEditDataViewPermission: () => servicesMock.dataViewFieldEditor.userPermissions.editIndexPattern(), onFilter: () => {}, - columnTypes: { - var_test: 'number', + columnsMeta: { + var_test: { type: 'number' }, }, }); expect(gridColumns[1].schema).toBe('numeric'); diff --git a/packages/kbn-unified-data-table/src/components/data_table_columns.tsx b/packages/kbn-unified-data-table/src/components/data_table_columns.tsx index 985cf10a4e3091..90cba4f6be4f9e 100644 --- a/packages/kbn-unified-data-table/src/components/data_table_columns.tsx +++ b/packages/kbn-unified-data-table/src/components/data_table_columns.tsx @@ -18,7 +18,7 @@ import { ToastsStart, IUiSettingsClient } from '@kbn/core/public'; import { DocViewFilterFn } from '@kbn/unified-doc-viewer/types'; import { ExpandButton } from './data_table_expand_button'; import { ControlColumns, CustomGridColumnsConfiguration, UnifiedDataTableSettings } from '../types'; -import type { ValueToStringConverter, DataTableColumnTypes } from '../types'; +import type { ValueToStringConverter, DataTableColumnsMeta } from '../types'; import { buildCellActions } from './default_cell_actions'; import { getSchemaByKbnType } from './data_table_schema'; import { SelectButton } from './data_table_document_selection'; @@ -93,7 +93,7 @@ function buildEuiGridColumn({ editField, columnCellActions, visibleCellActions, - columnTypes, + columnsMeta, showColumnTokens, headerRowHeight, customGridColumnsConfiguration, @@ -113,7 +113,7 @@ function buildEuiGridColumn({ editField?: (fieldName: string) => void; columnCellActions?: EuiDataGridColumnCellAction[]; visibleCellActions?: number; - columnTypes?: DataTableColumnTypes; + columnsMeta?: DataTableColumnsMeta; showColumnTokens?: boolean; headerRowHeight?: number; customGridColumnsConfiguration?: CustomGridColumnsConfiguration; @@ -140,7 +140,7 @@ function buildEuiGridColumn({ : []; } - const columnType = columnTypes?.[columnName] ?? dataViewField?.type; + const columnType = columnsMeta?.[columnName]?.type ?? dataViewField?.type; const column: EuiDataGridColumn = { id: columnName, @@ -152,7 +152,7 @@ function buildEuiGridColumn({ dataView={dataView} columnName={columnName} columnDisplayName={columnDisplayName} - columnTypes={columnTypes} + columnsMeta={columnsMeta} showColumnTokens={showColumnTokens} headerRowHeight={headerRowHeight} /> @@ -241,7 +241,7 @@ export function getEuiGridColumns({ onFilter, editField, visibleCellActions, - columnTypes, + columnsMeta, showColumnTokens, headerRowHeightLines, customGridColumnsConfiguration, @@ -263,7 +263,7 @@ export function getEuiGridColumns({ onFilter: DocViewFilterFn; editField?: (fieldName: string) => void; visibleCellActions?: number; - columnTypes?: DataTableColumnTypes; + columnsMeta?: DataTableColumnsMeta; showColumnTokens?: boolean; headerRowHeightLines: number; customGridColumnsConfiguration?: CustomGridColumnsConfiguration; @@ -289,7 +289,7 @@ export function getEuiGridColumns({ onFilter, editField, visibleCellActions, - columnTypes, + columnsMeta, showColumnTokens, headerRowHeight, customGridColumnsConfiguration, @@ -300,7 +300,7 @@ export function getEuiGridColumns({ export function canPrependTimeFieldColumn( columns: string[], timeFieldName: string | undefined, - columnTypes: DataTableColumnTypes | undefined, + columnsMeta: DataTableColumnsMeta | undefined, showTimeCol: boolean, isPlainRecord: boolean ) { @@ -309,7 +309,7 @@ export function canPrependTimeFieldColumn( } if (isPlainRecord) { - return !!columnTypes && timeFieldName in columnTypes && columns.includes('_source'); + return !!columnsMeta && timeFieldName in columnsMeta && columns.includes('_source'); } return true; diff --git a/packages/kbn-unified-data-table/src/types.ts b/packages/kbn-unified-data-table/src/types.ts index 38d305d0c61777..8b958d5be3bfbb 100644 --- a/packages/kbn-unified-data-table/src/types.ts +++ b/packages/kbn-unified-data-table/src/types.ts @@ -12,6 +12,7 @@ import type { DataTableRecord } from '@kbn/discover-utils/src/types'; import type { DataView } from '@kbn/data-views-plugin/common'; import type { FieldFormatsStart } from '@kbn/field-formats-plugin/public'; import { EuiDataGridControlColumn } from '@elastic/eui/src/components/datagrid/data_grid_types'; +import type { DatatableColumnMeta } from '@kbn/expressions-plugin/common'; /** * User configurable state of data grid, persisted in saved search @@ -33,7 +34,13 @@ export type ValueToStringConverter = ( /** * Custom column types per column name */ -export type DataTableColumnTypes = Record; +export type DataTableColumnsMeta = Record< + string, + { + type: DatatableColumnMeta['type']; + esType?: DatatableColumnMeta['esType']; + } +>; export type DataGridCellValueElementProps = EuiDataGridCellValueElementProps & { row: DataTableRecord; diff --git a/packages/kbn-unified-data-table/src/utils/get_column_types.test.ts b/packages/kbn-unified-data-table/src/utils/get_columns_meta.test.ts similarity index 68% rename from packages/kbn-unified-data-table/src/utils/get_column_types.test.ts rename to packages/kbn-unified-data-table/src/utils/get_columns_meta.test.ts index f26086607a966c..ba06306422e95c 100644 --- a/packages/kbn-unified-data-table/src/utils/get_column_types.test.ts +++ b/packages/kbn-unified-data-table/src/utils/get_columns_meta.test.ts @@ -6,12 +6,12 @@ * Side Public License, v 1. */ -import { getTextBasedColumnTypes } from './get_column_types'; +import { getTextBasedColumnsMeta } from './get_columns_meta'; describe('getColumnTypes', () => { - describe('getTextBasedColumnTypes', () => { + describe('getTextBasedColumnsMeta', () => { test('returns a correct column types map', async () => { - const result = getTextBasedColumnTypes([ + const result = getTextBasedColumnsMeta([ { id: '@timestamp', name: '@timestamp', @@ -24,6 +24,7 @@ describe('getColumnTypes', () => { name: 'agent.keyword', meta: { type: 'string', + esType: 'keyword', }, }, { @@ -36,9 +37,16 @@ describe('getColumnTypes', () => { ]); expect(result).toMatchInlineSnapshot(` Object { - "@timestamp": "date", - "agent.keyword": "string", - "bytes": "number", + "@timestamp": Object { + "type": "date", + }, + "agent.keyword": Object { + "esType": "keyword", + "type": "string", + }, + "bytes": Object { + "type": "number", + }, } `); }); diff --git a/packages/kbn-unified-data-table/src/utils/get_column_types.ts b/packages/kbn-unified-data-table/src/utils/get_columns_meta.ts similarity index 70% rename from packages/kbn-unified-data-table/src/utils/get_column_types.ts rename to packages/kbn-unified-data-table/src/utils/get_columns_meta.ts index edc8c1eb49ce5d..0a1ab0bdc29cbe 100644 --- a/packages/kbn-unified-data-table/src/utils/get_column_types.ts +++ b/packages/kbn-unified-data-table/src/utils/get_columns_meta.ts @@ -6,19 +6,19 @@ * Side Public License, v 1. */ -import type { DatatableColumn, DatatableColumnType } from '@kbn/expressions-plugin/common'; +import type { DatatableColumn, DatatableColumnMeta } from '@kbn/expressions-plugin/common'; -type TextBasedColumnTypes = Record; +type TextBasedColumnTypes = Record; /** - * Column types for text based searches + * Columns meta for text based searches * @param textBasedColumns */ -export const getTextBasedColumnTypes = ( +export const getTextBasedColumnsMeta = ( textBasedColumns: DatatableColumn[] ): TextBasedColumnTypes => { return textBasedColumns.reduce((map, next) => { - map[next.name] = next.meta.type; + map[next.name] = next.meta; return map; }, {}); }; diff --git a/packages/kbn-unified-doc-viewer/src/services/types.ts b/packages/kbn-unified-doc-viewer/src/services/types.ts index b67c61de6ae882..bcf64c817e8873 100644 --- a/packages/kbn-unified-doc-viewer/src/services/types.ts +++ b/packages/kbn-unified-doc-viewer/src/services/types.ts @@ -9,6 +9,7 @@ import type { DataView, DataViewField } from '@kbn/data-views-plugin/public'; import type { AggregateQuery, Query } from '@kbn/es-query'; import type { DataTableRecord, IgnoredReason } from '@kbn/discover-utils/types'; +import type { DatatableColumnMeta } from '@kbn/expressions-plugin/common'; import { DocViewsRegistry } from './doc_views_registry'; export interface FieldMapping { @@ -34,7 +35,13 @@ export interface DocViewRenderProps { * If not provided, types will be derived by default from the dataView field types. * For displaying text-based search results, define column types (which are available separately in the fetch request) here. */ - columnTypes?: Record; + columnsMeta?: Record< + string, + { + type: DatatableColumnMeta['type']; + esType?: DatatableColumnMeta['esType']; + } + >; query?: Query | AggregateQuery; textBasedHits?: DataTableRecord[]; hideActionsColumn?: boolean; diff --git a/packages/kbn-unified-doc-viewer/tsconfig.json b/packages/kbn-unified-doc-viewer/tsconfig.json index e9429af74bd749..05bb4f1539598e 100644 --- a/packages/kbn-unified-doc-viewer/tsconfig.json +++ b/packages/kbn-unified-doc-viewer/tsconfig.json @@ -24,5 +24,6 @@ "@kbn/i18n", "@kbn/react-field", "@kbn/field-utils", + "@kbn/expressions-plugin", ] } diff --git a/packages/kbn-unified-field-list/src/components/field_popover/field_popover_header.tsx b/packages/kbn-unified-field-list/src/components/field_popover/field_popover_header.tsx index 9700bb3da1cf16..43bebba6e660b0 100644 --- a/packages/kbn-unified-field-list/src/components/field_popover/field_popover_header.tsx +++ b/packages/kbn-unified-field-list/src/components/field_popover/field_popover_header.tsx @@ -80,7 +80,7 @@ export const FieldPopoverHeader: React.FC = ({ <> - +
{field.displayName}
 diff --git a/packages/kbn-unified-field-list/src/components/field_stats/field_stats.test.tsx b/packages/kbn-unified-field-list/src/components/field_stats/field_stats.test.tsx index 40a0d52e6cd13c..fe60760e4bf449 100644 --- a/packages/kbn-unified-field-list/src/components/field_stats/field_stats.test.tsx +++ b/packages/kbn-unified-field-list/src/components/field_stats/field_stats.test.tsx @@ -33,7 +33,7 @@ const mockedServices = { uiSettings: coreMock.createStart().uiSettings, }; -describe('UnifiedFieldList ', () => { +describe('UnifiedFieldList FieldStats', () => { let defaultProps: FieldStatsWithKbnQuery; let dataView: DataView; @@ -467,6 +467,7 @@ describe('UnifiedFieldList ', () => { sampledDocuments: 1624, sampledValues: 3248, topValues: { + areExamples: true, buckets: [ { count: 1349, diff --git a/packages/kbn-unified-field-list/src/components/field_stats/field_stats.tsx b/packages/kbn-unified-field-list/src/components/field_stats/field_stats.tsx index c2d5fd0f326329..00c9d748515bf3 100755 --- a/packages/kbn-unified-field-list/src/components/field_stats/field_stats.tsx +++ b/packages/kbn-unified-field-list/src/components/field_stats/field_stats.tsx @@ -11,6 +11,7 @@ import type { DataView, DataViewField } from '@kbn/data-views-plugin/common'; import { getTimeZone } from '@kbn/visualization-utils'; import { ES_FIELD_TYPES, KBN_FIELD_TYPES } from '@kbn/field-types'; import { getEsQueryConfig } from '@kbn/data-service/src/es_query'; +import { isOfAggregateQueryType } from '@kbn/es-query'; import type { IUiSettingsClient } from '@kbn/core/public'; import type { DataViewsContract } from '@kbn/data-views-plugin/public'; import type { DataPublicPluginStart } from '@kbn/data-plugin/public'; @@ -33,14 +34,14 @@ import { } from '@elastic/charts'; import { i18n } from '@kbn/i18n'; import { buildEsQuery, Query, Filter, AggregateQuery } from '@kbn/es-query'; -import { showExamplesForField } from '../../services/field_stats/field_examples_calculator'; import { OverrideFieldTopValueBarCallback } from './field_top_values_bucket'; import type { BucketedAggregation, NumberSummary } from '../../types'; import { canProvideStatsForField, canProvideNumberSummaryForField, -} from '../../services/field_stats/field_stats_utils'; +} from '../../utils/can_provide_stats'; import { loadFieldStats } from '../../services/field_stats'; +import { loadFieldStatsTextBased } from '../../services/field_stats_text_based'; import type { AddFieldFilterHandler } from '../../types'; import { FieldTopValues, @@ -57,8 +58,8 @@ export interface FieldStatsState { totalDocuments?: number; sampledDocuments?: number; sampledValues?: number; - histogram?: BucketedAggregation; - topValues?: BucketedAggregation; + histogram?: BucketedAggregation; + topValues?: BucketedAggregation; numberSummary?: NumberSummary; } @@ -134,6 +135,7 @@ const FieldStatsComponent: React.FC = ({ const [dataView, changeDataView] = useState(null); const abortControllerRef = useRef(null); const isCanceledRef = useRef(false); + const isTextBased = !!query && isOfAggregateQueryType(query); const setState: typeof changeState = useCallback( (nextState) => { @@ -184,17 +186,32 @@ const FieldStatsComponent: React.FC = ({ abortControllerRef.current?.abort(); abortControllerRef.current = new AbortController(); - const results = await loadFieldStats({ - services: { data }, - dataView: loadedDataView, - field, - fromDate, - toDate, - dslQuery: - dslQuery ?? - buildEsQuery(loadedDataView, query ?? [], filters ?? [], getEsQueryConfig(uiSettings)), - abortController: abortControllerRef.current, - }); + const results = isTextBased + ? await loadFieldStatsTextBased({ + services: { data }, + dataView: loadedDataView, + field, + fromDate, + toDate, + baseQuery: query, + abortController: abortControllerRef.current, + }) + : await loadFieldStats({ + services: { data }, + dataView: loadedDataView, + field, + fromDate, + toDate, + dslQuery: + dslQuery ?? + buildEsQuery( + loadedDataView, + query ?? [], + filters ?? [], + getEsQueryConfig(uiSettings) + ), + abortController: abortControllerRef.current, + }); abortControllerRef.current = null; @@ -279,44 +296,7 @@ const FieldStatsComponent: React.FC = ({ let title = <>; function combineWithTitleAndFooter(el: React.ReactElement) { - const dataTestSubjDocsCount = 'unifiedFieldStats-statsFooter-docsCount'; - const countsElement = totalDocuments ? ( - - {sampledDocuments && sampledDocuments < totalDocuments ? ( - - {fieldFormats - .getDefaultInstance(KBN_FIELD_TYPES.NUMBER, [ES_FIELD_TYPES.INTEGER]) - .convert(sampledDocuments)} - - ), - }} - /> - ) : ( - - {fieldFormats - .getDefaultInstance(KBN_FIELD_TYPES.NUMBER, [ES_FIELD_TYPES.INTEGER]) - .convert(totalDocuments)} - - ), - }} - /> - )} - - ) : ( - <> - ); + const countsElement = getCountsElement(state, services, isTextBased, dataTestSubject); return ( <> @@ -338,7 +318,7 @@ const FieldStatsComponent: React.FC = ({ ); } - if (!canProvideStatsForField(field)) { + if (!canProvideStatsForField(field, isTextBased)) { const messageNoAnalysis = ( = ({ : messageNoAnalysis; } - if (canProvideNumberSummaryForField(field) && isNumberSummaryValid(numberSummary)) { + if (canProvideNumberSummaryForField(field, isTextBased) && isNumberSummaryValid(numberSummary)) { title = (
@@ -458,7 +438,7 @@ const FieldStatsComponent: React.FC = ({ title = (
- {showExamplesForField(field) + {topValues.areExamples ? i18n.translate('unifiedFieldList.fieldStats.examplesLabel', { defaultMessage: 'Examples', }) @@ -563,7 +543,7 @@ const FieldStatsComponent: React.FC = ({ if (topValues && topValues.buckets.length) { return combineWithTitleAndFooter( = ({ return null; }; +function getCountsElement( + state: FieldStatsState, + services: FieldStatsServices, + isTextBased: boolean, + dataTestSubject: string +): JSX.Element { + const dataTestSubjDocsCount = 'unifiedFieldStats-statsFooter-docsCount'; + const { fieldFormats } = services; + const { totalDocuments, sampledValues, sampledDocuments, topValues } = state; + + if (!totalDocuments) { + return <>; + } + + let labelElement; + + if (isTextBased) { + labelElement = topValues?.areExamples ? ( + + {fieldFormats + .getDefaultInstance(KBN_FIELD_TYPES.NUMBER, [ES_FIELD_TYPES.INTEGER]) + .convert(sampledDocuments)} + + ), + }} + /> + ) : ( + + {fieldFormats + .getDefaultInstance(KBN_FIELD_TYPES.NUMBER, [ES_FIELD_TYPES.INTEGER]) + .convert(sampledValues)} + + ), + }} + /> + ); + } else { + labelElement = + sampledDocuments && sampledDocuments < totalDocuments ? ( + + {fieldFormats + .getDefaultInstance(KBN_FIELD_TYPES.NUMBER, [ES_FIELD_TYPES.INTEGER]) + .convert(sampledDocuments)} + + ), + }} + /> + ) : ( + + {fieldFormats + .getDefaultInstance(KBN_FIELD_TYPES.NUMBER, [ES_FIELD_TYPES.INTEGER]) + .convert(totalDocuments)} + + ), + }} + /> + ); + } + + return ( + + {labelElement} + + ); +} + /** * Component which fetches and renders stats for a data view field * @param props diff --git a/packages/kbn-unified-field-list/src/components/field_stats/field_top_values.tsx b/packages/kbn-unified-field-list/src/components/field_stats/field_top_values.tsx index c899b624db0b7f..d2d506a3829015 100755 --- a/packages/kbn-unified-field-list/src/components/field_stats/field_top_values.tsx +++ b/packages/kbn-unified-field-list/src/components/field_stats/field_top_values.tsx @@ -14,8 +14,8 @@ import FieldTopValuesBucket from './field_top_values_bucket'; import type { OverrideFieldTopValueBarCallback } from './field_top_values_bucket'; export interface FieldTopValuesProps { - areExamples: boolean; // real top values or only examples - buckets: BucketedAggregation['buckets']; + areExamples: boolean | undefined; // real top values or only examples distributed in buckets + buckets: BucketedAggregation['buckets']; dataView: DataView; field: DataViewField; sampledValuesCount: number; @@ -58,7 +58,7 @@ export const FieldTopValues: React.FC = ({ const formatted = formatter.convert(fieldValue); return ( - + {index > 0 && } ['buckets'] + buckets?: BucketedAggregation['buckets'] ): number => { return buckets?.reduce((prev, bucket) => bucket.count + prev, 0) || 0; }; diff --git a/packages/kbn-unified-field-list/src/components/field_stats/field_top_values_bucket.tsx b/packages/kbn-unified-field-list/src/components/field_stats/field_top_values_bucket.tsx index 4a221bd08a13c3..d158d29ce80af4 100755 --- a/packages/kbn-unified-field-list/src/components/field_stats/field_top_values_bucket.tsx +++ b/packages/kbn-unified-field-list/src/components/field_stats/field_top_values_bucket.tsx @@ -112,6 +112,7 @@ const FieldTopValuesBucket: React.FC = ({ )} + - {multiFields && ( + {searchMode === 'documents' && multiFields && ( <> )} - {!!services.uiActions && ( + {searchMode === 'documents' && !!services.uiActions && ( )} - renderContent={searchMode === 'documents' ? renderPopover : undefined} + renderContent={ + (searchMode === 'text-based' && canProvideStatsForFieldTextBased(field)) || + searchMode === 'documents' + ? renderPopover + : undefined + } /> ); } diff --git a/packages/kbn-unified-field-list/src/services/field_stats/field_examples_calculator.test.ts b/packages/kbn-unified-field-list/src/services/field_examples_calculator/field_examples_calculator.test.ts similarity index 74% rename from packages/kbn-unified-field-list/src/services/field_stats/field_examples_calculator.test.ts rename to packages/kbn-unified-field-list/src/services/field_examples_calculator/field_examples_calculator.test.ts index 0e0e3c886066f9..f7241b1c6b1c41 100644 --- a/packages/kbn-unified-field-list/src/services/field_stats/field_examples_calculator.test.ts +++ b/packages/kbn-unified-field-list/src/services/field_examples_calculator/field_examples_calculator.test.ts @@ -7,9 +7,14 @@ */ import { keys, clone, uniq, filter, map, flatten } from 'lodash'; -import type { DataView } from '@kbn/data-views-plugin/public'; import { stubLogstashDataView as dataView } from '@kbn/data-views-plugin/common/data_view.stub'; -import { getFieldExampleBuckets, groupValues, getFieldValues } from './field_examples_calculator'; +import { + getFieldExampleBuckets, + groupValues, + getFieldValues, + type FieldValueCountsParams, +} from './field_examples_calculator'; +import type { DataViewField } from '@kbn/data-views-plugin/common'; const hitsAsValues: Array> = [ { @@ -211,13 +216,13 @@ describe('fieldExamplesCalculator', function () { }); describe('getFieldExampleBuckets', function () { - let params: { hits: any; field: any; count: number; dataView: DataView }; + let params: FieldValueCountsParams; beforeEach(function () { params = { - hits, - field: dataView.fields.getByName('extension'), + values: getFieldValues(hits, dataView.fields.getByName('extension')!, dataView), + field: dataView.fields.getByName('extension')!, count: 3, - dataView, + isTextBased: false, }; }); @@ -230,14 +235,16 @@ describe('fieldExamplesCalculator', function () { }); it('analyzes geo types', function () { - params.field = dataView.fields.getByName('point'); + params.field = dataView.fields.getByName('point')!; + params.values = getFieldValues(hits, params.field, dataView); expect(getFieldExampleBuckets(params)).toEqual({ buckets: [{ count: 3, key: { coordinates: [100, 20], type: 'Point' } }], sampledDocuments: 20, sampledValues: 3, }); - params.field = dataView.fields.getByName('area'); + params.field = dataView.fields.getByName('area')!; + params.values = getFieldValues(hits, params.field, dataView); expect(getFieldExampleBuckets(params)).toEqual({ buckets: [], sampledDocuments: 20, @@ -246,30 +253,67 @@ describe('fieldExamplesCalculator', function () { }); it('fails to analyze attachment types', function () { - params.field = dataView.fields.getByName('request_body'); + params.field = dataView.fields.getByName('request_body')!; + params.values = getFieldValues(hits, params.field, dataView); expect(() => getFieldExampleBuckets(params)).toThrowError(); - params.field = dataView.fields.getByName('_score'); + params.field = dataView.fields.getByName('_score')!; + params.values = getFieldValues(hits, params.field, dataView); expect(() => getFieldExampleBuckets(params)).toThrowError(); }); it('fails to analyze fields that are in the mapping, but not the hits', function () { - params.field = dataView.fields.getByName('machine.os'); + params.field = dataView.fields.getByName('machine.os')!; + params.values = getFieldValues(hits, params.field, dataView); expect(getFieldExampleBuckets(params).buckets).toHaveLength(0); expect(getFieldExampleBuckets(params).sampledValues).toBe(0); }); it('counts the total hits', function () { - expect(getFieldExampleBuckets(params).sampledDocuments).toBe(params.hits.length); + expect(getFieldExampleBuckets(params).sampledDocuments).toBe(params.values.length); }); it('counts total number of values', function () { - params.field = dataView.fields.getByName('@tags'); + params.field = dataView.fields.getByName('@tags')!; + params.values = getFieldValues(hits, params.field, dataView); expect(getFieldExampleBuckets(params).sampledValues).toBe(3); - params.field = dataView.fields.getByName('extension'); - expect(getFieldExampleBuckets(params).sampledValues).toBe(params.hits.length); - params.field = dataView.fields.getByName('phpmemory'); + params.field = dataView.fields.getByName('extension')!; + params.values = getFieldValues(hits, params.field, dataView); + expect(getFieldExampleBuckets(params).sampledValues).toBe(params.values.length); + params.field = dataView.fields.getByName('phpmemory')!; + params.values = getFieldValues(hits, params.field, dataView); expect(getFieldExampleBuckets(params).sampledValues).toBe(5); }); + + it('works for text-based', function () { + const result = getFieldExampleBuckets({ + values: [['a'], ['b'], ['a'], ['a']], + field: { name: 'message', type: 'string', esTypes: ['text'] } as DataViewField, + isTextBased: true, + }); + expect(result).toMatchInlineSnapshot(` + Object { + "buckets": Array [ + Object { + "count": 3, + "key": "a", + }, + Object { + "count": 1, + "key": "b", + }, + ], + "sampledDocuments": 4, + "sampledValues": 4, + } + `); + expect(() => + getFieldExampleBuckets({ + values: [['a'], ['b'], ['a'], ['a']], + field: { name: 'message', type: 'string', esTypes: ['keyword'] } as DataViewField, + isTextBased: true, + }) + ).toThrowError(); + }); }); }); diff --git a/packages/kbn-unified-field-list/src/services/field_stats/field_examples_calculator.ts b/packages/kbn-unified-field-list/src/services/field_examples_calculator/field_examples_calculator.ts similarity index 79% rename from packages/kbn-unified-field-list/src/services/field_stats/field_examples_calculator.ts rename to packages/kbn-unified-field-list/src/services/field_examples_calculator/field_examples_calculator.ts index 982321284ab166..ac03aff1387dbe 100644 --- a/packages/kbn-unified-field-list/src/services/field_stats/field_examples_calculator.ts +++ b/packages/kbn-unified-field-list/src/services/field_examples_calculator/field_examples_calculator.ts @@ -13,52 +13,30 @@ import * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { DataView, DataViewField } from '@kbn/data-views-plugin/common'; import { flattenHit } from '@kbn/data-service/src/search/tabify'; import { FieldFormat } from '@kbn/field-formats-plugin/common'; +import { canProvideExamplesForField } from '../../utils/can_provide_stats'; +import { DEFAULT_SIMPLE_EXAMPLES_SIZE } from '../../constants'; type FieldHitValue = any; -interface FieldValueCountsParams { - hits: estypes.SearchHit[]; - dataView: DataView; +export interface FieldValueCountsParams { + values: FieldHitValue[]; field: DataViewField; count?: number; + isTextBased: boolean; } -export const canProvideExamplesForField = (field: DataViewField): boolean => { - if (field.name === '_score') { - return false; - } - return [ - 'string', - 'text', - 'keyword', - 'version', - 'ip', - 'number', - 'geo_point', - 'geo_shape', - ].includes(field.type); -}; - -export const showExamplesForField = (field: DataViewField): boolean => { - return ( - (!field.aggregatable && canProvideExamplesForField(field)) || - field.type === 'geo_point' || - field.type === 'geo_shape' - ); -}; - export function getFieldExampleBuckets(params: FieldValueCountsParams, formatter?: FieldFormat) { params = defaults(params, { - count: 5, + count: DEFAULT_SIMPLE_EXAMPLES_SIZE, }); - if (!canProvideExamplesForField(params.field)) { + if (!canProvideExamplesForField(params.field, params.isTextBased)) { throw new Error( `Analysis is not available this field type: "${params.field.type}". Field name: "${params.field.name}"` ); } - const records = getFieldValues(params.hits, params.field, params.dataView); + const records = params.values; const { groups, sampledValues } = groupValues(records, formatter); const buckets = sortBy(groups, ['count', 'order']) @@ -69,7 +47,7 @@ export function getFieldExampleBuckets(params: FieldValueCountsParams, formatter return { buckets, sampledValues, - sampledDocuments: params.hits.length, + sampledDocuments: params.values.length, }; } @@ -78,6 +56,9 @@ export function getFieldValues( field: DataViewField, dataView: DataView ): FieldHitValue[] { + if (!field?.name) { + return []; + } return map(hits, function (hit) { return flattenHit(hit, dataView, { includeIgnoredValues: true })[field.name]; }); diff --git a/packages/kbn-unified-field-list/src/services/field_examples_calculator/index.ts b/packages/kbn-unified-field-list/src/services/field_examples_calculator/index.ts new file mode 100644 index 00000000000000..61d08bbd28ac01 --- /dev/null +++ b/packages/kbn-unified-field-list/src/services/field_examples_calculator/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export { getFieldExampleBuckets, getFieldValues } from './field_examples_calculator'; diff --git a/packages/kbn-unified-field-list/src/services/field_stats/field_stats_utils.test.ts b/packages/kbn-unified-field-list/src/services/field_stats/field_stats_utils.test.ts index 9f868b4c6b60aa..3b6cb8fa8f64b9 100644 --- a/packages/kbn-unified-field-list/src/services/field_stats/field_stats_utils.test.ts +++ b/packages/kbn-unified-field-list/src/services/field_stats/field_stats_utils.test.ts @@ -646,6 +646,7 @@ describe('fieldStatsUtils', function () { "sampledDocuments": 2, "sampledValues": 2, "topValues": Object { + "areExamples": true, "buckets": Array [ Object { "count": 1, diff --git a/packages/kbn-unified-field-list/src/services/field_stats/field_stats_utils.ts b/packages/kbn-unified-field-list/src/services/field_stats/field_stats_utils.ts index 7f99287f8e4b7a..4207c6fbdb2216 100644 --- a/packages/kbn-unified-field-list/src/services/field_stats/field_stats_utils.ts +++ b/packages/kbn-unified-field-list/src/services/field_stats/field_stats_utils.ts @@ -12,11 +12,18 @@ import type { DataView, DataViewField } from '@kbn/data-views-plugin/common'; import type { ESSearchResponse } from '@kbn/es-types'; import { FieldFormat } from '@kbn/field-formats-plugin/common'; import type { FieldStatsResponse } from '../../types'; +import { getFieldExampleBuckets, getFieldValues } from '../field_examples_calculator'; import { - getFieldExampleBuckets, canProvideExamplesForField, - showExamplesForField, -} from './field_examples_calculator'; + canProvideNumberSummaryForField, + canProvideAggregatedStatsForField, +} from '../../utils/can_provide_stats'; +import { + SHARD_SIZE, + DEFAULT_TOP_VALUES_SIZE, + SIMPLE_EXAMPLES_FETCH_SIZE, + DEFAULT_SIMPLE_EXAMPLES_SIZE, +} from '../../constants'; export type SearchHandler = ({ aggs, @@ -28,10 +35,6 @@ export type SearchHandler = ({ size?: number; }) => Promise>; -const SHARD_SIZE = 5000; -const DEFAULT_TOP_VALUES_SIZE = 10; -const SIMPLE_EXAMPLES_SIZE = 100; - export function buildSearchParams({ dataViewPattern, timeFieldName, @@ -110,7 +113,7 @@ export async function fetchAndCalculateFieldStats({ size?: number; }) { if (!field.aggregatable) { - return canProvideExamplesForField(field) + return canProvideExamplesForField(field, false) ? await getSimpleExamples(searchHandler, field, dataView) : {}; } @@ -119,7 +122,7 @@ export async function fetchAndCalculateFieldStats({ return await getGeoExamples(searchHandler, field, dataView); } - if (!canProvideAggregatedStatsForField(field)) { + if (!canProvideAggregatedStatsForField(field, false)) { return {}; } @@ -127,7 +130,7 @@ export async function fetchAndCalculateFieldStats({ return await getNumberHistogram(searchHandler, field, false); } - if (canProvideNumberSummaryForField(field)) { + if (canProvideNumberSummaryForField(field, false)) { return await getNumberSummary(searchHandler, field); } @@ -142,27 +145,6 @@ export async function fetchAndCalculateFieldStats({ return await getStringSamples(searchHandler, field, size); } -function canProvideAggregatedStatsForField(field: DataViewField): boolean { - return !( - field.type === 'document' || - field.type.includes('range') || - field.type === 'geo_point' || - field.type === 'geo_shape' || - field.type === 'murmur3' || - field.type === 'attachment' - ); -} - -export function canProvideStatsForField(field: DataViewField): boolean { - return ( - (field.aggregatable && canProvideAggregatedStatsForField(field)) || showExamplesForField(field) - ); -} - -export function canProvideNumberSummaryForField(field: DataViewField): boolean { - return field.timeSeriesMetric === 'counter'; -} - export async function getNumberSummary( aggSearchWithBody: SearchHandler, field: DataViewField @@ -423,17 +405,17 @@ export async function getSimpleExamples( const fieldRef = getFieldRef(field); const simpleExamplesBody = { - size: SIMPLE_EXAMPLES_SIZE, + size: SIMPLE_EXAMPLES_FETCH_SIZE, fields: [fieldRef], }; const simpleExamplesResult = await search(simpleExamplesBody); const fieldExampleBuckets = getFieldExampleBuckets( { - hits: simpleExamplesResult.hits.hits, + values: getFieldValues(simpleExamplesResult.hits.hits, field, dataView), field, - dataView, - count: DEFAULT_TOP_VALUES_SIZE, + count: DEFAULT_SIMPLE_EXAMPLES_SIZE, + isTextBased: false, }, formatter ); @@ -444,6 +426,7 @@ export async function getSimpleExamples( sampledValues: fieldExampleBuckets.sampledValues, topValues: { buckets: fieldExampleBuckets.buckets, + areExamples: true, }, }; } catch (error) { diff --git a/packages/kbn-unified-field-list/src/services/field_stats_text_based/field_stats_utils_text_based.test.ts b/packages/kbn-unified-field-list/src/services/field_stats_text_based/field_stats_utils_text_based.test.ts new file mode 100644 index 00000000000000..8ebd9640302a2b --- /dev/null +++ b/packages/kbn-unified-field-list/src/services/field_stats_text_based/field_stats_utils_text_based.test.ts @@ -0,0 +1,129 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { DataViewField } from '@kbn/data-views-plugin/common'; +import { buildSearchFilter, fetchAndCalculateFieldStats } from './field_stats_utils_text_based'; + +describe('fieldStatsUtilsTextBased', function () { + describe('buildSearchFilter()', () => { + it('should create a time range filter', () => { + expect( + buildSearchFilter({ + timeFieldName: 'timestamp', + fromDate: '2022-12-05T23:00:00.000Z', + toDate: '2023-01-05T09:33:05.359Z', + }) + ).toMatchInlineSnapshot(` + Object { + "range": Object { + "timestamp": Object { + "format": "strict_date_optional_time", + "gte": "2022-12-05T23:00:00.000Z", + "lte": "2023-01-05T09:33:05.359Z", + }, + }, + } + `); + }); + it('should not create a time range filter', () => { + expect( + buildSearchFilter({ + timeFieldName: undefined, + fromDate: '2022-12-05T23:00:00.000Z', + toDate: '2023-01-05T09:33:05.359Z', + }) + ).toBeNull(); + }); + }); + + describe('fetchAndCalculateFieldStats()', () => { + it('should provide top values', async () => { + const searchHandler = jest.fn().mockResolvedValue({ + values: [ + [3, 'a'], + [1, 'b'], + ], + }); + expect( + await fetchAndCalculateFieldStats({ + searchHandler, + esqlBaseQuery: 'from logs* | limit 1000', + field: { name: 'message', type: 'string', esTypes: ['keyword'] } as DataViewField, + }) + ).toMatchInlineSnapshot(` + Object { + "sampledDocuments": 4, + "sampledValues": 4, + "topValues": Object { + "buckets": Array [ + Object { + "count": 3, + "key": "a", + }, + Object { + "count": 1, + "key": "b", + }, + ], + }, + "totalDocuments": 4, + } + `); + + expect(searchHandler).toHaveBeenCalledWith( + expect.objectContaining({ + query: + 'from logs* | limit 1000| WHERE `message` IS NOT NULL\n | STATS `message_terms` = count(`message`) BY `message`\n | SORT `message_terms` DESC\n | LIMIT 10', + }) + ); + }); + + it('should provide text examples', async () => { + const searchHandler = jest.fn().mockResolvedValue({ + values: [[['programming', 'cool']], ['elastic', 'cool']], + }); + expect( + await fetchAndCalculateFieldStats({ + searchHandler, + esqlBaseQuery: 'from logs* | limit 1000', + field: { name: 'message', type: 'string', esTypes: ['text'] } as DataViewField, + }) + ).toMatchInlineSnapshot(` + Object { + "sampledDocuments": 2, + "sampledValues": 4, + "topValues": Object { + "areExamples": true, + "buckets": Array [ + Object { + "count": 2, + "key": "cool", + }, + Object { + "count": 1, + "key": "elastic", + }, + Object { + "count": 1, + "key": "programming", + }, + ], + }, + "totalDocuments": 2, + } + `); + + expect(searchHandler).toHaveBeenCalledWith( + expect.objectContaining({ + query: + 'from logs* | limit 1000| WHERE `message` IS NOT NULL\n | KEEP `message`\n | LIMIT 100', + }) + ); + }); + }); +}); diff --git a/packages/kbn-unified-field-list/src/services/field_stats_text_based/field_stats_utils_text_based.ts b/packages/kbn-unified-field-list/src/services/field_stats_text_based/field_stats_utils_text_based.ts new file mode 100644 index 00000000000000..a5f41853d6e336 --- /dev/null +++ b/packages/kbn-unified-field-list/src/services/field_stats_text_based/field_stats_utils_text_based.ts @@ -0,0 +1,151 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { ESQLSearchReponse } from '@kbn/es-types'; +import type { DataViewField } from '@kbn/data-views-plugin/common'; +import type { FieldStatsResponse } from '../../types'; +import { + DEFAULT_TOP_VALUES_SIZE, + DEFAULT_SIMPLE_EXAMPLES_SIZE, + SIMPLE_EXAMPLES_FETCH_SIZE, +} from '../../constants'; +import { + canProvideStatsForFieldTextBased, + canProvideTopValuesForFieldTextBased, + canProvideExamplesForField, +} from '../../utils/can_provide_stats'; +import { getFieldExampleBuckets } from '../field_examples_calculator'; + +export type SearchHandlerTextBased = ({ query }: { query: string }) => Promise; + +export function buildSearchFilter({ + timeFieldName, + fromDate, + toDate, +}: { + timeFieldName?: string; + fromDate: string; + toDate: string; +}) { + return timeFieldName + ? { + range: { + [timeFieldName]: { + gte: fromDate, + lte: toDate, + format: 'strict_date_optional_time', + }, + }, + } + : null; +} + +interface FetchAndCalculateFieldStatsParams { + searchHandler: SearchHandlerTextBased; + field: DataViewField; + esqlBaseQuery: string; +} + +export async function fetchAndCalculateFieldStats(params: FetchAndCalculateFieldStatsParams) { + const { field } = params; + if (!canProvideStatsForFieldTextBased(field)) { + return {}; + } + if (field.type === 'boolean') { + return await getStringTopValues(params, 3); + } + if (canProvideTopValuesForFieldTextBased(field)) { + return await getStringTopValues(params); + } + if (canProvideExamplesForField(field, true)) { + return await getSimpleTextExamples(params); + } + + return {}; +} + +export async function getStringTopValues( + params: FetchAndCalculateFieldStatsParams, + size = DEFAULT_TOP_VALUES_SIZE +): Promise> { + const { searchHandler, field, esqlBaseQuery } = params; + const esqlQuery = + esqlBaseQuery + + `| WHERE ${getSafeESQLFieldName(field.name)} IS NOT NULL + | STATS ${getSafeESQLFieldName(`${field.name}_terms`)} = count(${getSafeESQLFieldName( + field.name + )}) BY ${getSafeESQLFieldName(field.name)} + | SORT ${getSafeESQLFieldName(`${field.name}_terms`)} DESC + | LIMIT ${size}`; + + const result = await searchHandler({ query: esqlQuery }); + const values = result?.values as Array<[number, string]>; + + if (!values?.length) { + return {}; + } + + const sampledValues = values?.reduce((acc: number, row) => acc + row[0], 0); + + const topValues = { + buckets: values.map((value) => ({ + count: value[0], + key: value[1], + })), + }; + + return { + totalDocuments: sampledValues, + sampledDocuments: sampledValues, + sampledValues, + topValues, + }; +} + +export async function getSimpleTextExamples( + params: FetchAndCalculateFieldStatsParams +): Promise> { + const { searchHandler, field, esqlBaseQuery } = params; + const esqlQuery = + esqlBaseQuery + + `| WHERE ${getSafeESQLFieldName(field.name)} IS NOT NULL + | KEEP ${getSafeESQLFieldName(field.name)} + | LIMIT ${SIMPLE_EXAMPLES_FETCH_SIZE}`; + + const result = await searchHandler({ query: esqlQuery }); + const values = ((result?.values as Array<[string | string[]]>) || []).map((value) => + Array.isArray(value) && value.length === 1 ? value[0] : value + ); + + if (!values?.length) { + return {}; + } + + const sampledDocuments = values?.length; + + const fieldExampleBuckets = getFieldExampleBuckets({ + values, + field, + count: DEFAULT_SIMPLE_EXAMPLES_SIZE, + isTextBased: true, + }); + + return { + totalDocuments: sampledDocuments, + sampledDocuments: fieldExampleBuckets.sampledDocuments, + sampledValues: fieldExampleBuckets.sampledValues, + topValues: { + buckets: fieldExampleBuckets.buckets, + areExamples: true, + }, + }; +} + +function getSafeESQLFieldName(str: string): string { + return `\`${str}\``; +} diff --git a/packages/kbn-unified-field-list/src/services/field_stats_text_based/index.ts b/packages/kbn-unified-field-list/src/services/field_stats_text_based/index.ts new file mode 100644 index 00000000000000..4fc354e7575def --- /dev/null +++ b/packages/kbn-unified-field-list/src/services/field_stats_text_based/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export { loadFieldStatsTextBased } from './load_field_stats_text_based'; diff --git a/packages/kbn-unified-field-list/src/services/field_stats_text_based/load_field_stats_text_based.ts b/packages/kbn-unified-field-list/src/services/field_stats_text_based/load_field_stats_text_based.ts new file mode 100644 index 00000000000000..347bdba7238086 --- /dev/null +++ b/packages/kbn-unified-field-list/src/services/field_stats_text_based/load_field_stats_text_based.ts @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { lastValueFrom } from 'rxjs'; +import type { DataView, DataViewField } from '@kbn/data-views-plugin/common'; +import type { + DataPublicPluginStart, + IKibanaSearchRequest, + IKibanaSearchResponse, +} from '@kbn/data-plugin/public'; +import type { ESQLSearchParams, ESQLSearchReponse } from '@kbn/es-types'; +import type { AggregateQuery } from '@kbn/es-query'; +import { getESQLWithSafeLimit } from '@kbn/esql-utils'; +import type { FieldStatsResponse } from '../../types'; +import { + buildSearchFilter, + SearchHandlerTextBased, + fetchAndCalculateFieldStats, +} from './field_stats_utils_text_based'; +import { ESQL_SAFE_LIMIT } from '../../constants'; + +interface FetchFieldStatsParamsTextBased { + services: { + data: DataPublicPluginStart; + }; + dataView: DataView; + field: DataViewField; + fromDate: string; + toDate: string; + baseQuery: AggregateQuery; + abortController?: AbortController; +} + +export type LoadFieldStatsTextBasedHandler = ( + params: FetchFieldStatsParamsTextBased +) => Promise>; + +/** + * Loads and aggregates stats data for an ES|QL query field + * @param services + * @param dataView + * @param field + * @param fromDate + * @param toDate + * @param baseQuery + * @param abortController + */ +export const loadFieldStatsTextBased: LoadFieldStatsTextBasedHandler = async ({ + services, + dataView, + field, + fromDate, + toDate, + baseQuery, + abortController, +}) => { + const { data } = services; + + try { + if (!dataView?.id || !field?.type) { + return {}; + } + + const searchHandler: SearchHandlerTextBased = async (body) => { + const filter = buildSearchFilter({ timeFieldName: dataView.timeFieldName, fromDate, toDate }); + const result = await lastValueFrom( + data.search.search< + IKibanaSearchRequest, + IKibanaSearchResponse + >( + { + params: { + ...(filter ? { filter } : {}), + ...body, + }, + }, + { + abortSignal: abortController?.signal, + strategy: 'esql', + } + ) + ); + return result.rawResponse; + }; + + if (!('esql' in baseQuery)) { + throw new Error('query must be of type AggregateQuery'); + } + + return await fetchAndCalculateFieldStats({ + searchHandler, + field, + esqlBaseQuery: getESQLWithSafeLimit(baseQuery.esql, ESQL_SAFE_LIMIT), + }); + } catch (error) { + // console.error(error); + throw new Error('Could not provide field stats', { cause: error }); + } +}; diff --git a/packages/kbn-unified-field-list/src/types.ts b/packages/kbn-unified-field-list/src/types.ts index 7d9403eac1cf29..c4e1d4cd3f9993 100755 --- a/packages/kbn-unified-field-list/src/types.ts +++ b/packages/kbn-unified-field-list/src/types.ts @@ -15,6 +15,7 @@ export interface BucketedAggregation { key: KeyType; count: number; }>; + areExamples?: boolean; // whether `topValues` holds examples in buckets rather than top values } export interface NumberSummary { diff --git a/packages/kbn-unified-field-list/src/utils/can_provide_stats.test.ts b/packages/kbn-unified-field-list/src/utils/can_provide_stats.test.ts new file mode 100644 index 00000000000000..2ade697c5976a1 --- /dev/null +++ b/packages/kbn-unified-field-list/src/utils/can_provide_stats.test.ts @@ -0,0 +1,164 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { + canProvideStatsForField, + canProvideExamplesForField, + canProvideStatsForFieldTextBased, +} from './can_provide_stats'; +import type { DataViewField } from '@kbn/data-views-plugin/common'; +import { stubLogstashDataView as dataView } from '@kbn/data-views-plugin/common/data_view.stub'; + +describe('can_provide_stats', function () { + describe('canProvideStatsForField', function () { + it('works for data view fields', function () { + expect(canProvideStatsForField(dataView.fields.getByName('extension.keyword')!, false)).toBe( + true + ); + expect(canProvideStatsForField(dataView.fields.getByName('non-sortable')!, false)).toBe(true); + expect(canProvideStatsForField(dataView.fields.getByName('bytes')!, false)).toBe(true); + expect(canProvideStatsForField(dataView.fields.getByName('ip')!, false)).toBe(true); + expect(canProvideStatsForField(dataView.fields.getByName('ssl')!, false)).toBe(true); + expect(canProvideStatsForField(dataView.fields.getByName('@timestamp')!, false)).toBe(true); + expect(canProvideStatsForField(dataView.fields.getByName('geo.coordinates')!, false)).toBe( + true + ); + expect(canProvideStatsForField(dataView.fields.getByName('request_body')!, false)).toBe( + false + ); + }); + + it('works for text based columns', function () { + expect( + canProvideStatsForField( + { name: 'message', type: 'string', esTypes: ['text'] } as DataViewField, + true + ) + ).toBe(true); + expect( + canProvideStatsForField( + { name: 'message', type: 'string', esTypes: ['keyword'] } as DataViewField, + true + ) + ).toBe(true); + expect( + canProvideStatsForField({ name: 'message', type: 'number' } as DataViewField, true) + ).toBe(true); + expect( + canProvideStatsForField({ name: 'message', type: 'boolean' } as DataViewField, true) + ).toBe(true); + expect(canProvideStatsForField({ name: 'message', type: 'ip' } as DataViewField, true)).toBe( + true + ); + expect( + canProvideStatsForField({ name: 'message', type: 'geo_point' } as DataViewField, true) + ).toBe(true); + expect( + canProvideStatsForField( + { name: '_id', type: 'string', esTypes: ['keyword'] } as DataViewField, + true + ) + ).toBe(true); + + expect( + canProvideStatsForField({ name: 'message', type: 'date' } as DataViewField, true) + ).toBe(false); + }); + }); + + describe('canProvideExamplesForField', function () { + it('works for data view fields', function () { + expect(canProvideExamplesForField(dataView.fields.getByName('non-sortable')!, false)).toBe( + true + ); + expect(canProvideExamplesForField(dataView.fields.getByName('geo.coordinates')!, false)).toBe( + true + ); + }); + + it('works for text based columns', function () { + expect( + canProvideExamplesForField( + { name: 'message', type: 'string', esTypes: ['text'] } as DataViewField, + true + ) + ).toBe(true); + expect( + canProvideExamplesForField( + { name: 'message', type: 'string', esTypes: ['keyword'] } as DataViewField, + true + ) + ).toBe(false); + expect( + canProvideExamplesForField({ name: 'message', type: 'number' } as DataViewField, true) + ).toBe(false); + expect( + canProvideExamplesForField({ name: 'message', type: 'boolean' } as DataViewField, true) + ).toBe(false); + expect( + canProvideExamplesForField({ name: 'message', type: 'ip' } as DataViewField, true) + ).toBe(false); + expect( + canProvideExamplesForField({ name: 'message', type: 'geo_point' } as DataViewField, true) + ).toBe(true); + expect( + canProvideExamplesForField({ name: 'message', type: 'date' } as DataViewField, true) + ).toBe(false); + expect( + canProvideStatsForField( + { name: '_id', type: 'string', esTypes: ['keyword'] } as DataViewField, + true + ) + ).toBe(true); + }); + + describe('canProvideStatsForFieldTextBased', function () { + it('works for text based columns', function () { + expect( + canProvideStatsForFieldTextBased({ + name: 'message', + type: 'string', + esTypes: ['text'], + } as DataViewField) + ).toBe(true); + expect( + canProvideStatsForFieldTextBased({ + name: 'message', + type: 'string', + esTypes: ['keyword'], + } as DataViewField) + ).toBe(true); + expect( + canProvideStatsForFieldTextBased({ name: 'message', type: 'number' } as DataViewField) + ).toBe(true); + expect( + canProvideStatsForFieldTextBased({ name: 'message', type: 'boolean' } as DataViewField) + ).toBe(true); + expect( + canProvideStatsForFieldTextBased({ name: 'message', type: 'ip' } as DataViewField) + ).toBe(true); + expect( + canProvideStatsForFieldTextBased({ name: 'message', type: 'ip_range' } as DataViewField) + ).toBe(false); + expect( + canProvideStatsForFieldTextBased({ name: 'message', type: 'geo_point' } as DataViewField) + ).toBe(true); + expect( + canProvideStatsForFieldTextBased({ name: 'message', type: 'date' } as DataViewField) + ).toBe(false); + expect( + canProvideStatsForFieldTextBased({ + name: '_id', + type: 'string', + esTypes: ['keyword'], + } as DataViewField) + ).toBe(true); + }); + }); + }); +}); diff --git a/packages/kbn-unified-field-list/src/utils/can_provide_stats.ts b/packages/kbn-unified-field-list/src/utils/can_provide_stats.ts new file mode 100644 index 00000000000000..714dfb4dfeb194 --- /dev/null +++ b/packages/kbn-unified-field-list/src/utils/can_provide_stats.ts @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { DataViewField } from '@kbn/data-views-plugin/common'; + +export function canProvideStatsForField(field: DataViewField, isTextBased: boolean): boolean { + if (isTextBased) { + return canProvideStatsForFieldTextBased(field); + } + return ( + (field.aggregatable && canProvideAggregatedStatsForField(field, isTextBased)) || + ((!field.aggregatable || field.type === 'geo_point' || field.type === 'geo_shape') && + canProvideExamplesForField(field, isTextBased)) + ); +} + +export function canProvideAggregatedStatsForField( + field: DataViewField, + isTextBased: boolean +): boolean { + if (isTextBased) { + return false; + } + return !( + field.type === 'document' || + field.type.includes('range') || + field.type === 'geo_point' || + field.type === 'geo_shape' || + field.type === 'murmur3' || + field.type === 'attachment' + ); +} + +export function canProvideNumberSummaryForField( + field: DataViewField, + isTextBased: boolean +): boolean { + if (isTextBased) { + return false; + } + return field.timeSeriesMetric === 'counter'; +} + +export function canProvideExamplesForField(field: DataViewField, isTextBased: boolean): boolean { + if (isTextBased) { + return ( + (field.type === 'string' && !canProvideTopValuesForFieldTextBased(field)) || + ['geo_point', 'geo_shape'].includes(field.type) + ); + } + if (field.name === '_score') { + return false; + } + return [ + 'string', + 'text', + 'keyword', + 'version', + 'ip', + 'number', + 'geo_point', + 'geo_shape', + ].includes(field.type); +} + +export function canProvideTopValuesForFieldTextBased(field: DataViewField): boolean { + if (field.name === '_id') { + return false; + } + const esTypes = field.esTypes?.[0]; + return ( + Boolean(field.type === 'string' && esTypes && ['keyword', 'version'].includes(esTypes)) || + ['keyword', 'version', 'ip', 'number', 'boolean'].includes(field.type) + ); +} + +export function canProvideStatsForFieldTextBased(field: DataViewField): boolean { + return canProvideTopValuesForFieldTextBased(field) || canProvideExamplesForField(field, true); +} diff --git a/packages/kbn-unified-field-list/tsconfig.json b/packages/kbn-unified-field-list/tsconfig.json index dce08dcdebaf5d..657498734e3d8f 100644 --- a/packages/kbn-unified-field-list/tsconfig.json +++ b/packages/kbn-unified-field-list/tsconfig.json @@ -33,6 +33,7 @@ "@kbn/field-utils", "@kbn/ml-ui-actions", "@kbn/visualization-utils", + "@kbn/esql-utils" ], "exclude": ["target/**/*"] } diff --git a/src/plugins/data/common/search/expressions/esql.ts b/src/plugins/data/common/search/expressions/esql.ts index d01b77087bf043..8b063b7e734c33 100644 --- a/src/plugins/data/common/search/expressions/esql.ts +++ b/src/plugins/data/common/search/expressions/esql.ts @@ -233,7 +233,7 @@ export const getEsqlFn = ({ getStartDependencies }: EsqlFnArguments) => { (body.all_columns ?? body.columns)?.map(({ name, type }) => ({ id: name, name, - meta: { type: esFieldTypeToKibanaFieldType(type) }, + meta: { type: esFieldTypeToKibanaFieldType(type), esType: type }, isNull: hasEmptyColumns ? !lookup.has(name) : false, })) ?? []; diff --git a/src/plugins/discover/public/application/main/components/layout/discover_documents.tsx b/src/plugins/discover/public/application/main/components/layout/discover_documents.tsx index 22f174cd0dea7f..4c0f01d04b6ebb 100644 --- a/src/plugins/discover/public/application/main/components/layout/discover_documents.tsx +++ b/src/plugins/discover/public/application/main/components/layout/discover_documents.tsx @@ -24,8 +24,8 @@ import { SearchResponseWarningsCallout } from '@kbn/search-response-warnings'; import { DataLoadingState, useColumns, - type DataTableColumnTypes, - getTextBasedColumnTypes, + type DataTableColumnsMeta, + getTextBasedColumnsMeta, } from '@kbn/unified-data-table'; import { DOC_HIDE_TIME_COLUMN_SETTING, @@ -231,10 +231,10 @@ function DiscoverDocumentsComponent({ [uiSettings] ); - const columnTypes: DataTableColumnTypes | undefined = useMemo( + const columnsMeta: DataTableColumnsMeta | undefined = useMemo( () => documentState.textBasedQueryColumns - ? getTextBasedColumnTypes(documentState.textBasedQueryColumns) + ? getTextBasedColumnsMeta(documentState.textBasedQueryColumns) : undefined, [documentState.textBasedQueryColumns] ); @@ -244,7 +244,7 @@ function DiscoverDocumentsComponent({ hit: DataTableRecord, displayedRows: DataTableRecord[], displayedColumns: string[], - customColumnTypes?: DataTableColumnTypes + customColumnsMeta?: DataTableColumnsMeta ) => ( ; + columnsMeta?: DataTableColumnsMeta; hit: DataTableRecord; hits?: DataTableRecord[]; dataView: DataView; @@ -61,7 +62,7 @@ export function DiscoverGridFlyout({ hits, dataView, columns, - columnTypes, + columnsMeta, savedSearchId, filters, query, @@ -147,7 +148,7 @@ export function DiscoverGridFlyout({ () => ( ( ; * Datatable column meta information */ export interface DatatableColumnMeta { + /** + * The Kibana normalized type of the column + */ type: DatatableColumnType; + /** + * The original type of the column from ES + */ + esType?: string; /** * field this column is based on */ diff --git a/src/plugins/unified_doc_viewer/public/components/doc_viewer_table/table.tsx b/src/plugins/unified_doc_viewer/public/components/doc_viewer_table/table.tsx index b8e5b9cd379708..ad957053b7dd88 100644 --- a/src/plugins/unified_doc_viewer/public/components/doc_viewer_table/table.tsx +++ b/src/plugins/unified_doc_viewer/public/components/doc_viewer_table/table.tsx @@ -38,7 +38,11 @@ import { isNestedFieldParent, usePager, } from '@kbn/discover-utils'; -import { fieldNameWildcardMatcher, getFieldSearchMatchingHighlight } from '@kbn/field-utils'; +import { + fieldNameWildcardMatcher, + getFieldSearchMatchingHighlight, + getTextBasedColumnIconType, +} from '@kbn/field-utils'; import type { DocViewRenderProps, FieldRecordLegacy } from '@kbn/unified-doc-viewer/types'; import { FieldName } from '@kbn/unified-doc-viewer'; import { getUnifiedDocViewerServices } from '../../plugin'; @@ -106,7 +110,7 @@ const updateSearchText = debounce( export const DocViewerTable = ({ columns, - columnTypes, + columnsMeta, hit, dataView, hideActionsColumn, @@ -167,8 +171,10 @@ export const DocViewerTable = ({ (field: string) => { const fieldMapping = mapping(field); const displayName = fieldMapping?.displayName ?? field; - const fieldType = columnTypes - ? columnTypes[field] // for text-based results types come separately + const columnMeta = columnsMeta?.[field]; + const columnIconType = getTextBasedColumnIconType(columnMeta); + const fieldType = columnIconType + ? columnIconType // for text-based results types come separately : isNestedFieldParent(field, dataView) ? 'nested' : fieldMapping @@ -212,7 +218,7 @@ export const DocViewerTable = ({ onToggleColumn, filter, columns, - columnTypes, + columnsMeta, flattened, pinnedFields, onTogglePinned, diff --git a/test/functional/apps/discover/group2/_data_grid_field_tokens.ts b/test/functional/apps/discover/group2/_data_grid_field_tokens.ts index 27ba295e488986..1d60fe3cd8c79d 100644 --- a/test/functional/apps/discover/group2/_data_grid_field_tokens.ts +++ b/test/functional/apps/discover/group2/_data_grid_field_tokens.ts @@ -130,19 +130,19 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.unifiedFieldList.clickFieldListItemAdd('ip'); await PageObjects.unifiedFieldList.clickFieldListItemAdd('geo.coordinates'); - expect(await findFirstColumnTokens()).to.eql(['Number', 'String', 'IP address', 'Geo point']); + expect(await findFirstColumnTokens()).to.eql(['Number', 'Text', 'IP address', 'Geo point']); expect(await findFirstDocViewerTokens()).to.eql([ - 'String', - 'String', + 'Text', + 'Text', 'Date', - 'String', + 'Text', 'Number', 'IP address', - 'String', + 'Text', 'Geo point', - 'String', - 'String', + 'Keyword', + 'Keyword', ]); }); diff --git a/test/functional/apps/discover/group3/_sidebar.ts b/test/functional/apps/discover/group3/_sidebar.ts index c81722ec78cbfc..d536e099205901 100644 --- a/test/functional/apps/discover/group3/_sidebar.ts +++ b/test/functional/apps/discover/group3/_sidebar.ts @@ -96,9 +96,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); it('should show filters by type in text-based view', async function () { - await kibanaServer.uiSettings.update({ 'discover:enableESQL': true }); - await browser.refresh(); - await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded(); await PageObjects.unifiedFieldList.openSidebarFieldFilter(); let options = await find.allByCssSelector('[data-test-subj*="typeFilter"]'); @@ -114,7 +111,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded(); await PageObjects.unifiedFieldList.openSidebarFieldFilter(); options = await find.allByCssSelector('[data-test-subj*="typeFilter"]'); - expect(options).to.have.length(5); + expect(options).to.have.length(6); expect(await PageObjects.unifiedFieldList.getSidebarAriaDescription()).to.be( '82 available fields.' @@ -131,9 +128,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); it('should show empty fields in text-based view', async function () { - await kibanaServer.uiSettings.update({ 'discover:enableESQL': true }); - await browser.refresh(); - await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded(); await PageObjects.discover.selectTextBaseLang(); @@ -433,9 +427,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); it('should show selected and available fields in text-based mode', async function () { - await kibanaServer.uiSettings.update({ 'discover:enableESQL': true }); - await browser.refresh(); - await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded(); expect(await PageObjects.unifiedFieldList.getSidebarAriaDescription()).to.be( diff --git a/test/functional/apps/discover/group3/_sidebar_field_stats.ts b/test/functional/apps/discover/group3/_sidebar_field_stats.ts new file mode 100644 index 00000000000000..7923bc311dedcc --- /dev/null +++ b/test/functional/apps/discover/group3/_sidebar_field_stats.ts @@ -0,0 +1,305 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../ftr_provider_context'; + +export default function ({ getService, getPageObjects }: FtrProviderContext) { + const esArchiver = getService('esArchiver'); + const kibanaServer = getService('kibanaServer'); + const PageObjects = getPageObjects([ + 'common', + 'discover', + 'timePicker', + 'header', + 'unifiedFieldList', + ]); + const testSubjects = getService('testSubjects'); + const find = getService('find'); + const monacoEditor = getService('monacoEditor'); + const retry = getService('retry'); + + describe('discover sidebar field stats popover', function describeIndexTests() { + before(async function () { + await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/logstash_functional'); + await kibanaServer.importExport.load('test/functional/fixtures/kbn_archiver/discover'); + await kibanaServer.uiSettings.replace({ + defaultIndex: 'logstash-*', + }); + }); + + after(async () => { + await kibanaServer.importExport.unload('test/functional/fixtures/kbn_archiver/discover'); + await kibanaServer.savedObjects.cleanStandardList(); + await kibanaServer.uiSettings.replace({}); + }); + + describe('data view fields', function () { + before(async () => { + await PageObjects.timePicker.setDefaultAbsoluteRangeViaUiSettings(); + await PageObjects.common.navigateToApp('discover'); + await PageObjects.discover.waitUntilSearchingHasFinished(); + + await PageObjects.discover.addRuntimeField( + '_is_large', + 'emit(doc["bytes"].value > 1024)', + 'boolean' + ); + + await retry.waitFor('form to close', async () => { + return !(await testSubjects.exists('fieldEditor')); + }); + + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded(); + }); + + it('should show a top values popover for a boolean runtime field', async () => { + await PageObjects.unifiedFieldList.clickFieldListItem('_is_large'); + await testSubjects.existOrFail('dscFieldStats-topValues'); + expect(await testSubjects.getVisibleText('dscFieldStats-title')).to.be('Top values'); + const topValuesRows = await testSubjects.findAll('dscFieldStats-topValues-bucket'); + expect(topValuesRows.length).to.eql(2); + await testSubjects.missingOrFail('unifiedFieldStats-buttonGroup'); + await testSubjects.missingOrFail('unifiedFieldStats-histogram'); + expect(await testSubjects.getVisibleText('dscFieldStats-statsFooter')).to.contain( + '14,004 records' + ); + await PageObjects.unifiedFieldList.closeFieldPopover(); + }); + + it('should show a histogram and top values popover for numeric field', async () => { + await PageObjects.unifiedFieldList.clickFieldListItem('bytes'); + await testSubjects.existOrFail('dscFieldStats-topValues'); + expect(await testSubjects.getVisibleText('dscFieldStats-title')).to.contain('Top values'); + expect(await testSubjects.getVisibleText('dscFieldStats-title')).to.contain('Distribution'); + const topValuesRows = await testSubjects.findAll('dscFieldStats-topValues-bucket'); + expect(topValuesRows.length).to.eql(11); + await testSubjects.click('dscFieldStats-buttonGroup-distributionButton'); + expect( + await find.existsByCssSelector('[data-test-subj="unifiedFieldStats-histogram"] .echChart') + ).to.eql(true); + expect(await testSubjects.getVisibleText('dscFieldStats-statsFooter')).to.contain( + '14,004 records' + ); + await PageObjects.unifiedFieldList.closeFieldPopover(); + }); + + it('should show a top values popover for a keyword field', async () => { + await PageObjects.unifiedFieldList.clickFieldListItem('extension'); + await testSubjects.existOrFail('dscFieldStats-topValues'); + expect(await testSubjects.getVisibleText('dscFieldStats-title')).to.be('Top values'); + const topValuesRows = await testSubjects.findAll('dscFieldStats-topValues-bucket'); + expect(topValuesRows.length).to.eql(5); + await testSubjects.missingOrFail('unifiedFieldStats-buttonGroup'); + await testSubjects.missingOrFail('unifiedFieldStats-histogram'); + expect(await testSubjects.getVisibleText('dscFieldStats-statsFooter')).to.contain( + '14,004 records' + ); + await PageObjects.unifiedFieldList.closeFieldPopover(); + }); + + it('should show a top values popover for an ip field', async () => { + await PageObjects.unifiedFieldList.clickFieldListItem('clientip'); + await testSubjects.existOrFail('dscFieldStats-topValues'); + expect(await testSubjects.getVisibleText('dscFieldStats-title')).to.be('Top values'); + const topValuesRows = await testSubjects.findAll('dscFieldStats-topValues-bucket'); + expect(topValuesRows.length).to.eql(11); + await testSubjects.missingOrFail('unifiedFieldStats-buttonGroup'); + await testSubjects.missingOrFail('unifiedFieldStats-histogram'); + expect(await testSubjects.getVisibleText('dscFieldStats-statsFooter')).to.contain( + '14,004 records' + ); + await PageObjects.unifiedFieldList.closeFieldPopover(); + }); + + it('should show a date histogram popover for a date field', async () => { + await PageObjects.unifiedFieldList.clickFieldListItem('@timestamp'); + await testSubjects.existOrFail('unifiedFieldStats-timeDistribution'); + await testSubjects.missingOrFail('dscFieldStats-buttonGroup-topValuesButton'); + expect(await testSubjects.getVisibleText('dscFieldStats-statsFooter')).to.contain( + '14,004 records' + ); + await PageObjects.unifiedFieldList.closeFieldPopover(); + }); + + it('should show examples for geo points field', async () => { + await PageObjects.unifiedFieldList.clickFieldListItem('geo.coordinates'); + await testSubjects.existOrFail('dscFieldStats-topValues'); + expect(await testSubjects.getVisibleText('dscFieldStats-title')).to.be('Examples'); + const topValuesRows = await testSubjects.findAll('dscFieldStats-topValues-bucket'); + expect(topValuesRows.length).to.eql(11); + await testSubjects.missingOrFail('unifiedFieldStats-buttonGroup'); + await testSubjects.missingOrFail('unifiedFieldStats-histogram'); + expect(await testSubjects.getVisibleText('dscFieldStats-statsFooter')).to.contain( + '100 sample records' + ); + await PageObjects.unifiedFieldList.closeFieldPopover(); + }); + }); + + describe('text based columns', function () { + before(async () => { + const TEST_START_TIME = 'Sep 23, 2015 @ 06:31:44.000'; + const TEST_END_TIME = 'Sep 23, 2015 @ 18:31:44.000'; + await PageObjects.timePicker.setAbsoluteRange(TEST_START_TIME, TEST_END_TIME); + await PageObjects.common.navigateToApp('discover'); + await PageObjects.discover.waitUntilSearchingHasFinished(); + + await PageObjects.discover.selectTextBaseLang(); + + const testQuery = `from logstash-* [METADATA _index, _id] | sort @timestamp desc | limit 500`; + await monacoEditor.setCodeEditorValue(testQuery); + await testSubjects.click('querySubmitButton'); + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded(); + }); + + it('should show top values popover for numeric field', async () => { + await PageObjects.unifiedFieldList.clickFieldListItem('bytes'); + await testSubjects.existOrFail('dscFieldStats-topValues'); + expect(await testSubjects.getVisibleText('dscFieldStats-title')).to.be('Top values'); + const topValuesRows = await testSubjects.findAll('dscFieldStats-topValues-bucket'); + expect(topValuesRows.length).to.eql(10); + expect(await testSubjects.getVisibleText('dscFieldStats-statsFooter')).to.contain( + '42 sample values' + ); + await PageObjects.unifiedFieldList.closeFieldPopover(); + }); + + it('should show a top values popover for a keyword field', async () => { + await PageObjects.unifiedFieldList.clickFieldListItem('extension.raw'); + await testSubjects.existOrFail('dscFieldStats-topValues'); + expect(await testSubjects.getVisibleText('dscFieldStats-title')).to.be('Top values'); + const topValuesRows = await testSubjects.findAll('dscFieldStats-topValues-bucket'); + expect(topValuesRows.length).to.eql(5); + await testSubjects.missingOrFail('unifiedFieldStats-buttonGroup'); + await testSubjects.missingOrFail('unifiedFieldStats-histogram'); + expect(await testSubjects.getVisibleText('dscFieldStats-statsFooter')).to.contain( + '500 sample values' + ); + await PageObjects.unifiedFieldList.closeFieldPopover(); + }); + + it('should show a top values popover for an ip field', async () => { + await PageObjects.unifiedFieldList.clickFieldListItem('clientip'); + await testSubjects.existOrFail('dscFieldStats-topValues'); + expect(await testSubjects.getVisibleText('dscFieldStats-title')).to.be('Top values'); + const topValuesRows = await testSubjects.findAll('dscFieldStats-topValues-bucket'); + expect(topValuesRows.length).to.eql(10); + await testSubjects.missingOrFail('unifiedFieldStats-buttonGroup'); + await testSubjects.missingOrFail('unifiedFieldStats-histogram'); + expect(await testSubjects.getVisibleText('dscFieldStats-statsFooter')).to.contain( + '32 sample values' + ); + await PageObjects.unifiedFieldList.closeFieldPopover(); + }); + + it('should show a top values popover for _index field', async () => { + await PageObjects.unifiedFieldList.clickFieldListItem('_index'); + await testSubjects.existOrFail('dscFieldStats-topValues'); + expect(await testSubjects.getVisibleText('dscFieldStats-title')).to.be('Top values'); + const topValuesRows = await testSubjects.findAll('dscFieldStats-topValues-bucket'); + expect(topValuesRows.length).to.eql(1); + await testSubjects.missingOrFail('unifiedFieldStats-buttonGroup'); + await testSubjects.missingOrFail('unifiedFieldStats-histogram'); + expect(await testSubjects.getVisibleText('dscFieldStats-statsFooter')).to.contain( + '500 sample values' + ); + await PageObjects.unifiedFieldList.closeFieldPopover(); + }); + + it('should not have stats for a date field yet', async () => { + await PageObjects.unifiedFieldList.clickFieldListItem('@timestamp'); + await testSubjects.missingOrFail('dscFieldStats-statsFooter'); + await PageObjects.unifiedFieldList.closeFieldPopover(); + }); + + it('should show examples for geo points field', async () => { + await PageObjects.unifiedFieldList.clickFieldListItem('geo.coordinates'); + await testSubjects.existOrFail('dscFieldStats-topValues'); + expect(await testSubjects.getVisibleText('dscFieldStats-title')).to.be('Examples'); + const topValuesRows = await testSubjects.findAll('dscFieldStats-topValues-bucket'); + expect(topValuesRows.length).to.eql(11); + await testSubjects.missingOrFail('unifiedFieldStats-buttonGroup'); + await testSubjects.missingOrFail('unifiedFieldStats-histogram'); + expect(await testSubjects.getVisibleText('dscFieldStats-statsFooter')).to.contain( + '100 sample records' + ); + await PageObjects.unifiedFieldList.closeFieldPopover(); + }); + + it('should show examples for text field', async () => { + await PageObjects.unifiedFieldList.clickFieldListItem('extension'); + await testSubjects.existOrFail('dscFieldStats-topValues'); + expect(await testSubjects.getVisibleText('dscFieldStats-title')).to.be('Examples'); + const topValuesRows = await testSubjects.findAll('dscFieldStats-topValues-bucket'); + expect(topValuesRows.length).to.eql(5); + await testSubjects.missingOrFail('unifiedFieldStats-buttonGroup'); + await testSubjects.missingOrFail('unifiedFieldStats-histogram'); + expect(await testSubjects.getVisibleText('dscFieldStats-statsFooter')).to.contain( + '100 sample records' + ); + await PageObjects.unifiedFieldList.closeFieldPopover(); + }); + + it('should show examples for _id field', async () => { + await PageObjects.unifiedFieldList.clickFieldListItem('_id'); + await testSubjects.existOrFail('dscFieldStats-topValues'); + expect(await testSubjects.getVisibleText('dscFieldStats-title')).to.be('Examples'); + const topValuesRows = await testSubjects.findAll('dscFieldStats-topValues-bucket'); + expect(topValuesRows.length).to.eql(11); + await testSubjects.missingOrFail('unifiedFieldStats-buttonGroup'); + await testSubjects.missingOrFail('unifiedFieldStats-histogram'); + expect(await testSubjects.getVisibleText('dscFieldStats-statsFooter')).to.contain( + '100 sample records' + ); + await PageObjects.unifiedFieldList.closeFieldPopover(); + }); + + it('should show a top values popover for a more complex query', async () => { + const testQuery = `from logstash-* | sort @timestamp desc | limit 50 | stats avg(bytes) by geo.dest | limit 3`; + await monacoEditor.setCodeEditorValue(testQuery); + await testSubjects.click('querySubmitButton'); + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded(); + + await PageObjects.unifiedFieldList.clickFieldListItem('avg(bytes)'); + await testSubjects.existOrFail('dscFieldStats-topValues'); + expect(await testSubjects.getVisibleText('dscFieldStats-title')).to.be('Top values'); + const topValuesRows = await testSubjects.findAll('dscFieldStats-topValues-bucket'); + expect(topValuesRows.length).to.eql(3); + expect(await testSubjects.getVisibleText('dscFieldStats-statsFooter')).to.contain( + '3 sample values' + ); + await PageObjects.unifiedFieldList.closeFieldPopover(); + }); + + it('should show a top values popover for a boolean field', async () => { + const testQuery = `row enabled = true`; + await monacoEditor.setCodeEditorValue(testQuery); + await testSubjects.click('querySubmitButton'); + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded(); + + await PageObjects.unifiedFieldList.clickFieldListItem('enabled'); + await testSubjects.existOrFail('dscFieldStats-topValues'); + expect(await testSubjects.getVisibleText('dscFieldStats-title')).to.be('Top values'); + const topValuesRows = await testSubjects.findAll('dscFieldStats-topValues-bucket'); + expect(topValuesRows.length).to.eql(1); + expect(await PageObjects.unifiedFieldList.getFieldStatsTopValueBucketsVisibleText()).to.be( + 'true\n100%' + ); + expect(await testSubjects.getVisibleText('dscFieldStats-statsFooter')).to.contain( + '1 sample value' + ); + await PageObjects.unifiedFieldList.closeFieldPopover(); + }); + }); + }); +} diff --git a/test/functional/apps/discover/group3/index.ts b/test/functional/apps/discover/group3/index.ts index 60f835154d5e9f..2fe5a4ebb1db1a 100644 --- a/test/functional/apps/discover/group3/index.ts +++ b/test/functional/apps/discover/group3/index.ts @@ -24,6 +24,7 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./_time_field_column')); loadTestFile(require.resolve('./_drag_drop')); loadTestFile(require.resolve('./_sidebar')); + loadTestFile(require.resolve('./_sidebar_field_stats')); loadTestFile(require.resolve('./_request_counts')); loadTestFile(require.resolve('./_doc_viewer')); loadTestFile(require.resolve('./_view_mode_toggle')); diff --git a/test/functional/page_objects/discover_page.ts b/test/functional/page_objects/discover_page.ts index 18e15ea69bb1a1..5e1c74bc78d4c7 100644 --- a/test/functional/page_objects/discover_page.ts +++ b/test/functional/page_objects/discover_page.ts @@ -763,9 +763,12 @@ export class DiscoverPageObject extends FtrService { } } - public async addRuntimeField(name: string, script: string) { + public async addRuntimeField(name: string, script: string, type?: string) { await this.clickAddField(); await this.fieldEditor.setName(name); + if (type) { + await this.fieldEditor.setFieldType(type); + } await this.fieldEditor.enableValue(); await this.fieldEditor.typeScript(script); await this.fieldEditor.save(); diff --git a/test/functional/page_objects/unified_field_list.ts b/test/functional/page_objects/unified_field_list.ts index 378b20cc02e247..152a1b4c1c660c 100644 --- a/test/functional/page_objects/unified_field_list.ts +++ b/test/functional/page_objects/unified_field_list.ts @@ -116,6 +116,13 @@ export class UnifiedFieldListPageObject extends FtrService { }); } + public async closeFieldPopover() { + await this.browser.pressKeys(this.browser.keys.ESCAPE); + await this.retry.waitFor('popover is closed', async () => { + return !(await this.testSubjects.exists('fieldPopoverHeader_fieldDisplayName')); + }); + } + public async clickFieldListItem(field: string) { await this.testSubjects.moveMouseTo(`field-${field}`); await this.testSubjects.click(`field-${field}`);