diff --git a/packages/kbn-search-index-documents/components/document_list.tsx b/packages/kbn-search-index-documents/components/document_list.tsx index ec9efe7d6b1d7f..5491d228e7d074 100644 --- a/packages/kbn-search-index-documents/components/document_list.tsx +++ b/packages/kbn-search-index-documents/components/document_list.tsx @@ -9,7 +9,7 @@ import React, { useState } from 'react'; -import { MappingProperty, SearchHit } from '@elastic/elasticsearch/lib/api/types'; +import type { IndicesGetMappingResponse, SearchHit } from '@elastic/elasticsearch/lib/api/types'; import { EuiButtonEmpty, @@ -30,18 +30,22 @@ import { i18n } from '@kbn/i18n'; import { FormattedMessage, FormattedNumber } from '@kbn/i18n-react'; -import { resultMetaData, resultToField } from './result/result_metadata'; +import { resultMetaData, resultToFieldFromMappingResponse } from './result/result_metadata'; import { Result } from '..'; +import { type ResultProps } from './result/result'; + interface DocumentListProps { dataTelemetryIdPrefix: string; docs: SearchHit[]; docsPerPage: number; isLoading: boolean; - mappings: Record | undefined; + mappings: IndicesGetMappingResponse | undefined; meta: Pagination; onPaginate: (newPageIndex: number) => void; - setDocsPerPage: (docsPerPage: number) => void; + setDocsPerPage?: (docsPerPage: number) => void; + onDocumentClick?: (doc: SearchHit) => void; + resultProps?: Partial; } export const DocumentList: React.FC = ({ @@ -53,6 +57,8 @@ export const DocumentList: React.FC = ({ meta, onPaginate, setDocsPerPage, + onDocumentClick, + resultProps = {}, }) => { const [isPopoverOpen, setIsPopoverOpen] = useState(false); @@ -99,7 +105,12 @@ export const DocumentList: React.FC = ({ {docs.map((doc) => { return ( - + onDocumentClick(doc) : undefined} + {...resultProps} + /> ); @@ -116,81 +127,83 @@ export const DocumentList: React.FC = ({ onPageClick={onPaginate} /> - - { - setIsPopoverOpen(true); - }} - > - {i18n.translate('searchIndexDocuments.documentList.pagination.itemsPerPage', { - defaultMessage: 'Documents per page: {docPerPage}', - values: { docPerPage: docsPerPage }, - })} - - } - isOpen={isPopoverOpen} - closePopover={() => { - setIsPopoverOpen(false); - }} - panelPaddingSize="none" - anchorPosition="downLeft" - > - + { - setIsPopoverOpen(false); - setDocsPerPage(10); + setIsPopoverOpen(true); }} > - {i18n.translate('searchIndexDocuments.documentList.paginationOptions.option', { - defaultMessage: '{docCount} documents', - values: { docCount: 10 }, + {i18n.translate('searchIndexDocuments.documentList.pagination.itemsPerPage', { + defaultMessage: 'Documents per page: {docPerPage}', + values: { docPerPage: docsPerPage }, })} - , + + } + isOpen={isPopoverOpen} + closePopover={() => { + setIsPopoverOpen(false); + }} + panelPaddingSize="none" + anchorPosition="downLeft" + > + { + setIsPopoverOpen(false); + setDocsPerPage(10); + }} + > + {i18n.translate('searchIndexDocuments.documentList.paginationOptions.option', { + defaultMessage: '{docCount} documents', + values: { docCount: 10 }, + })} + , - { - setIsPopoverOpen(false); - setDocsPerPage(25); - }} - > - {i18n.translate('searchIndexDocuments.documentList.paginationOptions.option', { - defaultMessage: '{docCount} documents', - values: { docCount: 25 }, - })} - , - { - setIsPopoverOpen(false); - setDocsPerPage(50); - }} - > - {i18n.translate('searchIndexDocuments.documentList.paginationOptions.option', { - defaultMessage: '{docCount} documents', - values: { docCount: 50 }, - })} - , - ]} - /> - - + { + setIsPopoverOpen(false); + setDocsPerPage(25); + }} + > + {i18n.translate('searchIndexDocuments.documentList.paginationOptions.option', { + defaultMessage: '{docCount} documents', + values: { docCount: 25 }, + })} + , + { + setIsPopoverOpen(false); + setDocsPerPage(50); + }} + > + {i18n.translate('searchIndexDocuments.documentList.paginationOptions.option', { + defaultMessage: '{docCount} documents', + values: { docCount: 50 }, + })} + , + ]} + /> + + + )} diff --git a/packages/kbn-search-index-documents/components/documents_list.test.tsx b/packages/kbn-search-index-documents/components/documents_list.test.tsx index b445c5fa48711b..b97b36989ad62c 100644 --- a/packages/kbn-search-index-documents/components/documents_list.test.tsx +++ b/packages/kbn-search-index-documents/components/documents_list.test.tsx @@ -58,8 +58,14 @@ describe('DocumentList', () => { }, ], mappings: { - AvgTicketPrice: { - type: 'float' as const, + kibana_sample_data_flights: { + mappings: { + properties: { + AvgTicketPrice: { + type: 'float' as const, + }, + }, + }, }, }, }; diff --git a/packages/kbn-search-index-documents/components/result/index.ts b/packages/kbn-search-index-documents/components/result/index.ts index a5e613fbd83ec9..8c894f7e2ca3be 100644 --- a/packages/kbn-search-index-documents/components/result/index.ts +++ b/packages/kbn-search-index-documents/components/result/index.ts @@ -8,4 +8,8 @@ */ export { Result } from './result'; -export { resultMetaData, resultToField } from './result_metadata'; +export { + resultMetaData, + resultToFieldFromMappingResponse, + resultToFieldFromMappings as resultToField, +} from './result_metadata'; diff --git a/packages/kbn-search-index-documents/components/result/result.tsx b/packages/kbn-search-index-documents/components/result/result.tsx index 4e6f0ed8c6eb84..5e1c4db1041169 100644 --- a/packages/kbn-search-index-documents/components/result/result.tsx +++ b/packages/kbn-search-index-documents/components/result/result.tsx @@ -9,76 +9,147 @@ import React, { useState } from 'react'; -import { EuiButtonIcon, EuiFlexGroup, EuiFlexItem, EuiPanel, EuiToolTip } from '@elastic/eui'; +import { + EuiButtonIcon, + EuiFlexGroup, + EuiFlexItem, + EuiPanel, + EuiSpacer, + EuiToolTip, +} from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { ResultFields } from './results_fields'; -import { ResultHeader } from './result_header'; import './result.scss'; import { MetaDataProps, ResultFieldProps } from './result_types'; +import { RichResultHeader } from './rich_result_header'; +import { ResultHeader } from './result_header'; + +export const DEFAULT_VISIBLE_FIELDS = 3; -interface ResultProps { +export interface ResultProps { fields: ResultFieldProps[]; metaData: MetaDataProps; + defaultVisibleFields?: number; + showScore?: boolean; + compactCard?: boolean; + onDocumentClick?: () => void; } -export const Result: React.FC = ({ metaData, fields }) => { +export const Result: React.FC = ({ + metaData, + fields, + defaultVisibleFields = DEFAULT_VISIBLE_FIELDS, + compactCard = true, + showScore = false, + onDocumentClick, +}) => { const [isExpanded, setIsExpanded] = useState(false); const tooltipText = - fields.length <= 3 + fields.length <= defaultVisibleFields ? i18n.translate('searchIndexDocuments.result.expandTooltip.allVisible', { defaultMessage: 'All fields are visible', }) : isExpanded ? i18n.translate('searchIndexDocuments.result.expandTooltip.showFewer', { defaultMessage: 'Show {amount} fewer fields', - values: { amount: fields.length - 3 }, + values: { amount: fields.length - defaultVisibleFields }, }) : i18n.translate('searchIndexDocuments.result.expandTooltip.showMore', { defaultMessage: 'Show {amount} more fields', - values: { amount: fields.length - 3 }, + values: { amount: fields.length - defaultVisibleFields }, }); const toolTipContent = <>{tooltipText}; return ( - + - + - + {compactCard && ( + + )} + {!compactCard && ( + + + ) => { + e.stopPropagation(); + setIsExpanded(!isExpanded); + }} + aria-label={tooltipText} + /> + + + } + /> + )} + {!compactCard && + ((isExpanded && fields.length > 0) || + (!isExpanded && fields.slice(0, defaultVisibleFields).length > 0)) && ( + + )} - -
- - setIsExpanded(!isExpanded)} - aria-label={tooltipText} - /> - -
- + {compactCard && ( + +
+ + ) => { + e.stopPropagation(); + setIsExpanded(!isExpanded); + }} + aria-label={tooltipText} + /> + +
+ + )} ); diff --git a/packages/kbn-search-index-documents/components/result/result_metadata.ts b/packages/kbn-search-index-documents/components/result/result_metadata.ts index 783cd537b45350..ba50644cafc591 100644 --- a/packages/kbn-search-index-documents/components/result/result_metadata.ts +++ b/packages/kbn-search-index-documents/components/result/result_metadata.ts @@ -7,7 +7,11 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -import type { MappingProperty, SearchHit } from '@elastic/elasticsearch/lib/api/types'; +import type { + IndicesGetMappingResponse, + MappingProperty, + SearchHit, +} from '@elastic/elasticsearch/lib/api/types'; import type { MetaDataProps } from './result_types'; const TITLE_KEYS = ['title', 'name']; @@ -37,15 +41,19 @@ export const resultTitle = (result: SearchHit): string | undefined => { export const resultMetaData = (result: SearchHit): MetaDataProps => ({ id: result._id!, title: resultTitle(result), + score: result._score, }); -export const resultToField = (result: SearchHit, mappings?: Record) => { - if (mappings && result._source && !Array.isArray(result._source)) { +export const resultToFieldFromMappingResponse = ( + result: SearchHit, + mappings?: IndicesGetMappingResponse +) => { + if (mappings && mappings[result._index] && result._source && !Array.isArray(result._source)) { if (typeof result._source === 'object') { return Object.entries(result._source).map(([key, value]) => { return { fieldName: key, - fieldType: mappings[key]?.type ?? 'object', + fieldType: mappings[result._index]?.mappings?.properties?.[key]?.type ?? 'object', fieldValue: JSON.stringify(value, null, 2), }; }); @@ -53,3 +61,19 @@ export const resultToField = (result: SearchHit, mappings?: Record +) => { + if (mappings && result._source && !Array.isArray(result._source)) { + return Object.entries(result._source).map(([key, value]) => { + return { + fieldName: key, + fieldType: mappings[key]?.type ?? 'object', + fieldValue: JSON.stringify(value, null, 2), + }; + }); + } + return []; +}; diff --git a/packages/kbn-search-index-documents/components/result/result_types.ts b/packages/kbn-search-index-documents/components/result/result_types.ts index e04ccb1091c8ec..c7899874b27ee5 100644 --- a/packages/kbn-search-index-documents/components/result/result_types.ts +++ b/packages/kbn-search-index-documents/components/result/result_types.ts @@ -7,6 +7,7 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ +import { SearchHit } from '@elastic/elasticsearch/lib/api/types'; import { IconType } from '@elastic/eui'; export interface ResultFieldProps { @@ -20,4 +21,6 @@ export interface MetaDataProps { id: string; onDocumentDelete?: Function; title?: string; + score?: SearchHit['_score']; + showScore?: boolean; } diff --git a/packages/kbn-search-index-documents/components/result/rich_result_header.tsx b/packages/kbn-search-index-documents/components/result/rich_result_header.tsx new file mode 100644 index 00000000000000..7caff8514871fe --- /dev/null +++ b/packages/kbn-search-index-documents/components/result/rich_result_header.tsx @@ -0,0 +1,228 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import React, { useState } from 'react'; + +import { + EuiButton, + EuiButtonIcon, + EuiFlexGroup, + EuiFlexItem, + EuiIcon, + EuiLink, + EuiPanel, + EuiPopover, + EuiPopoverFooter, + EuiPopoverTitle, + EuiText, + EuiTextColor, + EuiTitle, + useEuiTheme, +} from '@elastic/eui'; + +import { i18n } from '@kbn/i18n'; +import { css } from '@emotion/react'; +import { MetaDataProps } from './result_types'; + +interface Props { + metaData: MetaDataProps; + title: string; + rightSideActions?: React.ReactNode; + showScore?: boolean; + onTitleClick?: () => void; +} + +interface TermDef { + label: string | number; +} + +const Term: React.FC = ({ label }) => ( + + + {label}: + + +); + +const Definition: React.FC = ({ label }) => ( + + {label} + +); +const MetadataPopover: React.FC = ({ + id, + onDocumentDelete, + score, + showScore = false, +}) => { + const [popoverIsOpen, setPopoverIsOpen] = useState(false); + const closePopover = () => setPopoverIsOpen(false); + + const metaDataIcon = ( + ) => { + e.stopPropagation(); + setPopoverIsOpen(!popoverIsOpen); + }} + aria-label={i18n.translate('searchIndexDocuments.result.header.metadata.icon.ariaLabel', { + defaultMessage: 'Metadata for document: {id}', + values: { id }, + })} + /> + ); + + return ( + + + {i18n.translate('searchIndexDocuments.result.header.metadata.title', { + defaultMessage: 'Document metadata', + })} + + + + + + + + + + {score && showScore && ( + + + + + + + )} + + {onDocumentDelete && ( + + ) => { + e.stopPropagation(); + closePopover(); + }} + fullWidth + > + {i18n.translate('searchIndexDocuments.result.header.metadata.deleteDocument', { + defaultMessage: 'Delete document', + })} + + + )} + + ); +}; + +const Score: React.FC<{ score: MetaDataProps['score'] }> = ({ score }) => { + return ( + + + + + + + + + {score ? score.toString().substring(0, 5) : '-'} + + + + + + ); +}; + +export const RichResultHeader: React.FC = ({ + title, + metaData, + rightSideActions = null, + showScore = false, + onTitleClick, +}) => { + const { euiTheme } = useEuiTheme(); + return ( + + + {showScore && ( + + + + )} + + + + + + + {onTitleClick ? ( + + +

{title}

+ + + ) : ( + +

{title}

+ + )} + + {!!metaData && ( + + + + )} + + + + + + {rightSideActions} + + + ); +}; diff --git a/test/functional/fixtures/es_archiver/kibana_sample_data_logs_logsdb/data.json.gz b/test/functional/fixtures/es_archiver/kibana_sample_data_logs_logsdb/data.json.gz new file mode 100644 index 00000000000000..349fa50d7989f9 Binary files /dev/null and b/test/functional/fixtures/es_archiver/kibana_sample_data_logs_logsdb/data.json.gz differ diff --git a/test/functional/fixtures/es_archiver/kibana_sample_data_logs_logsdb/mappings.json b/test/functional/fixtures/es_archiver/kibana_sample_data_logs_logsdb/mappings.json new file mode 100644 index 00000000000000..6b3dea1e0d7186 --- /dev/null +++ b/test/functional/fixtures/es_archiver/kibana_sample_data_logs_logsdb/mappings.json @@ -0,0 +1,171 @@ +{ + "type": "index", + "value": { + "aliases": {}, + "index": "kibana_sample_data_logslogsdb", + "mappings": { + "_data_stream_timestamp": { + "enabled": true + }, + "properties": { + "@timestamp": { + "type": "date" + }, + "agent": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "bytes": { + "type": "long" + }, + "bytes_counter": { + "time_series_metric": "counter", + "type": "long" + }, + "bytes_gauge": { + "time_series_metric": "gauge", + "type": "long" + }, + "clientip": { + "type": "ip" + }, + "event": { + "properties": { + "dataset": { + "type": "keyword" + } + } + }, + "extension": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "geo": { + "properties": { + "coordinates": { + "type": "geo_point" + }, + "dest": { + "type": "keyword" + }, + "src": { + "type": "keyword" + }, + "srcdest": { + "type": "keyword" + } + } + }, + "host": { + "properties": { + "name": { + "type": "keyword" + } + } + }, + "index": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "ip": { + "type": "ip" + }, + "machine": { + "properties": { + "os": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "ram": { + "type": "long" + } + } + }, + "memory": { + "type": "double" + }, + "message": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "phpmemory": { + "type": "long" + }, + "referer": { + "type": "keyword" + }, + "request": { + "time_series_dimension": true, + "type": "keyword" + }, + "response": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "tags": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "timestamp": { + "path": "@timestamp", + "type": "alias" + }, + "url": { + "time_series_dimension": true, + "type": "keyword" + }, + "utc_time": { + "type": "date" + } + } + }, + "settings": { + "index": { + "auto_expand_replicas": "0-1", + "mode": "time_series", + "number_of_replicas": "0", + "number_of_shards": "1", + "routing_path": "request", + "time_series": { + "end_time": "2023-06-28T09:17:00.283Z", + "start_time": "2023-03-28T09:17:00.283Z" + } + } + } + } +} \ No newline at end of file diff --git a/test/functional/fixtures/kbn_archiver/kibana_sample_data_logs_logsdb.json b/test/functional/fixtures/kbn_archiver/kibana_sample_data_logs_logsdb.json new file mode 100644 index 00000000000000..b5836dff4d7179 --- /dev/null +++ b/test/functional/fixtures/kbn_archiver/kibana_sample_data_logs_logsdb.json @@ -0,0 +1,22 @@ +{ + "attributes": { + "fieldFormatMap": "{\"hour_of_day\":{}}", + "name": "Kibana Sample Data Logs (LogsDB)", + "runtimeFieldMap": "{\"hour_of_day\":{\"type\":\"long\",\"script\":{\"source\":\"emit(doc['timestamp'].value.getHour());\"}}}", + "timeFieldName": "timestamp", + "title": "kibana_sample_data_logslogsdb" + }, + "coreMigrationVersion": "8.8.0", + "created_at": "2023-04-27T13:09:20.333Z", + "id": "90943e30-9a47-11e8-b64d-95841ca0c247", + "managed": false, + "references": [], + "sort": [ + 1682600960333, + 64 + ], + "type": "index-pattern", + "typeMigrationVersion": "7.11.0", + "updated_at": "2023-04-27T13:09:20.333Z", + "version": "WzIxLDFd" +} \ No newline at end of file diff --git a/x-pack/plugins/actions/common/connector_feature_config.test.ts b/x-pack/plugins/actions/common/connector_feature_config.test.ts index 1f18e923593d55..2c80203c5baf2a 100644 --- a/x-pack/plugins/actions/common/connector_feature_config.test.ts +++ b/x-pack/plugins/actions/common/connector_feature_config.test.ts @@ -44,7 +44,7 @@ describe('getConnectorCompatibility', () => { it('returns the compatibility list for valid feature ids', () => { expect( getConnectorCompatibility(['alerting', 'cases', 'uptime', 'siem', 'generativeAIForSecurity']) - ).toEqual(['Alerting Rules', 'Cases', 'Generative AI for Security']); + ).toEqual(['Alerting Rules', 'Cases', 'Security Solution', 'Generative AI for Security']); }); it('skips invalid feature ids', () => { diff --git a/x-pack/plugins/actions/common/connector_feature_config.ts b/x-pack/plugins/actions/common/connector_feature_config.ts index 588966b307db10..5ba316f47d59be 100644 --- a/x-pack/plugins/actions/common/connector_feature_config.ts +++ b/x-pack/plugins/actions/common/connector_feature_config.ts @@ -56,6 +56,12 @@ const compatibilityAlertingRules = i18n.translate( defaultMessage: 'Alerting Rules', } ); +const compatibilitySecuritySolution = i18n.translate( + 'xpack.actions.availableConnectorFeatures.compatibility.securitySolution', + { + defaultMessage: 'Security Solution', + } +); const compatibilityCases = i18n.translate( 'xpack.actions.availableConnectorFeatures.compatibility.cases', @@ -93,7 +99,7 @@ export const SecuritySolutionFeature: ConnectorFeatureConfig = { name: i18n.translate('xpack.actions.availableConnectorFeatures.securitySolution', { defaultMessage: 'Security Solution', }), - compatibility: compatibilityAlertingRules, + compatibility: compatibilitySecuritySolution, }; export const GenerativeAIForSecurityFeature: ConnectorFeatureConfig = { diff --git a/x-pack/plugins/actions/server/config.ts b/x-pack/plugins/actions/server/config.ts index 980144117e4e80..399f92090818b1 100644 --- a/x-pack/plugins/actions/server/config.ts +++ b/x-pack/plugins/actions/server/config.ts @@ -145,6 +145,15 @@ export const configSchema = schema.object({ max: schema.maybe(schema.number({ min: MIN_QUEUED_MAX, defaultValue: DEFAULT_QUEUED_MAX })), }) ), + usage: schema.maybe( + schema.object({ + cert: schema.maybe( + schema.object({ + path: schema.string(), + }) + ), + }) + ), }); export type ActionsConfig = TypeOf; diff --git a/x-pack/plugins/cases/public/components/all_cases/cases_metrics.tsx b/x-pack/plugins/cases/public/components/all_cases/cases_metrics.tsx index 7a9d175632f273..3d883550bd4a82 100644 --- a/x-pack/plugins/cases/public/components/all_cases/cases_metrics.tsx +++ b/x-pack/plugins/cases/public/components/all_cases/cases_metrics.tsx @@ -7,20 +7,20 @@ import React, { useMemo } from 'react'; import { - EuiDescriptionList, EuiFlexGroup, EuiFlexItem, EuiIconTip, EuiLoadingSpinner, EuiPanel, EuiSpacer, + EuiStat, } from '@elastic/eui'; import prettyMilliseconds from 'pretty-ms'; import { CaseStatuses } from '../../../common/types/domain'; import { useGetCasesStatus } from '../../containers/use_get_cases_status'; import { StatusStats } from '../status/status_stats'; import { useGetCasesMetrics } from '../../containers/use_get_cases_metrics'; -import { ATTC_DESCRIPTION, ATTC_STAT } from './translations'; +import { ATTC_DESCRIPTION, ATTC_STAT, ATTC_STAT_INFO_ARIA_LABEL } from './translations'; export const CasesMetrics: React.FC = () => { const { @@ -68,23 +68,28 @@ export const CasesMetrics: React.FC = () => { /> - - {ATTC_STAT} - - ), - description: isCasesMetricsLoading ? ( - - ) : ( - mttrValue - ), - }, - ]} + description={ + <> + {ATTC_STAT} +   + + + } + title={ + isCasesMetricsLoading ? ( + + ) : ( + mttrValue + ) + } + titleSize="xs" + text-align="left" /> diff --git a/x-pack/plugins/cases/public/components/all_cases/translations.ts b/x-pack/plugins/cases/public/components/all_cases/translations.ts index e29019516e911e..a69c9421406025 100644 --- a/x-pack/plugins/cases/public/components/all_cases/translations.ts +++ b/x-pack/plugins/cases/public/components/all_cases/translations.ts @@ -128,6 +128,13 @@ export const ATTC_STAT = i18n.translate('xpack.cases.casesStats.mttr', { defaultMessage: 'Average time to close', }); +export const ATTC_STAT_INFO_ARIA_LABEL = i18n.translate( + 'xpack.cases.casesStats.mttr.info.ariaLabel', + { + defaultMessage: 'More about average time to close', + } +); + export const ATTC_DESCRIPTION = i18n.translate('xpack.cases.casesStats.mttrDescription', { defaultMessage: 'The average duration (from creation to closure) for your current cases', }); diff --git a/x-pack/plugins/cases/public/components/status/status_stats.test.tsx b/x-pack/plugins/cases/public/components/status/status_stats.test.tsx index e24e3019f8b9c0..d6958a9d5654a5 100644 --- a/x-pack/plugins/cases/public/components/status/status_stats.test.tsx +++ b/x-pack/plugins/cases/public/components/status/status_stats.test.tsx @@ -27,9 +27,7 @@ describe('Stats', () => { it('shows the count', async () => { const wrapper = mount(); - expect( - wrapper.find(`[data-test-subj="test-stats"] .euiDescriptionList__description`).first().text() - ).toBe('2'); + expect(wrapper.find(`[data-test-subj="test-stats"] .euiStat__title`).first().text()).toBe('2'); }); it('shows the loading spinner', async () => { @@ -39,29 +37,29 @@ describe('Stats', () => { }); describe('Status title', () => { - it('shows the correct title for status open', async () => { + it('shows the correct description for status open', async () => { const wrapper = mount(); expect( - wrapper.find(`[data-test-subj="test-stats"] .euiDescriptionList__title`).first().text() + wrapper.find(`[data-test-subj="test-stats"] .euiStat__description`).first().text() ).toBe('Open cases'); }); - it('shows the correct title for status in-progress', async () => { + it('shows the correct description for status in-progress', async () => { const wrapper = mount( ); expect( - wrapper.find(`[data-test-subj="test-stats"] .euiDescriptionList__title`).first().text() + wrapper.find(`[data-test-subj="test-stats"] .euiStat__description`).first().text() ).toBe('In progress cases'); }); - it('shows the correct title for status closed', async () => { + it('shows the correct description for status closed', async () => { const wrapper = mount(); expect( - wrapper.find(`[data-test-subj="test-stats"] .euiDescriptionList__title`).first().text() + wrapper.find(`[data-test-subj="test-stats"] .euiStat__description`).first().text() ).toBe('Closed cases'); }); }); diff --git a/x-pack/plugins/cases/public/components/status/status_stats.tsx b/x-pack/plugins/cases/public/components/status/status_stats.tsx index 1f1686c26a4dbb..8f01c1e98aba28 100644 --- a/x-pack/plugins/cases/public/components/status/status_stats.tsx +++ b/x-pack/plugins/cases/public/components/status/status_stats.tsx @@ -6,7 +6,7 @@ */ import React, { memo, useMemo } from 'react'; -import { EuiDescriptionList, EuiLoadingSpinner } from '@elastic/eui'; +import { EuiStat, EuiLoadingSpinner } from '@elastic/eui'; import type { CaseStatuses } from '../../../common/types/domain'; import { statuses } from './config'; @@ -23,21 +23,25 @@ const StatusStatsComponent: React.FC = ({ isLoading, dataTestSubj, }) => { - const statusStats = useMemo( - () => [ - { - title: statuses[caseStatus].stats.title, - description: isLoading ? ( - - ) : ( - caseCount ?? '-' - ), - }, - ], + const { title, description } = useMemo( + () => ({ + description: statuses[caseStatus].stats.title, + title: isLoading ? ( + + ) : ( + caseCount ?? '-' + ), + }), [caseCount, caseStatus, dataTestSubj, isLoading] ); return ( - + ); }; diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/documents.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/documents.tsx index 353fc84546a9c1..592da5d044f2ab 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/documents.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/documents.tsx @@ -122,7 +122,7 @@ export const SearchIndexDocuments: React.FC = () => { docs={docs} docsPerPage={pagination.pageSize ?? 10} isLoading={status !== Status.SUCCESS && mappingStatus !== Status.SUCCESS} - mappings={mappingData?.mappings?.properties ?? {}} + mappings={mappingData ? { [indexName]: mappingData } : undefined} meta={data?.meta ?? DEFAULT_PAGINATION} onPaginate={(pageIndex) => setPagination({ ...pagination, pageIndex })} setDocsPerPage={(pageSize) => setPagination({ ...pagination, pageSize })} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/columns.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/columns.tsx index 68ec09897f7c21..99fdc25382bf21 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/columns.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/columns.tsx @@ -186,7 +186,11 @@ export const getDatasetQualityTableColumns = ({ const { integration, name, rawName } = dataStreamStat; return ( - + diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/dataset_quality_details_link.test.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/dataset_quality_details_link.test.tsx new file mode 100644 index 00000000000000..ca9dc9764e4b4e --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/dataset_quality_details_link.test.tsx @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { DATA_QUALITY_DETAILS_LOCATOR_ID } from '@kbn/deeplinks-observability'; +import { BrowserUrlService } from '@kbn/share-plugin/public'; +import { shallow } from 'enzyme'; +import React from 'react'; +import { DatasetQualityDetailsLink } from './dataset_quality_details_link'; + +const createMockLocator = (id: string) => ({ + navigate: jest.fn(), + getRedirectUrl: jest.fn().mockReturnValue(id), +}); + +describe('DatasetQualityDetailsLink', () => { + const mockDataQualityDetailsLocator = createMockLocator(DATA_QUALITY_DETAILS_LOCATOR_ID); + + const urlServiceMock = { + locators: { + get: jest.fn((id) => { + switch (id) { + case DATA_QUALITY_DETAILS_LOCATOR_ID: + return mockDataQualityDetailsLocator; + default: + throw new Error(`Unknown locator id: ${id}`); + } + }), + }, + } as any as BrowserUrlService; + + const dataStream = { + title: 'My data stream', + rawName: 'logs-my.data.stream-default', + }; + + const timeRange = { + from: 'now-7d/d', + refresh: { + pause: true, + value: 60000, + }, + to: 'now', + }; + + afterEach(() => { + jest.clearAllMocks(); + }); + + it('renders a link to dataset quality details', () => { + const wrapper = shallow( + + {dataStream.title} + + ); + + expect(mockDataQualityDetailsLocator.getRedirectUrl).toHaveBeenCalledWith({ + dataStream: dataStream.rawName, + timeRange, + }); + expect(wrapper.prop('href')).toBe(DATA_QUALITY_DETAILS_LOCATOR_ID); + }); +}); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/dataset_quality_details_link.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/dataset_quality_details_link.tsx index ac73f269d9f5ae..ec6c34ce1a7726 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/dataset_quality_details_link.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/dataset_quality_details_link.tsx @@ -5,31 +5,34 @@ * 2.0. */ -import React from 'react'; -import { BrowserUrlService } from '@kbn/share-plugin/public'; +import { EuiHeaderLink } from '@elastic/eui'; import { DATA_QUALITY_DETAILS_LOCATOR_ID, DataQualityDetailsLocatorParams, } from '@kbn/deeplinks-observability'; import { getRouterLinkProps } from '@kbn/router-utils'; -import { EuiHeaderLink } from '@elastic/eui'; +import { BrowserUrlService } from '@kbn/share-plugin/public'; +import React from 'react'; +import { TimeRangeConfig } from '../../../../common/types'; export const DatasetQualityDetailsLink = React.memo( ({ urlService, dataStream, + timeRange, children, }: { urlService: BrowserUrlService; dataStream: string; + timeRange: TimeRangeConfig; children: React.ReactNode; }) => { const locator = urlService.locators.get( DATA_QUALITY_DETAILS_LOCATOR_ID ); - const datasetQualityUrl = locator?.getRedirectUrl({ dataStream }); + const datasetQualityUrl = locator?.getRedirectUrl({ dataStream, timeRange }); const navigateToDatasetQuality = () => { - locator?.navigate({ dataStream }); + locator?.navigate({ dataStream, timeRange }); }; const datasetQualityLinkDetailsProps = getRouterLinkProps({ diff --git a/x-pack/plugins/observability_solution/infra/public/apps/logs_app.tsx b/x-pack/plugins/observability_solution/infra/public/apps/logs_app.tsx index 8bd243b0723d5b..329e059288e3e8 100644 --- a/x-pack/plugins/observability_solution/infra/public/apps/logs_app.tsx +++ b/x-pack/plugins/observability_solution/infra/public/apps/logs_app.tsx @@ -57,7 +57,7 @@ const LogsApp: React.FC<{ storage: Storage; theme$: AppMountParameters['theme$']; }> = ({ core, history, pluginStart, plugins, setHeaderActionMenu, storage, theme$ }) => { - const uiCapabilities = core.application.capabilities; + const { logs, discover, fleet } = core.application.capabilities; return ( @@ -74,19 +74,21 @@ const LogsApp: React.FC<{ toastsService={core.notifications.toasts} > - { - plugins.share.url.locators - .get(ALL_DATASETS_LOCATOR_ID) - ?.navigate({}); + {Boolean(discover?.show && fleet?.read) && ( + { + plugins.share.url.locators + .get(ALL_DATASETS_LOCATOR_ID) + ?.navigate({}); - return null; - }} - /> + return null; + }} + /> + )} - {uiCapabilities?.logs?.show && } + {logs?.show && } diff --git a/x-pack/plugins/observability_solution/infra/public/pages/logs/page_content.tsx b/x-pack/plugins/observability_solution/infra/public/pages/logs/page_content.tsx index 056c98513c2446..5b5965bb2d5eca 100644 --- a/x-pack/plugins/observability_solution/infra/public/pages/logs/page_content.tsx +++ b/x-pack/plugins/observability_solution/infra/public/pages/logs/page_content.tsx @@ -113,6 +113,7 @@ export const LogsPageContent: React.FunctionComponent = () => { )} + ( { + const { infrastructure, logs, discover, fleet } = capabilities; return [ - ...(capabilities.logs.show + ...(logs.show ? [ { label: 'Logs', sortKey: 200, entries: [ - { - label: 'Explorer', - app: 'observability-logs-explorer', - path: '/', - isBetaFeature: true, - }, + ...(discover?.show && fleet?.read + ? [ + { + label: 'Explorer', + app: 'observability-logs-explorer', + path: '/', + isBetaFeature: true, + }, + ] + : []), ...(this.config.featureFlags.logsUIEnabled ? [ { label: 'Stream', app: 'logs', path: '/stream' }, @@ -161,7 +166,7 @@ export class Plugin implements InfraClientPluginClass { }, ] : []), - ...(capabilities.infrastructure.show + ...(infrastructure.show ? [ { label: metricsTitle, diff --git a/x-pack/plugins/observability_solution/observability_logs_explorer/public/plugin.ts b/x-pack/plugins/observability_solution/observability_logs_explorer/public/plugin.ts index 906caf72a450a2..798a03da0ebdfc 100644 --- a/x-pack/plugins/observability_solution/observability_logs_explorer/public/plugin.ts +++ b/x-pack/plugins/observability_solution/observability_logs_explorer/public/plugin.ts @@ -7,6 +7,8 @@ import { AppMountParameters, + AppStatus, + AppUpdater, CoreSetup, CoreStart, DEFAULT_APP_CATEGORIES, @@ -14,6 +16,7 @@ import { PluginInitializerContext, } from '@kbn/core/public'; import { OBSERVABILITY_LOGS_EXPLORER_APP_ID } from '@kbn/deeplinks-observability'; +import { BehaviorSubject } from 'rxjs'; import { AllDatasetsLocatorDefinition, ObservabilityLogsExplorerLocators, @@ -35,6 +38,7 @@ export class ObservabilityLogsExplorerPlugin { private config: ObservabilityLogsExplorerConfig; private locators?: ObservabilityLogsExplorerLocators; + private appStateUpdater = new BehaviorSubject(() => ({})); constructor(context: PluginInitializerContext) { this.config = context.config.get(); @@ -56,6 +60,7 @@ export class ObservabilityLogsExplorerPlugin ? ['globalSearch', 'sideNav'] : ['globalSearch'], keywords: ['logs', 'log', 'explorer', 'logs explorer'], + updater$: this.appStateUpdater, mount: async (appMountParams: ObservabilityLogsExplorerAppMountParameters) => { const [coreStart, pluginsStart, ownPluginStart] = await core.getStartServices(); const { renderObservabilityLogsExplorer } = await import( @@ -123,7 +128,16 @@ export class ObservabilityLogsExplorerPlugin }; } - public start(_core: CoreStart, _pluginsStart: ObservabilityLogsExplorerStartDeps) { + public start(core: CoreStart, _pluginsStart: ObservabilityLogsExplorerStartDeps) { + const { discover, fleet, logs } = core.application.capabilities; + + if (!(discover?.show && fleet?.read && logs?.show)) { + this.appStateUpdater.next(() => ({ + status: AppStatus.inaccessible, + visibleIn: [], + })); + } + return {}; } } diff --git a/x-pack/plugins/observability_solution/slo/public/pages/slo_edit/components/common/group_by_field.tsx b/x-pack/plugins/observability_solution/slo/public/pages/slo_edit/components/common/group_by_field.tsx index 939755b8a9e3eb..a7450bfb2a4653 100644 --- a/x-pack/plugins/observability_solution/slo/public/pages/slo_edit/components/common/group_by_field.tsx +++ b/x-pack/plugins/observability_solution/slo/public/pages/slo_edit/components/common/group_by_field.tsx @@ -5,16 +5,13 @@ * 2.0. */ -import { ALL_VALUE, QuerySchema } from '@kbn/slo-schema'; -import { i18n } from '@kbn/i18n'; -import { EuiIconTip } from '@elastic/eui'; -import React from 'react'; import { DataView, FieldSpec } from '@kbn/data-views-plugin/common'; +import { QuerySchema } from '@kbn/slo-schema'; +import React from 'react'; import { useFormContext } from 'react-hook-form'; -import { OptionalText } from './optional_text'; import { CreateSLOForm } from '../../types'; -import { IndexFieldSelector } from './index_field_selector'; import { GroupByCardinality } from './group_by_cardinality'; +import { GroupByFieldSelector } from './group_by_field_selector'; export function GroupByField({ dataView, @@ -32,27 +29,8 @@ export function GroupByField({ return ( <> - - {i18n.translate('xpack.slo.sloEdit.groupBy.label', { - defaultMessage: 'Group by', - })}{' '} - - - } - labelAppend={} - placeholder={i18n.translate('xpack.slo.sloEdit.groupBy.placeholder', { - defaultMessage: 'Select an optional field to group by', - })} isLoading={!!index && isLoading} isDisabled={!index} /> diff --git a/x-pack/plugins/observability_solution/slo/public/pages/slo_edit/components/common/index_field_selector.tsx b/x-pack/plugins/observability_solution/slo/public/pages/slo_edit/components/common/group_by_field_selector.tsx similarity index 59% rename from x-pack/plugins/observability_solution/slo/public/pages/slo_edit/components/common/index_field_selector.tsx rename to x-pack/plugins/observability_solution/slo/public/pages/slo_edit/components/common/group_by_field_selector.tsx index 0a277900ac31f1..c45cc1d337aadd 100644 --- a/x-pack/plugins/observability_solution/slo/public/pages/slo_edit/components/common/index_field_selector.tsx +++ b/x-pack/plugins/observability_solution/slo/public/pages/slo_edit/components/common/group_by_field_selector.tsx @@ -5,35 +5,27 @@ * 2.0. */ -import { EuiComboBox, EuiComboBoxOptionOption, EuiFormRow } from '@elastic/eui'; -import React, { useEffect, useState, ReactNode } from 'react'; -import { Controller, useFormContext } from 'react-hook-form'; +import { EuiComboBox, EuiComboBoxOptionOption, EuiFormRow, EuiIconTip } from '@elastic/eui'; import { FieldSpec } from '@kbn/data-views-plugin/common'; -import { createOptionsFromFields, Option } from '../../helpers/create_options'; +import { i18n } from '@kbn/i18n'; +import { ALL_VALUE } from '@kbn/slo-schema'; +import React, { useEffect, useState } from 'react'; +import { Controller, useFormContext } from 'react-hook-form'; +import { Option, createOptionsFromFields } from '../../helpers/create_options'; import { CreateSLOForm } from '../../types'; +import { OptionalText } from './optional_text'; interface Props { indexFields: FieldSpec[]; - name: 'groupBy' | 'indicator.params.timestampField'; - label: ReactNode | string; - placeholder: string; isDisabled: boolean; isLoading: boolean; - isRequired?: boolean; - defaultValue?: string; - labelAppend?: ReactNode; } -export function IndexFieldSelector({ - indexFields, - name, - label, - labelAppend, - placeholder, - isDisabled, - isLoading, - isRequired = false, - defaultValue = '', -}: Props) { + +const placeholder = i18n.translate('xpack.slo.sloEdit.groupBy.placeholder', { + defaultMessage: 'Select an optional field to group by', +}); + +export function GroupByFieldSelector({ indexFields, isDisabled, isLoading }: Props) { const { control, getFieldState } = useFormContext(); const [options, setOptions] = useState(createOptionsFromFields(indexFields)); @@ -41,10 +33,10 @@ export function IndexFieldSelector({ setOptions(createOptionsFromFields(indexFields)); }, [indexFields]); - const getSelectedItems = (value: string | string[], fields: FieldSpec[]) => { + const getSelectedItems = (value: string | string[]) => { const values = [value].flat(); const selectedItems: Array> = []; - fields.forEach((field) => { + indexFields.forEach((field) => { if (values.includes(field.name)) { selectedItems.push({ value: field.name, label: field.name }); } @@ -53,12 +45,27 @@ export function IndexFieldSelector({ }; return ( - + + {i18n.translate('xpack.slo.sloEdit.groupBy.label', { + defaultMessage: 'Group by', + })}{' '} + + + } + isInvalid={getFieldState('groupBy').invalid} + labelAppend={} + > { return ( @@ -75,7 +82,7 @@ export function IndexFieldSelector({ return field.onChange(selected.map((selection) => selection.value)); } - field.onChange(defaultValue); + field.onChange([ALL_VALUE]); }} options={options} onSearchChange={(searchValue: string) => { @@ -83,9 +90,7 @@ export function IndexFieldSelector({ createOptionsFromFields(indexFields, ({ value }) => value.includes(searchValue)) ); }} - selectedOptions={ - !!indexFields && !!field.value ? getSelectedItems(field.value, indexFields) : [] - } + selectedOptions={!!indexFields && !!field.value ? getSelectedItems(field.value) : []} /> ); }} diff --git a/x-pack/plugins/observability_solution/slo/public/pages/slo_edit/components/common/timestamp_field_selector.tsx b/x-pack/plugins/observability_solution/slo/public/pages/slo_edit/components/common/timestamp_field_selector.tsx new file mode 100644 index 00000000000000..dc3289ca895c11 --- /dev/null +++ b/x-pack/plugins/observability_solution/slo/public/pages/slo_edit/components/common/timestamp_field_selector.tsx @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiComboBox, EuiComboBoxOptionOption, EuiFormRow } from '@elastic/eui'; +import { FieldSpec } from '@kbn/data-views-plugin/common'; +import { i18n } from '@kbn/i18n'; +import React, { useEffect, useState } from 'react'; +import { Controller, useFormContext } from 'react-hook-form'; +import { Option, createOptionsFromFields } from '../../helpers/create_options'; +import { CreateSLOForm } from '../../types'; + +interface Props { + fields: FieldSpec[]; + isDisabled: boolean; + isLoading: boolean; +} + +const placeholder = i18n.translate('xpack.slo.sloEdit.timestampField.placeholder', { + defaultMessage: 'Select a timestamp field', +}); + +export function TimestampFieldSelector({ fields, isDisabled, isLoading }: Props) { + const { control, getFieldState } = useFormContext(); + const [options, setOptions] = useState(createOptionsFromFields(fields)); + + useEffect(() => { + setOptions(createOptionsFromFields(fields)); + }, [fields]); + + return ( + + { + return ( + + {...field} + async + placeholder={placeholder} + aria-label={placeholder} + isClearable + isDisabled={isLoading || isDisabled} + isInvalid={fieldState.invalid} + isLoading={isLoading} + onChange={(selected: EuiComboBoxOptionOption[]) => { + if (selected.length) { + return field.onChange(selected[0].value); + } + + field.onChange(''); + }} + singleSelection={{ asPlainText: true }} + options={options} + onSearchChange={(searchValue: string) => { + setOptions( + createOptionsFromFields(fields, ({ value }) => value.includes(searchValue)) + ); + }} + selectedOptions={ + !!fields && !!field.value ? [{ value: field.value, label: field.value }] : [] + } + /> + ); + }} + /> + + ); +} diff --git a/x-pack/plugins/observability_solution/slo/public/pages/slo_edit/components/custom_common/index_and_timestamp_field.tsx b/x-pack/plugins/observability_solution/slo/public/pages/slo_edit/components/custom_common/index_and_timestamp_field.tsx index b4b5bdd4557d47..7059810a5aae07 100644 --- a/x-pack/plugins/observability_solution/slo/public/pages/slo_edit/components/custom_common/index_and_timestamp_field.tsx +++ b/x-pack/plugins/observability_solution/slo/public/pages/slo_edit/components/custom_common/index_and_timestamp_field.tsx @@ -6,13 +6,12 @@ */ import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; -import { i18n } from '@kbn/i18n'; -import React from 'react'; import { DataView } from '@kbn/data-views-plugin/public'; +import React from 'react'; import { useFormContext } from 'react-hook-form'; -import { IndexSelection } from './index_selection'; -import { IndexFieldSelector } from '../common/index_field_selector'; import { CreateSLOForm } from '../../types'; +import { TimestampFieldSelector } from '../common/timestamp_field_selector'; +import { IndexSelection } from './index_selection'; export function IndexAndTimestampField({ dataView, @@ -32,18 +31,10 @@ export function IndexAndTimestampField({ - diff --git a/x-pack/plugins/search_indices/public/components/index_documents/document_list.tsx b/x-pack/plugins/search_indices/public/components/index_documents/document_list.tsx index ddf4f27122ef99..22735cc86c05f9 100644 --- a/x-pack/plugins/search_indices/public/components/index_documents/document_list.tsx +++ b/x-pack/plugins/search_indices/public/components/index_documents/document_list.tsx @@ -8,7 +8,7 @@ import React from 'react'; import { MappingProperty, SearchHit } from '@elastic/elasticsearch/lib/api/types'; -import { Result, resultToField, resultMetaData } from '@kbn/search-index-documents'; +import { Result, resultMetaData, resultToField } from '@kbn/search-index-documents'; import { EuiSpacer } from '@elastic/eui'; diff --git a/x-pack/plugins/search_playground/common/types.ts b/x-pack/plugins/search_playground/common/types.ts index c0e3300fe7dff5..c239858b5b459a 100644 --- a/x-pack/plugins/search_playground/common/types.ts +++ b/x-pack/plugins/search_playground/common/types.ts @@ -51,6 +51,7 @@ export enum APIRoutes { POST_QUERY_SOURCE_FIELDS = '/internal/search_playground/query_source_fields', GET_INDICES = '/internal/search_playground/indices', POST_SEARCH_QUERY = '/internal/search_playground/search', + GET_INDEX_MAPPINGS = '/internal/search_playground/mappings', } export enum LLMs { diff --git a/x-pack/plugins/search_playground/public/components/app.tsx b/x-pack/plugins/search_playground/public/components/app.tsx index 34b89433ea7058..4f371ea5d15bb6 100644 --- a/x-pack/plugins/search_playground/public/components/app.tsx +++ b/x-pack/plugins/search_playground/public/components/app.tsx @@ -106,7 +106,11 @@ export const App: React.FC = ({ css={{ position: 'relative', }} - contentProps={{ css: { display: 'flex', flexGrow: 1, position: 'absolute', inset: 0 } }} + contentProps={ + selectedPageMode === PlaygroundPageMode.search && selectedMode === 'chat' + ? undefined + : { css: { display: 'flex', flexGrow: 1, position: 'absolute', inset: 0 } } + } paddingSize={paddingSize} className="eui-fullHeight" > diff --git a/x-pack/plugins/search_playground/public/components/search_mode/empty_results.tsx b/x-pack/plugins/search_playground/public/components/search_mode/empty_results.tsx deleted file mode 100644 index ab5779e85ddd5f..00000000000000 --- a/x-pack/plugins/search_playground/public/components/search_mode/empty_results.tsx +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import React from 'react'; - -import { EuiEmptyPrompt } from '@elastic/eui'; -import { i18n } from '@kbn/i18n'; - -export interface EmptyResultsArgs { - query?: string; -} - -export const EmptyResults: React.FC = ({ query }) => { - return ( - - {query - ? i18n.translate('xpack.searchPlayground.resultList.emptyWithQuery.text', { - defaultMessage: 'No result found for: {query}', - values: { query }, - }) - : i18n.translate('xpack.searchPlayground.resultList.empty.text', { - defaultMessage: 'No results found', - })} -

- } - /> - ); -}; diff --git a/x-pack/plugins/search_playground/public/components/search_mode/result_list.tsx b/x-pack/plugins/search_playground/public/components/search_mode/result_list.tsx index 02e1193e223321..87c7060c291515 100644 --- a/x-pack/plugins/search_playground/public/components/search_mode/result_list.tsx +++ b/x-pack/plugins/search_playground/public/components/search_mode/result_list.tsx @@ -7,40 +7,29 @@ import React, { useEffect, useState } from 'react'; -import { - EuiFlexGroup, - EuiFlexItem, - EuiHorizontalRule, - EuiPagination, - EuiPanel, - EuiText, - EuiTitle, -} from '@elastic/eui'; - -import { UnifiedDocViewerFlyout } from '@kbn/unified-doc-viewer-plugin/public'; +import { DocumentList, pageToPagination } from '@kbn/search-index-documents'; import type { DataView } from '@kbn/data-views-plugin/public'; import type { EsHitRecord } from '@kbn/discover-utils/types'; -import type { SearchHit } from '@elastic/elasticsearch/lib/api/types'; +import type { IndicesGetMappingResponse, SearchHit } from '@elastic/elasticsearch/lib/api/types'; import { buildDataTableRecord } from '@kbn/discover-utils'; -import { i18n } from '@kbn/i18n'; -import { Pagination } from '../../types'; -import { getPageCounts } from '../../utils/pagination_helper'; -import { EmptyResults } from './empty_results'; +import { UnifiedDocViewerFlyout } from '@kbn/unified-doc-viewer-plugin/public'; +import { Pagination as PaginationTypeEui } from '@elastic/eui'; import { useKibana } from '../../hooks/use_kibana'; +import { Pagination } from '../../types'; export interface ResultListArgs { searchResults: SearchHit[]; + mappings?: IndicesGetMappingResponse; pagination: Pagination; onPaginationChange: (nextPage: number) => void; - searchQuery?: string; } export const ResultList: React.FC = ({ searchResults, + mappings, pagination, onPaginationChange, - searchQuery = '', }) => { const { services: { data }, @@ -50,73 +39,42 @@ export const ResultList: React.FC = ({ data.dataViews.getDefaultDataView().then((d) => setDataView(d)); }, [data]); const [flyoutDocId, setFlyoutDocId] = useState(undefined); - const { totalPage, page } = getPageCounts(pagination); + const documentMeta: PaginationTypeEui = pageToPagination(pagination); const hit = flyoutDocId && buildDataTableRecord(searchResults.find((item) => item._id === flyoutDocId) as EsHitRecord); return ( - - - {searchResults.length === 0 && ( - - - - )} - {searchResults.length !== 0 && - searchResults.map((item, index) => { - return ( - <> - setFlyoutDocId(item._id)} - grow - > - - - -

ID:{item._id}

- - - - -

- {i18n.translate('xpack.searchPlayground.resultList.result.score', { - defaultMessage: 'Document score: {score}', - values: { score: item._score }, - })} -

- - - - - {index !== searchResults.length - 1 && } - - ); - })} - {searchResults.length !== 0 && ( - - - - )} - {flyoutDocId && dataView && hit && ( - setFlyoutDocId(undefined)} - isEsqlQuery={false} - columns={[]} - hit={hit} - dataView={dataView} - onAddColumn={() => {}} - onRemoveColumn={() => {}} - setExpandedDoc={() => {}} - flyoutType="overlay" - /> - )} - - + <> + setFlyoutDocId(searchHit._id)} + resultProps={{ + showScore: true, + compactCard: false, + defaultVisibleFields: 0, + }} + /> + + {flyoutDocId && dataView && hit && ( + setFlyoutDocId(undefined)} + isEsqlQuery={false} + columns={[]} + hit={hit} + dataView={dataView} + onAddColumn={() => {}} + onRemoveColumn={() => {}} + setExpandedDoc={() => {}} + flyoutType="overlay" + /> + )} + ); }; diff --git a/x-pack/plugins/search_playground/public/components/search_mode/search_mode.tsx b/x-pack/plugins/search_playground/public/components/search_mode/search_mode.tsx index 967c5786eed632..94e6337f1a03c3 100644 --- a/x-pack/plugins/search_playground/public/components/search_mode/search_mode.tsx +++ b/x-pack/plugins/search_playground/public/components/search_mode/search_mode.tsx @@ -23,6 +23,7 @@ import { ResultList } from './result_list'; import { ChatForm, ChatFormFields, Pagination } from '../../types'; import { useSearchPreview } from '../../hooks/use_search_preview'; import { getPaginationFromPage } from '../../utils/pagination_helper'; +import { useIndexMappings } from '../../hooks/use_index_mappings'; export const SearchMode: React.FC = () => { const { euiTheme } = useEuiTheme(); @@ -40,6 +41,7 @@ export const SearchMode: React.FC = () => { }>({ query: searchBarValue, pagination: DEFAULT_PAGINATION }); const { results, pagination } = useSearchPreview(searchQuery); + const { data: mappingData } = useIndexMappings(); const queryClient = useQueryClient(); const handleSearch = async (query = searchBarValue, paginationParam = DEFAULT_PAGINATION) => { @@ -81,15 +83,15 @@ export const SearchMode: React.FC = () => { /> - + {searchQuery.query ? ( ) : ( { + const mappings = await http.post<{ + mappings: IndicesGetMappingResponse; + }>(APIRoutes.GET_INDEX_MAPPINGS, { + body: JSON.stringify({ + indices, + }), + }); + return mappings; +}; +export const useIndexMappings = () => { + const { + services: { http }, + } = useKibana(); + const { getValues } = useFormContext(); + const indices = getValues(ChatFormFields.indices); + const { data } = useQuery({ + queryKey: ['search-playground-index-mappings'], + queryFn: () => fetchIndexMappings({ indices, http }), + refetchOnWindowFocus: false, + refetchOnReconnect: false, + refetchOnMount: false, + }); + + return { data: data?.mappings }; +}; diff --git a/x-pack/plugins/search_playground/public/hooks/use_search_preview.ts b/x-pack/plugins/search_playground/public/hooks/use_search_preview.ts index 54566563fcee58..f66f81b37cd2eb 100644 --- a/x-pack/plugins/search_playground/public/hooks/use_search_preview.ts +++ b/x-pack/plugins/search_playground/public/hooks/use_search_preview.ts @@ -8,6 +8,7 @@ import { SearchHit } from '@elastic/elasticsearch/lib/api/types'; import { useQuery } from '@tanstack/react-query'; import { useFormContext } from 'react-hook-form'; +import type { HttpSetup } from '@kbn/core-http-browser'; import { APIRoutes, ChatForm, ChatFormFields, Pagination } from '../types'; import { useKibana } from './use_kibana'; import { DEFAULT_PAGINATION } from '../../common'; @@ -17,7 +18,7 @@ export interface FetchSearchResultsArgs { pagination: Pagination; indices: ChatForm[ChatFormFields.indices]; elasticsearchQuery: ChatForm[ChatFormFields.elasticsearchQuery]; - http: ReturnType['services']['http']; + http: HttpSetup; } interface UseSearchPreviewData { @@ -64,9 +65,10 @@ export const useSearchPreview = ({ query: string; pagination: Pagination; }) => { - const { services } = useKibana(); + const { + services: { http }, + } = useKibana(); const { getValues } = useFormContext(); - const { http } = services; const indices = getValues(ChatFormFields.indices); const elasticsearchQuery = getValues(ChatFormFields.elasticsearchQuery); diff --git a/x-pack/plugins/search_playground/public/plugin.ts b/x-pack/plugins/search_playground/public/plugin.ts index 20142c807b6096..47bc8352b763ee 100644 --- a/x-pack/plugins/search_playground/public/plugin.ts +++ b/x-pack/plugins/search_playground/public/plugin.ts @@ -47,6 +47,9 @@ export class SearchPlaygroundPlugin async mount({ element, history }: AppMountParameters) { const { renderApp } = await import('./application'); const [coreStart, depsStart] = await core.getStartServices(); + + coreStart.chrome.docTitle.change(PLUGIN_NAME); + const startDeps: AppPluginStartDependencies = { ...depsStart, history, diff --git a/x-pack/plugins/search_playground/public/utils/pagination_helper.ts b/x-pack/plugins/search_playground/public/utils/pagination_helper.ts index 1379bbc257bd42..2602327b8c9683 100644 --- a/x-pack/plugins/search_playground/public/utils/pagination_helper.ts +++ b/x-pack/plugins/search_playground/public/utils/pagination_helper.ts @@ -7,13 +7,6 @@ import { Pagination } from '../../common/types'; -export const getPageCounts = (pagination: Pagination) => { - const { total, from, size } = pagination; - const totalPage = Math.ceil(total / size); - const page = Math.floor(from / size); - return { totalPage, total, page, size }; -}; - export const getPaginationFromPage = (page: number, size: number, previousValue: Pagination) => { const from = page < 0 ? 0 : page * size; return { ...previousValue, from, size, page }; diff --git a/x-pack/plugins/search_playground/server/routes.ts b/x-pack/plugins/search_playground/server/routes.ts index d30904214d8dff..cf6a139b1344bd 100644 --- a/x-pack/plugins/search_playground/server/routes.ts +++ b/x-pack/plugins/search_playground/server/routes.ts @@ -305,4 +305,47 @@ export function defineRoutes({ } }) ); + router.post( + { + path: APIRoutes.GET_INDEX_MAPPINGS, + validate: { + body: schema.object({ + indices: schema.arrayOf(schema.string()), + }), + }, + }, + errorHandler(logger)(async (context, request, response) => { + const { client } = (await context.core).elasticsearch; + const { indices } = request.body; + + try { + if (indices.length === 0) { + return response.badRequest({ + body: { + message: 'Indices cannot be empty', + }, + }); + } + + const mappings = await client.asCurrentUser.indices.getMapping({ + index: indices, + }); + return response.ok({ + body: { + mappings, + }, + }); + } catch (e) { + logger.error('Failed to get index mappings', e); + if (typeof e === 'object' && e.message) { + return response.badRequest({ + body: { + message: e.message, + }, + }); + } + throw e; + } + }) + ); } diff --git a/x-pack/plugins/search_playground/tsconfig.json b/x-pack/plugins/search_playground/tsconfig.json index eebfd0df9a7b3d..29c144ff4bac8d 100644 --- a/x-pack/plugins/search_playground/tsconfig.json +++ b/x-pack/plugins/search_playground/tsconfig.json @@ -44,7 +44,8 @@ "@kbn/unified-doc-viewer-plugin", "@kbn/data-views-plugin", "@kbn/discover-utils", - "@kbn/data-plugin" + "@kbn/data-plugin", + "@kbn/search-index-documents" ], "exclude": [ "target/**/*", diff --git a/x-pack/plugins/serverless_search/public/application/components/connectors/edit_connector.tsx b/x-pack/plugins/serverless_search/public/application/components/connectors/edit_connector.tsx index 78d4da85ab9096..2932ffc7bf79a9 100644 --- a/x-pack/plugins/serverless_search/public/application/components/connectors/edit_connector.tsx +++ b/x-pack/plugins/serverless_search/public/application/components/connectors/edit_connector.tsx @@ -12,6 +12,7 @@ import { EuiFlexItem, EuiPageTemplate, EuiPanel, + EuiForm, EuiPopover, EuiSpacer, EuiText, @@ -158,9 +159,11 @@ export const EditConnector: React.FC = () => { - - - + + + + + diff --git a/x-pack/plugins/serverless_search/public/application/components/connectors/edit_description.tsx b/x-pack/plugins/serverless_search/public/application/components/connectors/edit_description.tsx index 111d91b74d0435..1749e1673e2694 100644 --- a/x-pack/plugins/serverless_search/public/application/components/connectors/edit_description.tsx +++ b/x-pack/plugins/serverless_search/public/application/components/connectors/edit_description.tsx @@ -12,12 +12,11 @@ import { EuiFlexItem, EuiFlexGroup, EuiFieldText, - EuiForm, EuiButton, EuiSpacer, EuiFormRow, EuiText, - EuiButtonEmpty, + EuiLink, } from '@elastic/eui'; import { useMutation, useQueryClient } from '@tanstack/react-query'; import { Connector } from '@kbn/search-connectors'; @@ -56,43 +55,48 @@ export const EditDescription: React.FC = ({ connector }) = }); return ( - - - - setIsEditing(true)} - > - {EDIT_LABEL} - - } + + setIsEditing(true)} + role="button" > - {isEditing ? ( - setNewDescription(event.target.value)} - value={newDescription || ''} - /> - ) : ( - - {connector.description} - - )} - + {EDIT_LABEL} + + + } + fullWidth + > + + + {isEditing ? ( + setNewDescription(event.target.value)} + value={newDescription || ''} + fullWidth + /> + ) : ( + + {connector.description} + + )} {isEditing && ( - <> + - + = ({ connector }) = - + )} - - + + ); }; diff --git a/x-pack/plugins/serverless_search/public/application/components/connectors/edit_service_type.tsx b/x-pack/plugins/serverless_search/public/application/components/connectors/edit_service_type.tsx index fe4b6bd7e97683..a6598b4de15ea2 100644 --- a/x-pack/plugins/serverless_search/public/application/components/connectors/edit_service_type.tsx +++ b/x-pack/plugins/serverless_search/public/application/components/connectors/edit_service_type.tsx @@ -7,14 +7,7 @@ import { i18n } from '@kbn/i18n'; import React from 'react'; -import { - EuiFlexItem, - EuiFlexGroup, - EuiForm, - EuiFormLabel, - EuiIcon, - EuiSuperSelect, -} from '@elastic/eui'; +import { EuiFlexItem, EuiFlexGroup, EuiIcon, EuiFormRow, EuiSuperSelect } from '@elastic/eui'; import { useMutation, useQueryClient } from '@tanstack/react-query'; import { Connector } from '@kbn/search-connectors'; import { useKibanaServices } from '../../hooks/use_kibana'; @@ -69,12 +62,13 @@ export const EditServiceType: React.FC = ({ connector }) = }); return ( - - - {i18n.translate('xpack.serverlessSearch.connectors.serviceTypeLabel', { - defaultMessage: 'Connector type', - })} - + = ({ connector }) = onChange={(event) => mutate(event)} options={options} valueOfSelected={connector.service_type || undefined} + fullWidth /> - + ); }; diff --git a/x-pack/plugins/serverless_search/public/application/components/index_documents/documents.tsx b/x-pack/plugins/serverless_search/public/application/components/index_documents/documents.tsx index d74c3a479a68a3..0dc6e74d63ba62 100644 --- a/x-pack/plugins/serverless_search/public/application/components/index_documents/documents.tsx +++ b/x-pack/plugins/serverless_search/public/application/components/index_documents/documents.tsx @@ -63,7 +63,7 @@ export const IndexDocuments: React.FC = ({ indexName }) => docs={docs} docsPerPage={pagination.pageSize ?? 10} isLoading={false} - mappings={mappingData?.mappings?.properties ?? {}} + mappings={mappingData ? { [indexName]: mappingData } : undefined} meta={documentsMeta ?? DEFAULT_PAGINATION} onPaginate={(pageIndex) => setPagination({ ...pagination, pageIndex })} setDocsPerPage={(pageSize) => setPagination({ ...pagination, pageSize })} diff --git a/x-pack/plugins/serverless_search/public/plugin.ts b/x-pack/plugins/serverless_search/public/plugin.ts index 7953474a099bf7..d06a104d01fccc 100644 --- a/x-pack/plugins/serverless_search/public/plugin.ts +++ b/x-pack/plugins/serverless_search/public/plugin.ts @@ -72,18 +72,22 @@ export class ServerlessSearchPlugin }, }), }); + + const homeTitle = i18n.translate('xpack.serverlessSearch.app.home.title', { + defaultMessage: 'Home', + }); + if (useSearchHomepage) { core.application.register({ id: 'serverlessHomeRedirect', - title: i18n.translate('xpack.serverlessSearch.app.home.title', { - defaultMessage: 'Home', - }), + title: homeTitle, appRoute: '/app/elasticsearch', euiIconType: 'logoElastic', category: DEFAULT_APP_CATEGORIES.enterpriseSearch, visibleIn: [], async mount({}: AppMountParameters) { const [coreStart] = await core.getStartServices(); + coreStart.chrome.docTitle.change(homeTitle); coreStart.application.navigateToApp('searchHomepage'); return () => {}; }, @@ -102,6 +106,7 @@ export class ServerlessSearchPlugin const { renderApp } = await import('./application/elasticsearch'); const [coreStart, services] = await core.getStartServices(); docLinks.setDocLinks(coreStart.docLinks.links); + coreStart.chrome.docTitle.change(homeTitle); let user: AuthenticatedUser | undefined; try { const response = await coreStart.security.authc.getCurrentUser(); @@ -114,11 +119,13 @@ export class ServerlessSearchPlugin }, }); + const connectorsTitle = i18n.translate('xpack.serverlessSearch.app.connectors.title', { + defaultMessage: 'Connectors', + }); + core.application.register({ id: 'serverlessConnectors', - title: i18n.translate('xpack.serverlessSearch.app.connectors.title', { - defaultMessage: 'Connectors', - }), + title: connectorsTitle, appRoute: '/app/connectors', euiIconType: 'logoElastic', category: DEFAULT_APP_CATEGORIES.enterpriseSearch, @@ -126,8 +133,9 @@ export class ServerlessSearchPlugin async mount({ element, history }: AppMountParameters) { const { renderApp } = await import('./application/connectors'); const [coreStart, services] = await core.getStartServices(); - + coreStart.chrome.docTitle.change(connectorsTitle); docLinks.setDocLinks(coreStart.docLinks.links); + return await renderApp(element, coreStart, { history, ...services }, queryClient); }, }); diff --git a/x-pack/plugins/stack_connectors/public/connector_types/crowdstrike/crowdstrike_connector.tsx b/x-pack/plugins/stack_connectors/public/connector_types/crowdstrike/crowdstrike_connector.tsx index 3af2cd8c4648a8..b468cd3bbb712f 100644 --- a/x-pack/plugins/stack_connectors/public/connector_types/crowdstrike/crowdstrike_connector.tsx +++ b/x-pack/plugins/stack_connectors/public/connector_types/crowdstrike/crowdstrike_connector.tsx @@ -15,13 +15,11 @@ import { } from '@kbn/triggers-actions-ui-plugin/public'; import * as i18n from './translations'; -const CROWDSTRIKE_DEFAULT_API_URL = 'https://api.crowdstrike.com'; const configFormSchema: ConfigFieldSchema[] = [ { id: 'url', label: i18n.URL_LABEL, isUrlField: true, - defaultValue: CROWDSTRIKE_DEFAULT_API_URL, }, ]; diff --git a/x-pack/plugins/stack_connectors/public/connector_types/crowdstrike/translations.ts b/x-pack/plugins/stack_connectors/public/connector_types/crowdstrike/translations.ts index 1fb91cecfe9920..e10226f5329141 100644 --- a/x-pack/plugins/stack_connectors/public/connector_types/crowdstrike/translations.ts +++ b/x-pack/plugins/stack_connectors/public/connector_types/crowdstrike/translations.ts @@ -19,14 +19,14 @@ export const URL_LABEL = i18n.translate( export const CLIENT_ID_LABEL = i18n.translate( 'xpack.stackConnectors.security.crowdstrike.config.clientIdTextFieldLabel', { - defaultMessage: 'Crowdstrike client ID', + defaultMessage: 'Crowdstrike Client ID', } ); export const CLIENT_SECRET_LABEL = i18n.translate( 'xpack.stackConnectors.security.crowdstrike.config.clientSecretTextFieldLabel', { - defaultMessage: 'Client secret', + defaultMessage: 'Client Secret', } ); diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/actions_connectors_list/components/actions_connectors_list.test.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/actions_connectors_list/components/actions_connectors_list.test.tsx index a0742ebfd2bbdb..a938aa6b7e6e5f 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/actions_connectors_list/components/actions_connectors_list.test.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/actions_connectors_list/components/actions_connectors_list.test.tsx @@ -308,7 +308,7 @@ describe('actions_connectors_list', () => { .at(4) .find('div[data-test-subj="compatibility-content"]') .text() - ).toBe('Alerting RulesCases'); + ).toBe('Alerting RulesCasesSecurity Solution'); expect( wrapper diff --git a/x-pack/test/api_integration/apis/cloud_security_posture/helper.ts b/x-pack/test/api_integration/apis/cloud_security_posture/helper.ts index 9ab91cf0d60c0b..51f98b5389a9d3 100644 --- a/x-pack/test/api_integration/apis/cloud_security_posture/helper.ts +++ b/x-pack/test/api_integration/apis/cloud_security_posture/helper.ts @@ -29,6 +29,21 @@ export const deleteIndex = async (es: Client, indexToBeDeleted: string[]) => { ]); }; +export const bulkIndex = async (es: Client, findingsMock: T[], indexName: string) => { + const operations = findingsMock.flatMap((finding) => [ + { create: { _index: indexName } }, // Action description + { + ...finding, + '@timestamp': new Date().toISOString(), + }, // Data to index + ]); + + await es.bulk({ + body: operations, // Bulk API expects 'body' for operations + refresh: true, + }); +}; + export const addIndex = async (es: Client, findingsMock: T[], indexName: string) => { await Promise.all([ ...findingsMock.map((finding) => diff --git a/x-pack/test/functional/apps/lens/group4/index.ts b/x-pack/test/functional/apps/lens/group4/index.ts index 51b3bac4c519f0..61074d6aa6d787 100644 --- a/x-pack/test/functional/apps/lens/group4/index.ts +++ b/x-pack/test/functional/apps/lens/group4/index.ts @@ -82,5 +82,6 @@ export default ({ getService, loadTestFile, getPageObjects }: FtrProviderContext loadTestFile(require.resolve('./share')); // 1m 20s // keep it last in the group loadTestFile(require.resolve('./tsdb')); // 1m + loadTestFile(require.resolve('./logsdb')); // 1m }); }; diff --git a/x-pack/test/functional/apps/lens/group4/logsdb.ts b/x-pack/test/functional/apps/lens/group4/logsdb.ts new file mode 100644 index 00000000000000..a58b5c6bf806f5 --- /dev/null +++ b/x-pack/test/functional/apps/lens/group4/logsdb.ts @@ -0,0 +1,586 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import moment from 'moment'; +import { FtrProviderContext } from '../../../ftr_provider_context'; +import { + type ScenarioIndexes, + getDataMapping, + getDocsGenerator, + setupScenarioRunner, + TIME_PICKER_FORMAT, +} from './tsdb_logsdb_helpers'; + +export default function ({ getService, getPageObjects }: FtrProviderContext) { + const { common, lens, discover, header } = getPageObjects([ + 'common', + 'lens', + 'discover', + 'header', + ]); + const testSubjects = getService('testSubjects'); + const find = getService('find'); + const kibanaServer = getService('kibanaServer'); + const es = getService('es'); + const log = getService('log'); + const dataStreams = getService('dataStreams'); + const indexPatterns = getService('indexPatterns'); + const esArchiver = getService('esArchiver'); + const monacoEditor = getService('monacoEditor'); + const retry = getService('retry'); + + const createDocs = getDocsGenerator(log, es, 'logsdb'); + + describe('lens logsdb', function () { + const logsdbIndex = 'kibana_sample_data_logslogsdb'; + const logsdbDataView = logsdbIndex; + const logsdbEsArchive = 'test/functional/fixtures/es_archiver/kibana_sample_data_logs_logsdb'; + const fromTime = 'Apr 16, 2023 @ 00:00:00.000'; + const toTime = 'Jun 16, 2023 @ 00:00:00.000'; + + before(async () => { + log.info(`loading ${logsdbIndex} index...`); + await esArchiver.loadIfNeeded(logsdbEsArchive); + log.info(`creating a data view for "${logsdbDataView}"...`); + await indexPatterns.create( + { + title: logsdbDataView, + timeFieldName: '@timestamp', + }, + { override: true } + ); + log.info(`updating settings to use the "${logsdbDataView}" dataView...`); + await kibanaServer.uiSettings.update({ + 'dateFormat:tz': 'UTC', + defaultIndex: '0ae0bc7a-e4ca-405c-ab67-f2b5913f2a51', + 'timepicker:timeDefaults': `{ "from": "${fromTime}", "to": "${toTime}" }`, + }); + }); + + after(async () => { + await kibanaServer.savedObjects.cleanStandardList(); + await kibanaServer.uiSettings.replace({}); + await es.indices.delete({ index: [logsdbIndex] }); + }); + + describe('smoke testing functions support', () => { + before(async () => { + await common.navigateToApp('lens'); + await lens.switchDataPanelIndexPattern(logsdbDataView); + await lens.goToTimeRange(); + }); + + afterEach(async () => { + await lens.removeLayer(); + }); + + // skip count for now as it's a special function and will + // change automatically the unsupported field to Records when detected + const allOperations = [ + 'average', + 'max', + 'last_value', + 'median', + 'percentile', + 'percentile_rank', + 'standard_deviation', + 'sum', + 'unique_count', + 'min', + 'max', + 'counter_rate', + 'last_value', + ]; + + it(`should work with all operations`, async () => { + // start from a count() over a date histogram + await lens.configureDimension({ + dimension: 'lnsXY_xDimensionPanel > lns-empty-dimension', + operation: 'date_histogram', + field: '@timestamp', + }); + + // minimum supports all logsdb field types + await lens.configureDimension({ + dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension', + operation: 'count', + field: 'bytes', + keepOpen: true, + }); + + // now check that operations won't show the incompatibility tooltip + for (const operation of allOperations) { + expect( + testSubjects.exists(`lns-indexPatternDimension-${operation} incompatible`, { + timeout: 500, + }) + ).to.eql(false); + } + + for (const operation of allOperations) { + // try to change to the provided function and check all is ok + await lens.selectOperation(operation); + + expect( + await find.existsByCssSelector( + '[data-test-subj="indexPattern-field-selection-row"] .euiFormErrorText' + ) + ).to.be(false); + } + await lens.closeDimensionEditor(); + }); + + describe('Scenarios with changing stream type', () => { + const getScenarios = ( + initialIndex: string + ): Array<{ + name: string; + indexes: ScenarioIndexes[]; + }> => [ + { + name: 'LogsDB stream with no additional stream/index', + indexes: [{ index: initialIndex }], + }, + { + name: 'LogsDB stream with no additional stream/index and no host.name field', + indexes: [ + { + index: `${initialIndex}_no_host`, + removeLogsDBFields: true, + create: true, + mode: 'logsdb', + }, + ], + }, + { + name: 'LogsDB stream with an additional regular index', + indexes: [{ index: initialIndex }, { index: 'regular_index', create: true }], + }, + { + name: 'LogsDB stream with an additional LogsDB stream', + indexes: [ + { index: initialIndex }, + { index: 'logsdb_index_2', create: true, mode: 'logsdb' }, + ], + }, + { + name: 'LogsDB stream with an additional TSDB stream', + indexes: [{ index: initialIndex }, { index: 'tsdb_index', create: true, mode: 'tsdb' }], + }, + { + name: 'LogsDB stream with an additional TSDB stream downsampled', + indexes: [ + { index: initialIndex }, + { index: 'tsdb_index_downsampled', create: true, mode: 'tsdb', downsample: true }, + ], + }, + ]; + + const { runTestsForEachScenario, toTimeForScenarios, fromTimeForScenarios } = + setupScenarioRunner(getService, getPageObjects, getScenarios); + + describe('Data-stream upgraded to LogsDB scenarios', () => { + const streamIndex = 'data_stream'; + // rollover does not allow to change name, it will just change backing index underneath + const streamConvertedToLogsDBIndex = streamIndex; + + before(async () => { + log.info(`Creating "${streamIndex}" data stream...`); + await dataStreams.createDataStream( + streamIndex, + getDataMapping({ mode: 'logsdb' }), + undefined + ); + + // add some data to the stream + await createDocs(streamIndex, { isStream: true }, fromTimeForScenarios); + + log.info(`Update settings for "${streamIndex}" dataView...`); + await kibanaServer.uiSettings.update({ + 'dateFormat:tz': 'UTC', + 'timepicker:timeDefaults': '{ "from": "now-1y", "to": "now" }', + }); + log.info(`Upgrade "${streamIndex}" stream to LogsDB...`); + + const logsdbMapping = getDataMapping({ mode: 'logsdb' }); + await dataStreams.upgradeStream(streamIndex, logsdbMapping, 'logsdb'); + log.info( + `Add more data to new "${streamConvertedToLogsDBIndex}" dataView (now with LogsDB backing index)...` + ); + // add some more data when upgraded + await createDocs(streamConvertedToLogsDBIndex, { isStream: true }, toTimeForScenarios); + }); + + after(async () => { + await dataStreams.deleteDataStream(streamIndex); + }); + + runTestsForEachScenario(streamConvertedToLogsDBIndex, 'logsdb', (indexes) => { + it(`should visualize a date histogram chart`, async () => { + await lens.configureDimension({ + dimension: 'lnsXY_xDimensionPanel > lns-empty-dimension', + operation: 'date_histogram', + field: '@timestamp', + }); + + // check that a basic agg on a field works + await lens.configureDimension({ + dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension', + operation: 'min', + field: `bytes`, + }); + + await lens.waitForVisualization('xyVisChart'); + const data = await lens.getCurrentChartDebugState('xyVisChart'); + const bars = data?.bars![0].bars; + + log.info('Check counter data before the upgrade'); + // check there's some data before the upgrade + expect(bars?.[0].y).to.be.above(0); + log.info('Check counter data after the upgrade'); + // check there's some data after the upgrade + expect(bars?.[bars.length - 1].y).to.be.above(0); + }); + + it(`should visualize a date histogram chart using a different date field`, async () => { + await lens.configureDimension({ + dimension: 'lnsXY_xDimensionPanel > lns-empty-dimension', + operation: 'date_histogram', + field: 'utc_time', + }); + + // check the counter field works + await lens.configureDimension({ + dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension', + operation: 'min', + field: `bytes`, + }); + + await lens.waitForVisualization('xyVisChart'); + const data = await lens.getCurrentChartDebugState('xyVisChart'); + const bars = data?.bars![0].bars; + + log.info('Check counter data before the upgrade'); + // check there's some data before the upgrade + expect(bars?.[0].y).to.be.above(0); + log.info('Check counter data after the upgrade'); + // check there's some data after the upgrade + expect(bars?.[bars.length - 1].y).to.be.above(0); + }); + + it('should visualize an annotation layer from a logsDB stream', async () => { + await lens.configureDimension({ + dimension: 'lnsXY_xDimensionPanel > lns-empty-dimension', + operation: 'date_histogram', + field: 'utc_time', + }); + + // check the counter field works + await lens.configureDimension({ + dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension', + operation: 'min', + field: `bytes`, + }); + await lens.createLayer('annotations'); + + expect( + (await find.allByCssSelector(`[data-test-subj^="lns-layerPanel-"]`)).length + ).to.eql(2); + expect( + await ( + await testSubjects.find('lnsXY_xAnnotationsPanel > lns-dimensionTrigger') + ).getVisibleText() + ).to.eql('Event'); + await testSubjects.click('lnsXY_xAnnotationsPanel > lns-dimensionTrigger'); + await testSubjects.click('lnsXY_annotation_query'); + await lens.configureQueryAnnotation({ + queryString: 'host.name: *', + timeField: '@timestamp', + textDecoration: { type: 'name' }, + extraFields: ['host.name', 'utc_time'], + }); + await lens.closeDimensionEditor(); + + await testSubjects.existOrFail('xyVisGroupedAnnotationIcon'); + await lens.removeLayer(1); + }); + + it('should visualize an annotation layer from a logsDB stream using another time field', async () => { + await lens.configureDimension({ + dimension: 'lnsXY_xDimensionPanel > lns-empty-dimension', + operation: 'date_histogram', + field: 'utc_time', + }); + + // check the counter field works + await lens.configureDimension({ + dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension', + operation: 'min', + field: `bytes`, + }); + await lens.createLayer('annotations'); + + expect( + (await find.allByCssSelector(`[data-test-subj^="lns-layerPanel-"]`)).length + ).to.eql(2); + expect( + await ( + await testSubjects.find('lnsXY_xAnnotationsPanel > lns-dimensionTrigger') + ).getVisibleText() + ).to.eql('Event'); + await testSubjects.click('lnsXY_xAnnotationsPanel > lns-dimensionTrigger'); + await testSubjects.click('lnsXY_annotation_query'); + await lens.configureQueryAnnotation({ + queryString: 'host.name: *', + timeField: 'utc_time', + textDecoration: { type: 'name' }, + extraFields: ['host.name', '@timestamp'], + }); + await lens.closeDimensionEditor(); + + await testSubjects.existOrFail('xyVisGroupedAnnotationIcon'); + await lens.removeLayer(1); + }); + + it('should visualize correctly ES|QL queries based on a LogsDB stream', async () => { + await common.navigateToApp('discover'); + await discover.selectTextBaseLang(); + await header.waitUntilLoadingHasFinished(); + await monacoEditor.setCodeEditorValue( + `from ${indexes + .map(({ index }) => index) + .join(', ')} | stats averageB = avg(bytes) by extension` + ); + await testSubjects.click('querySubmitButton'); + await header.waitUntilLoadingHasFinished(); + await testSubjects.click('unifiedHistogramEditFlyoutVisualization'); + + await header.waitUntilLoadingHasFinished(); + + await retry.waitFor('lens flyout', async () => { + const dimensions = await testSubjects.findAll('lns-dimensionTrigger-textBased'); + return ( + dimensions.length === 2 && (await dimensions[1].getVisibleText()) === 'averageB' + ); + }); + + // go back to Lens to not break the wrapping function + await common.navigateToApp('lens'); + }); + }); + }); + + describe('LogsDB downgraded to regular data stream scenarios', () => { + const logsdbStream = 'logsdb_stream_dowgradable'; + // rollover does not allow to change name, it will just change backing index underneath + const logsdbConvertedToStream = logsdbStream; + + before(async () => { + log.info(`Creating "${logsdbStream}" data stream...`); + await dataStreams.createDataStream( + logsdbStream, + getDataMapping({ mode: 'logsdb' }), + 'logsdb' + ); + + // add some data to the stream + await createDocs(logsdbStream, { isStream: true }, fromTimeForScenarios); + + log.info(`Update settings for "${logsdbStream}" dataView...`); + await kibanaServer.uiSettings.update({ + 'dateFormat:tz': 'UTC', + 'timepicker:timeDefaults': '{ "from": "now-1y", "to": "now" }', + }); + log.info( + `Dowgrade "${logsdbStream}" stream into regular stream "${logsdbConvertedToStream}"...` + ); + + await dataStreams.downgradeStream( + logsdbStream, + getDataMapping({ mode: 'logsdb' }), + 'logsdb' + ); + log.info( + `Add more data to new "${logsdbConvertedToStream}" dataView (no longer LogsDB)...` + ); + // add some more data when upgraded + await createDocs(logsdbConvertedToStream, { isStream: true }, toTimeForScenarios); + }); + + after(async () => { + await dataStreams.deleteDataStream(logsdbConvertedToStream); + }); + + runTestsForEachScenario(logsdbConvertedToStream, 'logsdb', (indexes) => { + it(`should visualize a date histogram chart`, async () => { + await lens.configureDimension({ + dimension: 'lnsXY_xDimensionPanel > lns-empty-dimension', + operation: 'date_histogram', + field: '@timestamp', + }); + + // check that a basic agg on a field works + await lens.configureDimension({ + dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension', + operation: 'min', + field: `bytes`, + }); + + await lens.waitForVisualization('xyVisChart'); + const data = await lens.getCurrentChartDebugState('xyVisChart'); + const bars = data?.bars![0].bars; + + log.info('Check counter data before the upgrade'); + // check there's some data before the upgrade + expect(bars?.[0].y).to.be.above(0); + log.info('Check counter data after the upgrade'); + // check there's some data after the upgrade + expect(bars?.[bars.length - 1].y).to.be.above(0); + }); + + it(`should visualize a date histogram chart using a different date field`, async () => { + await lens.configureDimension({ + dimension: 'lnsXY_xDimensionPanel > lns-empty-dimension', + operation: 'date_histogram', + field: 'utc_time', + }); + + // check the counter field works + await lens.configureDimension({ + dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension', + operation: 'min', + field: `bytes`, + }); + + await lens.waitForVisualization('xyVisChart'); + const data = await lens.getCurrentChartDebugState('xyVisChart'); + const bars = data?.bars![0].bars; + + log.info('Check counter data before the upgrade'); + // check there's some data before the upgrade + expect(bars?.[0].y).to.be.above(0); + log.info('Check counter data after the upgrade'); + // check there's some data after the upgrade + expect(bars?.[bars.length - 1].y).to.be.above(0); + }); + + it('should visualize an annotation layer from a logsDB stream', async () => { + await lens.configureDimension({ + dimension: 'lnsXY_xDimensionPanel > lns-empty-dimension', + operation: 'date_histogram', + field: 'utc_time', + }); + + // check the counter field works + await lens.configureDimension({ + dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension', + operation: 'min', + field: `bytes`, + }); + await lens.createLayer('annotations'); + + expect( + (await find.allByCssSelector(`[data-test-subj^="lns-layerPanel-"]`)).length + ).to.eql(2); + expect( + await ( + await testSubjects.find('lnsXY_xAnnotationsPanel > lns-dimensionTrigger') + ).getVisibleText() + ).to.eql('Event'); + await testSubjects.click('lnsXY_xAnnotationsPanel > lns-dimensionTrigger'); + await testSubjects.click('lnsXY_annotation_query'); + await lens.configureQueryAnnotation({ + queryString: 'host.name: *', + timeField: '@timestamp', + textDecoration: { type: 'name' }, + extraFields: ['host.name', 'utc_time'], + }); + await lens.closeDimensionEditor(); + + await testSubjects.existOrFail('xyVisGroupedAnnotationIcon'); + await lens.removeLayer(1); + }); + + it('should visualize an annotation layer from a logsDB stream using another time field', async () => { + await lens.configureDimension({ + dimension: 'lnsXY_xDimensionPanel > lns-empty-dimension', + operation: 'date_histogram', + field: 'utc_time', + }); + + // check the counter field works + await lens.configureDimension({ + dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension', + operation: 'min', + field: `bytes`, + }); + await lens.createLayer('annotations'); + + expect( + (await find.allByCssSelector(`[data-test-subj^="lns-layerPanel-"]`)).length + ).to.eql(2); + expect( + await ( + await testSubjects.find('lnsXY_xAnnotationsPanel > lns-dimensionTrigger') + ).getVisibleText() + ).to.eql('Event'); + await testSubjects.click('lnsXY_xAnnotationsPanel > lns-dimensionTrigger'); + await testSubjects.click('lnsXY_annotation_query'); + await lens.configureQueryAnnotation({ + queryString: 'host.name: *', + timeField: 'utc_time', + textDecoration: { type: 'name' }, + extraFields: ['host.name', '@timestamp'], + }); + await lens.closeDimensionEditor(); + + await testSubjects.existOrFail('xyVisGroupedAnnotationIcon'); + await lens.removeLayer(1); + }); + + it('should visualize correctly ES|QL queries based on a LogsDB stream', async () => { + await common.navigateToApp('discover'); + await discover.selectTextBaseLang(); + + // Use the lens page object here also for discover: both use the same timePicker object + await lens.goToTimeRange( + fromTimeForScenarios, + moment + .utc(toTimeForScenarios, TIME_PICKER_FORMAT) + .add(2, 'hour') + .format(TIME_PICKER_FORMAT) + ); + + await header.waitUntilLoadingHasFinished(); + await monacoEditor.setCodeEditorValue( + `from ${indexes + .map(({ index }) => index) + .join(', ')} | stats averageB = avg(bytes) by extension` + ); + await testSubjects.click('querySubmitButton'); + await header.waitUntilLoadingHasFinished(); + await testSubjects.click('unifiedHistogramEditFlyoutVisualization'); + + await header.waitUntilLoadingHasFinished(); + + await retry.waitFor('lens flyout', async () => { + const dimensions = await testSubjects.findAll('lns-dimensionTrigger-textBased'); + return ( + dimensions.length === 2 && (await dimensions[1].getVisibleText()) === 'averageB' + ); + }); + + // go back to Lens to not break the wrapping function + await common.navigateToApp('lens'); + }); + }); + }); + }); + }); + }); +} diff --git a/x-pack/test/functional/apps/lens/group4/tsdb.ts b/x-pack/test/functional/apps/lens/group4/tsdb.ts index bbe1eef8a442c5..3a6aac5ffa39bb 100644 --- a/x-pack/test/functional/apps/lens/group4/tsdb.ts +++ b/x-pack/test/functional/apps/lens/group4/tsdb.ts @@ -8,234 +8,16 @@ import expect from '@kbn/expect'; import { partition } from 'lodash'; import moment from 'moment'; -import { MappingProperty } from '@elastic/elasticsearch/lib/api/types'; import { FtrProviderContext } from '../../../ftr_provider_context'; - -const TEST_DOC_COUNT = 100; -const TIME_PICKER_FORMAT = 'MMM D, YYYY [@] HH:mm:ss.SSS'; -const timeSeriesMetrics: Record = { - bytes_gauge: 'gauge', - bytes_counter: 'counter', -}; -const timeSeriesDimensions = ['request', 'url']; - -type TestDoc = Record>; - -const testDocTemplate: TestDoc = { - agent: 'Mozilla/5.0 (X11; Linux x86_64; rv:6.0a1) Gecko/20110421 Firefox/6.0a1', - bytes: 6219, - clientip: '223.87.60.27', - extension: 'deb', - geo: { - srcdest: 'US:US', - src: 'US', - dest: 'US', - coordinates: { lat: 39.41042861, lon: -88.8454325 }, - }, - host: 'artifacts.elastic.co', - index: 'kibana_sample_data_logs', - ip: '223.87.60.27', - machine: { ram: 8589934592, os: 'win 8' }, - memory: null, - message: - '223.87.60.27 - - [2018-07-22T00:39:02.912Z] "GET /elasticsearch/elasticsearch-6.3.2.deb_1 HTTP/1.1" 200 6219 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:6.0a1) Gecko/20110421 Firefox/6.0a1"', - phpmemory: null, - referer: 'http://twitter.com/success/wendy-lawrence', - request: '/elasticsearch/elasticsearch-6.3.2.deb', - response: 200, - tags: ['success', 'info'], - '@timestamp': '2018-07-22T00:39:02.912Z', - url: 'https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.3.2.deb_1', - utc_time: '2018-07-22T00:39:02.912Z', - event: { dataset: 'sample_web_logs' }, - bytes_gauge: 0, - bytes_counter: 0, -}; - -function getDataMapping( - { tsdb, removeTSDBFields }: { tsdb: boolean; removeTSDBFields?: boolean } = { - tsdb: false, - } -): Record { - const dataStreamMapping: Record = { - '@timestamp': { - type: 'date', - }, - agent: { - fields: { - keyword: { - ignore_above: 256, - type: 'keyword', - }, - }, - type: 'text', - }, - bytes: { - type: 'long', - }, - bytes_counter: { - type: 'long', - }, - bytes_gauge: { - type: 'long', - }, - clientip: { - type: 'ip', - }, - event: { - properties: { - dataset: { - type: 'keyword', - }, - }, - }, - extension: { - fields: { - keyword: { - ignore_above: 256, - type: 'keyword', - }, - }, - type: 'text', - }, - geo: { - properties: { - coordinates: { - type: 'geo_point', - }, - dest: { - type: 'keyword', - }, - src: { - type: 'keyword', - }, - srcdest: { - type: 'keyword', - }, - }, - }, - host: { - fields: { - keyword: { - ignore_above: 256, - type: 'keyword', - }, - }, - type: 'text', - }, - index: { - fields: { - keyword: { - ignore_above: 256, - type: 'keyword', - }, - }, - type: 'text', - }, - ip: { - type: 'ip', - }, - machine: { - properties: { - os: { - fields: { - keyword: { - ignore_above: 256, - type: 'keyword', - }, - }, - type: 'text', - }, - ram: { - type: 'long', - }, - }, - }, - memory: { - type: 'double', - }, - message: { - fields: { - keyword: { - ignore_above: 256, - type: 'keyword', - }, - }, - type: 'text', - }, - phpmemory: { - type: 'long', - }, - referer: { - type: 'keyword', - }, - request: { - type: 'keyword', - }, - response: { - fields: { - keyword: { - ignore_above: 256, - type: 'keyword', - }, - }, - type: 'text', - }, - tags: { - fields: { - keyword: { - ignore_above: 256, - type: 'keyword', - }, - }, - type: 'text', - }, - timestamp: { - path: '@timestamp', - type: 'alias', - }, - url: { - type: 'keyword', - }, - utc_time: { - type: 'date', - }, - }; - - if (tsdb) { - // augment the current mapping - for (const [fieldName, fieldMapping] of Object.entries(dataStreamMapping || {})) { - if ( - timeSeriesMetrics[fieldName] && - (fieldMapping.type === 'double' || fieldMapping.type === 'long') - ) { - fieldMapping.time_series_metric = timeSeriesMetrics[fieldName]; - } - - if (timeSeriesDimensions.includes(fieldName) && fieldMapping.type === 'keyword') { - fieldMapping.time_series_dimension = true; - } - } - } else if (removeTSDBFields) { - for (const fieldName of Object.keys(timeSeriesMetrics)) { - delete dataStreamMapping[fieldName]; - } - } - return dataStreamMapping; -} - -function sumFirstNValues(n: number, bars: Array<{ y: number }> | undefined): number { - const indexes = Array(n) - .fill(1) - .map((_, i) => i); - let countSum = 0; - for (const index of indexes) { - if (bars?.[index]) { - countSum += bars[index].y; - } - } - return countSum; -} +import { + type ScenarioIndexes, + TEST_DOC_COUNT, + TIME_PICKER_FORMAT, + getDataMapping, + getDocsGenerator, + setupScenarioRunner, + sumFirstNValues, +} from './tsdb_logsdb_helpers'; export default function ({ getService, getPageObjects }: FtrProviderContext) { const { common, lens, dashboard } = getPageObjects(['common', 'lens', 'dashboard']); @@ -245,71 +27,11 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const es = getService('es'); const log = getService('log'); const dataStreams = getService('dataStreams'); - const elasticChart = getService('elasticChart'); const indexPatterns = getService('indexPatterns'); const esArchiver = getService('esArchiver'); const comboBox = getService('comboBox'); - const createDocs = async ( - esIndex: string, - { isStream, removeTSDBFields }: { isStream: boolean; removeTSDBFields?: boolean }, - startTime: string - ) => { - log.info( - `Adding ${TEST_DOC_COUNT} to ${esIndex} with starting time from ${moment - .utc(startTime, TIME_PICKER_FORMAT) - .format(TIME_PICKER_FORMAT)} to ${moment - .utc(startTime, TIME_PICKER_FORMAT) - .add(2 * TEST_DOC_COUNT, 'seconds') - .format(TIME_PICKER_FORMAT)}` - ); - const docs = Array(TEST_DOC_COUNT) - .fill(testDocTemplate) - .map((templateDoc, i) => { - const timestamp = moment - .utc(startTime, TIME_PICKER_FORMAT) - .add(TEST_DOC_COUNT + i, 'seconds') - .format(); - const doc: TestDoc = { - ...templateDoc, - '@timestamp': timestamp, - utc_time: timestamp, - bytes_gauge: Math.floor(Math.random() * 10000 * i), - bytes_counter: 5000, - }; - if (removeTSDBFields) { - for (const field of Object.keys(timeSeriesMetrics)) { - delete doc[field]; - } - } - return doc; - }); - - const result = await es.bulk( - { - index: esIndex, - body: docs.map((d) => `{"${isStream ? 'create' : 'index'}": {}}\n${JSON.stringify(d)}\n`), - }, - { meta: true } - ); - - const res = result.body; - - if (res.errors) { - const resultsWithErrors = res.items - .filter(({ index }) => index?.error) - .map(({ index }) => index?.error); - for (const error of resultsWithErrors) { - log.error(`Error: ${JSON.stringify(error)}`); - } - const [indexExists, dataStreamExists] = await Promise.all([ - es.indices.exists({ index: esIndex }), - es.indices.getDataStream({ name: esIndex }), - ]); - log.debug(`Index exists: ${indexExists} - Data stream exists: ${dataStreamExists}`); - } - log.info(`Indexed ${res.items.length} test data docs.`); - }; + const createDocs = getDocsGenerator(log, es, 'tsdb'); describe('lens tsdb', function () { const tsdbIndex = 'kibana_sample_data_logstsdb'; @@ -592,23 +314,11 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); describe('Scenarios with changing stream type', () => { - const now = moment().utc(); - const fromMoment = now.clone().subtract(1, 'hour'); - const toMoment = now.clone(); - const fromTimeForScenarios = fromMoment.format(TIME_PICKER_FORMAT); - const toTimeForScenarios = toMoment.format(TIME_PICKER_FORMAT); - const getScenarios = ( initialIndex: string ): Array<{ name: string; - indexes: Array<{ - index: string; - create?: boolean; - downsample?: boolean; - tsdb?: boolean; - removeTSDBFields?: boolean; - }>; + indexes: ScenarioIndexes[]; }> => [ { name: 'Dataview with no additional stream/index', @@ -625,7 +335,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { name: 'Dataview with an additional downsampled TSDB stream', indexes: [ { index: initialIndex }, - { index: 'tsdb_index_2', create: true, tsdb: true, downsample: true }, + { index: 'tsdb_index_2', create: true, mode: 'tsdb', downsample: true }, ], }, { @@ -633,112 +343,17 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { indexes: [ { index: initialIndex }, { index: 'regular_index', create: true, removeTSDBFields: true }, - { index: 'tsdb_index_2', create: true, tsdb: true, downsample: true }, + { index: 'tsdb_index_2', create: true, mode: 'tsdb', downsample: true }, ], }, { name: 'Dataview with an additional TSDB stream', - indexes: [{ index: initialIndex }, { index: 'tsdb_index_2', create: true, tsdb: true }], + indexes: [{ index: initialIndex }, { index: 'tsdb_index_2', create: true, mode: 'tsdb' }], }, ]; - function runTestsForEachScenario( - initialIndex: string, - testingFn: ( - indexes: Array<{ - index: string; - create?: boolean; - downsample?: boolean; - tsdb?: boolean; - removeTSDBFields?: boolean; - }> - ) => void - ): void { - for (const { name, indexes } of getScenarios(initialIndex)) { - describe(name, () => { - let dataViewName: string; - let downsampledTargetIndex: string = ''; - - before(async () => { - for (const { index, create, downsample, tsdb, removeTSDBFields } of indexes) { - if (create) { - if (tsdb) { - await dataStreams.createDataStream( - index, - getDataMapping({ tsdb, removeTSDBFields }), - tsdb - ); - } else { - log.info(`creating a index "${index}" with mapping...`); - await es.indices.create({ - index, - mappings: { - properties: getDataMapping({ tsdb: Boolean(tsdb), removeTSDBFields }), - }, - }); - } - // add data to the newly created index - await createDocs( - index, - { isStream: Boolean(tsdb), removeTSDBFields }, - fromTimeForScenarios - ); - } - if (downsample) { - downsampledTargetIndex = await dataStreams.downsampleTSDBIndex(index, { - isStream: Boolean(tsdb), - }); - } - } - dataViewName = `${indexes.map(({ index }) => index).join(',')}${ - downsampledTargetIndex ? `,${downsampledTargetIndex}` : '' - }`; - log.info(`creating a data view for "${dataViewName}"...`); - await indexPatterns.create( - { - title: dataViewName, - timeFieldName: '@timestamp', - }, - { override: true } - ); - await common.navigateToApp('lens'); - await elasticChart.setNewChartUiDebugFlag(true); - // go to the - await lens.goToTimeRange( - fromTimeForScenarios, - moment - .utc(toTimeForScenarios, TIME_PICKER_FORMAT) - .add(2, 'hour') - .format(TIME_PICKER_FORMAT) // consider also new documents - ); - }); - - after(async () => { - for (const { index, create, tsdb } of indexes) { - if (create) { - if (tsdb) { - await dataStreams.deleteDataStream(index); - } else { - log.info(`deleting the index "${index}"...`); - await es.indices.delete({ - index, - }); - } - } - // no need to cleant he specific downsample index as everything linked to the stream - // is cleaned up automatically - } - }); - - beforeEach(async () => { - await lens.switchDataPanelIndexPattern(dataViewName); - await lens.removeLayer(); - }); - - testingFn(indexes); - }); - } - } + const { runTestsForEachScenario, toTimeForScenarios, fromTimeForScenarios } = + setupScenarioRunner(getService, getPageObjects, getScenarios); describe('Data-stream upgraded to TSDB scenarios', () => { const streamIndex = 'data_stream'; @@ -747,7 +362,11 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { before(async () => { log.info(`Creating "${streamIndex}" data stream...`); - await dataStreams.createDataStream(streamIndex, getDataMapping(), false); + await dataStreams.createDataStream( + streamIndex, + getDataMapping({ mode: 'tsdb' }), + undefined + ); // add some data to the stream await createDocs(streamIndex, { isStream: true }, fromTimeForScenarios); @@ -759,8 +378,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); log.info(`Upgrade "${streamIndex}" stream to TSDB...`); - const tsdbMapping = getDataMapping({ tsdb: true }); - await dataStreams.upgradeStreamToTSDB(streamIndex, tsdbMapping); + const tsdbMapping = getDataMapping({ mode: 'tsdb' }); + await dataStreams.upgradeStream(streamIndex, tsdbMapping, 'tsdb'); log.info( `Add more data to new "${streamConvertedToTsdbIndex}" dataView (now with TSDB backing index)...` ); @@ -772,7 +391,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await dataStreams.deleteDataStream(streamIndex); }); - runTestsForEachScenario(streamConvertedToTsdbIndex, (indexes) => { + runTestsForEachScenario(streamConvertedToTsdbIndex, 'tsdb', (indexes) => { it('should detect the data stream has now been upgraded to TSDB', async () => { await lens.configureDimension({ dimension: 'lnsXY_xDimensionPanel > lns-empty-dimension', @@ -850,7 +469,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { before(async () => { log.info(`Creating "${tsdbStream}" data stream...`); - await dataStreams.createDataStream(tsdbStream, getDataMapping({ tsdb: true }), true); + await dataStreams.createDataStream(tsdbStream, getDataMapping({ mode: 'tsdb' }), 'tsdb'); // add some data to the stream await createDocs(tsdbStream, { isStream: true }, fromTimeForScenarios); @@ -864,7 +483,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { `Dowgrade "${tsdbStream}" stream into regular stream "${tsdbConvertedToStream}"...` ); - await dataStreams.downgradeTSDBtoStream(tsdbStream, getDataMapping({ tsdb: true })); + await dataStreams.downgradeStream(tsdbStream, getDataMapping({ mode: 'tsdb' }), 'tsdb'); log.info(`Add more data to new "${tsdbConvertedToStream}" dataView (no longer TSDB)...`); // add some more data when upgraded await createDocs(tsdbConvertedToStream, { isStream: true }, toTimeForScenarios); @@ -874,7 +493,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await dataStreams.deleteDataStream(tsdbConvertedToStream); }); - runTestsForEachScenario(tsdbConvertedToStream, (indexes) => { + runTestsForEachScenario(tsdbConvertedToStream, 'tsdb', (indexes) => { it('should keep TSDB restrictions only if a tsdb stream is in the dataView mix', async () => { await lens.configureDimension({ dimension: 'lnsXY_xDimensionPanel > lns-empty-dimension', @@ -893,7 +512,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { testSubjects.exists(`lns-indexPatternDimension-average incompatible`, { timeout: 500, }) - ).to.eql(indexes.some(({ tsdb }) => tsdb)); + ).to.eql(indexes.some(({ mode }) => mode === 'tsdb')); await lens.closeDimensionEditor(); }); diff --git a/x-pack/test/functional/apps/lens/group4/tsdb_logsdb_helpers.ts b/x-pack/test/functional/apps/lens/group4/tsdb_logsdb_helpers.ts new file mode 100644 index 00000000000000..e0169ebbae5753 --- /dev/null +++ b/x-pack/test/functional/apps/lens/group4/tsdb_logsdb_helpers.ts @@ -0,0 +1,480 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import expect from '@kbn/expect'; +import { Client } from '@elastic/elasticsearch'; +import { MappingProperty } from '@elastic/elasticsearch/lib/api/types'; +import { ToolingLog } from '@kbn/tooling-log'; +import moment from 'moment'; +import type { FtrProviderContext } from '../../../ftr_provider_context'; + +export const TEST_DOC_COUNT = 100; +export const TIME_PICKER_FORMAT = 'MMM D, YYYY [@] HH:mm:ss.SSS'; +export const timeSeriesMetrics: Record = { + bytes_gauge: 'gauge', + bytes_counter: 'counter', +}; +export const timeSeriesDimensions = ['request', 'url']; +export const logsDBSpecialFields = ['host']; + +export const sharedESArchive = + 'test/functional/fixtures/es_archiver/kibana_sample_data_logs_logsdb'; +export const fromTime = 'Apr 16, 2023 @ 00:00:00.000'; +export const toTime = 'Jun 16, 2023 @ 00:00:00.000'; + +export type TestDoc = Record>; + +export function testDocTemplate(mode: 'tsdb' | 'logsdb'): TestDoc { + return { + agent: 'Mozilla/5.0 (X11; Linux x86_64; rv:6.0a1) Gecko/20110421 Firefox/6.0a1', + bytes: 6219, + clientip: '223.87.60.27', + extension: 'deb', + geo: { + srcdest: 'US:US', + src: 'US', + dest: 'US', + coordinates: { lat: 39.41042861, lon: -88.8454325 }, + }, + host: mode === 'tsdb' ? 'artifacts.elastic.co' : { name: 'artifacts.elastic.co' }, + index: 'kibana_sample_data_logs', + ip: '223.87.60.27', + machine: { ram: 8589934592, os: 'win 8' }, + memory: null, + message: + '223.87.60.27 - - [2018-07-22T00:39:02.912Z] "GET /elasticsearch/elasticsearch-6.3.2.deb_1 HTTP/1.1" 200 6219 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:6.0a1) Gecko/20110421 Firefox/6.0a1"', + phpmemory: null, + referer: 'http://twitter.com/success/wendy-lawrence', + request: '/elasticsearch/elasticsearch-6.3.2.deb', + response: 200, + tags: ['success', 'info'], + '@timestamp': '2018-07-22T00:39:02.912Z', + url: 'https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.3.2.deb_1', + utc_time: '2018-07-22T00:39:02.912Z', + event: { dataset: 'sample_web_logs' }, + bytes_gauge: 0, + bytes_counter: 0, + }; +} + +export function getDataMapping({ + mode, + removeTSDBFields, + removeLogsDBFields, +}: { + mode: 'tsdb' | 'logsdb'; + removeTSDBFields?: boolean; + removeLogsDBFields?: boolean; +}): Record { + const dataStreamMapping: Record = { + '@timestamp': { + type: 'date', + }, + agent: { + fields: { + keyword: { + ignore_above: 256, + type: 'keyword', + }, + }, + type: 'text', + }, + bytes: { + type: 'long', + }, + bytes_counter: { + type: 'long', + }, + bytes_gauge: { + type: 'long', + }, + clientip: { + type: 'ip', + }, + event: { + properties: { + dataset: { + type: 'keyword', + }, + }, + }, + extension: { + fields: { + keyword: { + ignore_above: 256, + type: 'keyword', + }, + }, + type: 'text', + }, + geo: { + properties: { + coordinates: { + type: 'geo_point', + }, + dest: { + type: 'keyword', + }, + src: { + type: 'keyword', + }, + srcdest: { + type: 'keyword', + }, + }, + }, + host: + mode === 'tsdb' + ? { + fields: { + keyword: { + ignore_above: 256, + type: 'keyword', + }, + }, + type: 'text', + } + : { + properties: { + name: { + type: 'keyword', + }, + }, + }, + index: { + fields: { + keyword: { + ignore_above: 256, + type: 'keyword', + }, + }, + type: 'text', + }, + ip: { + type: 'ip', + }, + machine: { + properties: { + os: { + fields: { + keyword: { + ignore_above: 256, + type: 'keyword', + }, + }, + type: 'text', + }, + ram: { + type: 'long', + }, + }, + }, + memory: { + type: 'double', + }, + message: { + fields: { + keyword: { + ignore_above: 256, + type: 'keyword', + }, + }, + type: 'text', + }, + phpmemory: { + type: 'long', + }, + referer: { + type: 'keyword', + }, + request: { + type: 'keyword', + }, + response: { + fields: { + keyword: { + ignore_above: 256, + type: 'keyword', + }, + }, + type: 'text', + }, + tags: { + fields: { + keyword: { + ignore_above: 256, + type: 'keyword', + }, + }, + type: 'text', + }, + timestamp: { + path: '@timestamp', + type: 'alias', + }, + url: { + type: 'keyword', + }, + utc_time: { + type: 'date', + }, + }; + + if (mode === 'tsdb') { + // augment the current mapping + for (const [fieldName, fieldMapping] of Object.entries(dataStreamMapping || {})) { + if ( + timeSeriesMetrics[fieldName] && + (fieldMapping.type === 'double' || fieldMapping.type === 'long') + ) { + fieldMapping.time_series_metric = timeSeriesMetrics[fieldName]; + } + + if (timeSeriesDimensions.includes(fieldName) && fieldMapping.type === 'keyword') { + fieldMapping.time_series_dimension = true; + } + } + } + if (removeTSDBFields) { + for (const fieldName of Object.keys(timeSeriesMetrics)) { + delete dataStreamMapping[fieldName]; + } + } + if (removeLogsDBFields) { + for (const fieldName of logsDBSpecialFields) { + delete dataStreamMapping[fieldName]; + } + } + return dataStreamMapping; +} + +export function sumFirstNValues(n: number, bars: Array<{ y: number }> | undefined): number { + const indexes = Array(n) + .fill(1) + .map((_, i) => i); + let countSum = 0; + for (const index of indexes) { + if (bars?.[index]) { + countSum += bars[index].y; + } + } + return countSum; +} + +export const getDocsGenerator = + (log: ToolingLog, es: Client, mode: 'tsdb' | 'logsdb') => + async ( + esIndex: string, + { + isStream, + removeTSDBFields, + removeLogsDBFields, + }: { isStream: boolean; removeTSDBFields?: boolean; removeLogsDBFields?: boolean }, + startTime: string + ) => { + log.info( + `Adding ${TEST_DOC_COUNT} to ${esIndex} with starting time from ${moment + .utc(startTime, TIME_PICKER_FORMAT) + .format(TIME_PICKER_FORMAT)} to ${moment + .utc(startTime, TIME_PICKER_FORMAT) + .add(2 * TEST_DOC_COUNT, 'seconds') + .format(TIME_PICKER_FORMAT)}` + ); + const docs = Array(TEST_DOC_COUNT) + .fill(testDocTemplate(mode)) + .map((templateDoc, i) => { + const timestamp = moment + .utc(startTime, TIME_PICKER_FORMAT) + .add(TEST_DOC_COUNT + i, 'seconds') + .format(); + const doc: TestDoc = { + ...templateDoc, + '@timestamp': timestamp, + utc_time: timestamp, + bytes_gauge: Math.floor(Math.random() * 10000 * i), + bytes_counter: 5000, + }; + if (removeTSDBFields) { + for (const field of Object.keys(timeSeriesMetrics)) { + delete doc[field]; + } + } + // do not remove the fields for logsdb - ignore the flag + return doc; + }); + + const result = await es.bulk( + { + index: esIndex, + body: docs.map((d) => `{"${isStream ? 'create' : 'index'}": {}}\n${JSON.stringify(d)}\n`), + }, + { meta: true } + ); + + const res = result.body; + + if (res.errors) { + const resultsWithErrors = res.items + .filter(({ index }) => index?.error) + .map(({ index }) => index?.error); + for (const error of resultsWithErrors) { + log.error(`Error: ${JSON.stringify(error)}`); + } + const [indexExists, dataStreamExists] = await Promise.all([ + es.indices.exists({ index: esIndex }), + es.indices.getDataStream({ name: esIndex }), + ]); + log.debug(`Index exists: ${indexExists} - Data stream exists: ${dataStreamExists}`); + } + log.info(`Indexed ${res.items.length} test data docs.`); + }; + +export interface ScenarioIndexes { + index: string; + create?: boolean; + downsample?: boolean; + removeTSDBFields?: boolean; + removeLogsDBFields?: boolean; + mode?: 'tsdb' | 'logsdb'; +} +type GetScenarioFn = (initialIndex: string) => Array<{ + name: string; + indexes: ScenarioIndexes[]; +}>; + +export function setupScenarioRunner( + getService: FtrProviderContext['getService'], + getPageObjects: FtrProviderContext['getPageObjects'], + getScenario: GetScenarioFn +) { + const now = moment().utc(); + const fromMoment = now.clone().subtract(1, 'hour'); + const toMoment = now.clone(); + const fromTimeForScenarios = fromMoment.format(TIME_PICKER_FORMAT); + const toTimeForScenarios = toMoment.format(TIME_PICKER_FORMAT); + + function runTestsForEachScenario( + initialIndex: string, + scenarioMode: 'tsdb' | 'logsdb', + testingFn: (indexes: ScenarioIndexes[]) => void + ): void { + const { common, lens } = getPageObjects(['common', 'lens', 'dashboard']); + const es = getService('es'); + const log = getService('log'); + const dataStreams = getService('dataStreams'); + const elasticChart = getService('elasticChart'); + const indexPatterns = getService('indexPatterns'); + const createDocs = getDocsGenerator(log, es, scenarioMode); + + for (const { name, indexes } of getScenario(initialIndex)) { + describe(name, () => { + let dataViewName: string; + let downsampledTargetIndex: string = ''; + + before(async () => { + for (const { + index, + create, + downsample, + mode, + removeTSDBFields, + removeLogsDBFields, + } of indexes) { + // Validate the scenario config + if (downsample && mode !== 'tsdb') { + expect().fail('Cannot create a scenario with downsampled stream without tsdb'); + } + // Kick off the creation + const isStream = mode !== undefined; + if (create) { + if (isStream) { + await dataStreams.createDataStream( + index, + getDataMapping({ + mode, + removeTSDBFields: Boolean(removeTSDBFields || mode === 'logsdb'), + removeLogsDBFields, + }), + mode + ); + } else { + log.info(`creating a index "${index}" with mapping...`); + await es.indices.create({ + index, + mappings: { + properties: getDataMapping({ + mode: mode === 'logsdb' ? 'logsdb' : 'tsdb', // use tsdb by default in regular index is specified + removeTSDBFields, + removeLogsDBFields, + }), + }, + }); + } + // add data to the newly created index + await createDocs( + index, + { isStream, removeTSDBFields, removeLogsDBFields }, + fromTimeForScenarios + ); + } + if (downsample) { + downsampledTargetIndex = await dataStreams.downsampleTSDBIndex(index, { + isStream: mode === 'tsdb', + }); + } + } + dataViewName = `${indexes.map(({ index }) => index).join(',')}${ + downsampledTargetIndex ? `,${downsampledTargetIndex}` : '' + }`; + log.info(`creating a data view for "${dataViewName}"...`); + await indexPatterns.create( + { + title: dataViewName, + timeFieldName: '@timestamp', + }, + { override: true } + ); + await common.navigateToApp('lens'); + await elasticChart.setNewChartUiDebugFlag(true); + // go to the + await lens.goToTimeRange( + fromTimeForScenarios, + moment + .utc(toTimeForScenarios, TIME_PICKER_FORMAT) + .add(2, 'hour') + .format(TIME_PICKER_FORMAT) // consider also new documents + ); + }); + + after(async () => { + for (const { index, create, mode: indexMode } of indexes) { + if (create) { + if (indexMode === 'tsdb' || indexMode === 'logsdb') { + await dataStreams.deleteDataStream(index); + } else { + log.info(`deleting the index "${index}"...`); + await es.indices.delete({ + index, + }); + } + } + // no need to cleant he specific downsample index as everything linked to the stream + // is cleaned up automatically + } + }); + + beforeEach(async () => { + await lens.switchDataPanelIndexPattern(dataViewName); + await lens.removeLayer(); + }); + + testingFn(indexes); + }); + } + } + + return { runTestsForEachScenario, fromTimeForScenarios, toTimeForScenarios }; +} diff --git a/x-pack/test/functional/services/data_stream.ts b/x-pack/test/functional/services/data_stream.ts index 2864be1e0dc2b6..f4b33213e62ddf 100644 --- a/x-pack/test/functional/services/data_stream.ts +++ b/x-pack/test/functional/services/data_stream.ts @@ -9,7 +9,7 @@ import type { MappingProperty } from '@elastic/elasticsearch/lib/api/types'; import type { FtrProviderContext } from '../ftr_provider_context'; /** - * High level interface to operate with Elasticsearch data stream and TSDS. + * High level interface to operate with Elasticsearch data stream and TSDS/LogsDB. */ export function DataStreamProvider({ getService, getPageObject }: FtrProviderContext) { const es = getService('es'); @@ -112,23 +112,45 @@ export function DataStreamProvider({ getService, getPageObject }: FtrProviderCon async function updateDataStreamTemplate( stream: string, mapping: Record, - tsdb?: boolean + mode?: 'tsdb' | 'logsdb' ) { await es.cluster.putComponentTemplate({ name: `${stream}_mapping`, template: { - settings: tsdb - ? { + settings: !mode + ? { mode: undefined } + : mode === 'logsdb' + ? { mode: 'logsdb' } + : { mode: 'time_series', routing_path: 'request', - } - : { mode: undefined }, + }, mappings: { properties: mapping, }, }, }); - log.info(`Updating ${stream} index template${tsdb ? ' for TSDB' : ''}...`); + // Uncomment only when needed + // log.debug(` + // PUT _component_template/${stream}_mappings + // ${JSON.stringify({ + // name: `${stream}_mapping`, + // template: { + // settings: !mode + // ? { mode: undefined } + // : mode === 'logsdb' + // ? { mode: 'logsdb' } + // : { + // mode: 'time_series', + // routing_path: 'request', + // }, + // mappings: { + // properties: mapping, + // }, + // }, + // }, null, 2)} + // `); + log.info(`Updating ${stream} index template${mode ? ` for ${mode.toUpperCase()}` : ''}...`); await es.indices.putIndexTemplate({ name: `${stream}_index_template`, index_patterns: [stream], @@ -138,71 +160,98 @@ export function DataStreamProvider({ getService, getPageObject }: FtrProviderCon description: `Template for ${stream} testing index`, }, }); + // Uncomment only when needed + // log.verbose(` + // PUT _index_template/${stream}-index-template + // ${JSON.stringify({ + // name: `${stream}_index_template`, + // index_patterns: [stream], + // data_stream: {}, + // composed_of: [`${stream}_mapping`], + // _meta: { + // description: `Template for ${stream} testing index`, + // }, + // }, null, 2)} + // `); } /** - * "Upgrade" a given data stream into a time series data series (TSDB/TSDS) + * "Upgrade" a given data stream into a TSDB or LogsDB data series * @param stream the data stream name * @param newMapping the new mapping already with time series metrics/dimensions configured */ - async function upgradeStreamToTSDB(stream: string, newMapping: Record) { - // rollover to upgrade the index type to time_series + async function upgradeStream( + stream: string, + newMapping: Record, + mode: 'tsdb' | 'logsdb' + ) { + // rollover to upgrade the index type // uploading a new mapping for the stream index using the provided metric/dimension list - log.info(`Updating ${stream} data stream component template with TSDB stuff...`); - await updateDataStreamTemplate(stream, newMapping, true); + log.info(`Updating ${stream} data stream component template with ${mode} stuff...`); + await updateDataStreamTemplate(stream, newMapping, mode); - log.info('Rolling over the backing index for TSDB'); + log.info(`Rolling over the backing index for ${mode}`); await es.indices.rollover({ alias: stream, }); + // Uncomment only when needed + // log.verbose(`POST ${stream}/_rollover`); } /** - * "Downgrade" a TSDB/TSDS data stream into a regular data stream - * @param tsdbStream the TSDB/TSDS data stream to "downgrade" + * "Downgrade" a TSDB/TSDS/LogsDB data stream into a regular data stream + * @param stream the TSDB/TSDS/LogsDB data stream to "downgrade" * @param oldMapping the new mapping already with time series metrics/dimensions already removed */ - async function downgradeTSDBtoStream( - tsdbStream: string, - newMapping: Record + async function downgradeStream( + stream: string, + newMapping: Record, + mode: 'tsdb' | 'logsdb' ) { - // strip out any time-series specific mapping - for (const fieldMapping of Object.values(newMapping || {})) { - if ('time_series_metric' in fieldMapping) { - delete fieldMapping.time_series_metric; - } - if ('time_series_dimension' in fieldMapping) { - delete fieldMapping.time_series_dimension; + if (mode === 'tsdb') { + // strip out any time-series specific mapping + for (const fieldMapping of Object.values(newMapping || {})) { + if ('time_series_metric' in fieldMapping) { + delete fieldMapping.time_series_metric; + } + if ('time_series_dimension' in fieldMapping) { + delete fieldMapping.time_series_dimension; + } } + log.info(`Updating ${stream} data stream component template with TSDB stuff...`); + await updateDataStreamTemplate(stream, newMapping); } - log.info(`Updating ${tsdbStream} data stream component template with TSDB stuff...`); - await updateDataStreamTemplate(tsdbStream, newMapping, false); + // rollover to downgrade the index type to regular stream - log.info(`Rolling over the ${tsdbStream} data stream into a regular data stream...`); + log.info(`Rolling over the ${stream} data stream into a regular data stream...`); await es.indices.rollover({ - alias: tsdbStream, + alias: stream, }); + // Uncomment only when needed + // log.debug(`POST ${stream}/_rollover`); } /** * Takes care of the entire process to create a data stream * @param streamIndex name of the new data stream to create * @param mappings the mapping to associate with the data stream - * @param tsdb when enabled it will configure the data stream as a TSDB/TSDS + * @param tsdb when enabled it will configure the data stream as a TSDB/TSDS/LogsDB */ async function createDataStream( streamIndex: string, mappings: Record, - tsdb: boolean = true + mode: 'tsdb' | 'logsdb' | undefined ) { log.info(`Creating ${streamIndex} data stream component template...`); - await updateDataStreamTemplate(streamIndex, mappings, tsdb); + await updateDataStreamTemplate(streamIndex, mappings, mode); log.info(`Creating ${streamIndex} data stream index...`); await es.indices.createDataStream({ name: streamIndex, }); + // Uncomment only when needed + // log.debug(`PUT _data_stream/${streamIndex}`); } /** @@ -212,21 +261,27 @@ export function DataStreamProvider({ getService, getPageObject }: FtrProviderCon async function deleteDataStream(streamIndex: string) { log.info(`Delete ${streamIndex} data stream index...`); await es.indices.deleteDataStream({ name: streamIndex }); + // Uncomment only when needed + // log.debug(`DELETE _data_stream/${streamIndex}`); log.info(`Delete ${streamIndex} index template...`); await es.indices.deleteIndexTemplate({ name: `${streamIndex}_index_template`, }); + // Uncomment only when needed + // log.debug(`DELETE _index_template/${streamIndex}-index-template`); log.info(`Delete ${streamIndex} data stream component template...`); await es.cluster.deleteComponentTemplate({ name: `${streamIndex}_mapping`, }); + // Uncomment only when needed + // log.debug(`DELETE _component_template/${streamIndex}_mappings`); } return { createDataStream, deleteDataStream, downsampleTSDBIndex, - upgradeStreamToTSDB, - downgradeTSDBtoStream, + upgradeStream, + downgradeStream, }; } diff --git a/x-pack/test/security_solution_cypress/cypress/screens/all_cases.ts b/x-pack/test/security_solution_cypress/cypress/screens/all_cases.ts index 8dbc850fb017c2..dbc9cd7c83f0e0 100644 --- a/x-pack/test/security_solution_cypress/cypress/screens/all_cases.ts +++ b/x-pack/test/security_solution_cypress/cypress/screens/all_cases.ts @@ -5,8 +5,7 @@ * 2.0. */ -export const ALL_CASES_CLOSED_CASES_STATS = - '[data-test-subj="closedStatsHeader"] .euiDescriptionList__description'; +export const ALL_CASES_CLOSED_CASES_STATS = '[data-test-subj="closedStatsHeader"] .euiStat__title'; export const ALL_CASES_COMMENTS_COUNT = '[data-test-subj="case-table-column-commentCount"]'; @@ -15,7 +14,7 @@ export const ALL_CASES_CREATE_NEW_CASE_BTN = '[data-test-subj="createNewCaseBtn" export const ALL_CASES_CREATE_NEW_CASE_TABLE_BTN = '[data-test-subj="cases-table-add-case"]'; export const ALL_CASES_IN_PROGRESS_CASES_STATS = - '[data-test-subj="inProgressStatsHeader"] .euiDescriptionList__description'; + '[data-test-subj="inProgressStatsHeader"] .euiStat__title'; export const ALL_CASES_NAME = '[data-test-subj="case-details-link"]'; @@ -27,8 +26,7 @@ export const ALL_CASES_STATUS_FILTER = '[data-test-subj="options-filter-popover- export const ALL_CASES_OPEN_FILTER = '[data-test-subj="options-filter-popover-item-open"]'; -export const ALL_CASES_OPEN_CASES_STATS = - '[data-test-subj="openStatsHeader"] .euiDescriptionList__description'; +export const ALL_CASES_OPEN_CASES_STATS = '[data-test-subj="openStatsHeader"] .euiStat__title'; export const ALL_CASES_OPENED_ON = '[data-test-subj="case-table-column-createdAt"]'; diff --git a/x-pack/test_serverless/api_integration/test_suites/security/cloud_security_posture/serverless_metering/cloud_security_metering.ts b/x-pack/test_serverless/api_integration/test_suites/security/cloud_security_posture/serverless_metering/cloud_security_metering.ts index 01c2ebf9a64a7b..b3db98c829afd5 100644 --- a/x-pack/test_serverless/api_integration/test_suites/security/cloud_security_posture/serverless_metering/cloud_security_metering.ts +++ b/x-pack/test_serverless/api_integration/test_suites/security/cloud_security_posture/serverless_metering/cloud_security_metering.ts @@ -11,9 +11,9 @@ import { LATEST_FINDINGS_INDEX_DEFAULT_NS } from '@kbn/cloud-security-posture-pl import * as http from 'http'; import { deleteIndex, - addIndex, createPackagePolicy, createCloudDefendPackagePolicy, + bulkIndex, } from '@kbn/test-suites-xpack/api_integration/apis/cloud_security_posture/helper'; import { RoleCredentials } from '../../../../../shared/services'; import { getMockFindings, getMockDefendForContainersHeartbeats } from './mock_data'; @@ -38,8 +38,7 @@ export default function (providerContext: FtrProviderContext) { The task manager is running by default in security serverless project in the background and sending usage API requests to the usage API. This test mocks the usage API server and intercepts the usage API request sent by the metering background task manager. */ - // FLAKY: https://github.com/elastic/kibana/issues/188660 - describe.skip('Intercept the usage API request sent by the metering background task manager', function () { + describe('Intercept the usage API request sent by the metering background task manager', function () { this.tags(['skipMKI']); let mockUsageApiServer: http.Server; @@ -117,7 +116,7 @@ export default function (providerContext: FtrProviderContext) { numberOfFindings: 10, }); - await addIndex( + await bulkIndex( es, [...billableFindings, ...notBillableFindings], LATEST_FINDINGS_INDEX_DEFAULT_NS @@ -161,7 +160,7 @@ export default function (providerContext: FtrProviderContext) { numberOfFindings: 11, }); - await addIndex( + await bulkIndex( es, [...billableFindings, ...notBillableFindings], LATEST_FINDINGS_INDEX_DEFAULT_NS @@ -200,7 +199,7 @@ export default function (providerContext: FtrProviderContext) { numberOfFindings: 2, }); - await addIndex(es, billableFindings, CDR_LATEST_NATIVE_VULNERABILITIES_INDEX_PATTERN); + await bulkIndex(es, billableFindings, CDR_LATEST_NATIVE_VULNERABILITIES_INDEX_PATTERN); let interceptedRequestBody: UsageRecord[] = []; @@ -234,7 +233,7 @@ export default function (providerContext: FtrProviderContext) { isBlockActionEnables: false, numberOfHearbeats: 2, }); - await addIndex( + await bulkIndex( es, [...blockActionEnabledHeartbeats, ...blockActionDisabledHeartbeats], CLOUD_DEFEND_HEARTBEAT_INDEX_DEFAULT_NS @@ -316,7 +315,7 @@ export default function (providerContext: FtrProviderContext) { }); await Promise.all([ - addIndex( + bulkIndex( es, [ ...billableFindingsCSPM, @@ -326,8 +325,8 @@ export default function (providerContext: FtrProviderContext) { ], LATEST_FINDINGS_INDEX_DEFAULT_NS ), - addIndex(es, [...billableFindingsCNVM], CDR_LATEST_NATIVE_VULNERABILITIES_INDEX_PATTERN), - addIndex( + bulkIndex(es, [...billableFindingsCNVM], CDR_LATEST_NATIVE_VULNERABILITIES_INDEX_PATTERN), + bulkIndex( es, [...blockActionEnabledHeartbeats, ...blockActionDisabledHeartbeats], CLOUD_DEFEND_HEARTBEAT_INDEX_DEFAULT_NS diff --git a/x-pack/test_serverless/api_integration/test_suites/security/cloud_security_posture/serverless_metering/mock_data.ts b/x-pack/test_serverless/api_integration/test_suites/security/cloud_security_posture/serverless_metering/mock_data.ts index 2455c8e1762cc3..5e5844eaaf3b53 100644 --- a/x-pack/test_serverless/api_integration/test_suites/security/cloud_security_posture/serverless_metering/mock_data.ts +++ b/x-pack/test_serverless/api_integration/test_suites/security/cloud_security_posture/serverless_metering/mock_data.ts @@ -95,13 +95,13 @@ export const getMockDefendForContainersHeartbeats = ({ mockDefendForContainersHeartbeats(isBlockActionEnables) ); }; -const mockDefendForContainersHeartbeats = (isBlockActionEnables: boolean) => { +const mockDefendForContainersHeartbeats = (isBlockActionEnabled: boolean) => { return { agent: { id: chance.guid(), }, cloud_defend: { - block_action_enabled: isBlockActionEnables, + block_action_enabled: isBlockActionEnabled, }, event: { ingested: new Date().toISOString(), diff --git a/x-pack/test_serverless/functional/page_objects/svl_search_connectors_page.ts b/x-pack/test_serverless/functional/page_objects/svl_search_connectors_page.ts index 1f834cce9d847d..615e3397a45cef 100644 --- a/x-pack/test_serverless/functional/page_objects/svl_search_connectors_page.ts +++ b/x-pack/test_serverless/functional/page_objects/svl_search_connectors_page.ts @@ -50,7 +50,7 @@ export function SvlSearchConnectorsPageProvider({ getService }: FtrProviderConte expect(await testSubjects.getVisibleText('serverlessSearchConnectorName')).to.be(name); }, async editType(type: string) { - await testSubjects.existOrFail('serverlessSearchEditConnectorTypeLabel'); + await testSubjects.existOrFail('serverlessSearchEditConnectorType'); await testSubjects.existOrFail('serverlessSearchEditConnectorTypeChoices'); await testSubjects.click('serverlessSearchEditConnectorTypeChoices'); await testSubjects.exists('serverlessSearchConnectorServiceType-zoom'); diff --git a/x-pack/test_serverless/functional/test_suites/common/visualizations/group1/index.ts b/x-pack/test_serverless/functional/test_suites/common/visualizations/group1/index.ts index b6eeef5a70f906..db58761270144c 100644 --- a/x-pack/test_serverless/functional/test_suites/common/visualizations/group1/index.ts +++ b/x-pack/test_serverless/functional/test_suites/common/visualizations/group1/index.ts @@ -75,6 +75,7 @@ export default ({ getService, loadTestFile, getPageObjects }: FtrProviderContext loadTestFile(require.resolve('./smokescreen.ts')); loadTestFile(require.resolve('./tsdb.ts')); + loadTestFile(require.resolve('./logsdb.ts')); loadTestFile(require.resolve('./vega_chart.ts')); }); }; diff --git a/x-pack/test_serverless/functional/test_suites/common/visualizations/group1/logsdb.ts b/x-pack/test_serverless/functional/test_suites/common/visualizations/group1/logsdb.ts new file mode 100644 index 00000000000000..4fe3046aa5dbe9 --- /dev/null +++ b/x-pack/test_serverless/functional/test_suites/common/visualizations/group1/logsdb.ts @@ -0,0 +1,586 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import moment from 'moment'; +import { FtrProviderContext } from '../../../../ftr_provider_context'; +import { + type ScenarioIndexes, + getDataMapping, + getDocsGenerator, + setupScenarioRunner, + TIME_PICKER_FORMAT, +} from './tsdb_logsdb_helpers'; + +export default function ({ getService, getPageObjects }: FtrProviderContext) { + const { common, lens, discover, header } = getPageObjects([ + 'common', + 'lens', + 'discover', + 'header', + ]); + const testSubjects = getService('testSubjects'); + const find = getService('find'); + const kibanaServer = getService('kibanaServer'); + const es = getService('es'); + const log = getService('log'); + const dataStreams = getService('dataStreams'); + const indexPatterns = getService('indexPatterns'); + const esArchiver = getService('esArchiver'); + const monacoEditor = getService('monacoEditor'); + const retry = getService('retry'); + + const createDocs = getDocsGenerator(log, es, 'logsdb'); + + describe('lens logsdb', function () { + const logsdbIndex = 'kibana_sample_data_logslogsdb'; + const logsdbDataView = logsdbIndex; + const logsdbEsArchive = 'test/functional/fixtures/es_archiver/kibana_sample_data_logs_logsdb'; + const fromTime = 'Apr 16, 2023 @ 00:00:00.000'; + const toTime = 'Jun 16, 2023 @ 00:00:00.000'; + + before(async () => { + log.info(`loading ${logsdbIndex} index...`); + await esArchiver.loadIfNeeded(logsdbEsArchive); + log.info(`creating a data view for "${logsdbDataView}"...`); + await indexPatterns.create( + { + title: logsdbDataView, + timeFieldName: '@timestamp', + }, + { override: true } + ); + log.info(`updating settings to use the "${logsdbDataView}" dataView...`); + await kibanaServer.uiSettings.update({ + 'dateFormat:tz': 'UTC', + defaultIndex: '0ae0bc7a-e4ca-405c-ab67-f2b5913f2a51', + 'timepicker:timeDefaults': `{ "from": "${fromTime}", "to": "${toTime}" }`, + }); + }); + + after(async () => { + await kibanaServer.savedObjects.cleanStandardList(); + await kibanaServer.uiSettings.replace({}); + await es.indices.delete({ index: [logsdbIndex] }); + }); + + describe('smoke testing functions support', () => { + before(async () => { + await common.navigateToApp('lens'); + await lens.switchDataPanelIndexPattern(logsdbDataView); + await lens.goToTimeRange(); + }); + + afterEach(async () => { + await lens.removeLayer(); + }); + + // skip count for now as it's a special function and will + // change automatically the unsupported field to Records when detected + const allOperations = [ + 'average', + 'max', + 'last_value', + 'median', + 'percentile', + 'percentile_rank', + 'standard_deviation', + 'sum', + 'unique_count', + 'min', + 'max', + 'counter_rate', + 'last_value', + ]; + + it(`should work with all operations`, async () => { + // start from a count() over a date histogram + await lens.configureDimension({ + dimension: 'lnsXY_xDimensionPanel > lns-empty-dimension', + operation: 'date_histogram', + field: '@timestamp', + }); + + // minimum supports all logsdb field types + await lens.configureDimension({ + dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension', + operation: 'count', + field: 'bytes', + keepOpen: true, + }); + + // now check that operations won't show the incompatibility tooltip + for (const operation of allOperations) { + expect( + testSubjects.exists(`lns-indexPatternDimension-${operation} incompatible`, { + timeout: 500, + }) + ).to.eql(false); + } + + for (const operation of allOperations) { + // try to change to the provided function and check all is ok + await lens.selectOperation(operation); + + expect( + await find.existsByCssSelector( + '[data-test-subj="indexPattern-field-selection-row"] .euiFormErrorText' + ) + ).to.be(false); + } + await lens.closeDimensionEditor(); + }); + + describe('Scenarios with changing stream type', () => { + const getScenarios = ( + initialIndex: string + ): Array<{ + name: string; + indexes: ScenarioIndexes[]; + }> => [ + { + name: 'LogsDB stream with no additional stream/index', + indexes: [{ index: initialIndex }], + }, + { + name: 'LogsDB stream with no additional stream/index and no host.name field', + indexes: [ + { + index: `${initialIndex}_no_host`, + removeLogsDBFields: true, + create: true, + mode: 'logsdb', + }, + ], + }, + { + name: 'LogsDB stream with an additional regular index', + indexes: [{ index: initialIndex }, { index: 'regular_index', create: true }], + }, + { + name: 'LogsDB stream with an additional LogsDB stream', + indexes: [ + { index: initialIndex }, + { index: 'logsdb_index_2', create: true, mode: 'logsdb' }, + ], + }, + { + name: 'LogsDB stream with an additional TSDB stream', + indexes: [{ index: initialIndex }, { index: 'tsdb_index', create: true, mode: 'tsdb' }], + }, + { + name: 'LogsDB stream with an additional TSDB stream downsampled', + indexes: [ + { index: initialIndex }, + { index: 'tsdb_index_downsampled', create: true, mode: 'tsdb', downsample: true }, + ], + }, + ]; + + const { runTestsForEachScenario, toTimeForScenarios, fromTimeForScenarios } = + setupScenarioRunner(getService, getPageObjects, getScenarios); + + describe('Data-stream upgraded to LogsDB scenarios', () => { + const streamIndex = 'data_stream'; + // rollover does not allow to change name, it will just change backing index underneath + const streamConvertedToLogsDBIndex = streamIndex; + + before(async () => { + log.info(`Creating "${streamIndex}" data stream...`); + await dataStreams.createDataStream( + streamIndex, + getDataMapping({ mode: 'logsdb' }), + undefined + ); + + // add some data to the stream + await createDocs(streamIndex, { isStream: true }, fromTimeForScenarios); + + log.info(`Update settings for "${streamIndex}" dataView...`); + await kibanaServer.uiSettings.update({ + 'dateFormat:tz': 'UTC', + 'timepicker:timeDefaults': '{ "from": "now-1y", "to": "now" }', + }); + log.info(`Upgrade "${streamIndex}" stream to LogsDB...`); + + const logsdbMapping = getDataMapping({ mode: 'logsdb' }); + await dataStreams.upgradeStream(streamIndex, logsdbMapping, 'logsdb'); + log.info( + `Add more data to new "${streamConvertedToLogsDBIndex}" dataView (now with LogsDB backing index)...` + ); + // add some more data when upgraded + await createDocs(streamConvertedToLogsDBIndex, { isStream: true }, toTimeForScenarios); + }); + + after(async () => { + await dataStreams.deleteDataStream(streamIndex); + }); + + runTestsForEachScenario(streamConvertedToLogsDBIndex, 'logsdb', (indexes) => { + it(`should visualize a date histogram chart`, async () => { + await lens.configureDimension({ + dimension: 'lnsXY_xDimensionPanel > lns-empty-dimension', + operation: 'date_histogram', + field: '@timestamp', + }); + + // check that a basic agg on a field works + await lens.configureDimension({ + dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension', + operation: 'min', + field: `bytes`, + }); + + await lens.waitForVisualization('xyVisChart'); + const data = await lens.getCurrentChartDebugState('xyVisChart'); + const bars = data?.bars![0].bars; + + log.info('Check counter data before the upgrade'); + // check there's some data before the upgrade + expect(bars?.[0].y).to.be.above(0); + log.info('Check counter data after the upgrade'); + // check there's some data after the upgrade + expect(bars?.[bars.length - 1].y).to.be.above(0); + }); + + it(`should visualize a date histogram chart using a different date field`, async () => { + await lens.configureDimension({ + dimension: 'lnsXY_xDimensionPanel > lns-empty-dimension', + operation: 'date_histogram', + field: 'utc_time', + }); + + // check the counter field works + await lens.configureDimension({ + dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension', + operation: 'min', + field: `bytes`, + }); + + await lens.waitForVisualization('xyVisChart'); + const data = await lens.getCurrentChartDebugState('xyVisChart'); + const bars = data?.bars![0].bars; + + log.info('Check counter data before the upgrade'); + // check there's some data before the upgrade + expect(bars?.[0].y).to.be.above(0); + log.info('Check counter data after the upgrade'); + // check there's some data after the upgrade + expect(bars?.[bars.length - 1].y).to.be.above(0); + }); + + it('should visualize an annotation layer from a logsDB stream', async () => { + await lens.configureDimension({ + dimension: 'lnsXY_xDimensionPanel > lns-empty-dimension', + operation: 'date_histogram', + field: 'utc_time', + }); + + // check the counter field works + await lens.configureDimension({ + dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension', + operation: 'min', + field: `bytes`, + }); + await lens.createLayer('annotations'); + + expect( + (await find.allByCssSelector(`[data-test-subj^="lns-layerPanel-"]`)).length + ).to.eql(2); + expect( + await ( + await testSubjects.find('lnsXY_xAnnotationsPanel > lns-dimensionTrigger') + ).getVisibleText() + ).to.eql('Event'); + await testSubjects.click('lnsXY_xAnnotationsPanel > lns-dimensionTrigger'); + await testSubjects.click('lnsXY_annotation_query'); + await lens.configureQueryAnnotation({ + queryString: 'host.name: *', + timeField: '@timestamp', + textDecoration: { type: 'name' }, + extraFields: ['host.name', 'utc_time'], + }); + await lens.closeDimensionEditor(); + + await testSubjects.existOrFail('xyVisGroupedAnnotationIcon'); + await lens.removeLayer(1); + }); + + it('should visualize an annotation layer from a logsDB stream using another time field', async () => { + await lens.configureDimension({ + dimension: 'lnsXY_xDimensionPanel > lns-empty-dimension', + operation: 'date_histogram', + field: 'utc_time', + }); + + // check the counter field works + await lens.configureDimension({ + dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension', + operation: 'min', + field: `bytes`, + }); + await lens.createLayer('annotations'); + + expect( + (await find.allByCssSelector(`[data-test-subj^="lns-layerPanel-"]`)).length + ).to.eql(2); + expect( + await ( + await testSubjects.find('lnsXY_xAnnotationsPanel > lns-dimensionTrigger') + ).getVisibleText() + ).to.eql('Event'); + await testSubjects.click('lnsXY_xAnnotationsPanel > lns-dimensionTrigger'); + await testSubjects.click('lnsXY_annotation_query'); + await lens.configureQueryAnnotation({ + queryString: 'host.name: *', + timeField: 'utc_time', + textDecoration: { type: 'name' }, + extraFields: ['host.name', '@timestamp'], + }); + await lens.closeDimensionEditor(); + + await testSubjects.existOrFail('xyVisGroupedAnnotationIcon'); + await lens.removeLayer(1); + }); + + it('should visualize correctly ES|QL queries based on a LogsDB stream', async () => { + await common.navigateToApp('discover'); + await discover.selectTextBaseLang(); + await header.waitUntilLoadingHasFinished(); + await monacoEditor.setCodeEditorValue( + `from ${indexes + .map(({ index }) => index) + .join(', ')} | stats averageB = avg(bytes) by extension` + ); + await testSubjects.click('querySubmitButton'); + await header.waitUntilLoadingHasFinished(); + await testSubjects.click('unifiedHistogramEditFlyoutVisualization'); + + await header.waitUntilLoadingHasFinished(); + + await retry.waitFor('lens flyout', async () => { + const dimensions = await testSubjects.findAll('lns-dimensionTrigger-textBased'); + return ( + dimensions.length === 2 && (await dimensions[1].getVisibleText()) === 'averageB' + ); + }); + + // go back to Lens to not break the wrapping function + await common.navigateToApp('lens'); + }); + }); + }); + + describe('LogsDB downgraded to regular data stream scenarios', () => { + const logsdbStream = 'logsdb_stream_dowgradable'; + // rollover does not allow to change name, it will just change backing index underneath + const logsdbConvertedToStream = logsdbStream; + + before(async () => { + log.info(`Creating "${logsdbStream}" data stream...`); + await dataStreams.createDataStream( + logsdbStream, + getDataMapping({ mode: 'logsdb' }), + 'logsdb' + ); + + // add some data to the stream + await createDocs(logsdbStream, { isStream: true }, fromTimeForScenarios); + + log.info(`Update settings for "${logsdbStream}" dataView...`); + await kibanaServer.uiSettings.update({ + 'dateFormat:tz': 'UTC', + 'timepicker:timeDefaults': '{ "from": "now-1y", "to": "now" }', + }); + log.info( + `Dowgrade "${logsdbStream}" stream into regular stream "${logsdbConvertedToStream}"...` + ); + + await dataStreams.downgradeStream( + logsdbStream, + getDataMapping({ mode: 'logsdb' }), + 'logsdb' + ); + log.info( + `Add more data to new "${logsdbConvertedToStream}" dataView (no longer LogsDB)...` + ); + // add some more data when upgraded + await createDocs(logsdbConvertedToStream, { isStream: true }, toTimeForScenarios); + }); + + after(async () => { + await dataStreams.deleteDataStream(logsdbConvertedToStream); + }); + + runTestsForEachScenario(logsdbConvertedToStream, 'logsdb', (indexes) => { + it(`should visualize a date histogram chart`, async () => { + await lens.configureDimension({ + dimension: 'lnsXY_xDimensionPanel > lns-empty-dimension', + operation: 'date_histogram', + field: '@timestamp', + }); + + // check that a basic agg on a field works + await lens.configureDimension({ + dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension', + operation: 'min', + field: `bytes`, + }); + + await lens.waitForVisualization('xyVisChart'); + const data = await lens.getCurrentChartDebugState('xyVisChart'); + const bars = data?.bars![0].bars; + + log.info('Check counter data before the upgrade'); + // check there's some data before the upgrade + expect(bars?.[0].y).to.be.above(0); + log.info('Check counter data after the upgrade'); + // check there's some data after the upgrade + expect(bars?.[bars.length - 1].y).to.be.above(0); + }); + + it(`should visualize a date histogram chart using a different date field`, async () => { + await lens.configureDimension({ + dimension: 'lnsXY_xDimensionPanel > lns-empty-dimension', + operation: 'date_histogram', + field: 'utc_time', + }); + + // check the counter field works + await lens.configureDimension({ + dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension', + operation: 'min', + field: `bytes`, + }); + + await lens.waitForVisualization('xyVisChart'); + const data = await lens.getCurrentChartDebugState('xyVisChart'); + const bars = data?.bars![0].bars; + + log.info('Check counter data before the upgrade'); + // check there's some data before the upgrade + expect(bars?.[0].y).to.be.above(0); + log.info('Check counter data after the upgrade'); + // check there's some data after the upgrade + expect(bars?.[bars.length - 1].y).to.be.above(0); + }); + + it('should visualize an annotation layer from a logsDB stream', async () => { + await lens.configureDimension({ + dimension: 'lnsXY_xDimensionPanel > lns-empty-dimension', + operation: 'date_histogram', + field: 'utc_time', + }); + + // check the counter field works + await lens.configureDimension({ + dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension', + operation: 'min', + field: `bytes`, + }); + await lens.createLayer('annotations'); + + expect( + (await find.allByCssSelector(`[data-test-subj^="lns-layerPanel-"]`)).length + ).to.eql(2); + expect( + await ( + await testSubjects.find('lnsXY_xAnnotationsPanel > lns-dimensionTrigger') + ).getVisibleText() + ).to.eql('Event'); + await testSubjects.click('lnsXY_xAnnotationsPanel > lns-dimensionTrigger'); + await testSubjects.click('lnsXY_annotation_query'); + await lens.configureQueryAnnotation({ + queryString: 'host.name: *', + timeField: '@timestamp', + textDecoration: { type: 'name' }, + extraFields: ['host.name', 'utc_time'], + }); + await lens.closeDimensionEditor(); + + await testSubjects.existOrFail('xyVisGroupedAnnotationIcon'); + await lens.removeLayer(1); + }); + + it('should visualize an annotation layer from a logsDB stream using another time field', async () => { + await lens.configureDimension({ + dimension: 'lnsXY_xDimensionPanel > lns-empty-dimension', + operation: 'date_histogram', + field: 'utc_time', + }); + + // check the counter field works + await lens.configureDimension({ + dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension', + operation: 'min', + field: `bytes`, + }); + await lens.createLayer('annotations'); + + expect( + (await find.allByCssSelector(`[data-test-subj^="lns-layerPanel-"]`)).length + ).to.eql(2); + expect( + await ( + await testSubjects.find('lnsXY_xAnnotationsPanel > lns-dimensionTrigger') + ).getVisibleText() + ).to.eql('Event'); + await testSubjects.click('lnsXY_xAnnotationsPanel > lns-dimensionTrigger'); + await testSubjects.click('lnsXY_annotation_query'); + await lens.configureQueryAnnotation({ + queryString: 'host.name: *', + timeField: 'utc_time', + textDecoration: { type: 'name' }, + extraFields: ['host.name', '@timestamp'], + }); + await lens.closeDimensionEditor(); + + await testSubjects.existOrFail('xyVisGroupedAnnotationIcon'); + await lens.removeLayer(1); + }); + + it('should visualize correctly ES|QL queries based on a LogsDB stream', async () => { + await common.navigateToApp('discover'); + await discover.selectTextBaseLang(); + + // Use the lens page object here also for discover: both use the same timePicker object + await lens.goToTimeRange( + fromTimeForScenarios, + moment + .utc(toTimeForScenarios, TIME_PICKER_FORMAT) + .add(2, 'hour') + .format(TIME_PICKER_FORMAT) + ); + + await header.waitUntilLoadingHasFinished(); + await monacoEditor.setCodeEditorValue( + `from ${indexes + .map(({ index }) => index) + .join(', ')} | stats averageB = avg(bytes) by extension` + ); + await testSubjects.click('querySubmitButton'); + await header.waitUntilLoadingHasFinished(); + await testSubjects.click('unifiedHistogramEditFlyoutVisualization'); + + await header.waitUntilLoadingHasFinished(); + + await retry.waitFor('lens flyout', async () => { + const dimensions = await testSubjects.findAll('lns-dimensionTrigger-textBased'); + return ( + dimensions.length === 2 && (await dimensions[1].getVisibleText()) === 'averageB' + ); + }); + + // go back to Lens to not break the wrapping function + await common.navigateToApp('lens'); + }); + }); + }); + }); + }); + }); +} diff --git a/x-pack/test_serverless/functional/test_suites/common/visualizations/group1/tsdb.ts b/x-pack/test_serverless/functional/test_suites/common/visualizations/group1/tsdb.ts index 99633e01940c19..111cf30e919c57 100644 --- a/x-pack/test_serverless/functional/test_suites/common/visualizations/group1/tsdb.ts +++ b/x-pack/test_serverless/functional/test_suites/common/visualizations/group1/tsdb.ts @@ -8,239 +8,20 @@ import expect from '@kbn/expect'; import { partition } from 'lodash'; import moment from 'moment'; -import { MappingProperty } from '@elastic/elasticsearch/lib/api/types'; import { FtrProviderContext } from '../../../../ftr_provider_context'; - -const TEST_DOC_COUNT = 100; -const TIME_PICKER_FORMAT = 'MMM D, YYYY [@] HH:mm:ss.SSS'; -const timeSeriesMetrics: Record = { - bytes_gauge: 'gauge', - bytes_counter: 'counter', -}; -const timeSeriesDimensions = ['request', 'url']; - -type TestDoc = Record>; - -const testDocTemplate: TestDoc = { - agent: 'Mozilla/5.0 (X11; Linux x86_64; rv:6.0a1) Gecko/20110421 Firefox/6.0a1', - bytes: 6219, - clientip: '223.87.60.27', - extension: 'deb', - geo: { - srcdest: 'US:US', - src: 'US', - dest: 'US', - coordinates: { lat: 39.41042861, lon: -88.8454325 }, - }, - host: 'artifacts.elastic.co', - index: 'kibana_sample_data_logs', - ip: '223.87.60.27', - machine: { ram: 8589934592, os: 'win 8' }, - memory: null, - message: - '223.87.60.27 - - [2018-07-22T00:39:02.912Z] "GET /elasticsearch/elasticsearch-6.3.2.deb_1 HTTP/1.1" 200 6219 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:6.0a1) Gecko/20110421 Firefox/6.0a1"', - phpmemory: null, - referer: 'http://twitter.com/success/wendy-lawrence', - request: '/elasticsearch/elasticsearch-6.3.2.deb', - response: 200, - tags: ['success', 'info'], - '@timestamp': '2018-07-22T00:39:02.912Z', - url: 'https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.3.2.deb_1', - utc_time: '2018-07-22T00:39:02.912Z', - event: { dataset: 'sample_web_logs' }, - bytes_gauge: 0, - bytes_counter: 0, -}; - -function getDataMapping( - { tsdb, removeTSDBFields }: { tsdb: boolean; removeTSDBFields?: boolean } = { - tsdb: false, - } -): Record { - const dataStreamMapping: Record = { - '@timestamp': { - type: 'date', - }, - agent: { - fields: { - keyword: { - ignore_above: 256, - type: 'keyword', - }, - }, - type: 'text', - }, - bytes: { - type: 'long', - }, - bytes_counter: { - type: 'long', - }, - bytes_gauge: { - type: 'long', - }, - clientip: { - type: 'ip', - }, - event: { - properties: { - dataset: { - type: 'keyword', - }, - }, - }, - extension: { - fields: { - keyword: { - ignore_above: 256, - type: 'keyword', - }, - }, - type: 'text', - }, - geo: { - properties: { - coordinates: { - type: 'geo_point', - }, - dest: { - type: 'keyword', - }, - src: { - type: 'keyword', - }, - srcdest: { - type: 'keyword', - }, - }, - }, - host: { - fields: { - keyword: { - ignore_above: 256, - type: 'keyword', - }, - }, - type: 'text', - }, - index: { - fields: { - keyword: { - ignore_above: 256, - type: 'keyword', - }, - }, - type: 'text', - }, - ip: { - type: 'ip', - }, - machine: { - properties: { - os: { - fields: { - keyword: { - ignore_above: 256, - type: 'keyword', - }, - }, - type: 'text', - }, - ram: { - type: 'long', - }, - }, - }, - memory: { - type: 'double', - }, - message: { - fields: { - keyword: { - ignore_above: 256, - type: 'keyword', - }, - }, - type: 'text', - }, - phpmemory: { - type: 'long', - }, - referer: { - type: 'keyword', - }, - request: { - type: 'keyword', - }, - response: { - fields: { - keyword: { - ignore_above: 256, - type: 'keyword', - }, - }, - type: 'text', - }, - tags: { - fields: { - keyword: { - ignore_above: 256, - type: 'keyword', - }, - }, - type: 'text', - }, - timestamp: { - path: '@timestamp', - type: 'alias', - }, - url: { - type: 'keyword', - }, - utc_time: { - type: 'date', - }, - }; - - if (tsdb) { - // augment the current mapping - for (const [fieldName, fieldMapping] of Object.entries(dataStreamMapping || {})) { - if ( - timeSeriesMetrics[fieldName] && - (fieldMapping.type === 'double' || fieldMapping.type === 'long') - ) { - fieldMapping.time_series_metric = timeSeriesMetrics[fieldName]; - } - - if (timeSeriesDimensions.includes(fieldName) && fieldMapping.type === 'keyword') { - fieldMapping.time_series_dimension = true; - } - } - } else if (removeTSDBFields) { - for (const fieldName of Object.keys(timeSeriesMetrics)) { - delete dataStreamMapping[fieldName]; - } - } - return dataStreamMapping; -} - -function sumFirstNValues(n: number, bars: Array<{ y: number }>): number { - const indexes = Array(n) - .fill(1) - .map((_, i) => i); - let countSum = 0; - for (const index of indexes) { - if (bars[index]) { - countSum += bars[index].y; - } - } - return countSum; -} +import { + type ScenarioIndexes, + TEST_DOC_COUNT, + TIME_PICKER_FORMAT, + getDataMapping, + getDocsGenerator, + setupScenarioRunner, + sumFirstNValues, +} from './tsdb_logsdb_helpers'; export default function ({ getService, getPageObjects }: FtrProviderContext) { - const PageObjects = getPageObjects([ + const { common, lens, dashboard, svlCommonPage } = getPageObjects([ 'common', - 'timePicker', 'lens', 'dashboard', 'svlCommonPage', @@ -251,71 +32,11 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const es = getService('es'); const log = getService('log'); const dataStreams = getService('dataStreams'); - const elasticChart = getService('elasticChart'); const indexPatterns = getService('indexPatterns'); const esArchiver = getService('esArchiver'); const comboBox = getService('comboBox'); - const createDocs = async ( - esIndex: string, - { isStream, removeTSDBFields }: { isStream: boolean; removeTSDBFields?: boolean }, - startTime: string - ) => { - log.info( - `Adding ${TEST_DOC_COUNT} to ${esIndex} with starting time from ${moment - .utc(startTime, TIME_PICKER_FORMAT) - .format(TIME_PICKER_FORMAT)} to ${moment - .utc(startTime, TIME_PICKER_FORMAT) - .add(2 * TEST_DOC_COUNT, 'seconds') - .format(TIME_PICKER_FORMAT)}` - ); - const docs = Array(TEST_DOC_COUNT) - .fill(testDocTemplate) - .map((templateDoc, i) => { - const timestamp = moment - .utc(startTime, TIME_PICKER_FORMAT) - .add(TEST_DOC_COUNT + i, 'seconds') - .format(); - const doc: TestDoc = { - ...templateDoc, - '@timestamp': timestamp, - utc_time: timestamp, - bytes_gauge: Math.floor(Math.random() * 10000 * i), - bytes_counter: 5000, - }; - if (removeTSDBFields) { - for (const field of Object.keys(timeSeriesMetrics)) { - delete doc[field]; - } - } - return doc; - }); - - const result = await es.bulk( - { - index: esIndex, - body: docs.map((d) => `{"${isStream ? 'create' : 'index'}": {}}\n${JSON.stringify(d)}\n`), - }, - { meta: true } - ); - - const res = result.body; - - if (res.errors) { - const resultsWithErrors = res.items - .filter(({ index }) => index?.error) - .map(({ index }) => index?.error); - for (const error of resultsWithErrors) { - log.error(`Error: ${JSON.stringify(error)}`); - } - const [indexExists, dataStreamExists] = await Promise.all([ - es.indices.exists({ index: esIndex }), - es.indices.getDataStream({ name: esIndex }), - ]); - log.debug(`Index exists: ${indexExists} - Data stream exists: ${dataStreamExists}`); - } - log.info(`Indexed ${res.items.length} test data docs.`); - }; + const createDocs = getDocsGenerator(log, es, 'tsdb'); describe('lens tsdb', function () { const tsdbIndex = 'kibana_sample_data_logstsdb'; @@ -325,7 +46,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const toTime = 'Jun 16, 2023 @ 00:00:00.000'; before(async () => { - await PageObjects.svlCommonPage.loginAsAdmin(); + await svlCommonPage.loginAsAdmin(); log.info(`loading ${tsdbIndex} index...`); await esArchiver.loadIfNeeded(tsdbEsArchive); log.info(`creating a data view for "${tsdbDataView}"...`); @@ -375,48 +96,48 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { describe('for regular metric', () => { it('defaults to median for non-rolled up metric', async () => { - await PageObjects.common.navigateToApp('lens'); - await PageObjects.lens.switchDataPanelIndexPattern(tsdbDataView); - await PageObjects.lens.waitForField('bytes_gauge'); - await PageObjects.lens.dragFieldToWorkspace('bytes_gauge', 'xyVisChart'); - expect(await PageObjects.lens.getDimensionTriggerText('lnsXY_yDimensionPanel')).to.eql( + await common.navigateToApp('lens'); + await lens.switchDataPanelIndexPattern(tsdbDataView); + await lens.waitForField('bytes_gauge'); + await lens.dragFieldToWorkspace('bytes_gauge', 'xyVisChart'); + expect(await lens.getDimensionTriggerText('lnsXY_yDimensionPanel')).to.eql( 'Median of bytes_gauge' ); }); it('does not show a warning', async () => { - await PageObjects.lens.openDimensionEditor('lnsXY_yDimensionPanel'); + await lens.openDimensionEditor('lnsXY_yDimensionPanel'); await testSubjects.missingOrFail('median-partial-warning'); - await PageObjects.lens.assertNoEditorWarning(); - await PageObjects.lens.closeDimensionEditor(); + await lens.assertNoEditorWarning(); + await lens.closeDimensionEditor(); }); }); describe('for rolled up metric (downsampled)', () => { it('defaults to average for rolled up metric', async () => { - await PageObjects.lens.switchDataPanelIndexPattern(downsampleDataView.dataView); - await PageObjects.lens.removeLayer(); - await PageObjects.lens.waitForField('bytes_gauge'); - await PageObjects.lens.dragFieldToWorkspace('bytes_gauge', 'xyVisChart'); - expect(await PageObjects.lens.getDimensionTriggerText('lnsXY_yDimensionPanel')).to.eql( + await lens.switchDataPanelIndexPattern(downsampleDataView.dataView); + await lens.removeLayer(); + await lens.waitForField('bytes_gauge'); + await lens.dragFieldToWorkspace('bytes_gauge', 'xyVisChart'); + expect(await lens.getDimensionTriggerText('lnsXY_yDimensionPanel')).to.eql( 'Average of bytes_gauge' ); }); it('shows warnings in editor when using median', async () => { - await PageObjects.lens.openDimensionEditor('lnsXY_yDimensionPanel'); + await lens.openDimensionEditor('lnsXY_yDimensionPanel'); await testSubjects.existOrFail('median-partial-warning'); await testSubjects.click('lns-indexPatternDimension-median'); - await PageObjects.lens.waitForVisualization('xyVisChart'); - await PageObjects.lens.assertMessageListContains( + await lens.waitForVisualization('xyVisChart'); + await lens.assertMessageListContains( 'Median of bytes_gauge uses a function that is unsupported by rolled up data. Select a different function or change the time range.', 'warning' ); }); it('shows warnings in dashboards as well', async () => { - await PageObjects.lens.save('New', false, false, false, 'new'); + await lens.save('New', false, false, false, 'new'); - await PageObjects.dashboard.waitForRenderComplete(); - await PageObjects.lens.assertMessageListContains( + await dashboard.waitForRenderComplete(); + await lens.assertMessageListContains( 'Median of bytes_gauge uses a function that is unsupported by rolled up data. Select a different function or change the time range.', 'warning' ); @@ -426,13 +147,13 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { describe('time series special field types support', () => { before(async () => { - await PageObjects.common.navigateToApp('lens'); - await PageObjects.lens.switchDataPanelIndexPattern(tsdbDataView); - await PageObjects.lens.goToTimeRange(); + await common.navigateToApp('lens'); + await lens.switchDataPanelIndexPattern(tsdbDataView); + await lens.goToTimeRange(); }); afterEach(async () => { - await PageObjects.lens.removeLayer(); + await lens.removeLayer(); }); // skip count for now as it's a special function and will @@ -467,14 +188,14 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { if (supportedOperations.length) { it(`should allow operations when supported by ${fieldType} field type`, async () => { // Counter rate requires a date histogram dimension configured to work - await PageObjects.lens.configureDimension({ + await lens.configureDimension({ dimension: 'lnsXY_xDimensionPanel > lns-empty-dimension', operation: 'date_histogram', field: '@timestamp', }); // minimum supports all tsdb field types - await PageObjects.lens.configureDimension({ + await lens.configureDimension({ dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension', operation: 'min', field: `bytes_${fieldType}`, @@ -492,7 +213,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { for (const supportedOp of supportedOperations) { // try to change to the provided function and check all is ok - await PageObjects.lens.selectOperation(supportedOp.name); + await lens.selectOperation(supportedOp.name); expect( await find.existsByCssSelector( @@ -501,22 +222,22 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { ).to.be(false); // return in a clean state before checking the next operation - await PageObjects.lens.selectOperation('min'); + await lens.selectOperation('min'); } - await PageObjects.lens.closeDimensionEditor(); + await lens.closeDimensionEditor(); }); } if (unsupportedOperatons.length) { it(`should notify the incompatibility of unsupported operations for the ${fieldType} field type`, async () => { // Counter rate requires a date histogram dimension configured to work - await PageObjects.lens.configureDimension({ + await lens.configureDimension({ dimension: 'lnsXY_xDimensionPanel > lns-empty-dimension', operation: 'date_histogram', field: '@timestamp', }); // minimum supports all tsdb field types - await PageObjects.lens.configureDimension({ + await lens.configureDimension({ dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension', operation: 'min', field: `bytes_${fieldType}`, @@ -537,7 +258,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { for (const unsupportedOp of unsupportedOperatons) { // try to change to the provided function and check if it's in an incompatibility state - await PageObjects.lens.selectOperation(unsupportedOp.name, true); + await lens.selectOperation(unsupportedOp.name, true); const fieldSelectErrorEl = await find.byCssSelector( '[data-test-subj="indexPattern-field-selection-row"] .euiFormErrorText' @@ -548,28 +269,28 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { ); // return in a clean state before checking the next operation - await PageObjects.lens.selectOperation('min'); + await lens.selectOperation('min'); } - await PageObjects.lens.closeDimensionEditor(); + await lens.closeDimensionEditor(); }); } } describe('show time series dimension groups within breakdown', () => { it('should show the time series dimension group on field picker when configuring a breakdown', async () => { - await PageObjects.lens.configureDimension({ + await lens.configureDimension({ dimension: 'lnsXY_xDimensionPanel > lns-empty-dimension', operation: 'date_histogram', field: '@timestamp', }); - await PageObjects.lens.configureDimension({ + await lens.configureDimension({ dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension', operation: 'min', field: 'bytes_counter', }); - await PageObjects.lens.configureDimension({ + await lens.configureDimension({ dimension: 'lnsXY_splitDimensionPanel > lns-empty-dimension', operation: 'terms', keepOpen: true, @@ -577,46 +298,34 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const list = await comboBox.getOptionsList('indexPattern-dimension-field'); expect(list).to.contain('Time series dimensions'); - await PageObjects.lens.closeDimensionEditor(); + await lens.closeDimensionEditor(); }); it("should not show the time series dimension group on field picker if it's not a breakdown", async () => { - await PageObjects.lens.configureDimension({ + await lens.configureDimension({ dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension', operation: 'min', field: 'bytes_counter', }); - await PageObjects.lens.configureDimension({ + await lens.configureDimension({ dimension: 'lnsXY_xDimensionPanel > lns-empty-dimension', operation: 'date_histogram', keepOpen: true, }); const list = await comboBox.getOptionsList('indexPattern-dimension-field'); expect(list).to.not.contain('Time series dimensions'); - await PageObjects.lens.closeDimensionEditor(); + await lens.closeDimensionEditor(); }); }); }); describe('Scenarios with changing stream type', () => { - const now = moment().utc(); - const fromMoment = now.clone().subtract(1, 'hour'); - const toMoment = now.clone(); - const fromTimeForScenarios = fromMoment.format(TIME_PICKER_FORMAT); - const toTimeForScenarios = toMoment.format(TIME_PICKER_FORMAT); - const getScenarios = ( initialIndex: string ): Array<{ name: string; - indexes: Array<{ - index: string; - create?: boolean; - downsample?: boolean; - tsdb?: boolean; - removeTSDBFields?: boolean; - }>; + indexes: ScenarioIndexes[]; }> => [ { name: 'Dataview with no additional stream/index', @@ -633,7 +342,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { name: 'Dataview with an additional downsampled TSDB stream', indexes: [ { index: initialIndex }, - { index: 'tsdb_index_2', create: true, tsdb: true, downsample: true }, + { index: 'tsdb_index_2', create: true, mode: 'tsdb', downsample: true }, ], }, { @@ -641,112 +350,17 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { indexes: [ { index: initialIndex }, { index: 'regular_index', create: true, removeTSDBFields: true }, - { index: 'tsdb_index_2', create: true, tsdb: true, downsample: true }, + { index: 'tsdb_index_2', create: true, mode: 'tsdb', downsample: true }, ], }, { name: 'Dataview with an additional TSDB stream', - indexes: [{ index: initialIndex }, { index: 'tsdb_index_2', create: true, tsdb: true }], + indexes: [{ index: initialIndex }, { index: 'tsdb_index_2', create: true, mode: 'tsdb' }], }, ]; - function runTestsForEachScenario( - initialIndex: string, - testingFn: ( - indexes: Array<{ - index: string; - create?: boolean; - downsample?: boolean; - tsdb?: boolean; - removeTSDBFields?: boolean; - }> - ) => void - ): void { - for (const { name, indexes } of getScenarios(initialIndex)) { - describe(name, () => { - let dataViewName: string; - let downsampledTargetIndex: string = ''; - - before(async () => { - for (const { index, create, downsample, tsdb, removeTSDBFields } of indexes) { - if (create) { - if (tsdb) { - await dataStreams.createDataStream( - index, - getDataMapping({ tsdb, removeTSDBFields }), - tsdb - ); - } else { - log.info(`creating a index "${index}" with mapping...`); - await es.indices.create({ - index, - mappings: { - properties: getDataMapping({ tsdb: Boolean(tsdb), removeTSDBFields }), - }, - }); - } - // add data to the newly created index - await createDocs( - index, - { isStream: Boolean(tsdb), removeTSDBFields }, - fromTimeForScenarios - ); - } - if (downsample) { - downsampledTargetIndex = await dataStreams.downsampleTSDBIndex(index, { - isStream: Boolean(tsdb), - }); - } - } - dataViewName = `${indexes.map(({ index }) => index).join(',')}${ - downsampledTargetIndex ? `,${downsampledTargetIndex}` : '' - }`; - log.info(`creating a data view for "${dataViewName}"...`); - await indexPatterns.create( - { - title: dataViewName, - timeFieldName: '@timestamp', - }, - { override: true } - ); - await PageObjects.common.navigateToApp('lens'); - await elasticChart.setNewChartUiDebugFlag(true); - // go to the - await PageObjects.lens.goToTimeRange( - fromTimeForScenarios, - moment - .utc(toTimeForScenarios, TIME_PICKER_FORMAT) - .add(2, 'hour') - .format(TIME_PICKER_FORMAT) // consider also new documents - ); - }); - - after(async () => { - for (const { index, create, tsdb } of indexes) { - if (create) { - if (tsdb) { - await dataStreams.deleteDataStream(index); - } else { - log.info(`deleting the index "${index}"...`); - await es.indices.delete({ - index, - }); - } - } - // no need to cleant he specific downsample index as everything linked to the stream - // is cleaned up automatically - } - }); - - beforeEach(async () => { - await PageObjects.lens.switchDataPanelIndexPattern(dataViewName); - await PageObjects.lens.removeLayer(); - }); - - testingFn(indexes); - }); - } - } + const { runTestsForEachScenario, toTimeForScenarios, fromTimeForScenarios } = + setupScenarioRunner(getService, getPageObjects, getScenarios); describe('Data-stream upgraded to TSDB scenarios', () => { const streamIndex = 'data_stream'; @@ -755,7 +369,11 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { before(async () => { log.info(`Creating "${streamIndex}" data stream...`); - await dataStreams.createDataStream(streamIndex, getDataMapping(), false); + await dataStreams.createDataStream( + streamIndex, + getDataMapping({ mode: 'tsdb' }), + undefined + ); // add some data to the stream await createDocs(streamIndex, { isStream: true }, fromTimeForScenarios); @@ -767,8 +385,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); log.info(`Upgrade "${streamIndex}" stream to TSDB...`); - const tsdbMapping = getDataMapping({ tsdb: true }); - await dataStreams.upgradeStreamToTSDB(streamIndex, tsdbMapping); + const tsdbMapping = getDataMapping({ mode: 'tsdb' }); + await dataStreams.upgradeStream(streamIndex, tsdbMapping, 'tsdb'); log.info( `Add more data to new "${streamConvertedToTsdbIndex}" dataView (now with TSDB backing index)...` ); @@ -780,15 +398,15 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await dataStreams.deleteDataStream(streamIndex); }); - runTestsForEachScenario(streamConvertedToTsdbIndex, (indexes) => { + runTestsForEachScenario(streamConvertedToTsdbIndex, 'tsdb', (indexes) => { it('should detect the data stream has now been upgraded to TSDB', async () => { - await PageObjects.lens.configureDimension({ + await lens.configureDimension({ dimension: 'lnsXY_xDimensionPanel > lns-empty-dimension', operation: 'date_histogram', field: '@timestamp', }); - await PageObjects.lens.configureDimension({ + await lens.configureDimension({ dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension', operation: 'min', field: `bytes_counter`, @@ -800,53 +418,53 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { timeout: 500, }) ).to.eql(false); - await PageObjects.lens.closeDimensionEditor(); + await lens.closeDimensionEditor(); }); it(`should visualize a date histogram chart for counter field`, async () => { - await PageObjects.lens.configureDimension({ + await lens.configureDimension({ dimension: 'lnsXY_xDimensionPanel > lns-empty-dimension', operation: 'date_histogram', field: '@timestamp', }); // check the counter field works - await PageObjects.lens.configureDimension({ + await lens.configureDimension({ dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension', operation: 'min', field: `bytes_counter`, }); // and also that the count of documents should be "indexes.length" times overall - await PageObjects.lens.configureDimension({ + await lens.configureDimension({ dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension', operation: 'count', }); - await PageObjects.lens.waitForVisualization('xyVisChart'); - const data = await PageObjects.lens.getCurrentChartDebugState('xyVisChart'); - const counterBars = data.bars![0].bars; - const countBars = data.bars![1].bars; + await lens.waitForVisualization('xyVisChart'); + const data = await lens.getCurrentChartDebugState('xyVisChart'); + const counterBars = data?.bars![0].bars; + const countBars = data?.bars![1].bars; log.info('Check counter data before the upgrade'); // check there's some data before the upgrade - expect(counterBars[0].y).to.eql(5000); + expect(counterBars?.[0].y).to.eql(5000); log.info('Check counter data after the upgrade'); // check there's some data after the upgrade - expect(counterBars[counterBars.length - 1].y).to.eql(5000); + expect(counterBars?.[counterBars.length - 1].y).to.eql(5000); // due to the flaky nature of exact check here, we're going to relax it // as long as there's data before and after it is ok log.info('Check count before the upgrade'); - const columnsToCheck = countBars.length / 2; + const columnsToCheck = countBars ? countBars.length / 2 : 0; // Before the upgrade the count is N times the indexes expect(sumFirstNValues(columnsToCheck, countBars)).to.be.greaterThan( indexes.length * TEST_DOC_COUNT - 1 ); log.info('Check count after the upgrade'); // later there are only documents for the upgraded stream - expect(sumFirstNValues(columnsToCheck, [...countBars].reverse())).to.be.greaterThan( - TEST_DOC_COUNT - 1 - ); + expect( + sumFirstNValues(columnsToCheck, [...(countBars ?? [])].reverse()) + ).to.be.greaterThan(TEST_DOC_COUNT - 1); }); }); }); @@ -858,7 +476,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { before(async () => { log.info(`Creating "${tsdbStream}" data stream...`); - await dataStreams.createDataStream(tsdbStream, getDataMapping({ tsdb: true }), true); + await dataStreams.createDataStream(tsdbStream, getDataMapping({ mode: 'tsdb' }), 'tsdb'); // add some data to the stream await createDocs(tsdbStream, { isStream: true }, fromTimeForScenarios); @@ -872,7 +490,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { `Dowgrade "${tsdbStream}" stream into regular stream "${tsdbConvertedToStream}"...` ); - await dataStreams.downgradeTSDBtoStream(tsdbStream, getDataMapping({ tsdb: true })); + await dataStreams.downgradeStream(tsdbStream, getDataMapping({ mode: 'tsdb' }), 'tsdb'); log.info(`Add more data to new "${tsdbConvertedToStream}" dataView (no longer TSDB)...`); // add some more data when upgraded await createDocs(tsdbConvertedToStream, { isStream: true }, toTimeForScenarios); @@ -882,15 +500,15 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await dataStreams.deleteDataStream(tsdbConvertedToStream); }); - runTestsForEachScenario(tsdbConvertedToStream, (indexes) => { + runTestsForEachScenario(tsdbConvertedToStream, 'tsdb', (indexes) => { it('should keep TSDB restrictions only if a tsdb stream is in the dataView mix', async () => { - await PageObjects.lens.configureDimension({ + await lens.configureDimension({ dimension: 'lnsXY_xDimensionPanel > lns-empty-dimension', operation: 'date_histogram', field: '@timestamp', }); - await PageObjects.lens.configureDimension({ + await lens.configureDimension({ dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension', operation: 'min', field: `bytes_counter`, @@ -901,28 +519,28 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { testSubjects.exists(`lns-indexPatternDimension-average incompatible`, { timeout: 500, }) - ).to.eql(indexes.some(({ tsdb }) => tsdb)); - await PageObjects.lens.closeDimensionEditor(); + ).to.eql(indexes.some(({ mode }) => mode === 'tsdb')); + await lens.closeDimensionEditor(); }); it(`should visualize a date histogram chart for counter field`, async () => { - await PageObjects.lens.configureDimension({ + await lens.configureDimension({ dimension: 'lnsXY_xDimensionPanel > lns-empty-dimension', operation: 'date_histogram', field: '@timestamp', }); // just check the data is shown - await PageObjects.lens.configureDimension({ + await lens.configureDimension({ dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension', operation: 'count', }); + await lens.waitForVisualization('xyVisChart'); + const data = await lens.getCurrentChartDebugState('xyVisChart'); + const bars = data?.bars![0].bars; + const columnsToCheck = bars ? bars.length / 2 : 0; // due to the flaky nature of exact check here, we're going to relax it // as long as there's data before and after it is ok - await PageObjects.lens.waitForVisualization('xyVisChart'); - const data = await PageObjects.lens.getCurrentChartDebugState('xyVisChart'); - const bars = data.bars![0].bars; - const columnsToCheck = bars.length / 2; log.info('Check count before the downgrade'); // Before the upgrade the count is N times the indexes expect(sumFirstNValues(columnsToCheck, bars)).to.be.greaterThan( @@ -930,14 +548,14 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { ); log.info('Check count after the downgrade'); // later there are only documents for the upgraded stream - expect(sumFirstNValues(columnsToCheck, [...bars].reverse())).to.be.greaterThan( + expect(sumFirstNValues(columnsToCheck, [...(bars ?? [])].reverse())).to.be.greaterThan( TEST_DOC_COUNT - 1 ); }); it('should visualize data when moving the time window around the downgrade moment', async () => { // check after the downgrade - await PageObjects.lens.goToTimeRange( + await lens.goToTimeRange( moment .utc(fromTimeForScenarios, TIME_PICKER_FORMAT) .subtract(1, 'hour') @@ -948,23 +566,23 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { .format(TIME_PICKER_FORMAT) // consider only new documents ); - await PageObjects.lens.configureDimension({ + await lens.configureDimension({ dimension: 'lnsXY_xDimensionPanel > lns-empty-dimension', operation: 'date_histogram', field: '@timestamp', }); - await PageObjects.lens.configureDimension({ + await lens.configureDimension({ dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension', operation: 'count', }); - await PageObjects.lens.waitForVisualization('xyVisChart'); - const dataBefore = await PageObjects.lens.getCurrentChartDebugState('xyVisChart'); - const barsBefore = dataBefore.bars![0].bars; - expect(barsBefore.some(({ y }) => y)).to.eql(true); + await lens.waitForVisualization('xyVisChart'); + const dataBefore = await lens.getCurrentChartDebugState('xyVisChart'); + const barsBefore = dataBefore?.bars![0].bars; + expect(barsBefore?.some(({ y }) => y)).to.eql(true); // check after the downgrade - await PageObjects.lens.goToTimeRange( + await lens.goToTimeRange( moment .utc(toTimeForScenarios, TIME_PICKER_FORMAT) .add(1, 'second') @@ -975,10 +593,10 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { .format(TIME_PICKER_FORMAT) // consider also new documents ); - await PageObjects.lens.waitForVisualization('xyVisChart'); - const dataAfter = await PageObjects.lens.getCurrentChartDebugState('xyVisChart'); - const barsAfter = dataAfter.bars![0].bars; - expect(barsAfter.some(({ y }) => y)).to.eql(true); + await lens.waitForVisualization('xyVisChart'); + const dataAfter = await lens.getCurrentChartDebugState('xyVisChart'); + const barsAfter = dataAfter?.bars![0].bars; + expect(barsAfter?.some(({ y }) => y)).to.eql(true); }); }); }); diff --git a/x-pack/test_serverless/functional/test_suites/common/visualizations/group1/tsdb_logsdb_helpers.ts b/x-pack/test_serverless/functional/test_suites/common/visualizations/group1/tsdb_logsdb_helpers.ts new file mode 100644 index 00000000000000..23822aa1395a98 --- /dev/null +++ b/x-pack/test_serverless/functional/test_suites/common/visualizations/group1/tsdb_logsdb_helpers.ts @@ -0,0 +1,480 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import expect from '@kbn/expect'; +import { Client } from '@elastic/elasticsearch'; +import { MappingProperty } from '@elastic/elasticsearch/lib/api/types'; +import { ToolingLog } from '@kbn/tooling-log'; +import moment from 'moment'; +import { FtrProviderContext } from '../../../../ftr_provider_context'; + +export const TEST_DOC_COUNT = 100; +export const TIME_PICKER_FORMAT = 'MMM D, YYYY [@] HH:mm:ss.SSS'; +export const timeSeriesMetrics: Record = { + bytes_gauge: 'gauge', + bytes_counter: 'counter', +}; +export const timeSeriesDimensions = ['request', 'url']; +export const logsDBSpecialFields = ['host']; + +export const sharedESArchive = + 'test/functional/fixtures/es_archiver/kibana_sample_data_logs_logsdb'; +export const fromTime = 'Apr 16, 2023 @ 00:00:00.000'; +export const toTime = 'Jun 16, 2023 @ 00:00:00.000'; + +export type TestDoc = Record>; + +export function testDocTemplate(mode: 'tsdb' | 'logsdb'): TestDoc { + return { + agent: 'Mozilla/5.0 (X11; Linux x86_64; rv:6.0a1) Gecko/20110421 Firefox/6.0a1', + bytes: 6219, + clientip: '223.87.60.27', + extension: 'deb', + geo: { + srcdest: 'US:US', + src: 'US', + dest: 'US', + coordinates: { lat: 39.41042861, lon: -88.8454325 }, + }, + host: mode === 'tsdb' ? 'artifacts.elastic.co' : { name: 'artifacts.elastic.co' }, + index: 'kibana_sample_data_logs', + ip: '223.87.60.27', + machine: { ram: 8589934592, os: 'win 8' }, + memory: null, + message: + '223.87.60.27 - - [2018-07-22T00:39:02.912Z] "GET /elasticsearch/elasticsearch-6.3.2.deb_1 HTTP/1.1" 200 6219 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:6.0a1) Gecko/20110421 Firefox/6.0a1"', + phpmemory: null, + referer: 'http://twitter.com/success/wendy-lawrence', + request: '/elasticsearch/elasticsearch-6.3.2.deb', + response: 200, + tags: ['success', 'info'], + '@timestamp': '2018-07-22T00:39:02.912Z', + url: 'https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.3.2.deb_1', + utc_time: '2018-07-22T00:39:02.912Z', + event: { dataset: 'sample_web_logs' }, + bytes_gauge: 0, + bytes_counter: 0, + }; +} + +export function getDataMapping({ + mode, + removeTSDBFields, + removeLogsDBFields, +}: { + mode: 'tsdb' | 'logsdb'; + removeTSDBFields?: boolean; + removeLogsDBFields?: boolean; +}): Record { + const dataStreamMapping: Record = { + '@timestamp': { + type: 'date', + }, + agent: { + fields: { + keyword: { + ignore_above: 256, + type: 'keyword', + }, + }, + type: 'text', + }, + bytes: { + type: 'long', + }, + bytes_counter: { + type: 'long', + }, + bytes_gauge: { + type: 'long', + }, + clientip: { + type: 'ip', + }, + event: { + properties: { + dataset: { + type: 'keyword', + }, + }, + }, + extension: { + fields: { + keyword: { + ignore_above: 256, + type: 'keyword', + }, + }, + type: 'text', + }, + geo: { + properties: { + coordinates: { + type: 'geo_point', + }, + dest: { + type: 'keyword', + }, + src: { + type: 'keyword', + }, + srcdest: { + type: 'keyword', + }, + }, + }, + host: + mode === 'tsdb' + ? { + fields: { + keyword: { + ignore_above: 256, + type: 'keyword', + }, + }, + type: 'text', + } + : { + properties: { + name: { + type: 'keyword', + }, + }, + }, + index: { + fields: { + keyword: { + ignore_above: 256, + type: 'keyword', + }, + }, + type: 'text', + }, + ip: { + type: 'ip', + }, + machine: { + properties: { + os: { + fields: { + keyword: { + ignore_above: 256, + type: 'keyword', + }, + }, + type: 'text', + }, + ram: { + type: 'long', + }, + }, + }, + memory: { + type: 'double', + }, + message: { + fields: { + keyword: { + ignore_above: 256, + type: 'keyword', + }, + }, + type: 'text', + }, + phpmemory: { + type: 'long', + }, + referer: { + type: 'keyword', + }, + request: { + type: 'keyword', + }, + response: { + fields: { + keyword: { + ignore_above: 256, + type: 'keyword', + }, + }, + type: 'text', + }, + tags: { + fields: { + keyword: { + ignore_above: 256, + type: 'keyword', + }, + }, + type: 'text', + }, + timestamp: { + path: '@timestamp', + type: 'alias', + }, + url: { + type: 'keyword', + }, + utc_time: { + type: 'date', + }, + }; + + if (mode === 'tsdb') { + // augment the current mapping + for (const [fieldName, fieldMapping] of Object.entries(dataStreamMapping || {})) { + if ( + timeSeriesMetrics[fieldName] && + (fieldMapping.type === 'double' || fieldMapping.type === 'long') + ) { + fieldMapping.time_series_metric = timeSeriesMetrics[fieldName]; + } + + if (timeSeriesDimensions.includes(fieldName) && fieldMapping.type === 'keyword') { + fieldMapping.time_series_dimension = true; + } + } + } + if (removeTSDBFields) { + for (const fieldName of Object.keys(timeSeriesMetrics)) { + delete dataStreamMapping[fieldName]; + } + } + if (removeLogsDBFields) { + for (const fieldName of logsDBSpecialFields) { + delete dataStreamMapping[fieldName]; + } + } + return dataStreamMapping; +} + +export function sumFirstNValues(n: number, bars: Array<{ y: number }> | undefined): number { + const indexes = Array(n) + .fill(1) + .map((_, i) => i); + let countSum = 0; + for (const index of indexes) { + if (bars?.[index]) { + countSum += bars[index].y; + } + } + return countSum; +} + +export const getDocsGenerator = + (log: ToolingLog, es: Client, mode: 'tsdb' | 'logsdb') => + async ( + esIndex: string, + { + isStream, + removeTSDBFields, + removeLogsDBFields, + }: { isStream: boolean; removeTSDBFields?: boolean; removeLogsDBFields?: boolean }, + startTime: string + ) => { + log.info( + `Adding ${TEST_DOC_COUNT} to ${esIndex} with starting time from ${moment + .utc(startTime, TIME_PICKER_FORMAT) + .format(TIME_PICKER_FORMAT)} to ${moment + .utc(startTime, TIME_PICKER_FORMAT) + .add(2 * TEST_DOC_COUNT, 'seconds') + .format(TIME_PICKER_FORMAT)}` + ); + const docs = Array(TEST_DOC_COUNT) + .fill(testDocTemplate(mode)) + .map((templateDoc, i) => { + const timestamp = moment + .utc(startTime, TIME_PICKER_FORMAT) + .add(TEST_DOC_COUNT + i, 'seconds') + .format(); + const doc: TestDoc = { + ...templateDoc, + '@timestamp': timestamp, + utc_time: timestamp, + bytes_gauge: Math.floor(Math.random() * 10000 * i), + bytes_counter: 5000, + }; + if (removeTSDBFields) { + for (const field of Object.keys(timeSeriesMetrics)) { + delete doc[field]; + } + } + // do not remove the fields for logsdb - ignore the flag + return doc; + }); + + const result = await es.bulk( + { + index: esIndex, + body: docs.map((d) => `{"${isStream ? 'create' : 'index'}": {}}\n${JSON.stringify(d)}\n`), + }, + { meta: true } + ); + + const res = result.body; + + if (res.errors) { + const resultsWithErrors = res.items + .filter(({ index }) => index?.error) + .map(({ index }) => index?.error); + for (const error of resultsWithErrors) { + log.error(`Error: ${JSON.stringify(error)}`); + } + const [indexExists, dataStreamExists] = await Promise.all([ + es.indices.exists({ index: esIndex }), + es.indices.getDataStream({ name: esIndex }), + ]); + log.debug(`Index exists: ${indexExists} - Data stream exists: ${dataStreamExists}`); + } + log.info(`Indexed ${res.items.length} test data docs.`); + }; + +export interface ScenarioIndexes { + index: string; + create?: boolean; + downsample?: boolean; + removeTSDBFields?: boolean; + removeLogsDBFields?: boolean; + mode?: 'tsdb' | 'logsdb'; +} +type GetScenarioFn = (initialIndex: string) => Array<{ + name: string; + indexes: ScenarioIndexes[]; +}>; + +export function setupScenarioRunner( + getService: FtrProviderContext['getService'], + getPageObjects: FtrProviderContext['getPageObjects'], + getScenario: GetScenarioFn +) { + const now = moment().utc(); + const fromMoment = now.clone().subtract(1, 'hour'); + const toMoment = now.clone(); + const fromTimeForScenarios = fromMoment.format(TIME_PICKER_FORMAT); + const toTimeForScenarios = toMoment.format(TIME_PICKER_FORMAT); + + function runTestsForEachScenario( + initialIndex: string, + scenarioMode: 'tsdb' | 'logsdb', + testingFn: (indexes: ScenarioIndexes[]) => void + ): void { + const { common, lens } = getPageObjects(['common', 'lens', 'dashboard']); + const es = getService('es'); + const log = getService('log'); + const dataStreams = getService('dataStreams'); + const elasticChart = getService('elasticChart'); + const indexPatterns = getService('indexPatterns'); + const createDocs = getDocsGenerator(log, es, scenarioMode); + + for (const { name, indexes } of getScenario(initialIndex)) { + describe(name, () => { + let dataViewName: string; + let downsampledTargetIndex: string = ''; + + before(async () => { + for (const { + index, + create, + downsample, + mode, + removeTSDBFields, + removeLogsDBFields, + } of indexes) { + // Validate the scenario config + if (downsample && mode !== 'tsdb') { + expect().fail('Cannot create a scenario with downsampled stream without tsdb'); + } + // Kick off the creation + const isStream = mode !== undefined; + if (create) { + if (isStream) { + await dataStreams.createDataStream( + index, + getDataMapping({ + mode, + removeTSDBFields: Boolean(removeTSDBFields || mode === 'logsdb'), + removeLogsDBFields, + }), + mode + ); + } else { + log.info(`creating a index "${index}" with mapping...`); + await es.indices.create({ + index, + mappings: { + properties: getDataMapping({ + mode: mode === 'logsdb' ? 'logsdb' : 'tsdb', // use tsdb by default in regular index is specified + removeTSDBFields, + removeLogsDBFields, + }), + }, + }); + } + // add data to the newly created index + await createDocs( + index, + { isStream, removeTSDBFields, removeLogsDBFields }, + fromTimeForScenarios + ); + } + if (downsample) { + downsampledTargetIndex = await dataStreams.downsampleTSDBIndex(index, { + isStream: mode === 'tsdb', + }); + } + } + dataViewName = `${indexes.map(({ index }) => index).join(',')}${ + downsampledTargetIndex ? `,${downsampledTargetIndex}` : '' + }`; + log.info(`creating a data view for "${dataViewName}"...`); + await indexPatterns.create( + { + title: dataViewName, + timeFieldName: '@timestamp', + }, + { override: true } + ); + await common.navigateToApp('lens'); + await elasticChart.setNewChartUiDebugFlag(true); + // go to the + await lens.goToTimeRange( + fromTimeForScenarios, + moment + .utc(toTimeForScenarios, TIME_PICKER_FORMAT) + .add(2, 'hour') + .format(TIME_PICKER_FORMAT) // consider also new documents + ); + }); + + after(async () => { + for (const { index, create, mode: indexMode } of indexes) { + if (create) { + if (indexMode === 'tsdb' || indexMode === 'logsdb') { + await dataStreams.deleteDataStream(index); + } else { + log.info(`deleting the index "${index}"...`); + await es.indices.delete({ + index, + }); + } + } + // no need to cleant he specific downsample index as everything linked to the stream + // is cleaned up automatically + } + }); + + beforeEach(async () => { + await lens.switchDataPanelIndexPattern(dataViewName); + await lens.removeLayer(); + }); + + testingFn(indexes); + }); + } + } + + return { runTestsForEachScenario, fromTimeForScenarios, toTimeForScenarios }; +}