From cd2164b450c6b1c9264346468b64963debb3fe08 Mon Sep 17 00:00:00 2001 From: "Quynh Nguyen (Quinn)" <43350163+qn895@users.noreply.github.com> Date: Wed, 21 Dec 2022 10:48:49 -0600 Subject: [PATCH] [ML][Fleet] Update Transform installation mechanism to support upgrade paths (#142920) ## Summary This PR: - Adds index aliases to the destination index for Transforms: - It will automatically appends `{{package-version}}` name to the `destination_index_name` specified in transform.yml - Create a `{destination_index_name}.all` that points to all the destination indices from all the previous versions and new version - Create a `{destination_index_name}.latest` that points to just the destination index of the new version - Upgrading package to a new version no longer deletes the destination index - Downgrading package to an older version (e.g. from v3 to v1) will: - Delete the transform from the newer version (v3), create transform for the older version (v1) - If the older version was previously installed and the destination index of the older version v1 already exists: update the alias `{destination_index_name}.latest` to point to destination index v1. - If the older version was never installed and destination index of the older version v1 does not exist: create the destination index with `{destination_index_name}.all` and `{destination_index_name}.latest` alias. - Support installing transforms concurrently and sequentially. - If the `order` is specified in the `transform.yml`'s `_meta` section, and all the numerical order are unique, transforms will be created and started sequentially. If not, they will be created and started concurrently. - Support versioning of transforms. If `fleet_transform_version` is specified in `transform.yml`'s `_meta` section: - If `fleet_transform_version` changed (either incremented or decremented): delete old transform, keep the old destination index, install new index templates, component templates, and transform - If `fleet_transform_version` remains the same: keep old transform, keep the old destination index, do nothing new - Fixes an issue with the mappings and template not being applied to the destination index correctly when the destination index has an ingest pipeline. Previously, when the transform is associated with an ingest pipeline, we add the ingest pipeline to the settings when calling `PUT index/{transform-destination-index}`. This in turns makes the settings and mappings from the component templates not apply correctly to the destination. This PR changes so that it will add the pipeline to the component template. Technical changes: - [Adds a new `ElasticsearchAssetType` for `index` ](https://github.com/elastic/kibana/pull/142920/files#diff-395b753abcf65cdc07993651d6211a49194a76c0497e5f234ea13736cf24a2c0) - [Adds a new `version` for `PACKAGES_SAVED_OBJECT_TYPE` ](https://github.com/elastic/kibana/pull/142920/files#diff-4e164e3802d5171bf96a2cf9c91c20e97c5e0b74b2f93187a072d9a3139f1c18) ### Checklist Delete any items that are not applicable to this PR. - [ ] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md) - [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [ ] Any UI touched in this PR is usable by keyboard only (learn more about [keyboard accessibility](https://webaim.org/techniques/keyboard/)) - [ ] Any UI touched in this PR does not create any new axe failures (run axe in browser: [FF](https://addons.mozilla.org/en-US/firefox/addon/axe-devtools/), [Chrome](https://chrome.google.com/webstore/detail/axe-web-accessibility-tes/lhdoppojpmngadmnindnejefpokejbdd?hl=en-US)) - [ ] If a plugin configuration key changed, check if it needs to be allowlisted in the cloud and added to the [docker list](https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker) - [ ] This renders correctly on smaller devices using a responsive layout. (You can test this [in your browser](https://www.browserstack.com/guide/responsive-testing-on-local-server)) - [ ] This was checked for [cross-browser compatibility](https://www.elastic.co/support/matrix#matrix_browsers) ### Risk Matrix Delete this section if it is not applicable to this PR. Before closing this PR, invite QA, stakeholders, and other developers to identify risks that should be tested prior to the change/feature release. When forming the risk matrix, consider some of the following examples and how they may potentially impact the change: | Risk | Probability | Severity | Mitigation/Notes | |---------------------------|-------------|----------|-------------------------| | Multiple Spaces—unexpected behavior in non-default Kibana Space. | Low | High | Integration tests will verify that all features are still supported in non-default Kibana Space and when user switches between spaces. | | Multiple nodes—Elasticsearch polling might have race conditions when multiple Kibana nodes are polling for the same tasks. | High | Low | Tasks are idempotent, so executing them multiple times will not result in logical error, but will degrade performance. To test for this case we add plenty of unit tests around this logic and document manual testing procedure. | | Code should gracefully handle cases when feature X or plugin Y are disabled. | Medium | High | Unit tests will verify that any feature flag or plugin combination still results in our service operational. | | [See more potential risk examples](https://github.com/elastic/kibana/blob/main/RISK_MATRIX.mdx) | ### For maintainers - [ ] This was checked for breaking API changes and was [labeled appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../migrations/check_registered_types.test.ts | 2 +- .../plugins/fleet/common/types/models/epm.ts | 7 +- .../integrations/sections/epm/constants.tsx | 3 + .../fleet/server/saved_objects/index.ts | 1 + .../services/epm/elasticsearch/index/index.ts | 8 + .../elasticsearch/index/update_settings.ts | 31 + .../epm/elasticsearch/transform/common.ts | 5 + .../epm/elasticsearch/transform/install.ts | 370 ++++++-- .../epm/elasticsearch/transform/remove.ts | 8 +- .../transform/transforms.test.ts | 804 ++++++++++++++---- .../server/services/epm/packages/remove.ts | 22 +- 11 files changed, 1034 insertions(+), 227 deletions(-) create mode 100644 x-pack/plugins/fleet/server/services/epm/elasticsearch/index/index.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/elasticsearch/index/update_settings.ts diff --git a/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts b/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts index de9981618a01b42..b24f0acc004b022 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts @@ -85,7 +85,7 @@ describe('checking migration metadata changes on all registered SO types', () => "endpoint:user-artifact": "f94c250a52b30d0a2d32635f8b4c5bdabd1e25c0", "endpoint:user-artifact-manifest": "8c14d49a385d5d1307d956aa743ec78de0b2be88", "enterprise_search_telemetry": "fafcc8318528d34f721c42d1270787c52565bad5", - "epm-packages": "fe3716a54188b3c71327fa060dd6780a674d3994", + "epm-packages": "2915aee4302d4b00472ed05c21f59b7d498b5206", "epm-packages-assets": "9fd3d6726ac77369249e9a973902c2cd615fc771", "event_loop_delays_daily": "d2ed39cf669577d90921c176499908b4943fb7bd", "exception-list": "fe8cc004fd2742177cdb9300f4a67689463faf9c", diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts index d8d0dc250edc1ee..b9f471f1b4d437c 100644 --- a/x-pack/plugins/fleet/common/types/models/epm.ts +++ b/x-pack/plugins/fleet/common/types/models/epm.ts @@ -101,6 +101,7 @@ export enum KibanaSavedObjectType { } export enum ElasticsearchAssetType { + index = 'index', componentTemplate = 'component_template', ingestPipeline = 'ingest_pipeline', indexTemplate = 'index_template', @@ -109,6 +110,10 @@ export enum ElasticsearchAssetType { dataStreamIlmPolicy = 'data_stream_ilm_policy', mlModel = 'ml_model', } +export type FleetElasticsearchAssetType = Exclude< + ElasticsearchAssetType, + ElasticsearchAssetType.index +>; export type DataType = typeof dataTypes; export type MonitoringType = typeof monitoringTypes; @@ -313,7 +318,7 @@ export type ElasticsearchAssetParts = AssetParts & { export type KibanaAssetTypeToParts = Record; export type ElasticsearchAssetTypeToParts = Record< - ElasticsearchAssetType, + FleetElasticsearchAssetType, ElasticsearchAssetParts[] >; diff --git a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/constants.tsx b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/constants.tsx index 1fe4b7b38434d39..1b183f13b44260e 100644 --- a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/constants.tsx +++ b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/constants.tsx @@ -35,6 +35,9 @@ export const AssetTitleMap: Record = { transform: i18n.translate('xpack.fleet.epm.assetTitles.transforms', { defaultMessage: 'Transforms', }), + index: i18n.translate('xpack.fleet.epm.assetTitles.indices', { + defaultMessage: 'Indices', + }), index_pattern: i18n.translate('xpack.fleet.epm.assetTitles.indexPatterns', { defaultMessage: 'Index patterns', }), diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts index 8a061552a77edac..6c87e894e5777ed 100644 --- a/x-pack/plugins/fleet/server/saved_objects/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/index.ts @@ -262,6 +262,7 @@ const getSavedObjectTypes = ( properties: { id: { type: 'keyword' }, type: { type: 'keyword' }, + version: { type: 'keyword' }, }, }, installed_kibana: { diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/index/index.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/index/index.ts new file mode 100644 index 000000000000000..79d89d70ac67214 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/index/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { updateIndexSettings } from './update_settings'; diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/index/update_settings.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/index/update_settings.ts new file mode 100644 index 000000000000000..a381f19a3d9fa77 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/index/update_settings.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; + +import type { IndicesIndexSettings } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; + +import { retryTransientEsErrors } from '../retry'; + +export async function updateIndexSettings( + esClient: ElasticsearchClient, + index: string, + settings: IndicesIndexSettings +): Promise { + if (index) { + try { + await retryTransientEsErrors(() => + esClient.indices.putSettings({ + index, + body: settings, + }) + ); + } catch (err) { + throw new Error(`could not update index settings for ${index}`); + } + } +} diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/common.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/common.ts index e08d973f8df0e35..46e85d7f9df6245 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/common.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/common.ts @@ -6,3 +6,8 @@ */ export { getAsset } from '../../archive'; + +// Index alias that points to just one destination index from the latest package version +export const TRANSFORM_DEST_IDX_ALIAS_LATEST_SFX = '.latest'; +// Index alias that points to all of the destination indices from all the package versions +export const TRANSFORM_DEST_IDX_ALIAS_ALL_SFX = '.all'; diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts index 4999c07a2aec1b9..e8881bb247e12f9 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts @@ -9,6 +9,7 @@ import type { ElasticsearchClient, Logger, SavedObjectsClientContract } from '@k import { errors } from '@elastic/elasticsearch'; import { safeLoad } from 'js-yaml'; import { isPopulatedObject } from '@kbn/ml-is-populated-object'; +import { uniqBy } from 'lodash'; import { PACKAGE_TEMPLATE_SUFFIX, @@ -35,7 +36,7 @@ import { getInstallation } from '../../packages'; import { retryTransientEsErrors } from '../retry'; import { deleteTransforms } from './remove'; -import { getAsset } from './common'; +import { getAsset, TRANSFORM_DEST_IDX_ALIAS_LATEST_SFX } from './common'; const DEFAULT_TRANSFORM_TEMPLATES_PRIORITY = 250; enum TRANSFORM_SPECS_TYPES { @@ -55,6 +56,8 @@ interface DestinationIndexTemplateInstallation extends TransformModuleBase { interface TransformInstallation extends TransformModuleBase { installationName: string; content: any; + transformVersion?: string; + installationOrder?: number; } const installLegacyTransformsAssets = async ( @@ -67,6 +70,13 @@ const installLegacyTransformsAssets = async ( esReferences: EsAssetReference[] = [], previousInstalledTransformEsAssets: EsAssetReference[] = [] ) => { + await deleteTransforms( + esClient, + previousInstalledTransformEsAssets.map((asset) => asset.id), + // For legacy transforms, delete destination indices upon deleting transforms + true + ); + let installedTransforms: EsAssetReference[] = []; if (transformPaths.length > 0) { const transformRefs = transformPaths.reduce((acc, path) => { @@ -126,11 +136,16 @@ const installLegacyTransformsAssets = async ( const processTransformAssetsPerModule = ( installablePackage: InstallablePackage, installNameSuffix: string, - transformPaths: string[] + transformPaths: string[], + previousInstalledTransformEsAssets: EsAssetReference[] = [] ) => { const transformsSpecifications = new Map(); const destinationIndexTemplates: DestinationIndexTemplateInstallation[] = []; const transforms: TransformInstallation[] = []; + const aliasesRefs: string[] = []; + const transformsToRemove: EsAssetReference[] = []; + const transformsToRemoveWithDestIndex: EsAssetReference[] = []; + const indicesToAddRefs: EsAssetReference[] = []; transformPaths.forEach((path: string) => { const { transformModuleId, fileName } = getTransformFolderAndFileNames( @@ -150,38 +165,143 @@ const processTransformAssetsPerModule = ( if (fileName === TRANSFORM_SPECS_TYPES.FIELDS) { const validFields = processFields(content); const mappings = generateMappings(validFields); + const templateName = getTransformAssetNameForInstallation( + installablePackage, + transformModuleId, + 'template' + ); + const indexToModify = destinationIndexTemplates.findIndex( + (t) => t.transformModuleId === transformModuleId && t.installationName === templateName + ); + const template = { + transformModuleId, + _meta: getESAssetMetadata({ packageName: installablePackage.name }), + installationName: getTransformAssetNameForInstallation( + installablePackage, + transformModuleId, + 'template' + ), + template: {}, + } as DestinationIndexTemplateInstallation; + if (indexToModify === -1) { + destinationIndexTemplates.push(template); + } else { + destinationIndexTemplates[indexToModify] = template; + } packageAssets?.set('mappings', mappings); } if (fileName === TRANSFORM_SPECS_TYPES.TRANSFORM) { + const installationOrder = + isFinite(content._meta?.order) && content._meta?.order >= 0 ? content._meta?.order : 0; + const transformVersion = content._meta?.fleet_transform_version ?? '0.1.0'; + // The “all” alias for the transform destination indices will be adjusted to include the new transform destination index as well as everything it previously included + const allIndexAliasName = `${content.dest.index}.all`; + // The “latest” alias for the transform destination indices will point solely to the new transform destination index + const latestIndexAliasName = `${content.dest.index}.latest`; + + transformsSpecifications + .get(transformModuleId) + ?.set('originalDestinationIndexName', content.dest.index); + + // Create two aliases associated with the destination index + // for better handling during upgrades + const alias = { + [allIndexAliasName]: {}, + [latestIndexAliasName]: {}, + }; + + const versionedIndexName = `${content.dest.index}-${installNameSuffix}`; + content.dest.index = versionedIndexName; + indicesToAddRefs.push({ + id: versionedIndexName, + type: ElasticsearchAssetType.index, + }); transformsSpecifications.get(transformModuleId)?.set('destinationIndex', content.dest); + transformsSpecifications.get(transformModuleId)?.set('destinationIndexAlias', alias); transformsSpecifications.get(transformModuleId)?.set('transform', content); - content._meta = getESAssetMetadata({ packageName: installablePackage.name }); - transforms.push({ + transformsSpecifications.get(transformModuleId)?.set('transformVersion', transformVersion); + content._meta = { + ...(content._meta ?? {}), + ...getESAssetMetadata({ packageName: installablePackage.name }), + }; + + const installationName = getTransformAssetNameForInstallation( + installablePackage, transformModuleId, - installationName: getTransformAssetNameForInstallation( - installablePackage, + // transform_id is versioned by fleet_transform_version and not by package version + `default-${transformVersion}` + ); + + const currentTransformSameAsPrev = + previousInstalledTransformEsAssets.find((t) => t.id === installationName) !== undefined; + if (previousInstalledTransformEsAssets.length === 0) { + aliasesRefs.push(allIndexAliasName, latestIndexAliasName); + transforms.push({ transformModuleId, - `default-${installNameSuffix}` - ), - content, - }); + installationName, + installationOrder, + transformVersion, + content, + }); + transformsSpecifications.get(transformModuleId)?.set('transformVersionChanged', true); + } else { + if (!currentTransformSameAsPrev) { + // If upgrading from old json schema to new yml schema + // We need to make sure to delete those transforms by matching the legacy naming convention + const versionFromOldJsonSchema = previousInstalledTransformEsAssets.find((t) => + t.id.startsWith( + getLegacyTransformNameForInstallation( + installablePackage, + `${transformModuleId}/default.json` + ) + ) + ); + + if (versionFromOldJsonSchema !== undefined) { + transformsToRemoveWithDestIndex.push(versionFromOldJsonSchema); + } + + // If upgrading from yml to newer version of yaml + // Match using new naming convention + const installNameWithoutVersion = installationName.split(transformVersion)[0]; + const prevVersion = previousInstalledTransformEsAssets.find((t) => + t.id.startsWith(installNameWithoutVersion) + ); + if (prevVersion !== undefined) { + transformsToRemove.push(prevVersion); + } + transforms.push({ + transformModuleId, + installationName, + installationOrder, + transformVersion, + content, + }); + transformsSpecifications.get(transformModuleId)?.set('transformVersionChanged', true); + aliasesRefs.push(allIndexAliasName, latestIndexAliasName); + } else { + transformsSpecifications.get(transformModuleId)?.set('transformVersionChanged', false); + } + } } + // Create index templates for destination indices if destination_index_template OR fields are defined if (fileName === TRANSFORM_SPECS_TYPES.MANIFEST) { if (isPopulatedObject(content, ['start']) && content.start === false) { transformsSpecifications.get(transformModuleId)?.set('start', false); } - // If manifest.yml contains destination_index_template - // Combine the mappings and other index template settings from manifest.yml into a single index template - // Create the index template and track the template in EsAssetReferences - if ( - isPopulatedObject(content, ['destination_index_template']) || - isPopulatedObject(packageAssets.get('mappings')) - ) { - const destinationIndexTemplate = - (content.destination_index_template as Record) ?? {}; - destinationIndexTemplates.push({ + + if (content.destination_index_template) { + const templateName = getTransformAssetNameForInstallation( + installablePackage, + transformModuleId, + 'template' + ); + const indexToModify = destinationIndexTemplates.findIndex( + (t) => t.transformModuleId === transformModuleId && t.installationName === templateName + ); + const template = { transformModuleId, _meta: getESAssetMetadata({ packageName: installablePackage.name }), installationName: getTransformAssetNameForInstallation( @@ -189,9 +309,14 @@ const processTransformAssetsPerModule = ( transformModuleId, 'template' ), - template: destinationIndexTemplate, - } as DestinationIndexTemplateInstallation); - packageAssets.set('destinationIndexTemplate', destinationIndexTemplate); + template: content.destination_index_template, + } as DestinationIndexTemplateInstallation; + if (indexToModify === -1) { + destinationIndexTemplates.push(template); + } else { + destinationIndexTemplates[indexToModify] = template; + } + packageAssets.set('destinationIndexTemplate', template); } } }); @@ -199,30 +324,42 @@ const processTransformAssetsPerModule = ( const indexTemplatesRefs = destinationIndexTemplates.map((template) => ({ id: template.installationName, type: ElasticsearchAssetType.indexTemplate, + version: transformsSpecifications.get(template.transformModuleId)?.get('transformVersion'), })); const componentTemplatesRefs = [ ...destinationIndexTemplates.map((template) => ({ id: `${template.installationName}${USER_SETTINGS_TEMPLATE_SUFFIX}`, type: ElasticsearchAssetType.componentTemplate, + version: transformsSpecifications.get(template.transformModuleId)?.get('transformVersion'), })), ...destinationIndexTemplates.map((template) => ({ id: `${template.installationName}${PACKAGE_TEMPLATE_SUFFIX}`, type: ElasticsearchAssetType.componentTemplate, + version: transformsSpecifications.get(template.transformModuleId)?.get('transformVersion'), })), ]; - const transformRefs = transforms.map((t) => ({ + const sortedTransforms = transforms.sort( + (t1, t2) => (t1.installationOrder ?? 0) - (t2.installationOrder ?? 1) + ); + + const transformRefs = sortedTransforms.map((t) => ({ id: t.installationName, type: ElasticsearchAssetType.transform, + version: t.transformVersion, })); return { + indicesToAddRefs, indexTemplatesRefs, componentTemplatesRefs, transformRefs, - transforms, + transforms: sortedTransforms, destinationIndexTemplates, transformsSpecifications, + aliasesRefs, + transformsToRemove, + transformsToRemoveWithDestIndex, }; }; @@ -239,21 +376,60 @@ const installTransformsAssets = async ( let installedTransforms: EsAssetReference[] = []; if (transformPaths.length > 0) { const { + indicesToAddRefs, indexTemplatesRefs, componentTemplatesRefs, transformRefs, transforms, destinationIndexTemplates, transformsSpecifications, - } = processTransformAssetsPerModule(installablePackage, installNameSuffix, transformPaths); + aliasesRefs, + transformsToRemove, + transformsToRemoveWithDestIndex, + } = processTransformAssetsPerModule( + installablePackage, + installNameSuffix, + transformPaths, + previousInstalledTransformEsAssets + ); + + // ensure the .latest alias points to only the latest + // by removing any associate of old destination indices + await Promise.all( + aliasesRefs + .filter((a) => a.endsWith(TRANSFORM_DEST_IDX_ALIAS_LATEST_SFX)) + .map((alias) => deleteAliasFromIndices({ esClient, logger, alias })) + ); + + // delete all previous transform + await Promise.all([ + deleteTransforms( + esClient, + transformsToRemoveWithDestIndex.map((asset) => asset.id), + // Delete destination indices if specified or if from old json schema + true + ), + deleteTransforms( + esClient, + transformsToRemove.map((asset) => asset.id), + // Else, keep destination indices by default + false + ), + ]); + // get and save refs associated with the transforms before installing esReferences = await updateEsAssetReferences( savedObjectsClient, installablePackage.name, esReferences, { - assetsToAdd: [...indexTemplatesRefs, ...componentTemplatesRefs, ...transformRefs], - assetsToRemove: previousInstalledTransformEsAssets, + assetsToAdd: [ + ...indicesToAddRefs, + ...indexTemplatesRefs, + ...componentTemplatesRefs, + ...transformRefs, + ], + assetsToRemove: [...transformsToRemove, ...transformsToRemoveWithDestIndex], } ); @@ -261,10 +437,15 @@ const installTransformsAssets = async ( await Promise.all( destinationIndexTemplates .map((destinationIndexTemplate) => { - const customMappings = - transformsSpecifications - .get(destinationIndexTemplate.transformModuleId) - ?.get('mappings') ?? {}; + const transformSpec = transformsSpecifications.get( + destinationIndexTemplate.transformModuleId + ); + const customMappings = transformSpec?.get('mappings') ?? {}; + const pipelineId = transformSpec?.get('destinationIndex')?.pipeline; + const transformVersionChanged = transformSpec?.get('transformVersionChanged') ?? true; + + if (!transformVersionChanged) return; + const registryElasticsearch: RegistryElasticsearch = { 'index_template.settings': destinationIndexTemplate.template.settings, 'index_template.mappings': destinationIndexTemplate.template.mappings, @@ -275,7 +456,11 @@ const installTransformsAssets = async ( templateName: destinationIndexTemplate.installationName, registryElasticsearch, packageName: installablePackage.name, - defaultSettings: {}, + defaultSettings: { + // Adding destination pipeline here because else these templates will be overridden + // by index setting + ...(pipelineId ? { default_pipeline: pipelineId } : {}), + }, }); if (destinationIndexTemplate || customMappings) { @@ -285,10 +470,12 @@ const installTransformsAssets = async ( componentTemplates, indexTemplate: { templateName: destinationIndexTemplate.installationName, - // @ts-expect-error We don't need to pass data_stream property here - // as this template is applied to only an index and not a data stream + // @ts-expect-error data_stream property is not needed here indexTemplate: { - template: { settings: undefined, mappings: undefined }, + template: { + settings: undefined, + mappings: undefined, + }, priority: DEFAULT_TRANSFORM_TEMPLATES_PRIORITY, index_patterns: [ transformsSpecifications @@ -309,37 +496,81 @@ const installTransformsAssets = async ( await Promise.all( transforms.map(async (transform) => { const index = transform.content.dest.index; - const pipelineId = transform.content.dest.pipeline; + const aliases = transformsSpecifications + .get(transform.transformModuleId) + ?.get('destinationIndexAlias'); try { - await retryTransientEsErrors( + const resp = await retryTransientEsErrors( () => esClient.indices.create( { index, - ...(pipelineId ? { settings: { default_pipeline: pipelineId } } : {}), + aliases, }, { ignore: [400] } ), { logger } ); + logger.debug(`Created destination index: ${index}`); + + // If index already exists, we still need to update the destination index alias + // to point '{destinationIndexName}.latest' to the versioned index + // @ts-ignore status is a valid field of resp + if (resp.status === 400 && aliases) { + await retryTransientEsErrors( + () => + esClient.indices.updateAliases({ + body: { + actions: Object.keys(aliases).map((alias) => ({ add: { index, alias } })), + }, + }), + { logger } + ); + logger.debug(`Created aliases for destination index: ${index}`); + } } catch (err) { + logger.error( + `Error creating destination index: ${JSON.stringify({ + index, + aliases: transformsSpecifications + .get(transform.transformModuleId) + ?.get('destinationIndexAlias'), + })} with error ${err}` + ); + throw new Error(err.message); } }) ); - // create & optionally start transforms - const transformsPromises = transforms.map(async (transform) => { - return handleTransformInstall({ - esClient, - logger, - transform, - startTransform: transformsSpecifications.get(transform.transformModuleId)?.get('start'), + // If the transforms have specific installation order, install & optionally start transforms sequentially + const shouldInstallSequentially = + uniqBy(transforms, 'installationOrder').length === transforms.length; + + if (shouldInstallSequentially) { + for (const transform of transforms) { + const installTransform = await handleTransformInstall({ + esClient, + logger, + transform, + startTransform: transformsSpecifications.get(transform.transformModuleId)?.get('start'), + }); + installedTransforms.push(installTransform); + } + } else { + // Else, create & start all the transforms at once for speed + const transformsPromises = transforms.map(async (transform) => { + return handleTransformInstall({ + esClient, + logger, + transform, + startTransform: transformsSpecifications.get(transform.transformModuleId)?.get('start'), + }); }); - }); - installedTransforms = await Promise.all(transformsPromises).then((results) => results.flat()); + installedTransforms = await Promise.all(transformsPromises).then((results) => results.flat()); + } } return { installedTransforms, esReferences }; @@ -364,7 +595,7 @@ export const installTransforms = async ( previousInstalledTransformEsAssets = installation.installed_es.filter( ({ type, id }) => type === ElasticsearchAssetType.transform ); - if (previousInstalledTransformEsAssets.length) { + if (previousInstalledTransformEsAssets.length > 0) { logger.debug( `Found previous transform references:\n ${JSON.stringify( previousInstalledTransformEsAssets @@ -373,12 +604,6 @@ export const installTransforms = async ( } } - // delete all previous transform - await deleteTransforms( - esClient, - previousInstalledTransformEsAssets.map((asset) => asset.id) - ); - const installNameSuffix = `${installablePackage.version}`; // If package contains legacy transform specifications (i.e. with json instead of yml) @@ -412,6 +637,36 @@ export const isTransform = (path: string) => { return !path.endsWith('/') && pathParts.type === ElasticsearchAssetType.transform; }; +async function deleteAliasFromIndices({ + esClient, + logger, + alias, +}: { + esClient: ElasticsearchClient; + logger: Logger; + alias: string; +}) { + try { + const resp = await esClient.indices.getAlias({ name: alias }); + const indicesMatchingAlias = Object.keys(resp); + logger.debug(`Deleting alias: '${alias}' matching indices ${indicesMatchingAlias}`); + + if (indicesMatchingAlias.length > 0) { + await retryTransientEsErrors( + () => + // defer validation on put if the source index is not available + esClient.indices.deleteAlias( + { index: indicesMatchingAlias, name: alias }, + { ignore: [404] } + ), + { logger } + ); + logger.debug(`Deleted alias: '${alias}' matching indices ${indicesMatchingAlias}`); + } + } catch (err) { + logger.error(`Error deleting alias: ${alias}`); + } +} async function handleTransformInstall({ esClient, logger, @@ -434,6 +689,7 @@ async function handleTransformInstall({ }), { logger } ); + logger.debug(`Created transform: ${transform.installationName}`); } catch (err) { // swallow the error if the transform already exists. const isAlreadyExistError = @@ -460,12 +716,12 @@ async function handleTransformInstall({ const getLegacyTransformNameForInstallation = ( installablePackage: InstallablePackage, path: string, - suffix: string + suffix?: string ) => { const pathPaths = path.split('/'); const filename = pathPaths?.pop()?.split('.')[0]; const folderName = pathPaths?.pop(); - return `${installablePackage.name}.${folderName}-${filename}-${suffix}`; + return `${installablePackage.name}.${folderName}-${filename}${suffix ? '-' + suffix : ''}`; }; const getTransformAssetNameForInstallation = ( diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/remove.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/remove.ts index f10eddd3d5e65cf..77996674f402e60 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/remove.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/remove.ts @@ -21,7 +21,11 @@ export const stopTransforms = async (transformIds: string[], esClient: Elasticse } }; -export const deleteTransforms = async (esClient: ElasticsearchClient, transformIds: string[]) => { +export const deleteTransforms = async ( + esClient: ElasticsearchClient, + transformIds: string[], + deleteDestinationIndices = false +) => { const logger = appContextService.getLogger(); if (transformIds.length) { logger.info(`Deleting currently installed transform ids ${transformIds}`); @@ -40,7 +44,7 @@ export const deleteTransforms = async (esClient: ElasticsearchClient, transformI { ignore: [404] } ); logger.info(`Deleted: ${transformId}`); - if (transformResponse?.transforms) { + if (deleteDestinationIndices && transformResponse?.transforms) { // expect this to be 1 for (const transform of transformResponse.transforms) { await esClient.transport.request( diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transforms.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transforms.test.ts index aeeeb59e12b3868..77537f200628d76 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transforms.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transforms.test.ts @@ -42,7 +42,10 @@ describe('test transform install', () => { let esClient: ReturnType; let savedObjectsClient: jest.Mocked; - const getYamlTestData = (autoStart: boolean | undefined = undefined) => { + const getYamlTestData = ( + autoStart: boolean | undefined = undefined, + transformVersion: string = '0.1.0' + ) => { const start = autoStart === undefined ? '' @@ -93,6 +96,7 @@ pivot: field: agent.id description: Merges latest endpoint and Agent metadata documents. _meta: + fleet_transform_version: ${transformVersion} managed: true`, FIELDS: `- name: '@timestamp' type: date @@ -101,15 +105,15 @@ _meta: path: event.ingested`, }; }; - const getExpectedData = () => { + const getExpectedData = (transformVersion: string) => { return { TRANSFORM: { - transform_id: 'logs-endpoint.metadata_current-default-0.16.0-dev.0', + transform_id: `logs-endpoint.metadata_current-default-${transformVersion}`, defer_validation: true, body: { description: 'Merges latest endpoint and Agent metadata documents.', dest: { - index: '.metrics-endpoint.metadata_united_default', + index: '.metrics-endpoint.metadata_united_default-0.16.0-dev.0', }, frequency: '1s', pivot: { @@ -141,7 +145,7 @@ _meta: field: 'updated_at', }, }, - _meta: meta, + _meta: { fleet_transform_version: transformVersion, ...meta }, }, }, }; @@ -165,9 +169,10 @@ _meta: jest.clearAllMocks(); }); - test('can install new versions and removes older version when start is not defined', async () => { - const sourceData = getYamlTestData(); - const expectedData = getExpectedData(); + test('can install new versions and removes older version when fleet_transform_version increased', async () => { + // Old fleet_transform_version is 0.1.0, fleet_transform_version to be installed is 0.1.0 + const sourceData = getYamlTestData(undefined, '0.2.0'); + const expectedData = getExpectedData('0.2.0'); const previousInstallation: Installation = { installed_es: [ @@ -176,7 +181,7 @@ _meta: type: ElasticsearchAssetType.ingestPipeline, }, { - id: 'endpoint.metadata_current-default-0.15.0-dev.0', + id: 'logs-endpoint.metadata_current-default-0.1.0', type: ElasticsearchAssetType.transform, }, ], @@ -189,20 +194,17 @@ _meta: type: ElasticsearchAssetType.ingestPipeline, }, { - id: 'endpoint.metadata_current-default-0.15.0-dev.0', + id: 'logs-endpoint.metadata_current-default-0.2.0', type: ElasticsearchAssetType.transform, }, { - id: 'endpoint.metadata_current-default-0.16.0-dev.0', - type: ElasticsearchAssetType.transform, - }, - { - id: 'endpoint.metadata-default-0.16.0-dev.0', + id: 'logs-endpoint.metadata_current-default-0.2.0', type: ElasticsearchAssetType.transform, }, ], } as unknown as Installation; (getAsset as jest.MockedFunction) + .mockReturnValueOnce(Buffer.from(sourceData.FIELDS, 'utf8')) .mockReturnValueOnce(Buffer.from(sourceData.MANIFEST, 'utf8')) .mockReturnValueOnce(Buffer.from(sourceData.TRANSFORM, 'utf8')); @@ -239,6 +241,7 @@ _meta: version: '0.16.0-dev.0', } as unknown as RegistryPackage, [ + 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/fields/fields.yml', 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/manifest.yml', 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml', ], @@ -248,19 +251,11 @@ _meta: previousInstallation.installed_es ); - expect(esClient.transform.getTransform.mock.calls).toEqual([ - [ - { - transform_id: 'endpoint.metadata_current-default-0.15.0-dev.0', - }, - { ignore: [404] }, - ], - ]); // Stop and delete previously installed transforms expect(esClient.transform.stopTransform.mock.calls).toEqual([ [ { - transform_id: 'endpoint.metadata_current-default-0.15.0-dev.0', + transform_id: 'logs-endpoint.metadata_current-default-0.1.0', force: true, }, { ignore: [404] }, @@ -269,70 +264,62 @@ _meta: expect(esClient.transform.deleteTransform.mock.calls).toEqual([ [ { - transform_id: 'endpoint.metadata_current-default-0.15.0-dev.0', + transform_id: 'logs-endpoint.metadata_current-default-0.1.0', force: true, }, { ignore: [404] }, ], ]); - // Delete destination index - expect(esClient.transport.request.mock.calls).toEqual([ - [ - { - method: 'DELETE', - path: '/mock-old-destination-index', - }, - { ignore: [404] }, - ], - ]); + // Destination index should not be deleted when transform is deleted + expect(esClient.transport.request.mock.calls).toEqual([]); // Create a @package component template and an empty @custom component template expect(esClient.cluster.putComponentTemplate.mock.calls).toEqual([ [ { + name: 'logs-endpoint.metadata_current-template@package', body: { - _meta: meta, template: { + settings: { + index: { + codec: 'best_compression', + refresh_interval: '5s', + number_of_shards: 1, + number_of_routing_shards: 30, + hidden: true, + mapping: { total_fields: { limit: '10000' } }, + }, + }, mappings: { - _meta: {}, - date_detection: false, - dynamic: false, + properties: { '@timestamp': { type: 'date' } }, dynamic_templates: [ { strings_as_keyword: { - mapping: { ignore_above: 1024, type: 'keyword' }, match_mapping_type: 'string', + mapping: { ignore_above: 1024, type: 'keyword' }, }, }, ], - properties: {}, - }, - settings: { - index: { - codec: 'best_compression', - hidden: true, - mapping: { total_fields: { limit: '10000' } }, - number_of_routing_shards: 30, - number_of_shards: 1, - refresh_interval: '5s', - }, + dynamic: false, + _meta: {}, + date_detection: false, }, }, + _meta: { managed_by: 'fleet', managed: true, package: { name: 'endpoint' } }, }, create: false, - name: 'logs-endpoint.metadata_current-template@package', }, { ignore: [404] }, ], [ { + name: 'logs-endpoint.metadata_current-template@custom', body: { - _meta: meta, template: { settings: {} }, + _meta: { managed_by: 'fleet', managed: true, package: { name: 'endpoint' } }, }, create: true, - name: 'logs-endpoint.metadata_current-template@custom', }, { ignore: [404] }, ], @@ -349,7 +336,7 @@ _meta: 'logs-endpoint.metadata_current-template@package', 'logs-endpoint.metadata_current-template@custom', ], - index_patterns: ['.metrics-endpoint.metadata_united_default'], + index_patterns: ['.metrics-endpoint.metadata_united_default-0.16.0-dev.0'], priority: 250, template: { mappings: undefined, settings: undefined }, }, @@ -361,14 +348,23 @@ _meta: // Destination index is created before transform is created expect(esClient.indices.create.mock.calls).toEqual([ - [{ index: '.metrics-endpoint.metadata_united_default' }, { ignore: [400] }], + [ + { + aliases: { + '.metrics-endpoint.metadata_united_default.all': {}, + '.metrics-endpoint.metadata_united_default.latest': {}, + }, + index: '.metrics-endpoint.metadata_united_default-0.16.0-dev.0', + }, + { ignore: [400] }, + ], ]); expect(esClient.transform.putTransform.mock.calls).toEqual([[expectedData.TRANSFORM]]); expect(esClient.transform.startTransform.mock.calls).toEqual([ [ { - transform_id: 'logs-endpoint.metadata_current-default-0.16.0-dev.0', + transform_id: 'logs-endpoint.metadata_current-default-0.2.0', }, { ignore: [409] }, ], @@ -385,21 +381,29 @@ _meta: id: 'metrics-endpoint.policy-0.16.0-dev.0', type: ElasticsearchAssetType.ingestPipeline, }, + { + id: '.metrics-endpoint.metadata_united_default-0.16.0-dev.0', + type: ElasticsearchAssetType.index, + }, { id: 'logs-endpoint.metadata_current-template', type: ElasticsearchAssetType.indexTemplate, + version: '0.2.0', }, { id: 'logs-endpoint.metadata_current-template@custom', type: ElasticsearchAssetType.componentTemplate, + version: '0.2.0', }, { id: 'logs-endpoint.metadata_current-template@package', type: ElasticsearchAssetType.componentTemplate, + version: '0.2.0', }, { - id: 'logs-endpoint.metadata_current-default-0.16.0-dev.0', + id: 'logs-endpoint.metadata_current-default-0.2.0', type: ElasticsearchAssetType.transform, + version: '0.2.0', }, ], }, @@ -410,104 +414,39 @@ _meta: ]); }); - test('can install new version when no older version', async () => { - const sourceData = getYamlTestData(true); - const expectedData = getExpectedData(); + test('can install new versions and removes older version when upgraded from old json schema to new yml schema', async () => { + const sourceData = getYamlTestData(undefined, '0.2.0'); + const expectedData = getExpectedData('0.2.0'); const previousInstallation: Installation = { - installed_es: [], - } as unknown as Installation; - - const currentInstallation: Installation = { installed_es: [ { - id: 'metrics-endpoint.metadata-current-default-0.16.0-dev.0', - type: ElasticsearchAssetType.transform, - }, - ], - } as unknown as Installation; - (getAsset as jest.MockedFunction).mockReturnValueOnce( - Buffer.from(sourceData.TRANSFORM, 'utf8') - ); - (getInstallation as jest.MockedFunction) - .mockReturnValueOnce(Promise.resolve(previousInstallation)) - .mockReturnValueOnce(Promise.resolve(currentInstallation)); - - ( - getInstallationObject as jest.MockedFunction - ).mockReturnValueOnce( - Promise.resolve({ - attributes: { installed_es: [] }, - } as unknown as SavedObject) - ); - - await installTransforms( - { - name: 'endpoint', - version: '0.16.0-dev.0', - } as unknown as RegistryPackage, - ['endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml'], - esClient, - savedObjectsClient, - loggerMock.create(), - previousInstallation.installed_es - ); - - expect(esClient.transform.putTransform.mock.calls).toEqual([[expectedData.TRANSFORM]]); - expect(esClient.transform.startTransform.mock.calls).toEqual([ - [ - { - transform_id: 'logs-endpoint.metadata_current-default-0.16.0-dev.0', - }, - { ignore: [409] }, - ], - ]); - - expect(savedObjectsClient.update.mock.calls).toEqual([ - [ - 'epm-packages', - 'endpoint', - { - installed_es: [ - { id: 'logs-endpoint.metadata_current-default-0.16.0-dev.0', type: 'transform' }, - ], + id: 'metrics-endpoint.policy-0.1.0-dev.0', + type: ElasticsearchAssetType.ingestPipeline, }, { - refresh: false, + id: 'endpoint.metadata_current-default-0.1.0', + type: ElasticsearchAssetType.transform, }, ], - ]); - }); - - test('can combine settings fields.yml & manifest.yml and not start transform automatically', async () => { - const sourceData = getYamlTestData(false); - const expectedData = getExpectedData(); + } as unknown as Installation; - const previousInstallation: Installation = { + const currentInstallation: Installation = { installed_es: [ { - id: 'endpoint.metadata-current-default-0.15.0-dev.0', - type: ElasticsearchAssetType.transform, - }, - { - id: 'logs-endpoint.metadata_current-template', - type: ElasticsearchAssetType.indexTemplate, + id: 'metrics-endpoint.policy-0.16.0-dev.0', + type: ElasticsearchAssetType.ingestPipeline, }, { - id: 'logs-endpoint.metadata_current-template@custom', - type: ElasticsearchAssetType.componentTemplate, + id: 'logs-endpoint.metadata_current-default-0.2.0', + type: ElasticsearchAssetType.transform, }, { - id: 'logs-endpoint.metadata_current-template@package', - type: ElasticsearchAssetType.componentTemplate, + id: 'logs-endpoint.metadata_current-default-0.2.0', + type: ElasticsearchAssetType.transform, }, ], } as unknown as Installation; - - const currentInstallation: Installation = { - installed_es: [], - } as unknown as Installation; - (getAsset as jest.MockedFunction) .mockReturnValueOnce(Buffer.from(sourceData.FIELDS, 'utf8')) .mockReturnValueOnce(Buffer.from(sourceData.MANIFEST, 'utf8')) @@ -521,10 +460,13 @@ _meta: getInstallationObject as jest.MockedFunction ).mockReturnValueOnce( Promise.resolve({ - attributes: { installed_es: currentInstallation.installed_es }, + attributes: { + installed_es: previousInstallation.installed_es, + }, } as unknown as SavedObject) ); + // Mock transform from old version esClient.transform.getTransform.mockResponseOnce({ count: 1, transforms: [ @@ -553,43 +495,38 @@ _meta: previousInstallation.installed_es ); - expect(esClient.transform.getTransform.mock.calls).toEqual([ + // Stop and delete previously installed transforms + expect(esClient.transform.stopTransform.mock.calls).toEqual([ [ { - transform_id: 'endpoint.metadata-current-default-0.15.0-dev.0', + transform_id: 'endpoint.metadata_current-default-0.1.0', + force: true, }, { ignore: [404] }, ], ]); - - // Transform from old version is stopped & deleted - expect(esClient.transform.stopTransform.mock.calls).toEqual([ + expect(esClient.transform.deleteTransform.mock.calls).toEqual([ [ { - transform_id: 'endpoint.metadata-current-default-0.15.0-dev.0', + transform_id: 'endpoint.metadata_current-default-0.1.0', force: true, }, { ignore: [404] }, ], ]); - expect(esClient.transform.deleteTransform.mock.calls).toEqual([ + // Destination index from previous version using legacy schema should be deleted + expect(esClient.transport.request.mock.calls).toEqual([ [ { - transform_id: 'endpoint.metadata-current-default-0.15.0-dev.0', - force: true, + method: 'DELETE', + path: '/mock-old-destination-index', }, { ignore: [404] }, ], ]); - // Destination index from old version is also deleted - expect(esClient.transport.request.mock.calls).toEqual([ - [{ method: 'DELETE', path: '/mock-old-destination-index' }, { ignore: [404] }], - ]); - - // Component templates are created with mappings from fields.yml - // and template from manifest + // Create a @package component template and an empty @custom component template expect(esClient.cluster.putComponentTemplate.mock.calls).toEqual([ [ { @@ -621,7 +558,7 @@ _meta: date_detection: false, }, }, - _meta: meta, + _meta: { managed_by: 'fleet', managed: true, package: { name: 'endpoint' } }, }, create: false, }, @@ -632,13 +569,14 @@ _meta: name: 'logs-endpoint.metadata_current-template@custom', body: { template: { settings: {} }, - _meta: meta, + _meta: { managed_by: 'fleet', managed: true, package: { name: 'endpoint' } }, }, create: true, }, { ignore: [404] }, ], ]); + // Index template composed of the two component templates created // with index pattern matching the destination index expect(esClient.indices.putIndexTemplate.mock.calls).toEqual([ @@ -650,7 +588,7 @@ _meta: 'logs-endpoint.metadata_current-template@package', 'logs-endpoint.metadata_current-template@custom', ], - index_patterns: ['.metrics-endpoint.metadata_united_default'], + index_patterns: ['.metrics-endpoint.metadata_united_default-0.16.0-dev.0'], priority: 250, template: { mappings: undefined, settings: undefined }, }, @@ -662,11 +600,557 @@ _meta: // Destination index is created before transform is created expect(esClient.indices.create.mock.calls).toEqual([ - [{ index: '.metrics-endpoint.metadata_united_default' }, { ignore: [400] }], + [ + { + aliases: { + '.metrics-endpoint.metadata_united_default.all': {}, + '.metrics-endpoint.metadata_united_default.latest': {}, + }, + index: '.metrics-endpoint.metadata_united_default-0.16.0-dev.0', + }, + { ignore: [400] }, + ], ]); - // New transform created but not not started automatically if start: false in manifest.yml expect(esClient.transform.putTransform.mock.calls).toEqual([[expectedData.TRANSFORM]]); + expect(esClient.transform.startTransform.mock.calls).toEqual([ + [ + { + transform_id: 'logs-endpoint.metadata_current-default-0.2.0', + }, + { ignore: [409] }, + ], + ]); + + // Saved object is updated with newly created index templates, component templates, transform + expect(savedObjectsClient.update.mock.calls).toEqual([ + [ + 'epm-packages', + 'endpoint', + { + installed_es: [ + { + id: 'metrics-endpoint.policy-0.1.0-dev.0', + type: ElasticsearchAssetType.ingestPipeline, + }, + { + id: '.metrics-endpoint.metadata_united_default-0.16.0-dev.0', + type: ElasticsearchAssetType.index, + }, + { + id: 'logs-endpoint.metadata_current-template', + type: ElasticsearchAssetType.indexTemplate, + version: '0.2.0', + }, + { + id: 'logs-endpoint.metadata_current-template@custom', + type: ElasticsearchAssetType.componentTemplate, + version: '0.2.0', + }, + { + id: 'logs-endpoint.metadata_current-template@package', + type: ElasticsearchAssetType.componentTemplate, + version: '0.2.0', + }, + { + id: 'logs-endpoint.metadata_current-default-0.2.0', + type: ElasticsearchAssetType.transform, + version: '0.2.0', + }, + ], + }, + { + refresh: false, + }, + ], + ]); + }); + + test('creates index and component templates even if no manifest.yml', async () => { + // Old fleet_transform_version is 0.1.0, fleet_transform_version to be installed is 0.1.0 + const sourceData = getYamlTestData(false, '0.2.0'); + const expectedData = getExpectedData('0.2.0'); + + const previousInstallation: Installation = { + installed_es: [ + { + id: 'metrics-endpoint.policy-0.16.0-dev.0', + type: ElasticsearchAssetType.ingestPipeline, + }, + { + id: 'logs-endpoint.metadata_current-default-0.1.0', + type: ElasticsearchAssetType.transform, + }, + ], + } as unknown as Installation; + + const currentInstallation: Installation = { + installed_es: [ + { + id: 'metrics-endpoint.policy-0.16.0-dev.0', + type: ElasticsearchAssetType.ingestPipeline, + }, + { + id: 'logs-endpoint.metadata_current-default-0.2.0', + type: ElasticsearchAssetType.transform, + }, + { + id: 'logs-endpoint.metadata_current-default-0.2.0', + type: ElasticsearchAssetType.transform, + }, + ], + } as unknown as Installation; + (getAsset as jest.MockedFunction) + .mockReturnValueOnce(Buffer.from(sourceData.FIELDS, 'utf8')) + .mockReturnValueOnce(Buffer.from(sourceData.TRANSFORM, 'utf8')); + + (getInstallation as jest.MockedFunction) + .mockReturnValueOnce(Promise.resolve(previousInstallation)) + .mockReturnValueOnce(Promise.resolve(currentInstallation)); + + ( + getInstallationObject as jest.MockedFunction + ).mockReturnValueOnce( + Promise.resolve({ + attributes: { + installed_es: previousInstallation.installed_es, + }, + } as unknown as SavedObject) + ); + + // Mock transform from old version + esClient.transform.getTransform.mockResponseOnce({ + count: 1, + transforms: [ + // @ts-expect-error incomplete data + { + dest: { + index: 'mock-old-destination-index', + }, + }, + ], + }); + + await installTransforms( + { + name: 'endpoint', + version: '0.16.0-dev.0', + } as unknown as RegistryPackage, + [ + 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/fields/fields.yml', + 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml', + ], + esClient, + savedObjectsClient, + loggerMock.create(), + previousInstallation.installed_es + ); + + // Stop and delete previously installed transforms + expect(esClient.transform.stopTransform.mock.calls).toEqual([ + [ + { + transform_id: 'logs-endpoint.metadata_current-default-0.1.0', + force: true, + }, + { ignore: [404] }, + ], + ]); + expect(esClient.transform.deleteTransform.mock.calls).toEqual([ + [ + { + transform_id: 'logs-endpoint.metadata_current-default-0.1.0', + force: true, + }, + { ignore: [404] }, + ], + ]); + + // Destination index should not be deleted when transform is deleted + expect(esClient.transport.request.mock.calls).toEqual([]); + + // Create a @package component template and an empty @custom component template + expect(esClient.cluster.putComponentTemplate.mock.calls).toEqual([ + [ + { + name: 'logs-endpoint.metadata_current-template@package', + body: { + template: { + settings: { index: { mapping: { total_fields: { limit: '10000' } } } }, + mappings: { properties: { '@timestamp': { type: 'date' } } }, + }, + _meta: meta, + }, + create: false, + }, + { ignore: [404] }, + ], + [ + { + name: 'logs-endpoint.metadata_current-template@custom', + body: { + template: { settings: {} }, + _meta: { managed_by: 'fleet', managed: true, package: { name: 'endpoint' } }, + }, + create: true, + }, + { ignore: [404] }, + ], + ]); + + // Index template composed of the two component templates created + // with index pattern matching the destination index + expect(esClient.indices.putIndexTemplate.mock.calls).toEqual([ + [ + { + body: { + _meta: meta, + composed_of: [ + 'logs-endpoint.metadata_current-template@package', + 'logs-endpoint.metadata_current-template@custom', + ], + index_patterns: ['.metrics-endpoint.metadata_united_default-0.16.0-dev.0'], + priority: 250, + template: { mappings: undefined, settings: undefined }, + }, + name: 'logs-endpoint.metadata_current-template', + }, + { ignore: [404] }, + ], + ]); + + // Destination index is created before transform is created + expect(esClient.indices.create.mock.calls).toEqual([ + [ + { + aliases: { + '.metrics-endpoint.metadata_united_default.all': {}, + '.metrics-endpoint.metadata_united_default.latest': {}, + }, + index: '.metrics-endpoint.metadata_united_default-0.16.0-dev.0', + }, + { ignore: [400] }, + ], + ]); + + expect(esClient.transform.putTransform.mock.calls).toEqual([[expectedData.TRANSFORM]]); + expect(esClient.transform.startTransform.mock.calls).toEqual([ + [ + { + transform_id: 'logs-endpoint.metadata_current-default-0.2.0', + }, + { ignore: [409] }, + ], + ]); + + // Saved object is updated with newly created index templates, component templates, transform + expect(savedObjectsClient.update.mock.calls).toEqual([ + [ + 'epm-packages', + 'endpoint', + { + installed_es: [ + { + id: 'metrics-endpoint.policy-0.16.0-dev.0', + type: ElasticsearchAssetType.ingestPipeline, + }, + { + id: '.metrics-endpoint.metadata_united_default-0.16.0-dev.0', + type: ElasticsearchAssetType.index, + }, + { + id: 'logs-endpoint.metadata_current-template', + type: ElasticsearchAssetType.indexTemplate, + version: '0.2.0', + }, + { + id: 'logs-endpoint.metadata_current-template@custom', + type: ElasticsearchAssetType.componentTemplate, + version: '0.2.0', + }, + { + id: 'logs-endpoint.metadata_current-template@package', + type: ElasticsearchAssetType.componentTemplate, + version: '0.2.0', + }, + { + id: 'logs-endpoint.metadata_current-default-0.2.0', + type: ElasticsearchAssetType.transform, + version: '0.2.0', + }, + ], + }, + { + refresh: false, + }, + ], + ]); + }); + + test('can install new version when an older version does not exist', async () => { + const sourceData = getYamlTestData(false, '0.2.0'); + const expectedData = getExpectedData('0.2.0'); + + const previousInstallation: Installation = { + installed_es: [], + } as unknown as Installation; + + const currentInstallation: Installation = { + installed_es: [ + { + id: `logs-endpoint.metadata_current-default-0.2.0`, + type: ElasticsearchAssetType.transform, + }, + ], + } as unknown as Installation; + (getAsset as jest.MockedFunction) + .mockReturnValueOnce(Buffer.from(sourceData.MANIFEST, 'utf8')) + .mockReturnValueOnce(Buffer.from(sourceData.TRANSFORM, 'utf8')); + + (getInstallation as jest.MockedFunction) + .mockReturnValueOnce(Promise.resolve(previousInstallation)) + .mockReturnValueOnce(Promise.resolve(currentInstallation)); + + ( + getInstallationObject as jest.MockedFunction + ).mockReturnValueOnce( + Promise.resolve({ + attributes: { installed_es: [] }, + } as unknown as SavedObject) + ); + + await installTransforms( + { + name: 'endpoint', + version: '0.16.0-dev.0', + } as unknown as RegistryPackage, + [ + 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/manifest.yml', + 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml', + ], + esClient, + savedObjectsClient, + loggerMock.create(), + previousInstallation.installed_es + ); + + expect(esClient.transform.putTransform.mock.calls).toEqual([[expectedData.TRANSFORM]]); + // Does not start transform because start is set to false in manifest.yml + expect(esClient.transform.startTransform.mock.calls).toEqual([]); + }); + + test('can downgrade to older version when force: true', async () => { + const sourceData = getYamlTestData(false, '0.1.0'); + const expectedData = getExpectedData('0.1.0'); + + const previousInstallation: Installation = { + installed_es: [ + { + id: `logs-endpoint.metadata_current-default-0.2.0`, + type: ElasticsearchAssetType.transform, + }, + { + id: 'logs-endpoint.metadata_current-template', + type: ElasticsearchAssetType.indexTemplate, + }, + { + id: 'logs-endpoint.metadata_current-template@custom', + type: ElasticsearchAssetType.componentTemplate, + }, + { + id: 'logs-endpoint.metadata_current-template@package', + type: ElasticsearchAssetType.componentTemplate, + }, + ], + } as unknown as Installation; + + const currentInstallation: Installation = { + installed_es: [ + { + id: `logs-endpoint.metadata_current-default-0.1.0`, + type: ElasticsearchAssetType.transform, + }, + { + id: 'logs-endpoint.metadata_current-template', + type: ElasticsearchAssetType.indexTemplate, + }, + { + id: 'logs-endpoint.metadata_current-template@custom', + type: ElasticsearchAssetType.componentTemplate, + }, + { + id: 'logs-endpoint.metadata_current-template@package', + type: ElasticsearchAssetType.componentTemplate, + }, + ], + } as unknown as Installation; + (getAsset as jest.MockedFunction) + .mockReturnValueOnce(Buffer.from(sourceData.MANIFEST, 'utf8')) + .mockReturnValueOnce(Buffer.from(sourceData.TRANSFORM, 'utf8')); + + (getInstallation as jest.MockedFunction) + .mockReturnValueOnce(Promise.resolve(previousInstallation)) + .mockReturnValueOnce(Promise.resolve(currentInstallation)); + + ( + getInstallationObject as jest.MockedFunction + ).mockReturnValueOnce( + Promise.resolve({ + attributes: { installed_es: [] }, + } as unknown as SavedObject) + ); + + // Mock resp for when index from older version already exists + esClient.indices.create.mockReturnValueOnce( + // @ts-expect-error mock error instead of successful IndicesCreateResponse + Promise.resolve({ + error: { + type: 'resource_already_exists_exception', + }, + status: 400, + }) + ); + + await installTransforms( + { + name: 'endpoint', + version: '0.16.0-dev.0', + } as unknown as RegistryPackage, + [ + 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/manifest.yml', + 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml', + ], + esClient, + savedObjectsClient, + loggerMock.create(), + previousInstallation.installed_es + ); + + expect(esClient.indices.create.mock.calls).toEqual([ + [ + { + index: '.metrics-endpoint.metadata_united_default-0.16.0-dev.0', + aliases: { + '.metrics-endpoint.metadata_united_default.all': {}, + '.metrics-endpoint.metadata_united_default.latest': {}, + }, + }, + { ignore: [400] }, + ], + ]); + + // If downgrading to and older version, and destination index already exists + // aliases should still be updated to point .latest to this index + expect(esClient.indices.updateAliases.mock.calls).toEqual([ + [ + { + body: { + actions: [ + { + add: { + index: '.metrics-endpoint.metadata_united_default-0.16.0-dev.0', + alias: '.metrics-endpoint.metadata_united_default.all', + }, + }, + { + add: { + index: '.metrics-endpoint.metadata_united_default-0.16.0-dev.0', + alias: '.metrics-endpoint.metadata_united_default.latest', + }, + }, + ], + }, + }, + ], + ]); + + expect(esClient.transform.deleteTransform.mock.calls).toEqual([ + [ + { force: true, transform_id: 'logs-endpoint.metadata_current-default-0.2.0' }, + { ignore: [404] }, + ], + ]); + expect(esClient.transform.putTransform.mock.calls).toEqual([[expectedData.TRANSFORM]]); + }); + + test('retain old transforms and do nothing if fleet_transform_version is the same', async () => { + // Old fleet_transform_version is 0.1.0, fleet_transform_version to be installed is 0.1.0 + const sourceData = getYamlTestData(false, '0.1.0'); + + const previousInstallation: Installation = { + installed_es: [ + { + id: 'logs-endpoint.metadata_current-default-0.1.0', + type: ElasticsearchAssetType.transform, + }, + { + id: 'logs-endpoint.metadata_current-template', + type: ElasticsearchAssetType.indexTemplate, + }, + { + id: 'logs-endpoint.metadata_current-template@custom', + type: ElasticsearchAssetType.componentTemplate, + }, + { + id: 'logs-endpoint.metadata_current-template@package', + type: ElasticsearchAssetType.componentTemplate, + }, + ], + } as unknown as Installation; + + const currentInstallation: Installation = { + installed_es: [ + { + id: 'endpoint.metadata-current-default-0.1.0', + type: ElasticsearchAssetType.transform, + }, + ], + } as unknown as Installation; + + (getAsset as jest.MockedFunction) + .mockReturnValueOnce(Buffer.from(sourceData.FIELDS, 'utf8')) + .mockReturnValueOnce(Buffer.from(sourceData.MANIFEST, 'utf8')) + .mockReturnValueOnce(Buffer.from(sourceData.TRANSFORM, 'utf8')); + + (getInstallation as jest.MockedFunction) + .mockReturnValueOnce(Promise.resolve(previousInstallation)) + .mockReturnValueOnce(Promise.resolve(currentInstallation)); + + ( + getInstallationObject as jest.MockedFunction + ).mockReturnValueOnce( + Promise.resolve({ + attributes: { installed_es: currentInstallation.installed_es }, + } as unknown as SavedObject) + ); + + await installTransforms( + { + name: 'endpoint', + version: '0.16.0-dev.0', + } as unknown as RegistryPackage, + [ + 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/fields/fields.yml', + 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/manifest.yml', + 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml', + ], + esClient, + savedObjectsClient, + loggerMock.create(), + previousInstallation.installed_es + ); + + // Transform from old version is neither stopped nor deleted + expect(esClient.transform.stopTransform.mock.calls).toEqual([]); + expect(esClient.transform.deleteTransform.mock.calls).toEqual([]); + + // Destination index from old version is not deleted + expect(esClient.transport.request.mock.calls).toEqual([]); + + // No new destination index is created + expect(esClient.indices.create.mock.calls).toEqual([]); + // No new transform is created or started + expect(esClient.transform.putTransform.mock.calls).toEqual([]); expect(esClient.transform.startTransform.mock.calls).toEqual([]); }); }); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/remove.ts b/x-pack/plugins/fleet/server/services/epm/packages/remove.ts index c994320ab829a1c..61780c797716633 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/remove.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/remove.ts @@ -17,6 +17,8 @@ import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; import { SavedObjectsUtils, SavedObjectsErrorHelpers } from '@kbn/core/server'; +import { updateIndexSettings } from '../elasticsearch/index/update_settings'; + import { PACKAGE_POLICY_SAVED_OBJECT_TYPE, PACKAGES_SAVED_OBJECT_TYPE, @@ -143,7 +145,7 @@ function deleteESAssets( } else if (assetType === ElasticsearchAssetType.componentTemplate) { return deleteComponentTemplate(esClient, id); } else if (assetType === ElasticsearchAssetType.transform) { - return deleteTransforms(esClient, [id]); + return deleteTransforms(esClient, [id], true); } else if (assetType === ElasticsearchAssetType.dataStreamIlmPolicy) { return deleteIlms(esClient, [id]); } else if (assetType === ElasticsearchAssetType.ilmPolicy) { @@ -164,27 +166,35 @@ async function deleteAssets( esClient: ElasticsearchClient ) { const logger = appContextService.getLogger(); + // must unset default_pipelines settings in indices first, or pipelines associated with an index cannot not be deleted // must delete index templates first, or component templates which reference them cannot be deleted // must delete ingestPipelines first, or ml models referenced in them cannot be deleted. // separate the assets into Index Templates and other assets. - type Tuple = [EsAssetReference[], EsAssetReference[]]; - const [indexTemplatesAndPipelines, otherAssets] = installedEs.reduce( - ([indexAssetTypes, otherAssetTypes], asset) => { + type Tuple = [EsAssetReference[], EsAssetReference[], EsAssetReference[]]; + const [indexTemplatesAndPipelines, indexAssets, otherAssets] = installedEs.reduce( + ([indexTemplateAndPipelineTypes, indexAssetTypes, otherAssetTypes], asset) => { if ( asset.type === ElasticsearchAssetType.indexTemplate || asset.type === ElasticsearchAssetType.ingestPipeline ) { + indexTemplateAndPipelineTypes.push(asset); + } else if (asset.type === ElasticsearchAssetType.index) { indexAssetTypes.push(asset); } else { otherAssetTypes.push(asset); } - return [indexAssetTypes, otherAssetTypes]; + return [indexTemplateAndPipelineTypes, indexAssetTypes, otherAssetTypes]; }, - [[], []] + [[], [], []] ); try { + // must first unset any default pipeline associated with any existing indices + // by setting empty string + await Promise.all( + indexAssets.map((asset) => updateIndexSettings(esClient, asset.id, { default_pipeline: '' })) + ); // must delete index templates and pipelines first await Promise.all(deleteESAssets(indexTemplatesAndPipelines, esClient)); // then the other asset types