diff --git a/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts b/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts index de9981618a01b42..b24f0acc004b022 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts @@ -85,7 +85,7 @@ describe('checking migration metadata changes on all registered SO types', () => "endpoint:user-artifact": "f94c250a52b30d0a2d32635f8b4c5bdabd1e25c0", "endpoint:user-artifact-manifest": "8c14d49a385d5d1307d956aa743ec78de0b2be88", "enterprise_search_telemetry": "fafcc8318528d34f721c42d1270787c52565bad5", - "epm-packages": "fe3716a54188b3c71327fa060dd6780a674d3994", + "epm-packages": "2915aee4302d4b00472ed05c21f59b7d498b5206", "epm-packages-assets": "9fd3d6726ac77369249e9a973902c2cd615fc771", "event_loop_delays_daily": "d2ed39cf669577d90921c176499908b4943fb7bd", "exception-list": "fe8cc004fd2742177cdb9300f4a67689463faf9c", diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts index d8d0dc250edc1ee..b9f471f1b4d437c 100644 --- a/x-pack/plugins/fleet/common/types/models/epm.ts +++ b/x-pack/plugins/fleet/common/types/models/epm.ts @@ -101,6 +101,7 @@ export enum KibanaSavedObjectType { } export enum ElasticsearchAssetType { + index = 'index', componentTemplate = 'component_template', ingestPipeline = 'ingest_pipeline', indexTemplate = 'index_template', @@ -109,6 +110,10 @@ export enum ElasticsearchAssetType { dataStreamIlmPolicy = 'data_stream_ilm_policy', mlModel = 'ml_model', } +export type FleetElasticsearchAssetType = Exclude< + ElasticsearchAssetType, + ElasticsearchAssetType.index +>; export type DataType = typeof dataTypes; export type MonitoringType = typeof monitoringTypes; @@ -313,7 +318,7 @@ export type ElasticsearchAssetParts = AssetParts & { export type KibanaAssetTypeToParts = Record; export type ElasticsearchAssetTypeToParts = Record< - ElasticsearchAssetType, + FleetElasticsearchAssetType, ElasticsearchAssetParts[] >; diff --git a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/constants.tsx b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/constants.tsx index 1fe4b7b38434d39..1b183f13b44260e 100644 --- a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/constants.tsx +++ b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/constants.tsx @@ -35,6 +35,9 @@ export const AssetTitleMap: Record = { transform: i18n.translate('xpack.fleet.epm.assetTitles.transforms', { defaultMessage: 'Transforms', }), + index: i18n.translate('xpack.fleet.epm.assetTitles.indices', { + defaultMessage: 'Indices', + }), index_pattern: i18n.translate('xpack.fleet.epm.assetTitles.indexPatterns', { defaultMessage: 'Index patterns', }), diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts index 8a061552a77edac..6c87e894e5777ed 100644 --- a/x-pack/plugins/fleet/server/saved_objects/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/index.ts @@ -262,6 +262,7 @@ const getSavedObjectTypes = ( properties: { id: { type: 'keyword' }, type: { type: 'keyword' }, + version: { type: 'keyword' }, }, }, installed_kibana: { diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/index/index.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/index/index.ts new file mode 100644 index 000000000000000..79d89d70ac67214 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/index/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { updateIndexSettings } from './update_settings'; diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/index/update_settings.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/index/update_settings.ts new file mode 100644 index 000000000000000..a381f19a3d9fa77 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/index/update_settings.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; + +import type { IndicesIndexSettings } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; + +import { retryTransientEsErrors } from '../retry'; + +export async function updateIndexSettings( + esClient: ElasticsearchClient, + index: string, + settings: IndicesIndexSettings +): Promise { + if (index) { + try { + await retryTransientEsErrors(() => + esClient.indices.putSettings({ + index, + body: settings, + }) + ); + } catch (err) { + throw new Error(`could not update index settings for ${index}`); + } + } +} diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/common.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/common.ts index e08d973f8df0e35..46e85d7f9df6245 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/common.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/common.ts @@ -6,3 +6,8 @@ */ export { getAsset } from '../../archive'; + +// Index alias that points to just one destination index from the latest package version +export const TRANSFORM_DEST_IDX_ALIAS_LATEST_SFX = '.latest'; +// Index alias that points to all of the destination indices from all the package versions +export const TRANSFORM_DEST_IDX_ALIAS_ALL_SFX = '.all'; diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts index 4999c07a2aec1b9..e8881bb247e12f9 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts @@ -9,6 +9,7 @@ import type { ElasticsearchClient, Logger, SavedObjectsClientContract } from '@k import { errors } from '@elastic/elasticsearch'; import { safeLoad } from 'js-yaml'; import { isPopulatedObject } from '@kbn/ml-is-populated-object'; +import { uniqBy } from 'lodash'; import { PACKAGE_TEMPLATE_SUFFIX, @@ -35,7 +36,7 @@ import { getInstallation } from '../../packages'; import { retryTransientEsErrors } from '../retry'; import { deleteTransforms } from './remove'; -import { getAsset } from './common'; +import { getAsset, TRANSFORM_DEST_IDX_ALIAS_LATEST_SFX } from './common'; const DEFAULT_TRANSFORM_TEMPLATES_PRIORITY = 250; enum TRANSFORM_SPECS_TYPES { @@ -55,6 +56,8 @@ interface DestinationIndexTemplateInstallation extends TransformModuleBase { interface TransformInstallation extends TransformModuleBase { installationName: string; content: any; + transformVersion?: string; + installationOrder?: number; } const installLegacyTransformsAssets = async ( @@ -67,6 +70,13 @@ const installLegacyTransformsAssets = async ( esReferences: EsAssetReference[] = [], previousInstalledTransformEsAssets: EsAssetReference[] = [] ) => { + await deleteTransforms( + esClient, + previousInstalledTransformEsAssets.map((asset) => asset.id), + // For legacy transforms, delete destination indices upon deleting transforms + true + ); + let installedTransforms: EsAssetReference[] = []; if (transformPaths.length > 0) { const transformRefs = transformPaths.reduce((acc, path) => { @@ -126,11 +136,16 @@ const installLegacyTransformsAssets = async ( const processTransformAssetsPerModule = ( installablePackage: InstallablePackage, installNameSuffix: string, - transformPaths: string[] + transformPaths: string[], + previousInstalledTransformEsAssets: EsAssetReference[] = [] ) => { const transformsSpecifications = new Map(); const destinationIndexTemplates: DestinationIndexTemplateInstallation[] = []; const transforms: TransformInstallation[] = []; + const aliasesRefs: string[] = []; + const transformsToRemove: EsAssetReference[] = []; + const transformsToRemoveWithDestIndex: EsAssetReference[] = []; + const indicesToAddRefs: EsAssetReference[] = []; transformPaths.forEach((path: string) => { const { transformModuleId, fileName } = getTransformFolderAndFileNames( @@ -150,38 +165,143 @@ const processTransformAssetsPerModule = ( if (fileName === TRANSFORM_SPECS_TYPES.FIELDS) { const validFields = processFields(content); const mappings = generateMappings(validFields); + const templateName = getTransformAssetNameForInstallation( + installablePackage, + transformModuleId, + 'template' + ); + const indexToModify = destinationIndexTemplates.findIndex( + (t) => t.transformModuleId === transformModuleId && t.installationName === templateName + ); + const template = { + transformModuleId, + _meta: getESAssetMetadata({ packageName: installablePackage.name }), + installationName: getTransformAssetNameForInstallation( + installablePackage, + transformModuleId, + 'template' + ), + template: {}, + } as DestinationIndexTemplateInstallation; + if (indexToModify === -1) { + destinationIndexTemplates.push(template); + } else { + destinationIndexTemplates[indexToModify] = template; + } packageAssets?.set('mappings', mappings); } if (fileName === TRANSFORM_SPECS_TYPES.TRANSFORM) { + const installationOrder = + isFinite(content._meta?.order) && content._meta?.order >= 0 ? content._meta?.order : 0; + const transformVersion = content._meta?.fleet_transform_version ?? '0.1.0'; + // The “all” alias for the transform destination indices will be adjusted to include the new transform destination index as well as everything it previously included + const allIndexAliasName = `${content.dest.index}.all`; + // The “latest” alias for the transform destination indices will point solely to the new transform destination index + const latestIndexAliasName = `${content.dest.index}.latest`; + + transformsSpecifications + .get(transformModuleId) + ?.set('originalDestinationIndexName', content.dest.index); + + // Create two aliases associated with the destination index + // for better handling during upgrades + const alias = { + [allIndexAliasName]: {}, + [latestIndexAliasName]: {}, + }; + + const versionedIndexName = `${content.dest.index}-${installNameSuffix}`; + content.dest.index = versionedIndexName; + indicesToAddRefs.push({ + id: versionedIndexName, + type: ElasticsearchAssetType.index, + }); transformsSpecifications.get(transformModuleId)?.set('destinationIndex', content.dest); + transformsSpecifications.get(transformModuleId)?.set('destinationIndexAlias', alias); transformsSpecifications.get(transformModuleId)?.set('transform', content); - content._meta = getESAssetMetadata({ packageName: installablePackage.name }); - transforms.push({ + transformsSpecifications.get(transformModuleId)?.set('transformVersion', transformVersion); + content._meta = { + ...(content._meta ?? {}), + ...getESAssetMetadata({ packageName: installablePackage.name }), + }; + + const installationName = getTransformAssetNameForInstallation( + installablePackage, transformModuleId, - installationName: getTransformAssetNameForInstallation( - installablePackage, + // transform_id is versioned by fleet_transform_version and not by package version + `default-${transformVersion}` + ); + + const currentTransformSameAsPrev = + previousInstalledTransformEsAssets.find((t) => t.id === installationName) !== undefined; + if (previousInstalledTransformEsAssets.length === 0) { + aliasesRefs.push(allIndexAliasName, latestIndexAliasName); + transforms.push({ transformModuleId, - `default-${installNameSuffix}` - ), - content, - }); + installationName, + installationOrder, + transformVersion, + content, + }); + transformsSpecifications.get(transformModuleId)?.set('transformVersionChanged', true); + } else { + if (!currentTransformSameAsPrev) { + // If upgrading from old json schema to new yml schema + // We need to make sure to delete those transforms by matching the legacy naming convention + const versionFromOldJsonSchema = previousInstalledTransformEsAssets.find((t) => + t.id.startsWith( + getLegacyTransformNameForInstallation( + installablePackage, + `${transformModuleId}/default.json` + ) + ) + ); + + if (versionFromOldJsonSchema !== undefined) { + transformsToRemoveWithDestIndex.push(versionFromOldJsonSchema); + } + + // If upgrading from yml to newer version of yaml + // Match using new naming convention + const installNameWithoutVersion = installationName.split(transformVersion)[0]; + const prevVersion = previousInstalledTransformEsAssets.find((t) => + t.id.startsWith(installNameWithoutVersion) + ); + if (prevVersion !== undefined) { + transformsToRemove.push(prevVersion); + } + transforms.push({ + transformModuleId, + installationName, + installationOrder, + transformVersion, + content, + }); + transformsSpecifications.get(transformModuleId)?.set('transformVersionChanged', true); + aliasesRefs.push(allIndexAliasName, latestIndexAliasName); + } else { + transformsSpecifications.get(transformModuleId)?.set('transformVersionChanged', false); + } + } } + // Create index templates for destination indices if destination_index_template OR fields are defined if (fileName === TRANSFORM_SPECS_TYPES.MANIFEST) { if (isPopulatedObject(content, ['start']) && content.start === false) { transformsSpecifications.get(transformModuleId)?.set('start', false); } - // If manifest.yml contains destination_index_template - // Combine the mappings and other index template settings from manifest.yml into a single index template - // Create the index template and track the template in EsAssetReferences - if ( - isPopulatedObject(content, ['destination_index_template']) || - isPopulatedObject(packageAssets.get('mappings')) - ) { - const destinationIndexTemplate = - (content.destination_index_template as Record) ?? {}; - destinationIndexTemplates.push({ + + if (content.destination_index_template) { + const templateName = getTransformAssetNameForInstallation( + installablePackage, + transformModuleId, + 'template' + ); + const indexToModify = destinationIndexTemplates.findIndex( + (t) => t.transformModuleId === transformModuleId && t.installationName === templateName + ); + const template = { transformModuleId, _meta: getESAssetMetadata({ packageName: installablePackage.name }), installationName: getTransformAssetNameForInstallation( @@ -189,9 +309,14 @@ const processTransformAssetsPerModule = ( transformModuleId, 'template' ), - template: destinationIndexTemplate, - } as DestinationIndexTemplateInstallation); - packageAssets.set('destinationIndexTemplate', destinationIndexTemplate); + template: content.destination_index_template, + } as DestinationIndexTemplateInstallation; + if (indexToModify === -1) { + destinationIndexTemplates.push(template); + } else { + destinationIndexTemplates[indexToModify] = template; + } + packageAssets.set('destinationIndexTemplate', template); } } }); @@ -199,30 +324,42 @@ const processTransformAssetsPerModule = ( const indexTemplatesRefs = destinationIndexTemplates.map((template) => ({ id: template.installationName, type: ElasticsearchAssetType.indexTemplate, + version: transformsSpecifications.get(template.transformModuleId)?.get('transformVersion'), })); const componentTemplatesRefs = [ ...destinationIndexTemplates.map((template) => ({ id: `${template.installationName}${USER_SETTINGS_TEMPLATE_SUFFIX}`, type: ElasticsearchAssetType.componentTemplate, + version: transformsSpecifications.get(template.transformModuleId)?.get('transformVersion'), })), ...destinationIndexTemplates.map((template) => ({ id: `${template.installationName}${PACKAGE_TEMPLATE_SUFFIX}`, type: ElasticsearchAssetType.componentTemplate, + version: transformsSpecifications.get(template.transformModuleId)?.get('transformVersion'), })), ]; - const transformRefs = transforms.map((t) => ({ + const sortedTransforms = transforms.sort( + (t1, t2) => (t1.installationOrder ?? 0) - (t2.installationOrder ?? 1) + ); + + const transformRefs = sortedTransforms.map((t) => ({ id: t.installationName, type: ElasticsearchAssetType.transform, + version: t.transformVersion, })); return { + indicesToAddRefs, indexTemplatesRefs, componentTemplatesRefs, transformRefs, - transforms, + transforms: sortedTransforms, destinationIndexTemplates, transformsSpecifications, + aliasesRefs, + transformsToRemove, + transformsToRemoveWithDestIndex, }; }; @@ -239,21 +376,60 @@ const installTransformsAssets = async ( let installedTransforms: EsAssetReference[] = []; if (transformPaths.length > 0) { const { + indicesToAddRefs, indexTemplatesRefs, componentTemplatesRefs, transformRefs, transforms, destinationIndexTemplates, transformsSpecifications, - } = processTransformAssetsPerModule(installablePackage, installNameSuffix, transformPaths); + aliasesRefs, + transformsToRemove, + transformsToRemoveWithDestIndex, + } = processTransformAssetsPerModule( + installablePackage, + installNameSuffix, + transformPaths, + previousInstalledTransformEsAssets + ); + + // ensure the .latest alias points to only the latest + // by removing any associate of old destination indices + await Promise.all( + aliasesRefs + .filter((a) => a.endsWith(TRANSFORM_DEST_IDX_ALIAS_LATEST_SFX)) + .map((alias) => deleteAliasFromIndices({ esClient, logger, alias })) + ); + + // delete all previous transform + await Promise.all([ + deleteTransforms( + esClient, + transformsToRemoveWithDestIndex.map((asset) => asset.id), + // Delete destination indices if specified or if from old json schema + true + ), + deleteTransforms( + esClient, + transformsToRemove.map((asset) => asset.id), + // Else, keep destination indices by default + false + ), + ]); + // get and save refs associated with the transforms before installing esReferences = await updateEsAssetReferences( savedObjectsClient, installablePackage.name, esReferences, { - assetsToAdd: [...indexTemplatesRefs, ...componentTemplatesRefs, ...transformRefs], - assetsToRemove: previousInstalledTransformEsAssets, + assetsToAdd: [ + ...indicesToAddRefs, + ...indexTemplatesRefs, + ...componentTemplatesRefs, + ...transformRefs, + ], + assetsToRemove: [...transformsToRemove, ...transformsToRemoveWithDestIndex], } ); @@ -261,10 +437,15 @@ const installTransformsAssets = async ( await Promise.all( destinationIndexTemplates .map((destinationIndexTemplate) => { - const customMappings = - transformsSpecifications - .get(destinationIndexTemplate.transformModuleId) - ?.get('mappings') ?? {}; + const transformSpec = transformsSpecifications.get( + destinationIndexTemplate.transformModuleId + ); + const customMappings = transformSpec?.get('mappings') ?? {}; + const pipelineId = transformSpec?.get('destinationIndex')?.pipeline; + const transformVersionChanged = transformSpec?.get('transformVersionChanged') ?? true; + + if (!transformVersionChanged) return; + const registryElasticsearch: RegistryElasticsearch = { 'index_template.settings': destinationIndexTemplate.template.settings, 'index_template.mappings': destinationIndexTemplate.template.mappings, @@ -275,7 +456,11 @@ const installTransformsAssets = async ( templateName: destinationIndexTemplate.installationName, registryElasticsearch, packageName: installablePackage.name, - defaultSettings: {}, + defaultSettings: { + // Adding destination pipeline here because else these templates will be overridden + // by index setting + ...(pipelineId ? { default_pipeline: pipelineId } : {}), + }, }); if (destinationIndexTemplate || customMappings) { @@ -285,10 +470,12 @@ const installTransformsAssets = async ( componentTemplates, indexTemplate: { templateName: destinationIndexTemplate.installationName, - // @ts-expect-error We don't need to pass data_stream property here - // as this template is applied to only an index and not a data stream + // @ts-expect-error data_stream property is not needed here indexTemplate: { - template: { settings: undefined, mappings: undefined }, + template: { + settings: undefined, + mappings: undefined, + }, priority: DEFAULT_TRANSFORM_TEMPLATES_PRIORITY, index_patterns: [ transformsSpecifications @@ -309,37 +496,81 @@ const installTransformsAssets = async ( await Promise.all( transforms.map(async (transform) => { const index = transform.content.dest.index; - const pipelineId = transform.content.dest.pipeline; + const aliases = transformsSpecifications + .get(transform.transformModuleId) + ?.get('destinationIndexAlias'); try { - await retryTransientEsErrors( + const resp = await retryTransientEsErrors( () => esClient.indices.create( { index, - ...(pipelineId ? { settings: { default_pipeline: pipelineId } } : {}), + aliases, }, { ignore: [400] } ), { logger } ); + logger.debug(`Created destination index: ${index}`); + + // If index already exists, we still need to update the destination index alias + // to point '{destinationIndexName}.latest' to the versioned index + // @ts-ignore status is a valid field of resp + if (resp.status === 400 && aliases) { + await retryTransientEsErrors( + () => + esClient.indices.updateAliases({ + body: { + actions: Object.keys(aliases).map((alias) => ({ add: { index, alias } })), + }, + }), + { logger } + ); + logger.debug(`Created aliases for destination index: ${index}`); + } } catch (err) { + logger.error( + `Error creating destination index: ${JSON.stringify({ + index, + aliases: transformsSpecifications + .get(transform.transformModuleId) + ?.get('destinationIndexAlias'), + })} with error ${err}` + ); + throw new Error(err.message); } }) ); - // create & optionally start transforms - const transformsPromises = transforms.map(async (transform) => { - return handleTransformInstall({ - esClient, - logger, - transform, - startTransform: transformsSpecifications.get(transform.transformModuleId)?.get('start'), + // If the transforms have specific installation order, install & optionally start transforms sequentially + const shouldInstallSequentially = + uniqBy(transforms, 'installationOrder').length === transforms.length; + + if (shouldInstallSequentially) { + for (const transform of transforms) { + const installTransform = await handleTransformInstall({ + esClient, + logger, + transform, + startTransform: transformsSpecifications.get(transform.transformModuleId)?.get('start'), + }); + installedTransforms.push(installTransform); + } + } else { + // Else, create & start all the transforms at once for speed + const transformsPromises = transforms.map(async (transform) => { + return handleTransformInstall({ + esClient, + logger, + transform, + startTransform: transformsSpecifications.get(transform.transformModuleId)?.get('start'), + }); }); - }); - installedTransforms = await Promise.all(transformsPromises).then((results) => results.flat()); + installedTransforms = await Promise.all(transformsPromises).then((results) => results.flat()); + } } return { installedTransforms, esReferences }; @@ -364,7 +595,7 @@ export const installTransforms = async ( previousInstalledTransformEsAssets = installation.installed_es.filter( ({ type, id }) => type === ElasticsearchAssetType.transform ); - if (previousInstalledTransformEsAssets.length) { + if (previousInstalledTransformEsAssets.length > 0) { logger.debug( `Found previous transform references:\n ${JSON.stringify( previousInstalledTransformEsAssets @@ -373,12 +604,6 @@ export const installTransforms = async ( } } - // delete all previous transform - await deleteTransforms( - esClient, - previousInstalledTransformEsAssets.map((asset) => asset.id) - ); - const installNameSuffix = `${installablePackage.version}`; // If package contains legacy transform specifications (i.e. with json instead of yml) @@ -412,6 +637,36 @@ export const isTransform = (path: string) => { return !path.endsWith('/') && pathParts.type === ElasticsearchAssetType.transform; }; +async function deleteAliasFromIndices({ + esClient, + logger, + alias, +}: { + esClient: ElasticsearchClient; + logger: Logger; + alias: string; +}) { + try { + const resp = await esClient.indices.getAlias({ name: alias }); + const indicesMatchingAlias = Object.keys(resp); + logger.debug(`Deleting alias: '${alias}' matching indices ${indicesMatchingAlias}`); + + if (indicesMatchingAlias.length > 0) { + await retryTransientEsErrors( + () => + // defer validation on put if the source index is not available + esClient.indices.deleteAlias( + { index: indicesMatchingAlias, name: alias }, + { ignore: [404] } + ), + { logger } + ); + logger.debug(`Deleted alias: '${alias}' matching indices ${indicesMatchingAlias}`); + } + } catch (err) { + logger.error(`Error deleting alias: ${alias}`); + } +} async function handleTransformInstall({ esClient, logger, @@ -434,6 +689,7 @@ async function handleTransformInstall({ }), { logger } ); + logger.debug(`Created transform: ${transform.installationName}`); } catch (err) { // swallow the error if the transform already exists. const isAlreadyExistError = @@ -460,12 +716,12 @@ async function handleTransformInstall({ const getLegacyTransformNameForInstallation = ( installablePackage: InstallablePackage, path: string, - suffix: string + suffix?: string ) => { const pathPaths = path.split('/'); const filename = pathPaths?.pop()?.split('.')[0]; const folderName = pathPaths?.pop(); - return `${installablePackage.name}.${folderName}-${filename}-${suffix}`; + return `${installablePackage.name}.${folderName}-${filename}${suffix ? '-' + suffix : ''}`; }; const getTransformAssetNameForInstallation = ( diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/remove.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/remove.ts index f10eddd3d5e65cf..77996674f402e60 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/remove.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/remove.ts @@ -21,7 +21,11 @@ export const stopTransforms = async (transformIds: string[], esClient: Elasticse } }; -export const deleteTransforms = async (esClient: ElasticsearchClient, transformIds: string[]) => { +export const deleteTransforms = async ( + esClient: ElasticsearchClient, + transformIds: string[], + deleteDestinationIndices = false +) => { const logger = appContextService.getLogger(); if (transformIds.length) { logger.info(`Deleting currently installed transform ids ${transformIds}`); @@ -40,7 +44,7 @@ export const deleteTransforms = async (esClient: ElasticsearchClient, transformI { ignore: [404] } ); logger.info(`Deleted: ${transformId}`); - if (transformResponse?.transforms) { + if (deleteDestinationIndices && transformResponse?.transforms) { // expect this to be 1 for (const transform of transformResponse.transforms) { await esClient.transport.request( diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transforms.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transforms.test.ts index aeeeb59e12b3868..77537f200628d76 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transforms.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transforms.test.ts @@ -42,7 +42,10 @@ describe('test transform install', () => { let esClient: ReturnType; let savedObjectsClient: jest.Mocked; - const getYamlTestData = (autoStart: boolean | undefined = undefined) => { + const getYamlTestData = ( + autoStart: boolean | undefined = undefined, + transformVersion: string = '0.1.0' + ) => { const start = autoStart === undefined ? '' @@ -93,6 +96,7 @@ pivot: field: agent.id description: Merges latest endpoint and Agent metadata documents. _meta: + fleet_transform_version: ${transformVersion} managed: true`, FIELDS: `- name: '@timestamp' type: date @@ -101,15 +105,15 @@ _meta: path: event.ingested`, }; }; - const getExpectedData = () => { + const getExpectedData = (transformVersion: string) => { return { TRANSFORM: { - transform_id: 'logs-endpoint.metadata_current-default-0.16.0-dev.0', + transform_id: `logs-endpoint.metadata_current-default-${transformVersion}`, defer_validation: true, body: { description: 'Merges latest endpoint and Agent metadata documents.', dest: { - index: '.metrics-endpoint.metadata_united_default', + index: '.metrics-endpoint.metadata_united_default-0.16.0-dev.0', }, frequency: '1s', pivot: { @@ -141,7 +145,7 @@ _meta: field: 'updated_at', }, }, - _meta: meta, + _meta: { fleet_transform_version: transformVersion, ...meta }, }, }, }; @@ -165,9 +169,10 @@ _meta: jest.clearAllMocks(); }); - test('can install new versions and removes older version when start is not defined', async () => { - const sourceData = getYamlTestData(); - const expectedData = getExpectedData(); + test('can install new versions and removes older version when fleet_transform_version increased', async () => { + // Old fleet_transform_version is 0.1.0, fleet_transform_version to be installed is 0.1.0 + const sourceData = getYamlTestData(undefined, '0.2.0'); + const expectedData = getExpectedData('0.2.0'); const previousInstallation: Installation = { installed_es: [ @@ -176,7 +181,7 @@ _meta: type: ElasticsearchAssetType.ingestPipeline, }, { - id: 'endpoint.metadata_current-default-0.15.0-dev.0', + id: 'logs-endpoint.metadata_current-default-0.1.0', type: ElasticsearchAssetType.transform, }, ], @@ -189,20 +194,17 @@ _meta: type: ElasticsearchAssetType.ingestPipeline, }, { - id: 'endpoint.metadata_current-default-0.15.0-dev.0', + id: 'logs-endpoint.metadata_current-default-0.2.0', type: ElasticsearchAssetType.transform, }, { - id: 'endpoint.metadata_current-default-0.16.0-dev.0', - type: ElasticsearchAssetType.transform, - }, - { - id: 'endpoint.metadata-default-0.16.0-dev.0', + id: 'logs-endpoint.metadata_current-default-0.2.0', type: ElasticsearchAssetType.transform, }, ], } as unknown as Installation; (getAsset as jest.MockedFunction) + .mockReturnValueOnce(Buffer.from(sourceData.FIELDS, 'utf8')) .mockReturnValueOnce(Buffer.from(sourceData.MANIFEST, 'utf8')) .mockReturnValueOnce(Buffer.from(sourceData.TRANSFORM, 'utf8')); @@ -239,6 +241,7 @@ _meta: version: '0.16.0-dev.0', } as unknown as RegistryPackage, [ + 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/fields/fields.yml', 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/manifest.yml', 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml', ], @@ -248,19 +251,11 @@ _meta: previousInstallation.installed_es ); - expect(esClient.transform.getTransform.mock.calls).toEqual([ - [ - { - transform_id: 'endpoint.metadata_current-default-0.15.0-dev.0', - }, - { ignore: [404] }, - ], - ]); // Stop and delete previously installed transforms expect(esClient.transform.stopTransform.mock.calls).toEqual([ [ { - transform_id: 'endpoint.metadata_current-default-0.15.0-dev.0', + transform_id: 'logs-endpoint.metadata_current-default-0.1.0', force: true, }, { ignore: [404] }, @@ -269,70 +264,62 @@ _meta: expect(esClient.transform.deleteTransform.mock.calls).toEqual([ [ { - transform_id: 'endpoint.metadata_current-default-0.15.0-dev.0', + transform_id: 'logs-endpoint.metadata_current-default-0.1.0', force: true, }, { ignore: [404] }, ], ]); - // Delete destination index - expect(esClient.transport.request.mock.calls).toEqual([ - [ - { - method: 'DELETE', - path: '/mock-old-destination-index', - }, - { ignore: [404] }, - ], - ]); + // Destination index should not be deleted when transform is deleted + expect(esClient.transport.request.mock.calls).toEqual([]); // Create a @package component template and an empty @custom component template expect(esClient.cluster.putComponentTemplate.mock.calls).toEqual([ [ { + name: 'logs-endpoint.metadata_current-template@package', body: { - _meta: meta, template: { + settings: { + index: { + codec: 'best_compression', + refresh_interval: '5s', + number_of_shards: 1, + number_of_routing_shards: 30, + hidden: true, + mapping: { total_fields: { limit: '10000' } }, + }, + }, mappings: { - _meta: {}, - date_detection: false, - dynamic: false, + properties: { '@timestamp': { type: 'date' } }, dynamic_templates: [ { strings_as_keyword: { - mapping: { ignore_above: 1024, type: 'keyword' }, match_mapping_type: 'string', + mapping: { ignore_above: 1024, type: 'keyword' }, }, }, ], - properties: {}, - }, - settings: { - index: { - codec: 'best_compression', - hidden: true, - mapping: { total_fields: { limit: '10000' } }, - number_of_routing_shards: 30, - number_of_shards: 1, - refresh_interval: '5s', - }, + dynamic: false, + _meta: {}, + date_detection: false, }, }, + _meta: { managed_by: 'fleet', managed: true, package: { name: 'endpoint' } }, }, create: false, - name: 'logs-endpoint.metadata_current-template@package', }, { ignore: [404] }, ], [ { + name: 'logs-endpoint.metadata_current-template@custom', body: { - _meta: meta, template: { settings: {} }, + _meta: { managed_by: 'fleet', managed: true, package: { name: 'endpoint' } }, }, create: true, - name: 'logs-endpoint.metadata_current-template@custom', }, { ignore: [404] }, ], @@ -349,7 +336,7 @@ _meta: 'logs-endpoint.metadata_current-template@package', 'logs-endpoint.metadata_current-template@custom', ], - index_patterns: ['.metrics-endpoint.metadata_united_default'], + index_patterns: ['.metrics-endpoint.metadata_united_default-0.16.0-dev.0'], priority: 250, template: { mappings: undefined, settings: undefined }, }, @@ -361,14 +348,23 @@ _meta: // Destination index is created before transform is created expect(esClient.indices.create.mock.calls).toEqual([ - [{ index: '.metrics-endpoint.metadata_united_default' }, { ignore: [400] }], + [ + { + aliases: { + '.metrics-endpoint.metadata_united_default.all': {}, + '.metrics-endpoint.metadata_united_default.latest': {}, + }, + index: '.metrics-endpoint.metadata_united_default-0.16.0-dev.0', + }, + { ignore: [400] }, + ], ]); expect(esClient.transform.putTransform.mock.calls).toEqual([[expectedData.TRANSFORM]]); expect(esClient.transform.startTransform.mock.calls).toEqual([ [ { - transform_id: 'logs-endpoint.metadata_current-default-0.16.0-dev.0', + transform_id: 'logs-endpoint.metadata_current-default-0.2.0', }, { ignore: [409] }, ], @@ -385,21 +381,29 @@ _meta: id: 'metrics-endpoint.policy-0.16.0-dev.0', type: ElasticsearchAssetType.ingestPipeline, }, + { + id: '.metrics-endpoint.metadata_united_default-0.16.0-dev.0', + type: ElasticsearchAssetType.index, + }, { id: 'logs-endpoint.metadata_current-template', type: ElasticsearchAssetType.indexTemplate, + version: '0.2.0', }, { id: 'logs-endpoint.metadata_current-template@custom', type: ElasticsearchAssetType.componentTemplate, + version: '0.2.0', }, { id: 'logs-endpoint.metadata_current-template@package', type: ElasticsearchAssetType.componentTemplate, + version: '0.2.0', }, { - id: 'logs-endpoint.metadata_current-default-0.16.0-dev.0', + id: 'logs-endpoint.metadata_current-default-0.2.0', type: ElasticsearchAssetType.transform, + version: '0.2.0', }, ], }, @@ -410,104 +414,39 @@ _meta: ]); }); - test('can install new version when no older version', async () => { - const sourceData = getYamlTestData(true); - const expectedData = getExpectedData(); + test('can install new versions and removes older version when upgraded from old json schema to new yml schema', async () => { + const sourceData = getYamlTestData(undefined, '0.2.0'); + const expectedData = getExpectedData('0.2.0'); const previousInstallation: Installation = { - installed_es: [], - } as unknown as Installation; - - const currentInstallation: Installation = { installed_es: [ { - id: 'metrics-endpoint.metadata-current-default-0.16.0-dev.0', - type: ElasticsearchAssetType.transform, - }, - ], - } as unknown as Installation; - (getAsset as jest.MockedFunction).mockReturnValueOnce( - Buffer.from(sourceData.TRANSFORM, 'utf8') - ); - (getInstallation as jest.MockedFunction) - .mockReturnValueOnce(Promise.resolve(previousInstallation)) - .mockReturnValueOnce(Promise.resolve(currentInstallation)); - - ( - getInstallationObject as jest.MockedFunction - ).mockReturnValueOnce( - Promise.resolve({ - attributes: { installed_es: [] }, - } as unknown as SavedObject) - ); - - await installTransforms( - { - name: 'endpoint', - version: '0.16.0-dev.0', - } as unknown as RegistryPackage, - ['endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml'], - esClient, - savedObjectsClient, - loggerMock.create(), - previousInstallation.installed_es - ); - - expect(esClient.transform.putTransform.mock.calls).toEqual([[expectedData.TRANSFORM]]); - expect(esClient.transform.startTransform.mock.calls).toEqual([ - [ - { - transform_id: 'logs-endpoint.metadata_current-default-0.16.0-dev.0', - }, - { ignore: [409] }, - ], - ]); - - expect(savedObjectsClient.update.mock.calls).toEqual([ - [ - 'epm-packages', - 'endpoint', - { - installed_es: [ - { id: 'logs-endpoint.metadata_current-default-0.16.0-dev.0', type: 'transform' }, - ], + id: 'metrics-endpoint.policy-0.1.0-dev.0', + type: ElasticsearchAssetType.ingestPipeline, }, { - refresh: false, + id: 'endpoint.metadata_current-default-0.1.0', + type: ElasticsearchAssetType.transform, }, ], - ]); - }); - - test('can combine settings fields.yml & manifest.yml and not start transform automatically', async () => { - const sourceData = getYamlTestData(false); - const expectedData = getExpectedData(); + } as unknown as Installation; - const previousInstallation: Installation = { + const currentInstallation: Installation = { installed_es: [ { - id: 'endpoint.metadata-current-default-0.15.0-dev.0', - type: ElasticsearchAssetType.transform, - }, - { - id: 'logs-endpoint.metadata_current-template', - type: ElasticsearchAssetType.indexTemplate, + id: 'metrics-endpoint.policy-0.16.0-dev.0', + type: ElasticsearchAssetType.ingestPipeline, }, { - id: 'logs-endpoint.metadata_current-template@custom', - type: ElasticsearchAssetType.componentTemplate, + id: 'logs-endpoint.metadata_current-default-0.2.0', + type: ElasticsearchAssetType.transform, }, { - id: 'logs-endpoint.metadata_current-template@package', - type: ElasticsearchAssetType.componentTemplate, + id: 'logs-endpoint.metadata_current-default-0.2.0', + type: ElasticsearchAssetType.transform, }, ], } as unknown as Installation; - - const currentInstallation: Installation = { - installed_es: [], - } as unknown as Installation; - (getAsset as jest.MockedFunction) .mockReturnValueOnce(Buffer.from(sourceData.FIELDS, 'utf8')) .mockReturnValueOnce(Buffer.from(sourceData.MANIFEST, 'utf8')) @@ -521,10 +460,13 @@ _meta: getInstallationObject as jest.MockedFunction ).mockReturnValueOnce( Promise.resolve({ - attributes: { installed_es: currentInstallation.installed_es }, + attributes: { + installed_es: previousInstallation.installed_es, + }, } as unknown as SavedObject) ); + // Mock transform from old version esClient.transform.getTransform.mockResponseOnce({ count: 1, transforms: [ @@ -553,43 +495,38 @@ _meta: previousInstallation.installed_es ); - expect(esClient.transform.getTransform.mock.calls).toEqual([ + // Stop and delete previously installed transforms + expect(esClient.transform.stopTransform.mock.calls).toEqual([ [ { - transform_id: 'endpoint.metadata-current-default-0.15.0-dev.0', + transform_id: 'endpoint.metadata_current-default-0.1.0', + force: true, }, { ignore: [404] }, ], ]); - - // Transform from old version is stopped & deleted - expect(esClient.transform.stopTransform.mock.calls).toEqual([ + expect(esClient.transform.deleteTransform.mock.calls).toEqual([ [ { - transform_id: 'endpoint.metadata-current-default-0.15.0-dev.0', + transform_id: 'endpoint.metadata_current-default-0.1.0', force: true, }, { ignore: [404] }, ], ]); - expect(esClient.transform.deleteTransform.mock.calls).toEqual([ + // Destination index from previous version using legacy schema should be deleted + expect(esClient.transport.request.mock.calls).toEqual([ [ { - transform_id: 'endpoint.metadata-current-default-0.15.0-dev.0', - force: true, + method: 'DELETE', + path: '/mock-old-destination-index', }, { ignore: [404] }, ], ]); - // Destination index from old version is also deleted - expect(esClient.transport.request.mock.calls).toEqual([ - [{ method: 'DELETE', path: '/mock-old-destination-index' }, { ignore: [404] }], - ]); - - // Component templates are created with mappings from fields.yml - // and template from manifest + // Create a @package component template and an empty @custom component template expect(esClient.cluster.putComponentTemplate.mock.calls).toEqual([ [ { @@ -621,7 +558,7 @@ _meta: date_detection: false, }, }, - _meta: meta, + _meta: { managed_by: 'fleet', managed: true, package: { name: 'endpoint' } }, }, create: false, }, @@ -632,13 +569,14 @@ _meta: name: 'logs-endpoint.metadata_current-template@custom', body: { template: { settings: {} }, - _meta: meta, + _meta: { managed_by: 'fleet', managed: true, package: { name: 'endpoint' } }, }, create: true, }, { ignore: [404] }, ], ]); + // Index template composed of the two component templates created // with index pattern matching the destination index expect(esClient.indices.putIndexTemplate.mock.calls).toEqual([ @@ -650,7 +588,7 @@ _meta: 'logs-endpoint.metadata_current-template@package', 'logs-endpoint.metadata_current-template@custom', ], - index_patterns: ['.metrics-endpoint.metadata_united_default'], + index_patterns: ['.metrics-endpoint.metadata_united_default-0.16.0-dev.0'], priority: 250, template: { mappings: undefined, settings: undefined }, }, @@ -662,11 +600,557 @@ _meta: // Destination index is created before transform is created expect(esClient.indices.create.mock.calls).toEqual([ - [{ index: '.metrics-endpoint.metadata_united_default' }, { ignore: [400] }], + [ + { + aliases: { + '.metrics-endpoint.metadata_united_default.all': {}, + '.metrics-endpoint.metadata_united_default.latest': {}, + }, + index: '.metrics-endpoint.metadata_united_default-0.16.0-dev.0', + }, + { ignore: [400] }, + ], ]); - // New transform created but not not started automatically if start: false in manifest.yml expect(esClient.transform.putTransform.mock.calls).toEqual([[expectedData.TRANSFORM]]); + expect(esClient.transform.startTransform.mock.calls).toEqual([ + [ + { + transform_id: 'logs-endpoint.metadata_current-default-0.2.0', + }, + { ignore: [409] }, + ], + ]); + + // Saved object is updated with newly created index templates, component templates, transform + expect(savedObjectsClient.update.mock.calls).toEqual([ + [ + 'epm-packages', + 'endpoint', + { + installed_es: [ + { + id: 'metrics-endpoint.policy-0.1.0-dev.0', + type: ElasticsearchAssetType.ingestPipeline, + }, + { + id: '.metrics-endpoint.metadata_united_default-0.16.0-dev.0', + type: ElasticsearchAssetType.index, + }, + { + id: 'logs-endpoint.metadata_current-template', + type: ElasticsearchAssetType.indexTemplate, + version: '0.2.0', + }, + { + id: 'logs-endpoint.metadata_current-template@custom', + type: ElasticsearchAssetType.componentTemplate, + version: '0.2.0', + }, + { + id: 'logs-endpoint.metadata_current-template@package', + type: ElasticsearchAssetType.componentTemplate, + version: '0.2.0', + }, + { + id: 'logs-endpoint.metadata_current-default-0.2.0', + type: ElasticsearchAssetType.transform, + version: '0.2.0', + }, + ], + }, + { + refresh: false, + }, + ], + ]); + }); + + test('creates index and component templates even if no manifest.yml', async () => { + // Old fleet_transform_version is 0.1.0, fleet_transform_version to be installed is 0.1.0 + const sourceData = getYamlTestData(false, '0.2.0'); + const expectedData = getExpectedData('0.2.0'); + + const previousInstallation: Installation = { + installed_es: [ + { + id: 'metrics-endpoint.policy-0.16.0-dev.0', + type: ElasticsearchAssetType.ingestPipeline, + }, + { + id: 'logs-endpoint.metadata_current-default-0.1.0', + type: ElasticsearchAssetType.transform, + }, + ], + } as unknown as Installation; + + const currentInstallation: Installation = { + installed_es: [ + { + id: 'metrics-endpoint.policy-0.16.0-dev.0', + type: ElasticsearchAssetType.ingestPipeline, + }, + { + id: 'logs-endpoint.metadata_current-default-0.2.0', + type: ElasticsearchAssetType.transform, + }, + { + id: 'logs-endpoint.metadata_current-default-0.2.0', + type: ElasticsearchAssetType.transform, + }, + ], + } as unknown as Installation; + (getAsset as jest.MockedFunction) + .mockReturnValueOnce(Buffer.from(sourceData.FIELDS, 'utf8')) + .mockReturnValueOnce(Buffer.from(sourceData.TRANSFORM, 'utf8')); + + (getInstallation as jest.MockedFunction) + .mockReturnValueOnce(Promise.resolve(previousInstallation)) + .mockReturnValueOnce(Promise.resolve(currentInstallation)); + + ( + getInstallationObject as jest.MockedFunction + ).mockReturnValueOnce( + Promise.resolve({ + attributes: { + installed_es: previousInstallation.installed_es, + }, + } as unknown as SavedObject) + ); + + // Mock transform from old version + esClient.transform.getTransform.mockResponseOnce({ + count: 1, + transforms: [ + // @ts-expect-error incomplete data + { + dest: { + index: 'mock-old-destination-index', + }, + }, + ], + }); + + await installTransforms( + { + name: 'endpoint', + version: '0.16.0-dev.0', + } as unknown as RegistryPackage, + [ + 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/fields/fields.yml', + 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml', + ], + esClient, + savedObjectsClient, + loggerMock.create(), + previousInstallation.installed_es + ); + + // Stop and delete previously installed transforms + expect(esClient.transform.stopTransform.mock.calls).toEqual([ + [ + { + transform_id: 'logs-endpoint.metadata_current-default-0.1.0', + force: true, + }, + { ignore: [404] }, + ], + ]); + expect(esClient.transform.deleteTransform.mock.calls).toEqual([ + [ + { + transform_id: 'logs-endpoint.metadata_current-default-0.1.0', + force: true, + }, + { ignore: [404] }, + ], + ]); + + // Destination index should not be deleted when transform is deleted + expect(esClient.transport.request.mock.calls).toEqual([]); + + // Create a @package component template and an empty @custom component template + expect(esClient.cluster.putComponentTemplate.mock.calls).toEqual([ + [ + { + name: 'logs-endpoint.metadata_current-template@package', + body: { + template: { + settings: { index: { mapping: { total_fields: { limit: '10000' } } } }, + mappings: { properties: { '@timestamp': { type: 'date' } } }, + }, + _meta: meta, + }, + create: false, + }, + { ignore: [404] }, + ], + [ + { + name: 'logs-endpoint.metadata_current-template@custom', + body: { + template: { settings: {} }, + _meta: { managed_by: 'fleet', managed: true, package: { name: 'endpoint' } }, + }, + create: true, + }, + { ignore: [404] }, + ], + ]); + + // Index template composed of the two component templates created + // with index pattern matching the destination index + expect(esClient.indices.putIndexTemplate.mock.calls).toEqual([ + [ + { + body: { + _meta: meta, + composed_of: [ + 'logs-endpoint.metadata_current-template@package', + 'logs-endpoint.metadata_current-template@custom', + ], + index_patterns: ['.metrics-endpoint.metadata_united_default-0.16.0-dev.0'], + priority: 250, + template: { mappings: undefined, settings: undefined }, + }, + name: 'logs-endpoint.metadata_current-template', + }, + { ignore: [404] }, + ], + ]); + + // Destination index is created before transform is created + expect(esClient.indices.create.mock.calls).toEqual([ + [ + { + aliases: { + '.metrics-endpoint.metadata_united_default.all': {}, + '.metrics-endpoint.metadata_united_default.latest': {}, + }, + index: '.metrics-endpoint.metadata_united_default-0.16.0-dev.0', + }, + { ignore: [400] }, + ], + ]); + + expect(esClient.transform.putTransform.mock.calls).toEqual([[expectedData.TRANSFORM]]); + expect(esClient.transform.startTransform.mock.calls).toEqual([ + [ + { + transform_id: 'logs-endpoint.metadata_current-default-0.2.0', + }, + { ignore: [409] }, + ], + ]); + + // Saved object is updated with newly created index templates, component templates, transform + expect(savedObjectsClient.update.mock.calls).toEqual([ + [ + 'epm-packages', + 'endpoint', + { + installed_es: [ + { + id: 'metrics-endpoint.policy-0.16.0-dev.0', + type: ElasticsearchAssetType.ingestPipeline, + }, + { + id: '.metrics-endpoint.metadata_united_default-0.16.0-dev.0', + type: ElasticsearchAssetType.index, + }, + { + id: 'logs-endpoint.metadata_current-template', + type: ElasticsearchAssetType.indexTemplate, + version: '0.2.0', + }, + { + id: 'logs-endpoint.metadata_current-template@custom', + type: ElasticsearchAssetType.componentTemplate, + version: '0.2.0', + }, + { + id: 'logs-endpoint.metadata_current-template@package', + type: ElasticsearchAssetType.componentTemplate, + version: '0.2.0', + }, + { + id: 'logs-endpoint.metadata_current-default-0.2.0', + type: ElasticsearchAssetType.transform, + version: '0.2.0', + }, + ], + }, + { + refresh: false, + }, + ], + ]); + }); + + test('can install new version when an older version does not exist', async () => { + const sourceData = getYamlTestData(false, '0.2.0'); + const expectedData = getExpectedData('0.2.0'); + + const previousInstallation: Installation = { + installed_es: [], + } as unknown as Installation; + + const currentInstallation: Installation = { + installed_es: [ + { + id: `logs-endpoint.metadata_current-default-0.2.0`, + type: ElasticsearchAssetType.transform, + }, + ], + } as unknown as Installation; + (getAsset as jest.MockedFunction) + .mockReturnValueOnce(Buffer.from(sourceData.MANIFEST, 'utf8')) + .mockReturnValueOnce(Buffer.from(sourceData.TRANSFORM, 'utf8')); + + (getInstallation as jest.MockedFunction) + .mockReturnValueOnce(Promise.resolve(previousInstallation)) + .mockReturnValueOnce(Promise.resolve(currentInstallation)); + + ( + getInstallationObject as jest.MockedFunction + ).mockReturnValueOnce( + Promise.resolve({ + attributes: { installed_es: [] }, + } as unknown as SavedObject) + ); + + await installTransforms( + { + name: 'endpoint', + version: '0.16.0-dev.0', + } as unknown as RegistryPackage, + [ + 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/manifest.yml', + 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml', + ], + esClient, + savedObjectsClient, + loggerMock.create(), + previousInstallation.installed_es + ); + + expect(esClient.transform.putTransform.mock.calls).toEqual([[expectedData.TRANSFORM]]); + // Does not start transform because start is set to false in manifest.yml + expect(esClient.transform.startTransform.mock.calls).toEqual([]); + }); + + test('can downgrade to older version when force: true', async () => { + const sourceData = getYamlTestData(false, '0.1.0'); + const expectedData = getExpectedData('0.1.0'); + + const previousInstallation: Installation = { + installed_es: [ + { + id: `logs-endpoint.metadata_current-default-0.2.0`, + type: ElasticsearchAssetType.transform, + }, + { + id: 'logs-endpoint.metadata_current-template', + type: ElasticsearchAssetType.indexTemplate, + }, + { + id: 'logs-endpoint.metadata_current-template@custom', + type: ElasticsearchAssetType.componentTemplate, + }, + { + id: 'logs-endpoint.metadata_current-template@package', + type: ElasticsearchAssetType.componentTemplate, + }, + ], + } as unknown as Installation; + + const currentInstallation: Installation = { + installed_es: [ + { + id: `logs-endpoint.metadata_current-default-0.1.0`, + type: ElasticsearchAssetType.transform, + }, + { + id: 'logs-endpoint.metadata_current-template', + type: ElasticsearchAssetType.indexTemplate, + }, + { + id: 'logs-endpoint.metadata_current-template@custom', + type: ElasticsearchAssetType.componentTemplate, + }, + { + id: 'logs-endpoint.metadata_current-template@package', + type: ElasticsearchAssetType.componentTemplate, + }, + ], + } as unknown as Installation; + (getAsset as jest.MockedFunction) + .mockReturnValueOnce(Buffer.from(sourceData.MANIFEST, 'utf8')) + .mockReturnValueOnce(Buffer.from(sourceData.TRANSFORM, 'utf8')); + + (getInstallation as jest.MockedFunction) + .mockReturnValueOnce(Promise.resolve(previousInstallation)) + .mockReturnValueOnce(Promise.resolve(currentInstallation)); + + ( + getInstallationObject as jest.MockedFunction + ).mockReturnValueOnce( + Promise.resolve({ + attributes: { installed_es: [] }, + } as unknown as SavedObject) + ); + + // Mock resp for when index from older version already exists + esClient.indices.create.mockReturnValueOnce( + // @ts-expect-error mock error instead of successful IndicesCreateResponse + Promise.resolve({ + error: { + type: 'resource_already_exists_exception', + }, + status: 400, + }) + ); + + await installTransforms( + { + name: 'endpoint', + version: '0.16.0-dev.0', + } as unknown as RegistryPackage, + [ + 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/manifest.yml', + 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml', + ], + esClient, + savedObjectsClient, + loggerMock.create(), + previousInstallation.installed_es + ); + + expect(esClient.indices.create.mock.calls).toEqual([ + [ + { + index: '.metrics-endpoint.metadata_united_default-0.16.0-dev.0', + aliases: { + '.metrics-endpoint.metadata_united_default.all': {}, + '.metrics-endpoint.metadata_united_default.latest': {}, + }, + }, + { ignore: [400] }, + ], + ]); + + // If downgrading to and older version, and destination index already exists + // aliases should still be updated to point .latest to this index + expect(esClient.indices.updateAliases.mock.calls).toEqual([ + [ + { + body: { + actions: [ + { + add: { + index: '.metrics-endpoint.metadata_united_default-0.16.0-dev.0', + alias: '.metrics-endpoint.metadata_united_default.all', + }, + }, + { + add: { + index: '.metrics-endpoint.metadata_united_default-0.16.0-dev.0', + alias: '.metrics-endpoint.metadata_united_default.latest', + }, + }, + ], + }, + }, + ], + ]); + + expect(esClient.transform.deleteTransform.mock.calls).toEqual([ + [ + { force: true, transform_id: 'logs-endpoint.metadata_current-default-0.2.0' }, + { ignore: [404] }, + ], + ]); + expect(esClient.transform.putTransform.mock.calls).toEqual([[expectedData.TRANSFORM]]); + }); + + test('retain old transforms and do nothing if fleet_transform_version is the same', async () => { + // Old fleet_transform_version is 0.1.0, fleet_transform_version to be installed is 0.1.0 + const sourceData = getYamlTestData(false, '0.1.0'); + + const previousInstallation: Installation = { + installed_es: [ + { + id: 'logs-endpoint.metadata_current-default-0.1.0', + type: ElasticsearchAssetType.transform, + }, + { + id: 'logs-endpoint.metadata_current-template', + type: ElasticsearchAssetType.indexTemplate, + }, + { + id: 'logs-endpoint.metadata_current-template@custom', + type: ElasticsearchAssetType.componentTemplate, + }, + { + id: 'logs-endpoint.metadata_current-template@package', + type: ElasticsearchAssetType.componentTemplate, + }, + ], + } as unknown as Installation; + + const currentInstallation: Installation = { + installed_es: [ + { + id: 'endpoint.metadata-current-default-0.1.0', + type: ElasticsearchAssetType.transform, + }, + ], + } as unknown as Installation; + + (getAsset as jest.MockedFunction) + .mockReturnValueOnce(Buffer.from(sourceData.FIELDS, 'utf8')) + .mockReturnValueOnce(Buffer.from(sourceData.MANIFEST, 'utf8')) + .mockReturnValueOnce(Buffer.from(sourceData.TRANSFORM, 'utf8')); + + (getInstallation as jest.MockedFunction) + .mockReturnValueOnce(Promise.resolve(previousInstallation)) + .mockReturnValueOnce(Promise.resolve(currentInstallation)); + + ( + getInstallationObject as jest.MockedFunction + ).mockReturnValueOnce( + Promise.resolve({ + attributes: { installed_es: currentInstallation.installed_es }, + } as unknown as SavedObject) + ); + + await installTransforms( + { + name: 'endpoint', + version: '0.16.0-dev.0', + } as unknown as RegistryPackage, + [ + 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/fields/fields.yml', + 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/manifest.yml', + 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/transform.yml', + ], + esClient, + savedObjectsClient, + loggerMock.create(), + previousInstallation.installed_es + ); + + // Transform from old version is neither stopped nor deleted + expect(esClient.transform.stopTransform.mock.calls).toEqual([]); + expect(esClient.transform.deleteTransform.mock.calls).toEqual([]); + + // Destination index from old version is not deleted + expect(esClient.transport.request.mock.calls).toEqual([]); + + // No new destination index is created + expect(esClient.indices.create.mock.calls).toEqual([]); + // No new transform is created or started + expect(esClient.transform.putTransform.mock.calls).toEqual([]); expect(esClient.transform.startTransform.mock.calls).toEqual([]); }); }); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/remove.ts b/x-pack/plugins/fleet/server/services/epm/packages/remove.ts index c994320ab829a1c..61780c797716633 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/remove.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/remove.ts @@ -17,6 +17,8 @@ import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; import { SavedObjectsUtils, SavedObjectsErrorHelpers } from '@kbn/core/server'; +import { updateIndexSettings } from '../elasticsearch/index/update_settings'; + import { PACKAGE_POLICY_SAVED_OBJECT_TYPE, PACKAGES_SAVED_OBJECT_TYPE, @@ -143,7 +145,7 @@ function deleteESAssets( } else if (assetType === ElasticsearchAssetType.componentTemplate) { return deleteComponentTemplate(esClient, id); } else if (assetType === ElasticsearchAssetType.transform) { - return deleteTransforms(esClient, [id]); + return deleteTransforms(esClient, [id], true); } else if (assetType === ElasticsearchAssetType.dataStreamIlmPolicy) { return deleteIlms(esClient, [id]); } else if (assetType === ElasticsearchAssetType.ilmPolicy) { @@ -164,27 +166,35 @@ async function deleteAssets( esClient: ElasticsearchClient ) { const logger = appContextService.getLogger(); + // must unset default_pipelines settings in indices first, or pipelines associated with an index cannot not be deleted // must delete index templates first, or component templates which reference them cannot be deleted // must delete ingestPipelines first, or ml models referenced in them cannot be deleted. // separate the assets into Index Templates and other assets. - type Tuple = [EsAssetReference[], EsAssetReference[]]; - const [indexTemplatesAndPipelines, otherAssets] = installedEs.reduce( - ([indexAssetTypes, otherAssetTypes], asset) => { + type Tuple = [EsAssetReference[], EsAssetReference[], EsAssetReference[]]; + const [indexTemplatesAndPipelines, indexAssets, otherAssets] = installedEs.reduce( + ([indexTemplateAndPipelineTypes, indexAssetTypes, otherAssetTypes], asset) => { if ( asset.type === ElasticsearchAssetType.indexTemplate || asset.type === ElasticsearchAssetType.ingestPipeline ) { + indexTemplateAndPipelineTypes.push(asset); + } else if (asset.type === ElasticsearchAssetType.index) { indexAssetTypes.push(asset); } else { otherAssetTypes.push(asset); } - return [indexAssetTypes, otherAssetTypes]; + return [indexTemplateAndPipelineTypes, indexAssetTypes, otherAssetTypes]; }, - [[], []] + [[], [], []] ); try { + // must first unset any default pipeline associated with any existing indices + // by setting empty string + await Promise.all( + indexAssets.map((asset) => updateIndexSettings(esClient, asset.id, { default_pipeline: '' })) + ); // must delete index templates and pipelines first await Promise.all(deleteESAssets(indexTemplatesAndPipelines, esClient)); // then the other asset types