[EPM] Change PACKAGES_SAVED_OBJECT_TYPE id (#62818)

* changed PACKAGES_SAVED_OBJECT_TYPE id from packageName-version to packageName

* change references to keys to package and version

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts

@@ -102,7 +102,9 @@ export const getInfoHandler: RequestHandler<TypeOf<typeof GetInfoRequestSchema.p
   try {
     const { pkgkey } = request.params;
     const savedObjectsClient = context.core.savedObjects.client;
-    const res = await getPackageInfo({ savedObjectsClient, pkgkey });
+    // TODO: change epm API to /packageName/version so we don't need to do this
+    const [pkgName, pkgVersion] = pkgkey.split('-');
+    const res = await getPackageInfo({ savedObjectsClient, pkgName, pkgVersion });
     const body: GetInfoResponse = {
       response: res,
       success: true,

x-pack/plugins/ingest_manager/server/services/datasource.ts

@@ -9,7 +9,7 @@ import { DeleteDatasourcesResponse, packageToConfigDatasource } from '../../comm
 import { DATASOURCE_SAVED_OBJECT_TYPE } from '../constants';
 import { NewDatasource, Datasource, ListWithKuery } from '../types';
 import { agentConfigService } from './agent_config';
-import { findInstalledPackageByName, getPackageInfo } from './epm/packages';
+import { getPackageInfo, getInstallation } from './epm/packages';
 import { outputService } from './output';
 
 const SAVED_OBJECT_TYPE = DATASOURCE_SAVED_OBJECT_TYPE;
@@ -172,15 +172,13 @@ class DatasourceService {
     soClient: SavedObjectsClientContract,
     pkgName: string
   ): Promise<NewDatasource | undefined> {
-    const pkgInstall = await findInstalledPackageByName({
-      savedObjectsClient: soClient,
-      pkgName,
-    });
+    const pkgInstall = await getInstallation({ savedObjectsClient: soClient, pkgName });
     if (pkgInstall) {
       const [pkgInfo, defaultOutputId] = await Promise.all([
         getPackageInfo({
           savedObjectsClient: soClient,
-          pkgkey: `${pkgInstall.name}-${pkgInstall.version}`,
+          pkgName: pkgInstall.name,
+          pkgVersion: pkgInstall.version,
         }),
         outputService.getDefaultOutputId(soClient),
       ]);

x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/ilm/install.ts

@@ -7,9 +7,15 @@
 import { CallESAsCurrentUser, ElasticsearchAssetType } from '../../../../types';
 import * as Registry from '../../registry';
 
-export async function installILMPolicy(pkgkey: string, callCluster: CallESAsCurrentUser) {
-  const ilmPaths = await Registry.getArchiveInfo(pkgkey, (entry: Registry.ArchiveEntry) =>
-    isILMPolicy(entry)
+export async function installILMPolicy(
+  pkgName: string,
+  pkgVersion: string,
+  callCluster: CallESAsCurrentUser
+) {
+  const ilmPaths = await Registry.getArchiveInfo(
+    pkgName,
+    pkgVersion,
+    (entry: Registry.ArchiveEntry) => isILMPolicy(entry)
   );
   if (!ilmPaths.length) return;
   await Promise.all(

x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/ingest_pipeline/install.ts

@@ -30,11 +30,10 @@ export const installPipelines = async (
       if (dataset.ingest_pipeline) {
         acc.push(
           installPipelinesForDataset({
-            pkgkey: Registry.pkgToPkgKey(registryPackage),
             dataset,
             callCluster,
-            packageName: registryPackage.name,
-            packageVersion: registryPackage.version,
+            pkgName: registryPackage.name,
+            pkgVersion: registryPackage.version,
           })
         );
       }
@@ -68,19 +67,19 @@ export function rewriteIngestPipeline(
 
 export async function installPipelinesForDataset({
   callCluster,
-  pkgkey,
+  pkgName,
+  pkgVersion,
   dataset,
-  packageName,
-  packageVersion,
 }: {
   callCluster: CallESAsCurrentUser;
-  pkgkey: string;
+  pkgName: string;
+  pkgVersion: string;
   dataset: Dataset;
-  packageName: string;
-  packageVersion: string;
 }): Promise<AssetReference[]> {
-  const pipelinePaths = await Registry.getArchiveInfo(pkgkey, (entry: Registry.ArchiveEntry) =>
-    isDatasetPipeline(entry, dataset.path)
+  const pipelinePaths = await Registry.getArchiveInfo(
+    pkgName,
+    pkgVersion,
+    (entry: Registry.ArchiveEntry) => isDatasetPipeline(entry, dataset.path)
   );
   let pipelines: any[] = [];
   const substitutions: RewriteSubstitution[] = [];
@@ -90,7 +89,7 @@ export async function installPipelinesForDataset({
     const nameForInstallation = getPipelineNameForInstallation({
       pipelineName: name,
       dataset,
-      packageVersion,
+      packageVersion: pkgVersion,
     });
     const content = Registry.getAsset(path).toString('utf-8');
     pipelines.push({

x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/install.ts

@@ -20,11 +20,12 @@ import * as Registry from '../../registry';
 export const installTemplates = async (
   registryPackage: RegistryPackage,
   callCluster: CallESAsCurrentUser,
-  pkgkey: string
+  pkgName: string,
+  pkgVersion: string
 ) => {
   // install any pre-built index template assets,
   // atm, this is only the base package's global template
-  installPreBuiltTemplates(pkgkey, callCluster);
+  installPreBuiltTemplates(pkgName, pkgVersion, callCluster);
 
   // build templates per dataset from yml files
   const datasets = registryPackage.datasets;
@@ -45,9 +46,15 @@ export const installTemplates = async (
 };
 
 // this is temporary until we update the registry to use index templates v2 structure
-const installPreBuiltTemplates = async (pkgkey: string, callCluster: CallESAsCurrentUser) => {
-  const templatePaths = await Registry.getArchiveInfo(pkgkey, (entry: Registry.ArchiveEntry) =>
-    isTemplate(entry)
+const installPreBuiltTemplates = async (
+  pkgName: string,
+  pkgVersion: string,
+  callCluster: CallESAsCurrentUser
+) => {
+  const templatePaths = await Registry.getArchiveInfo(
+    pkgName,
+    pkgVersion,
+    (entry: Registry.ArchiveEntry) => isTemplate(entry)
   );
   templatePaths.forEach(async path => {
     const { file } = Registry.pathParts(path);

x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/install.ts

@@ -68,25 +68,32 @@ export enum IndexPatternType {
   metrics = 'metrics',
   events = 'events',
 }
-
+// TODO: use a function overload and make pkgName and pkgVersion required for install/update
+// and not for an update removal.  or separate out the functions
 export async function installIndexPatterns(
   savedObjectsClient: SavedObjectsClientContract,
-  pkgkey?: string
+  pkgName?: string,
+  pkgVersion?: string
 ) {
   // get all user installed packages
   const installedPackages = await getPackageKeysByStatus(
     savedObjectsClient,
     InstallationStatus.installed
   );
-  // add this package to the array if it doesn't already exist
-  // this should not happen because a user can't "reinstall" a package
-  // if it does because the install endpoint is called directly, the install continues
-  if (pkgkey && !installedPackages.includes(pkgkey)) {
-    installedPackages.push(pkgkey);
+  if (pkgName && pkgVersion) {
+    // add this package to the array if it doesn't already exist
+    const foundPkg = installedPackages.find(pkg => pkg.pkgName === pkgName);
+    // this may be removed if we add the packged to saved objects before installing index patterns
+    // otherwise this is a first time install
+    // TODO: handle update case when versions are different
+    if (!foundPkg) {
+      installedPackages.push({ pkgName, pkgVersion });
+    }
   }
-
   // get each package's registry info
-  const installedPackagesFetchInfoPromise = installedPackages.map(pkg => Registry.fetchInfo(pkg));
+  const installedPackagesFetchInfoPromise = installedPackages.map(pkg =>
+    Registry.fetchInfo(pkg.pkgName, pkg.pkgVersion)
+  );
   const installedPackagesInfo = await Promise.all(installedPackagesFetchInfoPromise);
 
   // for each index pattern type, create an index pattern
@@ -97,7 +104,7 @@ export async function installIndexPatterns(
   ];
   indexPatternTypes.forEach(async indexPatternType => {
     // if this is an update because a package is being unisntalled (no pkgkey argument passed) and no other packages are installed, remove the index pattern
-    if (!pkgkey && installedPackages.length === 0) {
+    if (!pkgName && installedPackages.length === 0) {
       try {
         await savedObjectsClient.delete(INDEX_PATTERN_SAVED_OBJECT_TYPE, `${indexPatternType}-*`);
       } catch (err) {

x-pack/plugins/ingest_manager/server/services/epm/packages/assets.ts

@@ -58,7 +58,7 @@ export async function getAssetsData(
 ): Promise<Registry.ArchiveEntry[]> {
   // TODO: Needs to be called to fill the cache but should not be required
   const pkgkey = packageInfo.name + '-' + packageInfo.version;
-  if (!cacheHas(pkgkey)) await Registry.getArchiveInfo(pkgkey);
+  if (!cacheHas(pkgkey)) await Registry.getArchiveInfo(packageInfo.name, packageInfo.version);
 
   // Gather all asset data
   const assets = getAssets(packageInfo, filter, datasetName);

x-pack/plugins/ingest_manager/server/services/epm/packages/get.ts

@@ -41,7 +41,7 @@ export async function getPackages(
     .map(item =>
       createInstallableFrom(
         item,
-        savedObjectsVisible.find(({ attributes }) => attributes.name === item.name)
+        savedObjectsVisible.find(({ id }) => id === item.name)
       )
     )
     .sort(sortByName);
@@ -53,23 +53,24 @@ export async function getPackageKeysByStatus(
   status: InstallationStatus
 ) {
   const allPackages = await getPackages({ savedObjectsClient });
-  return allPackages.reduce<string[]>((acc, pkg) => {
+  return allPackages.reduce<Array<{ pkgName: string; pkgVersion: string }>>((acc, pkg) => {
     if (pkg.status === status) {
-      acc.push(`${pkg.name}-${pkg.version}`);
+      acc.push({ pkgName: pkg.name, pkgVersion: pkg.version });
     }
     return acc;
   }, []);
 }
 
 export async function getPackageInfo(options: {
   savedObjectsClient: SavedObjectsClientContract;
-  pkgkey: string;
+  pkgName: string;
+  pkgVersion: string;
 }): Promise<PackageInfo> {
-  const { savedObjectsClient, pkgkey } = options;
+  const { savedObjectsClient, pkgName, pkgVersion } = options;
   const [item, savedObject] = await Promise.all([
-    Registry.fetchInfo(pkgkey),
-    getInstallationObject({ savedObjectsClient, pkgkey }),
-    Registry.getArchiveInfo(pkgkey),
+    Registry.fetchInfo(pkgName, pkgVersion),
+    getInstallationObject({ savedObjectsClient, pkgName }),
+    Registry.getArchiveInfo(pkgName, pkgVersion),
   ] as const);
   // adding `as const` due to regression in TS 3.7.2
   // see https://github.com/microsoft/TypeScript/issues/34925#issuecomment-550021453
@@ -86,37 +87,22 @@ export async function getPackageInfo(options: {
 
 export async function getInstallationObject(options: {
   savedObjectsClient: SavedObjectsClientContract;
-  pkgkey: string;
+  pkgName: string;
 }) {
-  const { savedObjectsClient, pkgkey } = options;
+  const { savedObjectsClient, pkgName } = options;
   return savedObjectsClient
-    .get<Installation>(PACKAGES_SAVED_OBJECT_TYPE, pkgkey)
+    .get<Installation>(PACKAGES_SAVED_OBJECT_TYPE, pkgName)
     .catch(e => undefined);
 }
 
 export async function getInstallation(options: {
   savedObjectsClient: SavedObjectsClientContract;
-  pkgkey: string;
+  pkgName: string;
 }) {
   const savedObject = await getInstallationObject(options);
   return savedObject?.attributes;
 }
 
-export async function findInstalledPackageByName(options: {
-  savedObjectsClient: SavedObjectsClientContract;
-  pkgName: string;
-}): Promise<Installation | undefined> {
-  const { savedObjectsClient, pkgName } = options;
-
-  const res = await savedObjectsClient.find<Installation>({
-    type: PACKAGES_SAVED_OBJECT_TYPE,
-    search: pkgName,
-    searchFields: ['name'],
-  });
-  if (res.saved_objects.length) return res.saved_objects[0].attributes;
-  return undefined;
-}
-
 function sortByName(a: { name: string }, b: { name: string }) {
   if (a.name > b.name) {
     return 1;

x-pack/plugins/ingest_manager/server/services/epm/packages/get_objects.ts

@@ -11,46 +11,6 @@ import * as Registry from '../registry';
 type ArchiveAsset = Pick<SavedObject, 'attributes' | 'migrationVersion' | 'references'>;
 type SavedObjectToBe = Required<SavedObjectsBulkCreateObject> & { type: AssetType };
 
-export async function getObjects(
-  pkgkey: string,
-  filter = (entry: Registry.ArchiveEntry): boolean => true
-): Promise<SavedObjectToBe[]> {
-  // Create a Map b/c some values, especially index-patterns, are referenced multiple times
-  const objects: Map<string, SavedObjectToBe> = new Map();
-
-  // Get paths which match the given filter
-  const paths = await Registry.getArchiveInfo(pkgkey, filter);
-
-  // Get all objects which matched filter. Add them to the Map
-  const rootObjects = await Promise.all(paths.map(getObject));
-  rootObjects.forEach(obj => objects.set(obj.id, obj));
-
-  // Each of those objects might have `references` property like [{id, type, name}]
-  for (const object of rootObjects) {
-    // For each of those objects, if they have references
-    for (const reference of object.references) {
-      // Get the referenced objects. Call same function with a new filter
-      const referencedObjects = await getObjects(pkgkey, (entry: Registry.ArchiveEntry) => {
-        // Skip anything we've already stored
-        if (objects.has(reference.id)) return false;
-
-        // Is the archive entry the reference we want?
-        const { type, file } = Registry.pathParts(entry.path);
-        const isType = type === reference.type;
-        const isJson = file === `${reference.id}.json`;
-
-        return isType && isJson;
-      });
-
-      // Add referenced objects to the Map
-      referencedObjects.forEach(ro => objects.set(ro.id, ro));
-    }
-  }
-
-  // return the array of unique objects
-  return Array.from(objects.values());
-}
-
 export async function getObject(key: string) {
   const buffer = Registry.getAsset(key);
 

x-pack/plugins/ingest_manager/server/services/epm/packages/index.ts

@@ -21,7 +21,6 @@ export {
   getPackageInfo,
   getPackages,
   SearchParams,
-  findInstalledPackageByName,
 } from './get';
 
 export { installKibanaAssets, installPackage, ensureInstalledPackage } from './install';