diff --git a/x-pack/plugins/fleet/server/routes/epm/handlers.ts b/x-pack/plugins/fleet/server/routes/epm/handlers.ts index 560af83619a484..47bdbd972b4c7e 100644 --- a/x-pack/plugins/fleet/server/routes/epm/handlers.ts +++ b/x-pack/plugins/fleet/server/routes/epm/handlers.ts @@ -142,9 +142,12 @@ export const getInstalledListHandler: FleetRequestHandler< TypeOf > = async (context, request, response) => { try { - const savedObjectsClient = (await context.fleet).internalSoClient; + const [fleetContext, coreContext] = await Promise.all([context.fleet, context.core]); + const savedObjectsClient = fleetContext.internalSoClient; + const esClient = coreContext.elasticsearch.client.asCurrentUser; const res = await getInstalledPackages({ savedObjectsClient, + esClient, ...request.query, }); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/get.test.ts index 31896033d24329..2dc295762e33ab 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/get.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/get.test.ts @@ -8,7 +8,7 @@ import type { SavedObjectsClientContract, SavedObjectsFindResult } from '@kbn/core/server'; import { SavedObjectsErrorHelpers } from '@kbn/core/server'; -import { savedObjectsClientMock } from '@kbn/core/server/mocks'; +import { savedObjectsClientMock, elasticsearchServiceMock } from '@kbn/core/server/mocks'; import { ASSETS_SAVED_OBJECT_TYPE, @@ -17,7 +17,7 @@ import { } from '../../../../common'; import type { RegistryPackage } from '../../../../common/types'; import type { PackagePolicySOAttributes } from '../../../types'; - +import { dataStreamService } from '../../data_streams'; import { createAppContextStartContractMock } from '../../../mocks'; import { appContextService } from '../../app_context'; import { PackageNotFoundError } from '../../../errors'; @@ -32,6 +32,7 @@ import { getInstalledPackages, getPackageInfo, getPackages, getPackageUsageStats jest.mock('../registry'); jest.mock('../../settings'); jest.mock('../../audit_logging'); +jest.mock('../../data_streams'); const MockRegistry = jest.mocked(Registry); @@ -643,6 +644,7 @@ owner: elastic`, }); await getInstalledPackages({ + esClient: elasticsearchServiceMock.createInternalClient(), savedObjectsClient: soClient, dataStreamType: 'logs', nameQuery: 'nginx', @@ -785,6 +787,7 @@ owner: elastic`, }); const results = await getInstalledPackages({ + esClient: elasticsearchServiceMock.createInternalClient(), savedObjectsClient: soClient, dataStreamType: 'logs', nameQuery: 'nginx', @@ -816,6 +819,84 @@ owner: elastic`, total: 5, }); }); + it('filter non active datastreams if flag is true', async () => { + const soClient = savedObjectsClientMock.create(); + + jest.mocked(dataStreamService.getAllFleetDataStreams).mockResolvedValue([ + { + name: `logs-elastic_agent.apm_server-production`, + }, + { + name: `metrics-elastic_agent.apm_server-production`, + }, + ] as any); + + soClient.find.mockImplementation(async (options) => { + if (options.type === PACKAGES_SAVED_OBJECT_TYPE) { + return { + total: 5, + saved_objects: [ + { + type: 'epm-packages', + id: 'elastic_agent', + attributes: { + es_index_patterns: { + fleet_server_logs: 'logs-elastic_agent.fleet_server-*', + apm_server_logs: 'logs-elastic_agent.apm_server-*', + apm_server_metrics: 'metrics-elastic_agent.apm_server-*', + }, + name: 'elastic_agent', + version: '1.8.0', + install_status: 'installed', + }, + references: [], + sort: ['elastic_agent'], + }, + ], + } as any; + } else if (options.type === ASSETS_SAVED_OBJECT_TYPE) { + return { + total: 5, + saved_objects: [ + { + type: 'epm-packages-assets', + id: '338b6f9e-e126-5f1e-abb9-afe017d4788b', + attributes: { + package_name: 'elastic_agent', + package_version: '1.8.0', + install_source: 'upload', + asset_path: 'elastic_agent-1.8.0/manifest.yml', + media_type: 'text/yaml; charset=utf-8', + data_utf8: + 'name: elastic_agent\ntitle: Elastic Agent\nversion: 1.8.0\ndescription: Collect logs and metrics from Elastic Agents.\ntype: integration\nformat_version: 1.0.0\nlicense: basic\ncategories: ["elastic_stack"]\nconditions:\n kibana.version: "^8.7.1"\nowner:\n github: elastic/elastic-agent\nicons:\n - src: /img/logo_elastic_agent.svg\n title: logo Elastic Agent\n size: 64x64\n type: image/svg+xml\nscreenshots:\n - src: /img/elastic_agent_overview.png\n title: Elastic Agent Overview\n size: 2560×1234\n type: image/png\n - src: /img/elastic_agent_metrics.png\n title: Elastic Agent Metrics\n size: 2560×1234\n type: image/png\n - src: /img/elastic_agent_info.png\n title: Elastic Agent Information\n size: 2560×1234\n type: image/png\n - src: /img/elastic_agent_integrations.png\n title: Elastic Agent Integrations\n size: 2560×1234\n type: image/png\n', + data_base64: '', + }, + references: [], + }, + ], + } as any; + } + }); + + const results = await getInstalledPackages({ + savedObjectsClient: soClient, + esClient: elasticsearchServiceMock.createInternalClient(), + perPage: 10, + sortOrder: 'asc', + showOnlyActiveDataStreams: true, + }); + + expect(results.items[0].dataStreams).toEqual([ + { + name: 'logs-elastic_agent.apm_server-*', + title: 'apm_server_logs', + }, + { + name: 'metrics-elastic_agent.apm_server-*', + title: 'apm_server_metrics', + }, + ]); + }); }); describe('getPackageInfo', () => { diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get.ts b/x-pack/plugins/fleet/server/services/epm/packages/get.ts index 2af0f1e9f1d2b7..e59f0ad4b5ae95 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/get.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/get.ts @@ -7,12 +7,17 @@ import { safeLoad } from 'js-yaml'; import pMap from 'p-map'; -import type { SavedObjectsClientContract, SavedObjectsFindOptions } from '@kbn/core/server'; +import minimatch from 'minimatch'; +import type { + ElasticsearchClient, + SavedObjectsClientContract, + SavedObjectsFindOptions, +} from '@kbn/core/server'; import semverGte from 'semver/functions/gte'; import type { Logger } from '@kbn/core/server'; import { withSpan } from '@kbn/apm-utils'; -import type { SortResults } from '@elastic/elasticsearch/lib/api/types'; +import type { IndicesDataStream, SortResults } from '@elastic/elasticsearch/lib/api/types'; import { nodeBuilder } from '@kbn/es-query'; @@ -50,6 +55,7 @@ import { PackageInvalidArchiveError, } from '../../../errors'; import { appContextService } from '../..'; +import { dataStreamService } from '../../data_streams'; import * as Registry from '../registry'; import type { PackageAsset } from '../archive/storage'; import { getEsPackage } from '../archive/storage'; @@ -180,20 +186,22 @@ export async function getPackages( interface GetInstalledPackagesOptions { savedObjectsClient: SavedObjectsClientContract; + esClient: ElasticsearchClient; dataStreamType?: PackageDataStreamTypes; nameQuery?: string; searchAfter?: SortResults; perPage: number; sortOrder: 'asc' | 'desc'; + showOnlyActiveDataStreams?: boolean; } export async function getInstalledPackages(options: GetInstalledPackagesOptions) { - const { savedObjectsClient, ...otherOptions } = options; + const { savedObjectsClient, esClient, showOnlyActiveDataStreams, ...otherOptions } = options; const { dataStreamType } = otherOptions; - const packageSavedObjects = await getInstalledPackageSavedObjects( - savedObjectsClient, - otherOptions - ); + const [packageSavedObjects, allFleetDataStreams] = await Promise.all([ + getInstalledPackageSavedObjects(savedObjectsClient, otherOptions), + showOnlyActiveDataStreams ? dataStreamService.getAllFleetDataStreams(esClient) : undefined, + ]); const integrations = packageSavedObjects.saved_objects.map((integrationSavedObject) => { const { @@ -203,7 +211,11 @@ export async function getInstalledPackages(options: GetInstalledPackagesOptions) es_index_patterns: esIndexPatterns, } = integrationSavedObject.attributes; - const dataStreams = getInstalledPackageSavedObjectDataStreams(esIndexPatterns, dataStreamType); + const dataStreams = getInstalledPackageSavedObjectDataStreams( + esIndexPatterns, + dataStreamType, + allFleetDataStreams + ); return { name, @@ -296,7 +308,7 @@ export async function getPackageSavedObjects( async function getInstalledPackageSavedObjects( savedObjectsClient: SavedObjectsClientContract, - options: Omit + options: Omit ) { const { searchAfter, sortOrder, perPage, nameQuery, dataStreamType } = options; @@ -385,8 +397,13 @@ export async function getInstalledPackageManifests( function getInstalledPackageSavedObjectDataStreams( indexPatterns: Record, - dataStreamType?: string + dataStreamType?: string, + filterActiveDatastreams?: IndicesDataStream[] ) { + const filterActiveDatastreamsName = filterActiveDatastreams + ? filterActiveDatastreams.map((ds) => ds.name) + : undefined; + return Object.entries(indexPatterns) .map(([key, value]) => { return { @@ -395,11 +412,22 @@ function getInstalledPackageSavedObjectDataStreams( }; }) .filter((stream) => { - if (!dataStreamType) { - return true; - } else { - return stream.name.startsWith(`${dataStreamType}-`); + if (dataStreamType && !stream.name.startsWith(`${dataStreamType}-`)) { + return false; } + + if (filterActiveDatastreamsName) { + const patternRegex = new minimatch.Minimatch(stream.name, { + noglobstar: true, + nonegate: true, + }).makeRe(); + + return filterActiveDatastreamsName.some((dataStreamName) => + dataStreamName.match(patternRegex) + ); + } + + return true; }); } diff --git a/x-pack/plugins/fleet/server/types/rest_spec/epm.ts b/x-pack/plugins/fleet/server/types/rest_spec/epm.ts index 358d2c8d6345dd..8f2ffe24fcf2a0 100644 --- a/x-pack/plugins/fleet/server/types/rest_spec/epm.ts +++ b/x-pack/plugins/fleet/server/types/rest_spec/epm.ts @@ -35,6 +35,7 @@ export const GetInstalledPackagesRequestSchema = { schema.literal('profiling'), ]) ), + showOnlyActiveDataStreams: schema.maybe(schema.boolean()), nameQuery: schema.maybe(schema.string()), searchAfter: schema.maybe(schema.arrayOf(schema.oneOf([schema.string(), schema.number()]))), perPage: schema.number({ defaultValue: 15 }), diff --git a/x-pack/test/fleet_api_integration/apis/epm/get.ts b/x-pack/test/fleet_api_integration/apis/epm/get.ts index 90247a730df177..6f1edcd6aca108 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/get.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/get.ts @@ -18,6 +18,7 @@ export default function (providerContext: FtrProviderContext) { const { getService } = providerContext; const supertest = getService('supertest'); + const es = getService('es'); const supertestWithoutAuth = getService('supertestWithoutAuth'); const fleetAndAgents = getService('fleetAndAgents'); @@ -117,12 +118,22 @@ export default function (providerContext: FtrProviderContext) { await installPackage('experimental', '0.1.0'); await bundlePackage('endpoint-8.6.1'); await installPackage('endpoint', '8.6.1'); + await es.index({ + index: 'logs-apache.access-default', + document: { + '@timestamp': new Date().toISOString(), + }, + refresh: 'wait_for', + }); }); after(async () => { await uninstallPackage(testPkgName, testPkgVersion); await uninstallPackage('experimental', '0.1.0'); await uninstallPackage('endpoint', '8.6.1'); await removeBundledPackages(log); + await es.indices.deleteDataStream({ + name: 'logs-apache.access-default', + }); }); it('Allows the fetching of installed packages', async () => { const res = await supertest.get(`/api/fleet/epm/packages/installed`).expect(200); @@ -173,6 +184,16 @@ export default function (providerContext: FtrProviderContext) { expect(packages.length).to.be(1); expect(packages[0].name).to.be('experimental'); }); + it('Can be to only return active datastreams', async () => { + const res = await supertest + .get(`/api/fleet/epm/packages/installed?nameQuery=apache&showOnlyActiveDataStreams=true`) + .expect(200); + const packages = res.body.items; + expect(packages.length).to.be(1); + expect(packages[0].name).to.be('apache'); + expect(packages[0].dataStreams.length).to.be(1); + expect(packages[0].dataStreams[0].name).to.be('logs-apache.access-*'); + }); }); it('returns a 404 for a package that do not exists', async function () { await supertest.get('/api/fleet/epm/packages/notexists/99.99.99').expect(404);