From 3aca0c2212d4caa16d8539e934210849d813aad1 Mon Sep 17 00:00:00 2001 From: Julia Bardi <90178898+juliaElastic@users.noreply.github.com> Date: Fri, 10 Feb 2023 16:54:27 +0100 Subject: [PATCH] [Fleet] added unit tests on parse package archive logic (#150888) ## Summary Related to https://github.com/elastic/kibana/issues/148599 Added unit tests on parse package archive logic ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --- .../server/services/epm/archive/parse.test.ts | 383 ++++++++++++++++++ .../server/services/epm/archive/parse.ts | 20 +- 2 files changed, 398 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/archive/parse.test.ts b/x-pack/plugins/fleet/server/services/epm/archive/parse.test.ts index 0c235ebb2c04d5..444c095ace4b30 100644 --- a/x-pack/plugins/fleet/server/services/epm/archive/parse.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/archive/parse.test.ts @@ -4,11 +4,23 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ +import type { ArchivePackage } from '../../../../common/types'; +import { PackageInvalidArchiveError } from '../../../errors'; + import { parseDefaultIngestPipeline, parseDataStreamElasticsearchEntry, parseTopLevelElasticsearchEntry, + _generatePackageInfoFromPaths, + parseAndVerifyArchive, + parseAndVerifyDataStreams, + parseAndVerifyStreams, + parseAndVerifyVars, + parseAndVerifyPolicyTemplates, + parseAndVerifyInputs, + parseAndVerifyReadme, } from './parse'; + describe('parseDefaultIngestPipeline', () => { it('Should return undefined for stream without any elasticsearch dir', () => { expect( @@ -268,3 +280,374 @@ describe('parseTopLevelElasticsearchEntry', () => { }); }); }); + +describe('parseAndVerifyArchive', () => { + it('should parse package successfully', async () => { + const packageInfo: ArchivePackage = await _generatePackageInfoFromPaths( + [ + 'x-pack/test/fleet_api_integration/apis/fixtures/package_verification/packages/src/input_only-0.1.0/docs/README.md', + 'x-pack/test/fleet_api_integration/apis/fixtures/package_verification/packages/src/input_only-0.1.0/manifest.yml', + ], + 'x-pack/test/fleet_api_integration/apis/fixtures/package_verification/packages/src/input_only-0.1.0' + ); + + expect(packageInfo).toEqual({ + categories: ['custom'], + description: 'Read lines from active log files with Elastic Agent.', + format_version: '1.0.0', + icons: [ + { + src: '/img/sample-logo.svg', + type: 'image/svg+xml', + }, + ], + license: 'basic', + name: 'input_only', + owner: { + github: 'elastic/integrations', + }, + policy_templates: [ + { + description: 'Collect your custom log files.', + input: 'logfile', + multiple: true, + name: 'first_policy_template', + template_path: 'input.yml.hbs', + title: 'Custom log file', + type: 'logs', + vars: [ + { + multi: true, + name: 'paths', + required: true, + show_user: true, + title: 'Paths', + type: 'text', + }, + { + multi: true, + name: 'tags', + required: true, + show_user: false, + title: 'Tags', + type: 'text', + }, + { + default: '72h', + name: 'ignore_older', + required: false, + title: 'Ignore events older than', + type: 'text', + }, + ], + }, + ], + release: 'beta', + screenshots: [ + { + size: '600x600', + src: '/img/sample-screenshot.png', + title: 'Sample screenshot', + type: 'image/png', + }, + ], + title: 'Custom Logs', + type: 'input', + version: '0.1.0', + }); + }); + + it('should throw on more than one top level dirs', () => { + expect(() => + parseAndVerifyArchive(['input_only-0.1.0/manifest.yml', 'dummy/manifest.yml'], {}) + ).toThrowError( + new PackageInvalidArchiveError('Package contains more than one top-level directory.') + ); + }); + + it('should throw on missing manifest file', () => { + expect(() => parseAndVerifyArchive(['input_only-0.1.0/test/manifest.yml'], {})).toThrowError( + new PackageInvalidArchiveError('Package must contain a top-level manifest.yml file.') + ); + }); + + it('should throw on invalid yml in manifest file', () => { + const buf = Buffer.alloc(1); + + expect(() => + parseAndVerifyArchive(['input_only-0.1.0/manifest.yml'], { + 'input_only-0.1.0/manifest.yml': buf, + }) + ).toThrowError('Could not parse top-level package manifest: YAMLException'); + }); + + it('should throw on missing required fields', () => { + const buf = Buffer.from( + ` +format_version: 1.0.0 +name: input_only +title: Custom Logs +description: >- + Read lines from active log files with Elastic Agent. +version: 0.1.0 + `, + 'utf8' + ); + + expect(() => + parseAndVerifyArchive(['input_only-0.1.0/manifest.yml'], { + 'input_only-0.1.0/manifest.yml': buf, + }) + ).toThrowError('Invalid top-level package manifest: one or more fields missing of '); + }); + + it('should throw on name or version mismatch', () => { + const buf = Buffer.from( + ` +format_version: 1.0.0 +name: input_only +title: Custom Logs +description: >- + Read lines from active log files with Elastic Agent. +version: 0.2.0 +owner: + github: elastic/integrations + `, + 'utf8' + ); + + expect(() => + parseAndVerifyArchive(['input_only-0.1.0/manifest.yml'], { + 'input_only-0.1.0/manifest.yml': buf, + }) + ).toThrowError( + 'Name input_only and version 0.2.0 do not match top-level directory input_only-0.1.0' + ); + }); +}); + +describe('parseAndVerifyDataStreams', () => { + it('should throw when data stream manifest file missing', async () => { + expect(() => + parseAndVerifyDataStreams({ + paths: ['input-only-0.1.0/data_stream/stream1/README.md'], + pkgName: 'input-only', + pkgVersion: '0.1.0', + manifests: {}, + }) + ).toThrowError("No manifest.yml file found for data stream 'stream1'"); + }); + + it('should throw when data stream manifest has invalid yaml', async () => { + expect(() => + parseAndVerifyDataStreams({ + paths: ['input-only-0.1.0/data_stream/stream1/manifest.yml'], + pkgName: 'input-only', + pkgVersion: '0.1.0', + manifests: { + 'input-only-0.1.0/data_stream/stream1/manifest.yml': Buffer.alloc(1), + }, + }) + ).toThrowError("Could not parse package manifest for data stream 'stream1': YAMLException"); + }); + + it('should throw when data stream manifest missing type', async () => { + expect(() => + parseAndVerifyDataStreams({ + paths: ['input-only-0.1.0/data_stream/stream1/manifest.yml'], + pkgName: 'input-only', + pkgVersion: '0.1.0', + manifests: { + 'input-only-0.1.0/data_stream/stream1/manifest.yml': Buffer.from( + ` + title: Custom Logs`, + 'utf8' + ), + }, + }) + ).toThrowError( + "Invalid manifest for data stream 'stream1': one or more fields missing of 'title', 'type'" + ); + }); + + it('should parse valid data stream', async () => { + expect( + parseAndVerifyDataStreams({ + paths: ['input-only-0.1.0/data_stream/stream1/manifest.yml'], + pkgName: 'input-only', + pkgVersion: '0.1.0', + manifests: { + 'input-only-0.1.0/data_stream/stream1/manifest.yml': Buffer.from( + ` + title: Custom Logs + type: logs + dataset: ds + version: 0.1.0`, + 'utf8' + ), + }, + }) + ).toEqual([ + { + dataset: 'ds', + elasticsearch: {}, + package: 'input-only', + path: 'stream1', + release: 'ga', + title: 'Custom Logs', + type: 'logs', + }, + ]); + }); +}); + +describe('parseAndVerifyStreams', () => { + it('should throw when stream manifest missing input', async () => { + expect(() => + parseAndVerifyStreams( + [ + { + title: 'stream', + }, + ], + 'input-only-0.1.0/data_stream/stream1' + ) + ).toThrowError( + 'Invalid manifest for data stream input-only-0.1.0/data_stream/stream1: stream is missing one or more fields of: input, title' + ); + }); + + it('should parse a valid stream', async () => { + expect( + parseAndVerifyStreams( + [ + { + title: 'stream', + input: 'logs', + description: 'desc', + vars: [ + { + name: 'var1', + type: 'string', + }, + ], + }, + ], + 'input-only-0.1.0/data_stream/stream1' + ) + ).toEqual([ + { + title: 'stream', + input: 'logs', + description: 'desc', + template_path: 'stream.yml.hbs', + vars: [ + { + name: 'var1', + type: 'string', + }, + ], + }, + ]); + }); +}); + +describe('parseAndVerifyVars', () => { + it('should throw when invalid var definition', () => { + expect(() => + parseAndVerifyVars( + [ + { + name: 'var1', + }, + ], + 'input-only-0.1.0/data_stream/stream1/var1' + ) + ).toThrowError( + 'Invalid var definition for input-only-0.1.0/data_stream/stream1/var1: one of mandatory fields \'name\' and \'type\' missing in var: {"name":"var1"}' + ); + }); + + it('should parse valid vars', () => { + expect( + parseAndVerifyVars( + [ + { + name: 'var1', + type: 'string', + title: 'Var', + }, + ], + 'input-only-0.1.0/data_stream/stream1/var1' + ) + ).toEqual([ + { + name: 'var1', + type: 'string', + title: 'Var', + }, + ]); + }); +}); + +describe('parseAndVerifyPolicyTemplates', () => { + it('should throw when missing mandatory fields', () => { + expect(() => + parseAndVerifyPolicyTemplates({ + policy_templates: [ + { + name: 'template1', + title: 'Template', + }, + ], + } as any) + ).toThrowError( + 'Invalid top-level manifest: one of mandatory fields \'name\', \'title\', \'description\' is missing in policy template: {"name":"template1","title":"Template"}' + ); + }); +}); + +describe('parseAndVerifyInputs', () => { + it('should throw when missing mandatory fields', () => { + expect(() => + parseAndVerifyInputs( + [ + { + type: 'logs', + }, + ], + '' + ) + ).toThrowError( + 'Invalid top-level manifest: one of mandatory fields \'type\', \'title\' missing in input: {"type":"logs"}' + ); + }); + + it('should return valid input', () => { + expect( + parseAndVerifyInputs( + [ + { + type: 'logs', + title: 'title', + vars: [ + { + name: 'var1', + type: 'string', + }, + ], + }, + ], + '' + ) + ).toEqual([{ title: 'title', type: 'logs', vars: [{ name: 'var1', type: 'string' }] }]); + }); +}); + +describe('parseAndVerifyReadme', () => { + it('should return readme path', () => { + expect( + parseAndVerifyReadme(['input-only-0.1.0/docs/README.md'], 'input-only', '0.1.0') + ).toEqual('/package/input-only/0.1.0/docs/README.md'); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/epm/archive/parse.ts b/x-pack/plugins/fleet/server/services/epm/archive/parse.ts index 87c58c31f6c36a..04ed584481205e 100644 --- a/x-pack/plugins/fleet/server/services/epm/archive/parse.ts +++ b/x-pack/plugins/fleet/server/services/epm/archive/parse.ts @@ -173,7 +173,7 @@ export async function _generatePackageInfoFromPaths( return parseAndVerifyArchive(paths, manifests, topLevelDir); } -function parseAndVerifyArchive( +export function parseAndVerifyArchive( paths: string[], manifests: ManifestMap, topLevelDirOverride?: string @@ -264,7 +264,11 @@ function parseAndVerifyArchive( return parsed; } -function parseAndVerifyReadme(paths: string[], pkgName: string, pkgVersion: string): string | null { +export function parseAndVerifyReadme( + paths: string[], + pkgName: string, + pkgVersion: string +): string | null { const readmeRelPath = `/docs/README.md`; const readmePath = `${pkgName}-${pkgVersion}${readmeRelPath}`; return paths.includes(readmePath) ? `/package/${pkgName}/${pkgVersion}${readmeRelPath}` : null; @@ -421,7 +425,9 @@ export function parseAndVerifyVars(manifestVars: any[], location: string): Regis const { name, type, ...restOfProps } = manifestVar; if (!(name && type)) { throw new PackageInvalidArchiveError( - `Invalid var definition for ${location}: one of mandatory fields 'name' and 'type' missing in var: ${manifestVar}` + `Invalid var definition for ${location}: one of mandatory fields 'name' and 'type' missing in var: ${JSON.stringify( + manifestVar + )}` ); } @@ -460,7 +466,9 @@ export function parseAndVerifyPolicyTemplates( } = policyTemplate; if (!(name && policyTemplateTitle && description)) { throw new PackageInvalidArchiveError( - `Invalid top-level manifest: one of mandatory fields 'name', 'title', 'description' is missing in policy template: ${policyTemplate}` + `Invalid top-level manifest: one of mandatory fields 'name', 'title', 'description' is missing in policy template: ${JSON.stringify( + policyTemplate + )}` ); } let parsedInputs: RegistryInput[] | undefined = []; @@ -503,7 +511,9 @@ export function parseAndVerifyInputs(manifestInputs: any, location: string): Reg const { title: inputTitle, vars, ...restOfProps } = input; if (!(input.type && inputTitle)) { throw new PackageInvalidArchiveError( - `Invalid top-level manifest: one of mandatory fields 'type', 'title' missing in input: ${input}` + `Invalid top-level manifest: one of mandatory fields 'type', 'title' missing in input: ${JSON.stringify( + input + )}` ); } const parsedVars = parseAndVerifyVars(vars, location);