From 4f5e279d75bfa04aed56f5870094bb8293c82e64 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner <56361221+jonathan-buttner@users.noreply.github.com> Date: Mon, 15 Jun 2020 17:42:18 -0400 Subject: [PATCH 01/11] Using correct index patterns (#69208) --- .../server/endpoint/routes/resolver/alerts.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/alerts.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/alerts.ts index 04171dd8137e54..7bfe101b920fb3 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/alerts.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/alerts.ts @@ -7,6 +7,7 @@ import { TypeOf } from '@kbn/config-schema'; import { RequestHandler, Logger } from 'kibana/server'; import { validateAlerts } from '../../../../common/endpoint/schema/resolver'; +import { eventsIndexPattern } from '../../../../common/endpoint/constants'; import { Fetcher } from './utils/fetch'; import { EndpointAppContext } from '../../types'; @@ -20,11 +21,9 @@ export function handleAlerts( query: { alerts, afterAlert, legacyEndpointID: endpointID }, } = req; try { - const indexRetriever = endpointAppContext.service.getIndexPatternRetriever(); const client = context.core.elasticsearch.legacy.client; - const indexPattern = await indexRetriever.getEventIndexPattern(context); - const fetcher = new Fetcher(client, id, indexPattern, endpointID); + const fetcher = new Fetcher(client, id, eventsIndexPattern, endpointID); return res.ok({ body: await fetcher.alerts(alerts, afterAlert), From 646a212b7bf550d2a557dc7c3f74e6cc0c46d559 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner <56361221+jonathan-buttner@users.noreply.github.com> Date: Mon, 15 Jun 2020 18:04:33 -0400 Subject: [PATCH 02/11] [Endpoint] [ES Archiver] Allowing create option to be passed through the cli for es archiver (#69191) * Allowing create option to be passed through the cli * Using kebab casing --- src/es_archiver/cli.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/es_archiver/cli.ts b/src/es_archiver/cli.ts index 98888b81d9a315..85e10b31a87ee3 100644 --- a/src/es_archiver/cli.ts +++ b/src/es_archiver/cli.ts @@ -67,9 +67,10 @@ cmd .action((name, indices) => execute((archiver, { raw }) => archiver.save(name, indices, { raw }))); cmd + .option('--use-create', 'use create instead of index for loading documents') .command('load ') .description('load the archive in --dir with ') - .action((name) => execute((archiver) => archiver.load(name))); + .action((name) => execute((archiver, { useCreate }) => archiver.load(name, { useCreate }))); cmd .command('unload ') From f7398f086b354e509998eabdb5468f15ec2b67ab Mon Sep 17 00:00:00 2001 From: Steph Milovic Date: Mon, 15 Jun 2020 16:18:03 -0600 Subject: [PATCH 03/11] [Security] siem to securitySolution SO migration (#68776) --- .../saved_objects/migrations.test.ts | 57 +++++++++++++++++++ .../ui_settings/saved_objects/migrations.ts | 42 ++++++++++++++ .../ui_settings/saved_objects/ui_settings.ts | 2 + .../apis/saved_objects/bulk_get.js | 1 + 4 files changed, 102 insertions(+) create mode 100644 src/core/server/ui_settings/saved_objects/migrations.test.ts create mode 100644 src/core/server/ui_settings/saved_objects/migrations.ts diff --git a/src/core/server/ui_settings/saved_objects/migrations.test.ts b/src/core/server/ui_settings/saved_objects/migrations.test.ts new file mode 100644 index 00000000000000..1620995c932c49 --- /dev/null +++ b/src/core/server/ui_settings/saved_objects/migrations.test.ts @@ -0,0 +1,57 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { SavedObjectUnsanitizedDoc } from 'kibana/server'; +import { migrations } from './migrations'; + +describe('ui_settings 7.9.0 migrations', () => { + const migration = migrations['7.9.0']; + + test('returns doc on empty object', () => { + expect(migration({} as SavedObjectUnsanitizedDoc)).toEqual({ + references: [], + }); + }); + test('properly renames siem attributes to securitySolution', () => { + const doc = { + type: 'config', + id: '8.0.0', + attributes: { + buildNum: 9007199254740991, + 'siem:defaultAnomalyScore': 59, + 'siem:enableNewsFeed': false, + }, + references: [], + updated_at: '2020-06-09T20:18:20.349Z', + migrationVersion: {}, + }; + expect(migration(doc)).toEqual({ + type: 'config', + id: '8.0.0', + attributes: { + buildNum: 9007199254740991, + 'securitySolution:defaultAnomalyScore': 59, + 'securitySolution:enableNewsFeed': false, + }, + references: [], + updated_at: '2020-06-09T20:18:20.349Z', + migrationVersion: {}, + }); + }); +}); diff --git a/src/core/server/ui_settings/saved_objects/migrations.ts b/src/core/server/ui_settings/saved_objects/migrations.ts new file mode 100644 index 00000000000000..750d4e6bc1ea6d --- /dev/null +++ b/src/core/server/ui_settings/saved_objects/migrations.ts @@ -0,0 +1,42 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { SavedObjectUnsanitizedDoc, SavedObjectSanitizedDoc } from 'kibana/server'; + +export const migrations = { + '7.9.0': (doc: SavedObjectUnsanitizedDoc): SavedObjectSanitizedDoc => ({ + ...doc, + ...(doc.attributes && { + attributes: Object.keys(doc.attributes).reduce( + (acc, key) => + key.startsWith('siem:') + ? { + ...acc, + [key.replace('siem', 'securitySolution')]: doc.attributes[key], + } + : { + ...acc, + [key]: doc.attributes[key], + }, + {} + ), + }), + references: doc.references || [], + }), +}; diff --git a/src/core/server/ui_settings/saved_objects/ui_settings.ts b/src/core/server/ui_settings/saved_objects/ui_settings.ts index 0eab40a7b3a5db..26704f46a509c4 100644 --- a/src/core/server/ui_settings/saved_objects/ui_settings.ts +++ b/src/core/server/ui_settings/saved_objects/ui_settings.ts @@ -18,6 +18,7 @@ */ import { SavedObjectsType } from '../../saved_objects'; +import { migrations } from './migrations'; export const uiSettingsType: SavedObjectsType = { name: 'config', @@ -46,4 +47,5 @@ export const uiSettingsType: SavedObjectsType = { return `Advanced Settings [${obj.id}]`; }, }, + migrations, }; diff --git a/test/api_integration/apis/saved_objects/bulk_get.js b/test/api_integration/apis/saved_objects/bulk_get.js index 23aa175740b67e..c802d529130651 100644 --- a/test/api_integration/apis/saved_objects/bulk_get.js +++ b/test/api_integration/apis/saved_objects/bulk_get.js @@ -94,6 +94,7 @@ export default function ({ getService }) { buildNum: 8467, defaultIndex: '91200a00-9efd-11e7-acb3-3dab96693fab', }, + migrationVersion: resp.body.saved_objects[2].migrationVersion, references: [], }, ], From 41e801f830f4ad7776d9e42187e514194c381ae5 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Mon, 15 Jun 2020 23:48:32 +0100 Subject: [PATCH 04/11] chore(NA): include hidden files when creating package with plugin helpers (#68247) Co-authored-by: Elastic Machine --- packages/kbn-plugin-helpers/src/tasks/build/create_package.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kbn-plugin-helpers/src/tasks/build/create_package.ts b/packages/kbn-plugin-helpers/src/tasks/build/create_package.ts index 9fa2305a94eabc..02c20049003471 100644 --- a/packages/kbn-plugin-helpers/src/tasks/build/create_package.ts +++ b/packages/kbn-plugin-helpers/src/tasks/build/create_package.ts @@ -36,7 +36,7 @@ export async function createPackage( // zip up the package await pipeline( - vfs.src(buildFiles, { cwd: buildTarget, base: buildTarget }), + vfs.src(buildFiles, { cwd: buildTarget, base: buildTarget, dot: true }), zip(`${buildId}.zip`), vfs.dest(buildTarget) ); From 7e2ab7fb823577fa05e6ae27fda7a2c4cb00c26f Mon Sep 17 00:00:00 2001 From: Tim Sullivan Date: Mon, 15 Jun 2020 16:19:25 -0700 Subject: [PATCH 05/11] [Reporting/Server] register plugin routes synchronously (#68976) * register routes synchronously * back out some refactoring * comment fix * fix tests * register route handler context provider * Add function level comments in core methods * fix tests * revert editor help * route context is the ReportingStart contract * Fix reporting job route tests * Fix generation tests Co-authored-by: Joel Griffith Co-authored-by: Elastic Machine --- x-pack/plugins/reporting/server/core.ts | 113 ++++++++++++------ .../csv/server/execute_job.test.ts | 25 ++-- .../export_types/csv/server/execute_job.ts | 2 +- .../server/lib/generate_csv_search.ts | 2 +- .../server/execute_job/index.test.ts | 36 +----- .../reporting/server/lib/jobs_query.ts | 12 +- .../plugins/reporting/server/plugin.test.ts | 3 +- x-pack/plugins/reporting/server/plugin.ts | 74 ++++++------ .../server/routes/generation.test.ts | 4 +- .../reporting/server/routes/generation.ts | 21 ++-- .../reporting/server/routes/jobs.test.ts | 4 +- .../plugins/reporting/server/routes/jobs.ts | 55 +++++++-- .../routes/lib/authorized_user_pre_routing.ts | 2 +- .../server/routes/lib/job_response_handler.ts | 21 ++-- .../create_mock_reportingplugin.ts | 81 +++++-------- .../usage/reporting_usage_collector.test.ts | 27 +++-- .../server/usage/reporting_usage_collector.ts | 14 +-- 17 files changed, 262 insertions(+), 234 deletions(-) diff --git a/x-pack/plugins/reporting/server/core.ts b/x-pack/plugins/reporting/server/core.ts index 94b138ffcae0b2..9acd359fa0db4e 100644 --- a/x-pack/plugins/reporting/server/core.ts +++ b/x-pack/plugins/reporting/server/core.ts @@ -5,7 +5,7 @@ */ import * as Rx from 'rxjs'; -import { first, map, mapTo } from 'rxjs/operators'; +import { first, map, take } from 'rxjs/operators'; import { BasePath, ElasticsearchServiceSetup, @@ -33,7 +33,8 @@ export interface ReportingInternalSetup { security?: SecurityPluginSetup; } -interface ReportingInternalStart { +export interface ReportingInternalStart { + browserDriverFactory: HeadlessChromiumDriverFactory; enqueueJob: EnqueueJobFn; esqueue: ESQueueInstance; savedObjects: SavedObjectsServiceStart; @@ -43,33 +44,83 @@ interface ReportingInternalStart { export class ReportingCore { private pluginSetupDeps?: ReportingInternalSetup; private pluginStartDeps?: ReportingInternalStart; - private browserDriverFactory?: HeadlessChromiumDriverFactory; - private readonly pluginSetup$ = new Rx.ReplaySubject(); - private readonly pluginStart$ = new Rx.ReplaySubject(); + private readonly pluginSetup$ = new Rx.ReplaySubject(); // observe async background setupDeps and config each are done + private readonly pluginStart$ = new Rx.ReplaySubject(); // observe async background startDeps private exportTypesRegistry = getExportTypesRegistry(); + private config?: ReportingConfig; - constructor(private config: ReportingConfig) {} + constructor() {} - public pluginSetup(reportingSetupDeps: ReportingInternalSetup) { - this.pluginSetupDeps = reportingSetupDeps; - this.pluginSetup$.next(reportingSetupDeps); + /* + * Register setupDeps + */ + public pluginSetup(setupDeps: ReportingInternalSetup) { + this.pluginSetup$.next(true); // trigger the observer + this.pluginSetupDeps = setupDeps; // cache } - public pluginStart(reportingStartDeps: ReportingInternalStart) { - this.pluginStart$.next(reportingStartDeps); + /* + * Register startDeps + */ + public pluginStart(startDeps: ReportingInternalStart) { + this.pluginStart$.next(startDeps); // trigger the observer + this.pluginStartDeps = startDeps; // cache } - public pluginHasStarted(): Promise { - return this.pluginStart$.pipe(first(), mapTo(true)).toPromise(); + /* + * Blocks the caller until setup is done + */ + public async pluginSetsUp(): Promise { + // use deps and config as a cached resolver + if (this.pluginSetupDeps && this.config) { + return true; + } + return await this.pluginSetup$.pipe(take(2)).toPromise(); // once for pluginSetupDeps (sync) and twice for config (async) } - public setBrowserDriverFactory(browserDriverFactory: HeadlessChromiumDriverFactory) { - this.browserDriverFactory = browserDriverFactory; + /* + * Blocks the caller until start is done + */ + public async pluginStartsUp(): Promise { + return await this.getPluginStartDeps().then(() => true); + } + + /* + * Synchronously checks if all async background setup and startup is completed + */ + public pluginIsStarted() { + return this.pluginSetupDeps != null && this.config != null && this.pluginStartDeps != null; } /* - * Internal module dependencies + * Allows config to be set in the background */ + public setConfig(config: ReportingConfig) { + this.config = config; + this.pluginSetup$.next(true); + } + + /* + * Gives synchronous access to the config + */ + public getConfig(): ReportingConfig { + if (!this.config) { + throw new Error('Config is not yet initialized'); + } + return this.config; + } + + /* + * Gives async access to the startDeps + */ + private async getPluginStartDeps() { + if (this.pluginStartDeps) { + return this.pluginStartDeps; + } + + return await this.pluginStart$.pipe(first()).toPromise(); + } + public getExportTypesRegistry() { return this.exportTypesRegistry; } @@ -92,18 +143,15 @@ export class ReportingCore { .toPromise(); } - public getConfig(): ReportingConfig { - return this.config; - } - - public getScreenshotsObservable(): ScreenshotsObservableFn { - const { browserDriverFactory } = this; - if (!browserDriverFactory) { - throw new Error(`"browserDriverFactory" dependency hasn't initialized yet`); - } - return screenshotsObservableFactory(this.config.get('capture'), browserDriverFactory); + public async getScreenshotsObservable(): Promise { + const config = this.getConfig(); + const { browserDriverFactory } = await this.getPluginStartDeps(); + return screenshotsObservableFactory(config.get('capture'), browserDriverFactory); } + /* + * Gives synchronous access to the setupDeps + */ public getPluginSetupDeps() { if (!this.pluginSetupDeps) { throw new Error(`"pluginSetupDeps" dependencies haven't initialized yet`); @@ -111,18 +159,7 @@ export class ReportingCore { return this.pluginSetupDeps; } - /* - * Outside dependencies - */ - - private async getPluginStartDeps() { - if (this.pluginStartDeps) { - return this.pluginStartDeps; - } - return await this.pluginStart$.pipe(first()).toPromise(); - } - - public async getElasticsearchService() { + public getElasticsearchService() { return this.getPluginSetupDeps().elasticsearch; } diff --git a/x-pack/plugins/reporting/server/export_types/csv/server/execute_job.test.ts b/x-pack/plugins/reporting/server/export_types/csv/server/execute_job.test.ts index ddcf94079ade42..4ce448e953bd17 100644 --- a/x-pack/plugins/reporting/server/export_types/csv/server/execute_job.test.ts +++ b/x-pack/plugins/reporting/server/export_types/csv/server/execute_job.test.ts @@ -5,10 +5,16 @@ */ import nodeCrypto from '@elastic/node-crypto'; +import { IUiSettingsClient, ElasticsearchServiceSetup } from 'kibana/server'; // @ts-ignore import Puid from 'puid'; import sinon from 'sinon'; +import { ReportingConfig, ReportingCore } from '../../../'; import { fieldFormats, UI_SETTINGS } from '../../../../../../../src/plugins/data/server'; +import { + CSV_QUOTE_VALUES_SETTING, + CSV_SEPARATOR_SETTING, +} from '../../../../../../../src/plugins/share/server'; import { CancellationToken } from '../../../../common'; import { CSV_BOM_CHARS } from '../../../../common/constants'; import { LevelLogger } from '../../../lib'; @@ -16,10 +22,6 @@ import { setFieldFormats } from '../../../services'; import { createMockReportingCore } from '../../../test_helpers'; import { JobDocPayloadDiscoverCsv } from '../types'; import { executeJobFactory } from './execute_job'; -import { - CSV_SEPARATOR_SETTING, - CSV_QUOTE_VALUES_SETTING, -} from '../../../../../../../src/plugins/share/server'; const delay = (ms: number) => new Promise((resolve) => setTimeout(() => resolve(), ms)); @@ -48,8 +50,8 @@ describe('CSV Execute Job', function () { let clusterStub: any; let configGetStub: any; - let mockReportingConfig: any; - let mockReportingCore: any; + let mockReportingConfig: ReportingConfig; + let mockReportingCore: ReportingCore; let callAsCurrentUserStub: any; let cancellationToken: any; @@ -78,9 +80,11 @@ describe('CSV Execute Job', function () { mockReportingConfig = { get: configGetStub, kbnConfig: { get: configGetStub } }; mockReportingCore = await createMockReportingCore(mockReportingConfig); - mockReportingCore.getUiSettingsServiceFactory = () => Promise.resolve(mockUiSettingsClient); - mockReportingCore.getElasticsearchService = () => Promise.resolve(mockElasticsearch); - mockReportingCore.config = mockReportingConfig; + mockReportingCore.getUiSettingsServiceFactory = () => + Promise.resolve((mockUiSettingsClient as unknown) as IUiSettingsClient); + mockReportingCore.getElasticsearchService = () => + mockElasticsearch as ElasticsearchServiceSetup; + mockReportingCore.setConfig(mockReportingConfig); cancellationToken = new CancellationToken(); @@ -995,7 +999,8 @@ describe('CSV Execute Job', function () { let maxSizeReached: boolean; beforeEach(async function () { - mockReportingCore.getUiSettingsServiceFactory = () => mockUiSettingsClient; + mockReportingCore.getUiSettingsServiceFactory = () => + Promise.resolve((mockUiSettingsClient as unknown) as IUiSettingsClient); configGetStub.withArgs('csv', 'maxSizeBytes').returns(18); callAsCurrentUserStub.onFirstCall().returns({ diff --git a/x-pack/plugins/reporting/server/export_types/csv/server/execute_job.ts b/x-pack/plugins/reporting/server/export_types/csv/server/execute_job.ts index 4b17cc669efe1b..91a4db0469fb53 100644 --- a/x-pack/plugins/reporting/server/export_types/csv/server/execute_job.ts +++ b/x-pack/plugins/reporting/server/export_types/csv/server/execute_job.ts @@ -33,7 +33,7 @@ export const executeJobFactory: ExecuteJobFactory callAsCurrentUser(...params); const uiSettings = await getUiSettings(uiConfig); diff --git a/x-pack/plugins/reporting/server/export_types/printable_pdf/server/execute_job/index.test.ts b/x-pack/plugins/reporting/server/export_types/printable_pdf/server/execute_job/index.test.ts index b8e1e5eebd9e79..2f4ca47cf739e9 100644 --- a/x-pack/plugins/reporting/server/export_types/printable_pdf/server/execute_job/index.test.ts +++ b/x-pack/plugins/reporting/server/export_types/printable_pdf/server/execute_job/index.test.ts @@ -84,6 +84,7 @@ test(`passes browserTimezone to generatePdf`, async () => { await executeJob( 'pdfJobId', getJobDocPayload({ + title: 'PDF Params Timezone Test', relativeUrl: '/app/kibana#/something', browserTimezone, headers: encryptedHeaders, @@ -91,39 +92,8 @@ test(`passes browserTimezone to generatePdf`, async () => { cancellationToken ); - expect(generatePdfObservable.mock.calls).toMatchInlineSnapshot(` - Array [ - Array [ - LevelLogger { - "_logger": Object { - "get": [MockFunction], - }, - "_tags": Array [ - "printable_pdf", - "execute", - "pdfJobId", - ], - "warning": [Function], - }, - undefined, - Array [ - "http://localhost:5601/sbp/app/kibana#/something", - ], - "UTC", - Object { - "conditions": Object { - "basePath": "/sbp", - "hostname": "localhost", - "port": 5601, - "protocol": "http", - }, - "headers": Object {}, - }, - undefined, - false, - ], - ] - `); + const tzParam = generatePdfObservable.mock.calls[0][3]; + expect(tzParam).toBe('UTC'); }); test(`returns content_type of application/pdf`, async () => { diff --git a/x-pack/plugins/reporting/server/lib/jobs_query.ts b/x-pack/plugins/reporting/server/lib/jobs_query.ts index 8784d8ff35d251..f4670847260ee6 100644 --- a/x-pack/plugins/reporting/server/lib/jobs_query.ts +++ b/x-pack/plugins/reporting/server/lib/jobs_query.ts @@ -6,10 +6,9 @@ import { i18n } from '@kbn/i18n'; import { errors as elasticsearchErrors } from 'elasticsearch'; -import { ElasticsearchServiceSetup } from 'kibana/server'; import { get } from 'lodash'; +import { ReportingCore } from '../'; import { AuthenticatedUser } from '../../../security/server'; -import { ReportingConfig } from '../'; import { JobSource } from '../types'; const esErrors = elasticsearchErrors as Record; @@ -42,11 +41,8 @@ interface CountAggResult { const getUsername = (user: AuthenticatedUser | null) => (user ? user.username : false); -export function jobsQueryFactory( - config: ReportingConfig, - elasticsearch: ElasticsearchServiceSetup -) { - const index = config.get('index'); +export function jobsQueryFactory(reportingCore: ReportingCore) { + const { elasticsearch } = reportingCore.getPluginSetupDeps(); const { callAsInternalUser } = elasticsearch.legacy.client; function execQuery(queryType: string, body: QueryBody) { @@ -60,6 +56,8 @@ export function jobsQueryFactory( }, }; + const config = reportingCore.getConfig(); + const index = config.get('index'); const query = { index: `${index}-*`, body: Object.assign(defaultBody[queryType] || {}, body), diff --git a/x-pack/plugins/reporting/server/plugin.test.ts b/x-pack/plugins/reporting/server/plugin.test.ts index b2bcd6b9c97ce3..420fa8347cdebb 100644 --- a/x-pack/plugins/reporting/server/plugin.test.ts +++ b/x-pack/plugins/reporting/server/plugin.test.ts @@ -3,6 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ + jest.mock('./browsers/install', () => ({ installBrowser: jest.fn().mockImplementation(() => ({ binaryPath$: { @@ -62,10 +63,10 @@ describe('Reporting Plugin', () => { }); it('logs setup issues', async () => { + initContext.config = null; const plugin = new ReportingPlugin(initContext); // @ts-ignore overloading error logger plugin.logger.error = jest.fn(); - coreSetup.elasticsearch = null; plugin.setup(coreSetup, pluginSetup); await sleep(5); diff --git a/x-pack/plugins/reporting/server/plugin.ts b/x-pack/plugins/reporting/server/plugin.ts index a3c89c7b8a8ce1..693b0917603fcd 100644 --- a/x-pack/plugins/reporting/server/plugin.ts +++ b/x-pack/plugins/reporting/server/plugin.ts @@ -4,7 +4,6 @@ * you may not use this file except in compliance with the Elastic License. */ -import * as Rx from 'rxjs'; import { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from 'src/core/server'; import { ReportingCore } from './'; import { initializeBrowserDriverFactory } from './browsers'; @@ -15,47 +14,57 @@ import { setFieldFormats } from './services'; import { ReportingSetup, ReportingSetupDeps, ReportingStart, ReportingStartDeps } from './types'; import { registerReportingUsageCollector } from './usage'; +declare module 'src/core/server' { + interface RequestHandlerContext { + reporting?: ReportingStart | null; + } +} + export class ReportingPlugin implements Plugin { private readonly initializerContext: PluginInitializerContext; private logger: LevelLogger; - private reportingCore?: ReportingCore; - - // Setup some observables for modules that need to await setup/start - public readonly setup$ = new Rx.Subject(); - public readonly start$ = new Rx.Subject(); + private reportingCore: ReportingCore; constructor(context: PluginInitializerContext) { this.logger = new LevelLogger(context.logger.get()); this.initializerContext = context; + this.reportingCore = new ReportingCore(); } public setup(core: CoreSetup, plugins: ReportingSetupDeps) { + // prevent throwing errors in route handlers about async deps not being initialized + core.http.registerRouteHandlerContext('reporting', () => { + if (this.reportingCore.pluginIsStarted()) { + return {}; // ReportingStart contract + } else { + return null; + } + }); + const { elasticsearch, http } = core; const { licensing, security } = plugins; - const { initializerContext: initContext } = this; + const { initializerContext: initContext, reportingCore } = this; + const router = http.createRouter(); const basePath = http.basePath.get; + reportingCore.pluginSetup({ + elasticsearch, + licensing, + basePath, + router, + security, + }); + + registerReportingUsageCollector(reportingCore, plugins); + registerRoutes(reportingCore, this.logger); + // async background setup (async () => { const config = await buildConfig(initContext, core, this.logger); - const reportingCore = new ReportingCore(config); - - reportingCore.pluginSetup({ - elasticsearch, - licensing, - basePath, - router, - security, - }); - - registerReportingUsageCollector(reportingCore, plugins); - registerRoutes(reportingCore, this.logger); - this.reportingCore = reportingCore; - + reportingCore.setConfig(config); this.logger.debug('Setup complete'); - this.setup$.next(true); })().catch((e) => { this.logger.error(`Error in Reporting setup, reporting may not function properly`); this.logger.error(e); @@ -68,20 +77,21 @@ export class ReportingPlugin // use data plugin for csv formats setFieldFormats(plugins.data.fieldFormats); - const { logger } = this; - const reportingCore = this.getReportingCore(); - const config = reportingCore.getConfig(); + const { logger, reportingCore } = this; const { elasticsearch } = reportingCore.getPluginSetupDeps(); // async background start (async () => { + await this.reportingCore.pluginSetsUp(); + const config = reportingCore.getConfig(); + const browserDriverFactory = await initializeBrowserDriverFactory(config, logger); - reportingCore.setBrowserDriverFactory(browserDriverFactory); - const esqueue = await createQueueFactory(reportingCore, logger); - const enqueueJob = enqueueJobFactory(reportingCore, logger); + const esqueue = await createQueueFactory(reportingCore, logger); // starts polling for pending jobs + const enqueueJob = enqueueJobFactory(reportingCore, logger); // called from generation routes reportingCore.pluginStart({ + browserDriverFactory, savedObjects: core.savedObjects, uiSettings: core.uiSettings, esqueue, @@ -92,7 +102,6 @@ export class ReportingPlugin runValidations(config, elasticsearch, browserDriverFactory, this.logger); this.logger.debug('Start complete'); - this.start$.next(true); })().catch((e) => { this.logger.error(`Error in Reporting start, reporting may not function properly`); this.logger.error(e); @@ -100,11 +109,4 @@ export class ReportingPlugin return {}; } - - public getReportingCore() { - if (!this.reportingCore) { - throw new Error('Setup is not ready'); - } - return this.reportingCore; - } } diff --git a/x-pack/plugins/reporting/server/routes/generation.test.ts b/x-pack/plugins/reporting/server/routes/generation.test.ts index f9b3e5446cfce6..4474f2c95e1c31 100644 --- a/x-pack/plugins/reporting/server/routes/generation.test.ts +++ b/x-pack/plugins/reporting/server/routes/generation.test.ts @@ -18,6 +18,7 @@ import { of } from 'rxjs'; type setupServerReturn = UnwrapPromise>; describe('POST /api/reporting/generate', () => { + const reportingSymbol = Symbol('reporting'); let server: setupServerReturn['server']; let httpSetup: setupServerReturn['httpSetup']; let exportTypesRegistry: ExportTypesRegistry; @@ -47,7 +48,8 @@ describe('POST /api/reporting/generate', () => { } as unknown) as jest.Mocked; beforeEach(async () => { - ({ server, httpSetup } = await setupServer()); + ({ server, httpSetup } = await setupServer(reportingSymbol)); + httpSetup.registerRouteHandlerContext(reportingSymbol, 'reporting', () => ({})); const mockDeps = ({ elasticsearch: { legacy: { diff --git a/x-pack/plugins/reporting/server/routes/generation.ts b/x-pack/plugins/reporting/server/routes/generation.ts index f2e616c0803a78..b4c81e698ce71f 100644 --- a/x-pack/plugins/reporting/server/routes/generation.ts +++ b/x-pack/plugins/reporting/server/routes/generation.ts @@ -17,15 +17,21 @@ import { HandlerFunction } from './types'; const esErrors = elasticsearchErrors as Record; -export function registerJobGenerationRoutes(reporting: ReportingCore, logger: Logger) { +const getDownloadBaseUrl = (reporting: ReportingCore) => { const config = reporting.getConfig(); - const downloadBaseUrl = - config.kbnConfig.get('server', 'basePath') + `${API_BASE_URL}/jobs/download`; + return config.kbnConfig.get('server', 'basePath') + `${API_BASE_URL}/jobs/download`; +}; +export function registerJobGenerationRoutes(reporting: ReportingCore, logger: Logger) { /* * Generates enqueued job details to use in responses */ const handler: HandlerFunction = async (user, exportTypeId, jobParams, context, req, res) => { + // ensure the async dependencies are loaded + if (!context.reporting) { + return res.custom({ statusCode: 503, body: 'Not Available' }); + } + const licenseInfo = await reporting.getLicenseInfo(); const licenseResults = licenseInfo[exportTypeId]; @@ -42,6 +48,7 @@ export function registerJobGenerationRoutes(reporting: ReportingCore, logger: Lo // return the queue's job information const jobJson = job.toJSON(); + const downloadBaseUrl = getDownloadBaseUrl(reporting); return res.ok({ headers: { @@ -86,10 +93,6 @@ export function registerJobGenerationRoutes(reporting: ReportingCore, logger: Lo } registerGenerateFromJobParams(reporting, handler, handleError); - - // Register beta panel-action download-related API's - if (config.get('csv', 'enablePanelActionDownload')) { - registerGenerateCsvFromSavedObject(reporting, handler, handleError); - registerGenerateCsvFromSavedObjectImmediate(reporting, handleError, logger); - } + registerGenerateCsvFromSavedObject(reporting, handler, handleError); // FIXME: remove this https://github.com/elastic/kibana/issues/62986 + registerGenerateCsvFromSavedObjectImmediate(reporting, handleError, logger); } diff --git a/x-pack/plugins/reporting/server/routes/jobs.test.ts b/x-pack/plugins/reporting/server/routes/jobs.test.ts index 22d60d62d5fdb5..35594474685b0d 100644 --- a/x-pack/plugins/reporting/server/routes/jobs.test.ts +++ b/x-pack/plugins/reporting/server/routes/jobs.test.ts @@ -19,6 +19,7 @@ import { registerJobInfoRoutes } from './jobs'; type setupServerReturn = UnwrapPromise>; describe('GET /api/reporting/jobs/download', () => { + const reportingSymbol = Symbol('reporting'); let server: setupServerReturn['server']; let httpSetup: setupServerReturn['httpSetup']; let exportTypesRegistry: ExportTypesRegistry; @@ -39,7 +40,8 @@ describe('GET /api/reporting/jobs/download', () => { }; beforeEach(async () => { - ({ server, httpSetup } = await setupServer()); + ({ server, httpSetup } = await setupServer(reportingSymbol)); + httpSetup.registerRouteHandlerContext(reportingSymbol, 'reporting', () => ({})); core = await createMockReportingCore(config, ({ elasticsearch: { legacy: { client: { callAsInternalUser: jest.fn() } }, diff --git a/x-pack/plugins/reporting/server/routes/jobs.ts b/x-pack/plugins/reporting/server/routes/jobs.ts index 29cf55bc5c72ec..90185f0736ed81 100644 --- a/x-pack/plugins/reporting/server/routes/jobs.ts +++ b/x-pack/plugins/reporting/server/routes/jobs.ts @@ -4,16 +4,16 @@ * you may not use this file except in compliance with the Elastic License. */ -import Boom from 'boom'; import { schema } from '@kbn/config-schema'; +import Boom from 'boom'; import { ReportingCore } from '../'; import { API_BASE_URL } from '../../common/constants'; import { jobsQueryFactory } from '../lib/jobs_query'; +import { authorizedUserPreRoutingFactory } from './lib/authorized_user_pre_routing'; import { deleteJobResponseHandlerFactory, downloadJobResponseHandlerFactory, } from './lib/job_response_handler'; -import { authorizedUserPreRoutingFactory } from './lib/authorized_user_pre_routing'; interface ListQuery { page: string; @@ -22,12 +22,14 @@ interface ListQuery { } const MAIN_ENTRY = `${API_BASE_URL}/jobs`; +const handleUnavailable = (res: any) => { + return res.custom({ statusCode: 503, body: 'Not Available' }); +}; + export function registerJobInfoRoutes(reporting: ReportingCore) { - const config = reporting.getConfig(); const setupDeps = reporting.getPluginSetupDeps(); const userHandler = authorizedUserPreRoutingFactory(reporting); - const { elasticsearch, router } = setupDeps; - const jobsQuery = jobsQueryFactory(config, elasticsearch); + const { router } = setupDeps; // list jobs in the queue, paginated router.get( @@ -36,6 +38,11 @@ export function registerJobInfoRoutes(reporting: ReportingCore) { validate: false, }, userHandler(async (user, context, req, res) => { + // ensure the async dependencies are loaded + if (!context.reporting) { + return handleUnavailable(res); + } + const { management: { jobTypes = [] }, } = await reporting.getLicenseInfo(); @@ -47,6 +54,7 @@ export function registerJobInfoRoutes(reporting: ReportingCore) { const page = parseInt(queryPage, 10) || 0; const size = Math.min(100, parseInt(querySize, 10) || 10); const jobIds = queryIds ? queryIds.split(',') : null; + const jobsQuery = jobsQueryFactory(reporting); const results = await jobsQuery.list(jobTypes, user, page, size, jobIds); return res.ok({ @@ -65,10 +73,16 @@ export function registerJobInfoRoutes(reporting: ReportingCore) { validate: false, }, userHandler(async (user, context, req, res) => { + // ensure the async dependencies are loaded + if (!context.reporting) { + return handleUnavailable(res); + } + const { management: { jobTypes = [] }, } = await reporting.getLicenseInfo(); + const jobsQuery = jobsQueryFactory(reporting); const count = await jobsQuery.count(jobTypes, user); return res.ok({ @@ -91,11 +105,17 @@ export function registerJobInfoRoutes(reporting: ReportingCore) { }, }, userHandler(async (user, context, req, res) => { + // ensure the async dependencies are loaded + if (!context.reporting) { + return handleUnavailable(res); + } + const { docId } = req.params as { docId: string }; const { management: { jobTypes = [] }, } = await reporting.getLicenseInfo(); + const jobsQuery = jobsQueryFactory(reporting); const result = await jobsQuery.get(user, docId, { includeContent: true }); if (!result) { @@ -130,11 +150,17 @@ export function registerJobInfoRoutes(reporting: ReportingCore) { }, }, userHandler(async (user, context, req, res) => { + // ensure the async dependencies are loaded + if (!context.reporting) { + return res.custom({ statusCode: 503 }); + } + const { docId } = req.params as { docId: string }; const { management: { jobTypes = [] }, } = await reporting.getLicenseInfo(); + const jobsQuery = jobsQueryFactory(reporting); const result = await jobsQuery.get(user, docId); if (!result) { @@ -164,12 +190,7 @@ export function registerJobInfoRoutes(reporting: ReportingCore) { ); // trigger a download of the output from a job - const exportTypesRegistry = reporting.getExportTypesRegistry(); - const downloadResponseHandler = downloadJobResponseHandlerFactory( - config, - elasticsearch, - exportTypesRegistry - ); + const downloadResponseHandler = downloadJobResponseHandlerFactory(reporting); router.get( { @@ -181,6 +202,11 @@ export function registerJobInfoRoutes(reporting: ReportingCore) { }, }, userHandler(async (user, context, req, res) => { + // ensure the async dependencies are loaded + if (!context.reporting) { + return handleUnavailable(res); + } + const { docId } = req.params as { docId: string }; const { management: { jobTypes = [] }, @@ -191,7 +217,7 @@ export function registerJobInfoRoutes(reporting: ReportingCore) { ); // allow a report to be deleted - const deleteResponseHandler = deleteJobResponseHandlerFactory(config, elasticsearch); + const deleteResponseHandler = deleteJobResponseHandlerFactory(reporting); router.delete( { path: `${MAIN_ENTRY}/delete/{docId}`, @@ -202,6 +228,11 @@ export function registerJobInfoRoutes(reporting: ReportingCore) { }, }, userHandler(async (user, context, req, res) => { + // ensure the async dependencies are loaded + if (!context.reporting) { + return handleUnavailable(res); + } + const { docId } = req.params as { docId: string }; const { management: { jobTypes = [] }, diff --git a/x-pack/plugins/reporting/server/routes/lib/authorized_user_pre_routing.ts b/x-pack/plugins/reporting/server/routes/lib/authorized_user_pre_routing.ts index 2ad974c9dd8e18..2f5d4ebe1419aa 100644 --- a/x-pack/plugins/reporting/server/routes/lib/authorized_user_pre_routing.ts +++ b/x-pack/plugins/reporting/server/routes/lib/authorized_user_pre_routing.ts @@ -19,7 +19,6 @@ export type RequestHandlerUser = RequestHandler extends (...a: infer U) => infer export const authorizedUserPreRoutingFactory = function authorizedUserPreRoutingFn( reporting: ReportingCore ) { - const config = reporting.getConfig(); const setupDeps = reporting.getPluginSetupDeps(); const getUser = getUserFactory(setupDeps.security); return (handler: RequestHandlerUser): RequestHandler => { @@ -36,6 +35,7 @@ export const authorizedUserPreRoutingFactory = function authorizedUserPreRouting if (user) { // check allowance with the configured set of roleas + "superuser" + const config = reporting.getConfig(); const allowedRoles = config.get('roles', 'allow') || []; const authorizedRoles = [superuserRole, ...allowedRoles]; diff --git a/x-pack/plugins/reporting/server/routes/lib/job_response_handler.ts b/x-pack/plugins/reporting/server/routes/lib/job_response_handler.ts index 1a2e10cf355a2c..a8492481e6b135 100644 --- a/x-pack/plugins/reporting/server/routes/lib/job_response_handler.ts +++ b/x-pack/plugins/reporting/server/routes/lib/job_response_handler.ts @@ -4,11 +4,10 @@ * you may not use this file except in compliance with the Elastic License. */ -import { ElasticsearchServiceSetup, kibanaResponseFactory } from 'kibana/server'; +import { kibanaResponseFactory } from 'kibana/server'; +import { ReportingCore } from '../../'; import { AuthenticatedUser } from '../../../../security/server'; -import { ReportingConfig } from '../../'; import { WHITELISTED_JOB_CONTENT_TYPES } from '../../../common/constants'; -import { ExportTypesRegistry } from '../../lib/export_types_registry'; import { jobsQueryFactory } from '../../lib/jobs_query'; import { getDocumentPayloadFactory } from './get_document_payload'; @@ -20,12 +19,9 @@ interface JobResponseHandlerOpts { excludeContent?: boolean; } -export function downloadJobResponseHandlerFactory( - config: ReportingConfig, - elasticsearch: ElasticsearchServiceSetup, - exportTypesRegistry: ExportTypesRegistry -) { - const jobsQuery = jobsQueryFactory(config, elasticsearch); +export function downloadJobResponseHandlerFactory(reporting: ReportingCore) { + const jobsQuery = jobsQueryFactory(reporting); + const exportTypesRegistry = reporting.getExportTypesRegistry(); const getDocumentPayload = getDocumentPayloadFactory(exportTypesRegistry); return async function jobResponseHandler( @@ -69,11 +65,8 @@ export function downloadJobResponseHandlerFactory( }; } -export function deleteJobResponseHandlerFactory( - config: ReportingConfig, - elasticsearch: ElasticsearchServiceSetup -) { - const jobsQuery = jobsQueryFactory(config, elasticsearch); +export function deleteJobResponseHandlerFactory(reporting: ReportingCore) { + const jobsQuery = jobsQueryFactory(reporting); return async function deleteJobResponseHander( res: typeof kibanaResponseFactory, diff --git a/x-pack/plugins/reporting/server/test_helpers/create_mock_reportingplugin.ts b/x-pack/plugins/reporting/server/test_helpers/create_mock_reportingplugin.ts index 669381a92c5227..579035a46f615e 100644 --- a/x-pack/plugins/reporting/server/test_helpers/create_mock_reportingplugin.ts +++ b/x-pack/plugins/reporting/server/test_helpers/create_mock_reportingplugin.ts @@ -11,18 +11,15 @@ jest.mock('../lib/create_queue'); jest.mock('../lib/enqueue_job'); jest.mock('../lib/validate'); -import { of } from 'rxjs'; -import { first } from 'rxjs/operators'; -import { coreMock } from 'src/core/server/mocks'; +import * as Rx from 'rxjs'; import { ReportingConfig, ReportingCore } from '../'; import { chromium, HeadlessChromiumDriverFactory, initializeBrowserDriverFactory, } from '../browsers'; -import { ReportingInternalSetup } from '../core'; -import { ReportingPlugin } from '../plugin'; -import { ReportingSetupDeps, ReportingStartDeps } from '../types'; +import { ReportingInternalSetup, ReportingInternalStart } from '../core'; +import { ReportingStartDeps } from '../types'; (initializeBrowserDriverFactory as jest.Mock< Promise @@ -30,32 +27,30 @@ import { ReportingSetupDeps, ReportingStartDeps } from '../types'; (chromium as any).createDriverFactory.mockImplementation(() => ({})); -const createMockSetupDeps = (setupMock?: any): ReportingSetupDeps => { +const createMockPluginSetup = (setupMock?: any): ReportingInternalSetup => { return { + elasticsearch: setupMock.elasticsearch || { legacy: { client: {} } }, + basePath: setupMock.basePath, + router: setupMock.router, security: setupMock.security, - licensing: { - license$: of({ isAvailable: true, isActive: true, type: 'basic' }), - } as any, - usageCollection: { - makeUsageCollector: jest.fn(), - registerCollector: jest.fn(), - } as any, + licensing: { license$: Rx.of({ isAvailable: true, isActive: true, type: 'basic' }) } as any, + }; +}; + +const createMockPluginStart = (startMock?: any): ReportingInternalStart => { + return { + browserDriverFactory: startMock.browserDriverFactory, + enqueueJob: startMock.enqueueJob, + esqueue: startMock.esqueue, + savedObjects: startMock.savedObjects || { getScopedClient: jest.fn() }, + uiSettings: startMock.uiSettings || { asScopedToClient: () => ({ get: jest.fn() }) }, }; }; export const createMockConfigSchema = (overrides?: any) => ({ index: '.reporting', - kibanaServer: { - hostname: 'localhost', - port: '80', - }, - capture: { - browser: { - chromium: { - disableSandbox: true, - }, - }, - }, + kibanaServer: { hostname: 'localhost', port: '80' }, + capture: { browser: { chromium: { disableSandbox: true } } }, ...overrides, }); @@ -63,36 +58,20 @@ export const createMockStartDeps = (startMock?: any): ReportingStartDeps => ({ data: startMock.data, }); -const createMockReportingPlugin = async (config: ReportingConfig): Promise => { - const mockConfigSchema = createMockConfigSchema(config); - const plugin = new ReportingPlugin(coreMock.createPluginInitializerContext(mockConfigSchema)); - const setupMock = coreMock.createSetup(); - const coreStartMock = coreMock.createStart(); - const startMock = { - ...coreStartMock, - data: { fieldFormats: {} }, - }; - - plugin.setup(setupMock, createMockSetupDeps(setupMock)); - await plugin.setup$.pipe(first()).toPromise(); - plugin.start(startMock, createMockStartDeps(startMock)); - await plugin.start$.pipe(first()).toPromise(); - - return plugin; -}; - export const createMockReportingCore = async ( config: ReportingConfig, - setupDepsMock?: ReportingInternalSetup -): Promise => { + setupDepsMock: ReportingInternalSetup | undefined = createMockPluginSetup({}), + startDepsMock: ReportingInternalStart | undefined = createMockPluginStart({}) +) => { config = config || {}; - const plugin = await createMockReportingPlugin(config); - const core = plugin.getReportingCore(); + const core = new ReportingCore(); + + core.pluginSetup(setupDepsMock); + core.setConfig(config); + await core.pluginSetsUp(); - if (setupDepsMock) { - // @ts-ignore overwriting private properties - core.pluginSetupDeps = setupDepsMock; - } + core.pluginStart(startDepsMock); + await core.pluginStartsUp(); return core; }; diff --git a/x-pack/plugins/reporting/server/usage/reporting_usage_collector.test.ts b/x-pack/plugins/reporting/server/usage/reporting_usage_collector.test.ts index d5dccaca3042a0..ed2abef2542deb 100644 --- a/x-pack/plugins/reporting/server/usage/reporting_usage_collector.test.ts +++ b/x-pack/plugins/reporting/server/usage/reporting_usage_collector.test.ts @@ -7,7 +7,7 @@ import * as Rx from 'rxjs'; import sinon from 'sinon'; import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; -import { ReportingConfig } from '../'; +import { ReportingConfig, ReportingCore } from '../'; import { createMockReportingCore } from '../test_helpers'; import { getExportTypesRegistry } from '../lib/export_types_registry'; import { ReportingSetupDeps } from '../types'; @@ -62,8 +62,10 @@ const getResponseMock = (base = {}) => base; describe('license checks', () => { let mockConfig: ReportingConfig; + let mockCore: ReportingCore; beforeAll(async () => { mockConfig = getMockReportingConfig(); + mockCore = await createMockReportingCore(mockConfig); }); describe('with a basic license', () => { @@ -72,7 +74,7 @@ describe('license checks', () => { const plugins = getPluginsMock({ license: 'basic' }); const callClusterMock = jest.fn(() => Promise.resolve(getResponseMock())); const { fetch } = getReportingUsageCollector( - mockConfig, + mockCore, plugins.usageCollection, getLicenseMock('basic'), exportTypesRegistry, @@ -102,7 +104,7 @@ describe('license checks', () => { const plugins = getPluginsMock({ license: 'none' }); const callClusterMock = jest.fn(() => Promise.resolve(getResponseMock())); const { fetch } = getReportingUsageCollector( - mockConfig, + mockCore, plugins.usageCollection, getLicenseMock('none'), exportTypesRegistry, @@ -132,7 +134,7 @@ describe('license checks', () => { const plugins = getPluginsMock({ license: 'platinum' }); const callClusterMock = jest.fn(() => Promise.resolve(getResponseMock())); const { fetch } = getReportingUsageCollector( - mockConfig, + mockCore, plugins.usageCollection, getLicenseMock('platinum'), exportTypesRegistry, @@ -162,7 +164,7 @@ describe('license checks', () => { const plugins = getPluginsMock({ license: 'basic' }); const callClusterMock = jest.fn(() => Promise.resolve({})); const { fetch } = getReportingUsageCollector( - mockConfig, + mockCore, plugins.usageCollection, getLicenseMock('basic'), exportTypesRegistry, @@ -184,11 +186,16 @@ describe('license checks', () => { }); describe('data modeling', () => { + let mockConfig: ReportingConfig; + let mockCore: ReportingCore; + beforeAll(async () => { + mockConfig = getMockReportingConfig(); + mockCore = await createMockReportingCore(mockConfig); + }); test('with normal looking usage data', async () => { - const mockConfig = getMockReportingConfig(); const plugins = getPluginsMock(); const { fetch } = getReportingUsageCollector( - mockConfig, + mockCore, plugins.usageCollection, getLicenseMock(), exportTypesRegistry, @@ -238,10 +245,9 @@ describe('data modeling', () => { }); test('with sparse data', async () => { - const mockConfig = getMockReportingConfig(); const plugins = getPluginsMock(); const { fetch } = getReportingUsageCollector( - mockConfig, + mockCore, plugins.usageCollection, getLicenseMock(), exportTypesRegistry, @@ -291,10 +297,9 @@ describe('data modeling', () => { }); test('with empty data', async () => { - const mockConfig = getMockReportingConfig(); const plugins = getPluginsMock(); const { fetch } = getReportingUsageCollector( - mockConfig, + mockCore, plugins.usageCollection, getLicenseMock(), exportTypesRegistry, diff --git a/x-pack/plugins/reporting/server/usage/reporting_usage_collector.ts b/x-pack/plugins/reporting/server/usage/reporting_usage_collector.ts index d77d1b5396844f..364f5187f056c0 100644 --- a/x-pack/plugins/reporting/server/usage/reporting_usage_collector.ts +++ b/x-pack/plugins/reporting/server/usage/reporting_usage_collector.ts @@ -9,7 +9,6 @@ import { CallCluster } from 'src/legacy/core_plugins/elasticsearch'; import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; import { ReportingCore } from '../'; import { KIBANA_REPORTING_TYPE } from '../../common/constants'; -import { ReportingConfig } from '../../server'; import { ExportTypesRegistry } from '../lib/export_types_registry'; import { ReportingSetupDeps } from '../types'; import { GetLicense } from './'; @@ -23,7 +22,7 @@ const METATYPE = 'kibana_stats'; * @return {Object} kibana usage stats type collection object */ export function getReportingUsageCollector( - config: ReportingConfig, + reporting: ReportingCore, usageCollection: UsageCollectionSetup, getLicense: GetLicense, exportTypesRegistry: ExportTypesRegistry, @@ -31,8 +30,10 @@ export function getReportingUsageCollector( ) { return usageCollection.makeUsageCollector({ type: KIBANA_REPORTING_TYPE, - fetch: (callCluster: CallCluster) => - getReportingUsage(config, getLicense, callCluster, exportTypesRegistry), + fetch: (callCluster: CallCluster) => { + const config = reporting.getConfig(); + return getReportingUsage(config, getLicense, callCluster, exportTypesRegistry); + }, isReady, /* @@ -63,7 +64,6 @@ export function registerReportingUsageCollector( return; } - const config = reporting.getConfig(); const exportTypesRegistry = reporting.getExportTypesRegistry(); const getLicense = async () => { return await licensing.license$ @@ -78,10 +78,10 @@ export function registerReportingUsageCollector( ) .toPromise(); }; - const collectionIsReady = reporting.pluginHasStarted.bind(reporting); + const collectionIsReady = reporting.pluginStartsUp.bind(reporting); const collector = getReportingUsageCollector( - config, + reporting, usageCollection, getLicense, exportTypesRegistry, From cc1758dd96c0152ecaf7bb2a8cd690536e34f6cf Mon Sep 17 00:00:00 2001 From: Spencer Date: Mon, 15 Jun 2020 16:27:56 -0700 Subject: [PATCH 06/11] [release-notes] add script to generate release notes from PRs (#68816) Co-authored-by: spalger Co-authored-by: Elastic Machine --- .gitignore | 4 + package.json | 1 + packages/kbn-release-notes/package.json | 23 ++ packages/kbn-release-notes/src/cli.ts | 162 ++++++++++ .../kbn-release-notes/src/formats/asciidoc.ts | 84 +++++ packages/kbn-release-notes/src/formats/csv.ts | 74 +++++ .../kbn-release-notes/src/formats/format.ts | 34 ++ .../kbn-release-notes/src/formats/index.ts | 25 ++ packages/kbn-release-notes/src/index.ts | 20 ++ .../kbn-release-notes/src/lib/classify_pr.ts | 66 ++++ .../src/lib/get_fix_references.test.ts | 68 ++++ .../src/lib/get_fix_references.ts | 29 ++ .../src/lib/get_note_from_description.test.ts | 79 +++++ .../src/lib/get_note_from_description.ts | 35 +++ packages/kbn-release-notes/src/lib/index.ts | 26 ++ .../src/lib/irrelevant_pr_summary.ts | 61 ++++ .../src/lib/is_pr_relevant.ts | 61 ++++ .../kbn-release-notes/src/lib/pull_request.ts | 206 ++++++++++++ packages/kbn-release-notes/src/lib/streams.ts | 34 ++ .../kbn-release-notes/src/lib/type_helpers.ts | 20 ++ .../kbn-release-notes/src/lib/version.test.ts | 146 +++++++++ packages/kbn-release-notes/src/lib/version.ts | 123 ++++++++ .../src/release_notes_config.ts | 294 ++++++++++++++++++ packages/kbn-release-notes/tsconfig.json | 12 + packages/kbn-release-notes/yarn.lock | 1 + scripts/release_notes.js | 21 ++ yarn.lock | 39 ++- 27 files changed, 1745 insertions(+), 3 deletions(-) create mode 100644 packages/kbn-release-notes/package.json create mode 100644 packages/kbn-release-notes/src/cli.ts create mode 100644 packages/kbn-release-notes/src/formats/asciidoc.ts create mode 100644 packages/kbn-release-notes/src/formats/csv.ts create mode 100644 packages/kbn-release-notes/src/formats/format.ts create mode 100644 packages/kbn-release-notes/src/formats/index.ts create mode 100644 packages/kbn-release-notes/src/index.ts create mode 100644 packages/kbn-release-notes/src/lib/classify_pr.ts create mode 100644 packages/kbn-release-notes/src/lib/get_fix_references.test.ts create mode 100644 packages/kbn-release-notes/src/lib/get_fix_references.ts create mode 100644 packages/kbn-release-notes/src/lib/get_note_from_description.test.ts create mode 100644 packages/kbn-release-notes/src/lib/get_note_from_description.ts create mode 100644 packages/kbn-release-notes/src/lib/index.ts create mode 100644 packages/kbn-release-notes/src/lib/irrelevant_pr_summary.ts create mode 100644 packages/kbn-release-notes/src/lib/is_pr_relevant.ts create mode 100644 packages/kbn-release-notes/src/lib/pull_request.ts create mode 100644 packages/kbn-release-notes/src/lib/streams.ts create mode 100644 packages/kbn-release-notes/src/lib/type_helpers.ts create mode 100644 packages/kbn-release-notes/src/lib/version.test.ts create mode 100644 packages/kbn-release-notes/src/lib/version.ts create mode 100644 packages/kbn-release-notes/src/release_notes_config.ts create mode 100644 packages/kbn-release-notes/tsconfig.json create mode 120000 packages/kbn-release-notes/yarn.lock create mode 100644 scripts/release_notes.js diff --git a/.gitignore b/.gitignore index c7c80fc48264d1..32377ec0f1ffe8 100644 --- a/.gitignore +++ b/.gitignore @@ -52,3 +52,7 @@ npm-debug.log* # apm plugin /x-pack/plugins/apm/tsconfig.json apm.tsconfig.json + +# release notes script output +report.csv +report.asciidoc diff --git a/package.json b/package.json index 887ffae755e37f..e8b07ae4abba25 100644 --- a/package.json +++ b/package.json @@ -306,6 +306,7 @@ "@kbn/expect": "1.0.0", "@kbn/optimizer": "1.0.0", "@kbn/plugin-generator": "1.0.0", + "@kbn/release-notes": "1.0.0", "@kbn/test": "1.0.0", "@kbn/utility-types": "1.0.0", "@microsoft/api-documenter": "7.7.2", diff --git a/packages/kbn-release-notes/package.json b/packages/kbn-release-notes/package.json new file mode 100644 index 00000000000000..25e1816b6cc1ed --- /dev/null +++ b/packages/kbn-release-notes/package.json @@ -0,0 +1,23 @@ +{ + "name": "@kbn/release-notes", + "version": "1.0.0", + "license": "Apache-2.0", + "main": "target/index.js", + "scripts": { + "kbn:bootstrap": "tsc", + "kbn:watch": "tsc --watch" + }, + "dependencies": { + "@kbn/dev-utils": "1.0.0", + "axios": "^0.19.2", + "cheerio": "0.22.0", + "dedent": "^0.7.0", + "graphql": "^14.0.0", + "graphql-tag": "^2.10.3", + "terminal-link": "^2.1.1" + }, + "devDependencies": { + "markdown-it": "^10.0.0", + "typescript": "3.9.5" + } +} \ No newline at end of file diff --git a/packages/kbn-release-notes/src/cli.ts b/packages/kbn-release-notes/src/cli.ts new file mode 100644 index 00000000000000..44b4a7a0282d20 --- /dev/null +++ b/packages/kbn-release-notes/src/cli.ts @@ -0,0 +1,162 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import Fs from 'fs'; +import Path from 'path'; +import { inspect } from 'util'; + +import { run, createFlagError, createFailError, REPO_ROOT } from '@kbn/dev-utils'; + +import { FORMATS, SomeFormat } from './formats'; +import { + iterRelevantPullRequests, + getPr, + Version, + ClassifiedPr, + streamFromIterable, + asyncPipeline, + IrrelevantPrSummary, + isPrRelevant, + classifyPr, +} from './lib'; + +const rootPackageJson = JSON.parse( + Fs.readFileSync(Path.resolve(REPO_ROOT, 'package.json'), 'utf8') +); +const extensions = FORMATS.map((f) => f.extension); + +export function runReleaseNotesCli() { + run( + async ({ flags, log }) => { + const token = flags.token; + if (!token || typeof token !== 'string') { + throw createFlagError('--token must be defined'); + } + + const version = Version.fromFlag(flags.version); + if (!version) { + throw createFlagError('unable to parse --version, use format "v{major}.{minor}.{patch}"'); + } + + const includeVersions = Version.fromFlags(flags.include || []); + if (!includeVersions) { + throw createFlagError('unable to parse --include, use format "v{major}.{minor}.{patch}"'); + } + + const Formats: SomeFormat[] = []; + for (const flag of Array.isArray(flags.format) ? flags.format : [flags.format]) { + const Format = FORMATS.find((F) => F.extension === flag); + if (!Format) { + throw createFlagError(`--format must be one of "${extensions.join('", "')}"`); + } + Formats.push(Format); + } + + const filename = flags.filename; + if (!filename || typeof filename !== 'string') { + throw createFlagError('--filename must be a string'); + } + + if (flags['debug-pr']) { + const number = parseInt(String(flags['debug-pr']), 10); + if (Number.isNaN(number)) { + throw createFlagError('--debug-pr must be a pr number when specified'); + } + + const summary = new IrrelevantPrSummary(log); + const pr = await getPr(token, number); + log.success( + inspect( + { + version: version.label, + includeVersions: includeVersions.map((v) => v.label), + isPrRelevant: isPrRelevant(pr, version, includeVersions, summary), + ...classifyPr(pr, log), + pr, + }, + { depth: 100 } + ) + ); + summary.logStats(); + return; + } + + log.info(`Loading all PRs with label [${version.label}] to build release notes...`); + + const summary = new IrrelevantPrSummary(log); + const prsToReport: ClassifiedPr[] = []; + const prIterable = iterRelevantPullRequests(token, version, log); + for await (const pr of prIterable) { + if (!isPrRelevant(pr, version, includeVersions, summary)) { + continue; + } + prsToReport.push(classifyPr(pr, log)); + } + summary.logStats(); + + if (!prsToReport.length) { + throw createFailError( + `All PRs with label [${version.label}] were filtered out by the config. Run again with --debug for more info.` + ); + } + + log.info(`Found ${prsToReport.length} prs to report on`); + + for (const Format of Formats) { + const format = new Format(version, prsToReport, log); + const outputPath = Path.resolve(`${filename}.${Format.extension}`); + await asyncPipeline(streamFromIterable(format.print()), Fs.createWriteStream(outputPath)); + log.success(`[${Format.extension}] report written to ${outputPath}`); + } + }, + { + usage: `node scripts/release_notes --token {token} --version {version}`, + flags: { + alias: { + version: 'v', + include: 'i', + }, + string: ['token', 'version', 'format', 'filename', 'include', 'debug-pr'], + default: { + filename: 'report', + version: rootPackageJson.version, + format: extensions, + }, + help: ` + --token (required) The Github access token to use for requests + --version, -v The version to fetch PRs by, PRs with version labels prior to + this one will be ignored (see --include-version) (default ${ + rootPackageJson.version + }) + --include, -i A version that is before --version but shouldn't be considered + "released" and cause PRs with a matching label to be excluded from + release notes. Use this when PRs are labeled with a version that + is less that --version and is expected to be released after + --version, can be specified multiple times. + --format Only produce a certain format, options: "${extensions.join('", "')}" + --filename Output filename, defaults to "report" + --debug-pr Fetch and print the details for a single PR, disabling reporting + `, + }, + description: ` + Fetch details from Github PRs for generating release notes + `, + } + ); +} diff --git a/packages/kbn-release-notes/src/formats/asciidoc.ts b/packages/kbn-release-notes/src/formats/asciidoc.ts new file mode 100644 index 00000000000000..d6c707f009f323 --- /dev/null +++ b/packages/kbn-release-notes/src/formats/asciidoc.ts @@ -0,0 +1,84 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import dedent from 'dedent'; + +import { Format } from './format'; +import { + ASCIIDOC_SECTIONS, + UNKNOWN_ASCIIDOC_SECTION, + AREAS, + UNKNOWN_AREA, +} from '../release_notes_config'; + +function* lines(body: string) { + for (const line of dedent(body).split('\n')) { + yield `${line}\n`; + } +} + +export class AsciidocFormat extends Format { + static extension = 'asciidoc'; + + *print() { + const sortedAreas = [ + ...AREAS.slice().sort((a, b) => a.title.localeCompare(b.title)), + UNKNOWN_AREA, + ]; + + yield* lines(` + [[release-notes-${this.version.label}]] + == ${this.version.label} Release Notes + + Also see <>. + `); + + for (const section of [...ASCIIDOC_SECTIONS, UNKNOWN_ASCIIDOC_SECTION]) { + const prsInSection = this.prs.filter((pr) => pr.asciidocSection === section); + if (!prsInSection.length) { + continue; + } + + yield '\n'; + yield* lines(` + [float] + [[${section.id}-${this.version.label}]] + === ${section.title} + `); + + for (const area of sortedAreas) { + const prsInArea = prsInSection.filter((pr) => pr.area === area); + + if (!prsInArea.length) { + continue; + } + + yield `${area.title}::\n`; + for (const pr of prsInArea) { + const fixes = pr.fixes.length ? `[Fixes ${pr.fixes.join(', ')}] ` : ''; + const strippedTitle = pr.title.replace(/^\s*\[[^\]]+\]\s*/, ''); + yield `* ${fixes}${strippedTitle} {pull}${pr.number}[#${pr.number}]\n`; + if (pr.note) { + yield ` - ${pr.note}\n`; + } + } + } + } + } +} diff --git a/packages/kbn-release-notes/src/formats/csv.ts b/packages/kbn-release-notes/src/formats/csv.ts new file mode 100644 index 00000000000000..0cf99edada696e --- /dev/null +++ b/packages/kbn-release-notes/src/formats/csv.ts @@ -0,0 +1,74 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Format } from './format'; + +/** + * Escape a value to conform to field and header encoding defined at https://tools.ietf.org/html/rfc4180 + */ +function esc(value: string | number) { + if (typeof value === 'number') { + return String(value); + } + + if (!value.includes(',') && !value.includes('\n') && !value.includes('"')) { + return value; + } + + return `"${value.split('"').join('""')}"`; +} + +function row(...fields: Array) { + return fields.map(esc).join(',') + '\r\n'; +} + +export class CsvFormat extends Format { + static extension = 'csv'; + + *print() { + // columns + yield row( + 'areas', + 'versions', + 'user', + 'title', + 'number', + 'url', + 'date', + 'fixes', + 'labels', + 'state' + ); + + for (const pr of this.prs) { + yield row( + pr.area.title, + pr.versions.map((v) => v.label).join(', '), + pr.user.name || pr.user.login, + pr.title, + pr.number, + pr.url, + pr.mergedAt, + pr.fixes.join(', '), + pr.labels.join(', '), + pr.state + ); + } + } +} diff --git a/packages/kbn-release-notes/src/formats/format.ts b/packages/kbn-release-notes/src/formats/format.ts new file mode 100644 index 00000000000000..41b769ab05de77 --- /dev/null +++ b/packages/kbn-release-notes/src/formats/format.ts @@ -0,0 +1,34 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { ToolingLog } from '@kbn/dev-utils'; + +import { Version, ClassifiedPr } from '../lib'; + +export abstract class Format { + static extension: string; + + constructor( + protected readonly version: Version, + protected readonly prs: ClassifiedPr[], + protected readonly log: ToolingLog + ) {} + + abstract print(): Iterator; +} diff --git a/packages/kbn-release-notes/src/formats/index.ts b/packages/kbn-release-notes/src/formats/index.ts new file mode 100644 index 00000000000000..3403e445a84ac7 --- /dev/null +++ b/packages/kbn-release-notes/src/formats/index.ts @@ -0,0 +1,25 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { ArrayItem } from '../lib'; +import { AsciidocFormat } from './asciidoc'; +import { CsvFormat } from './csv'; + +export const FORMATS = [CsvFormat, AsciidocFormat] as const; +export type SomeFormat = ArrayItem; diff --git a/packages/kbn-release-notes/src/index.ts b/packages/kbn-release-notes/src/index.ts new file mode 100644 index 00000000000000..a05bc698bde174 --- /dev/null +++ b/packages/kbn-release-notes/src/index.ts @@ -0,0 +1,20 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export * from './cli'; diff --git a/packages/kbn-release-notes/src/lib/classify_pr.ts b/packages/kbn-release-notes/src/lib/classify_pr.ts new file mode 100644 index 00000000000000..c567935ab7e480 --- /dev/null +++ b/packages/kbn-release-notes/src/lib/classify_pr.ts @@ -0,0 +1,66 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { ToolingLog } from '@kbn/dev-utils'; + +import { + Area, + AREAS, + UNKNOWN_AREA, + AsciidocSection, + ASCIIDOC_SECTIONS, + UNKNOWN_ASCIIDOC_SECTION, +} from '../release_notes_config'; +import { PullRequest } from './pull_request'; + +export interface ClassifiedPr extends PullRequest { + area: Area; + asciidocSection: AsciidocSection; +} + +export function classifyPr(pr: PullRequest, log: ToolingLog): ClassifiedPr { + const filter = (a: Area | AsciidocSection) => + a.labels.some((test) => + typeof test === 'string' ? pr.labels.includes(test) : pr.labels.some((l) => l.match(test)) + ); + + const areas = AREAS.filter(filter); + const asciidocSections = ASCIIDOC_SECTIONS.filter(filter); + + const pickOne = (name: string, options: T[]) => { + if (options.length > 1) { + const matches = options.map((o) => o.title).join(', '); + log.warning(`[${pr.terminalLink}] ambiguous ${name}, mulitple match [${matches}]`); + return options[0]; + } + + if (options.length === 0) { + log.error(`[${pr.terminalLink}] unable to determine ${name} because none match`); + return; + } + + return options[0]; + }; + + return { + ...pr, + area: pickOne('area', areas) || UNKNOWN_AREA, + asciidocSection: pickOne('asciidoc section', asciidocSections) || UNKNOWN_ASCIIDOC_SECTION, + }; +} diff --git a/packages/kbn-release-notes/src/lib/get_fix_references.test.ts b/packages/kbn-release-notes/src/lib/get_fix_references.test.ts new file mode 100644 index 00000000000000..bdac66f6cc02f0 --- /dev/null +++ b/packages/kbn-release-notes/src/lib/get_fix_references.test.ts @@ -0,0 +1,68 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { getFixReferences } from './get_fix_references'; + +it('returns all fixed issue mentions in the PR text', () => { + expect( + getFixReferences(` + clOses #1 + closes: #2 + clOse #3 + close: #4 + clOsed #5 + closed: #6 + fiX #7 + fix: #8 + fiXes #9 + fixes: #10 + fiXed #11 + fixed: #12 + reSolve #13 + resolve: #14 + reSolves #15 + resolves: #16 + reSolved #17 + resolved: #18 + fixed + #19 + `) + ).toMatchInlineSnapshot(` + Array [ + "#1", + "#2", + "#3", + "#4", + "#5", + "#6", + "#7", + "#8", + "#9", + "#10", + "#11", + "#12", + "#13", + "#14", + "#15", + "#16", + "#17", + "#18", + ] + `); +}); diff --git a/packages/kbn-release-notes/src/lib/get_fix_references.ts b/packages/kbn-release-notes/src/lib/get_fix_references.ts new file mode 100644 index 00000000000000..f45994e90ae899 --- /dev/null +++ b/packages/kbn-release-notes/src/lib/get_fix_references.ts @@ -0,0 +1,29 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +const FIXES_RE = /(?:closes|close|closed|fix|fixes|fixed|resolve|resolves|resolved)[ :]*(#\d*)/gi; + +export function getFixReferences(prText: string) { + const fixes: string[] = []; + let match; + while ((match = FIXES_RE.exec(prText))) { + fixes.push(match[1]); + } + return fixes; +} diff --git a/packages/kbn-release-notes/src/lib/get_note_from_description.test.ts b/packages/kbn-release-notes/src/lib/get_note_from_description.test.ts new file mode 100644 index 00000000000000..23dcb302f090d2 --- /dev/null +++ b/packages/kbn-release-notes/src/lib/get_note_from_description.test.ts @@ -0,0 +1,79 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import MarkdownIt from 'markdown-it'; +import dedent from 'dedent'; + +import { getNoteFromDescription } from './get_note_from_description'; + +it('extracts expected components from html', () => { + const mk = new MarkdownIt(); + + expect( + getNoteFromDescription( + mk.render(dedent` + My PR description + + Fixes: #1234 + + ## Release Note: + + Checkout this feature + `) + ) + ).toMatchInlineSnapshot(`"Checkout this feature"`); + + expect( + getNoteFromDescription( + mk.render(dedent` + My PR description + + Fixes: #1234 + + #### Release Note: + + We fixed an issue + `) + ) + ).toMatchInlineSnapshot(`"We fixed an issue"`); + + expect( + getNoteFromDescription( + mk.render(dedent` + My PR description + + Fixes: #1234 + + Release note: Checkout feature foo + `) + ) + ).toMatchInlineSnapshot(`"Checkout feature foo"`); + + expect( + getNoteFromDescription( + mk.render(dedent` + # Summary + + My PR description + + release note : bar + `) + ) + ).toMatchInlineSnapshot(`"bar"`); +}); diff --git a/packages/kbn-release-notes/src/lib/get_note_from_description.ts b/packages/kbn-release-notes/src/lib/get_note_from_description.ts new file mode 100644 index 00000000000000..57df203470a5a4 --- /dev/null +++ b/packages/kbn-release-notes/src/lib/get_note_from_description.ts @@ -0,0 +1,35 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import cheerio from 'cheerio'; + +export function getNoteFromDescription(descriptionHtml: string) { + const $ = cheerio.load(descriptionHtml); + for (const el of $('p,h1,h2,h3,h4,h5').toArray()) { + const text = $(el).text(); + const match = text.match(/^(\s*release note(?:s)?\s*:?\s*)/i); + + if (!match) { + continue; + } + + const note = text.replace(match[1], '').trim(); + return note || $(el).next().text().trim(); + } +} diff --git a/packages/kbn-release-notes/src/lib/index.ts b/packages/kbn-release-notes/src/lib/index.ts new file mode 100644 index 00000000000000..00d8f49cf763fa --- /dev/null +++ b/packages/kbn-release-notes/src/lib/index.ts @@ -0,0 +1,26 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export * from './pull_request'; +export * from './version'; +export * from './is_pr_relevant'; +export * from './streams'; +export * from './type_helpers'; +export * from './irrelevant_pr_summary'; +export * from './classify_pr'; diff --git a/packages/kbn-release-notes/src/lib/irrelevant_pr_summary.ts b/packages/kbn-release-notes/src/lib/irrelevant_pr_summary.ts new file mode 100644 index 00000000000000..1a458a04c7740d --- /dev/null +++ b/packages/kbn-release-notes/src/lib/irrelevant_pr_summary.ts @@ -0,0 +1,61 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { ToolingLog } from '@kbn/dev-utils'; + +import { PullRequest } from './pull_request'; +import { Version } from './version'; + +export class IrrelevantPrSummary { + private readonly stats = { + 'skipped by label': new Map(), + 'skipped by label regexp': new Map(), + 'skipped by version': new Map(), + }; + + constructor(private readonly log: ToolingLog) {} + + skippedByLabel(pr: PullRequest, label: string) { + this.log.debug(`${pr.terminalLink} skipped, label [${label}] is ignored`); + this.increment('skipped by label', label); + } + + skippedByLabelRegExp(pr: PullRequest, regexp: RegExp, label: string) { + this.log.debug(`${pr.terminalLink} skipped, label [${label}] matches regexp [${regexp}]`); + this.increment('skipped by label regexp', `${regexp}`); + } + + skippedByVersion(pr: PullRequest, earliestVersion: Version) { + this.log.debug(`${pr.terminalLink} skipped, earliest version is [${earliestVersion.label}]`); + this.increment('skipped by version', earliestVersion.label); + } + + private increment(stat: keyof IrrelevantPrSummary['stats'], key: string) { + const n = this.stats[stat].get(key) || 0; + this.stats[stat].set(key, n + 1); + } + + logStats() { + for (const [description, stats] of Object.entries(this.stats)) { + for (const [key, count] of stats) { + this.log.warning(`${count} ${count === 1 ? 'pr was' : 'prs were'} ${description} [${key}]`); + } + } + } +} diff --git a/packages/kbn-release-notes/src/lib/is_pr_relevant.ts b/packages/kbn-release-notes/src/lib/is_pr_relevant.ts new file mode 100644 index 00000000000000..af2ef9440dedeb --- /dev/null +++ b/packages/kbn-release-notes/src/lib/is_pr_relevant.ts @@ -0,0 +1,61 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Version } from './version'; +import { PullRequest } from './pull_request'; +import { IGNORE_LABELS } from '../release_notes_config'; +import { IrrelevantPrSummary } from './irrelevant_pr_summary'; + +export function isPrRelevant( + pr: PullRequest, + version: Version, + includeVersions: Version[], + summary: IrrelevantPrSummary +) { + for (const label of IGNORE_LABELS) { + if (typeof label === 'string') { + if (pr.labels.includes(label)) { + summary.skippedByLabel(pr, label); + return false; + } + } + + if (label instanceof RegExp) { + const matching = pr.labels.find((l) => label.test(l)); + if (matching) { + summary.skippedByLabelRegExp(pr, label, matching); + return false; + } + } + } + + const [earliestVersion] = Version.sort( + // filter out `includeVersions` so that they won't be considered the "earliest version", only + // versions which are actually before the current `version` or the `version` itself are eligible + pr.versions.filter((v) => !includeVersions.includes(v)), + 'asc' + ); + + if (version !== earliestVersion) { + summary.skippedByVersion(pr, earliestVersion); + return false; + } + + return true; +} diff --git a/packages/kbn-release-notes/src/lib/pull_request.ts b/packages/kbn-release-notes/src/lib/pull_request.ts new file mode 100644 index 00000000000000..e61e496642062a --- /dev/null +++ b/packages/kbn-release-notes/src/lib/pull_request.ts @@ -0,0 +1,206 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { inspect } from 'util'; + +import Axios from 'axios'; +import gql from 'graphql-tag'; +import * as GraphqlPrinter from 'graphql/language/printer'; +import { DocumentNode } from 'graphql/language/ast'; +import makeTerminalLink from 'terminal-link'; +import { ToolingLog } from '@kbn/dev-utils'; + +import { Version } from './version'; +import { getFixReferences } from './get_fix_references'; +import { getNoteFromDescription } from './get_note_from_description'; + +const PrNodeFragment = gql` + fragment PrNode on PullRequest { + number + url + title + bodyText + bodyHTML + mergedAt + baseRefName + state + author { + login + ... on User { + name + } + } + labels(first: 100) { + nodes { + name + } + } + } +`; + +export interface PullRequest { + number: number; + url: string; + title: string; + targetBranch: string; + mergedAt: string; + state: string; + labels: string[]; + fixes: string[]; + user: { + name: string; + login: string; + }; + versions: Version[]; + terminalLink: string; + note?: string; +} + +/** + * Send a single request to the Github v4 GraphQL API + */ +async function gqlRequest( + token: string, + query: DocumentNode, + variables: Record = {} +) { + const resp = await Axios.request({ + url: 'https://api.github.com/graphql', + method: 'POST', + headers: { + 'user-agent': '@kbn/release-notes', + authorization: `bearer ${token}`, + }, + data: { + query: GraphqlPrinter.print(query), + variables, + }, + }); + + return resp.data; +} + +/** + * Convert the Github API response into the structure used by this tool + * + * @param node A GraphQL response from Github using the PrNode fragment + */ +function parsePullRequestNode(node: any): PullRequest { + const terminalLink = makeTerminalLink(`#${node.number}`, node.url); + + const labels: string[] = node.labels.nodes.map((l: { name: string }) => l.name); + + return { + number: node.number, + url: node.url, + terminalLink, + title: node.title, + targetBranch: node.baseRefName, + state: node.state, + mergedAt: node.mergedAt, + labels, + fixes: getFixReferences(node.bodyText), + user: { + login: node.author?.login || 'deleted user', + name: node.author?.name, + }, + versions: labels + .map((l) => Version.fromLabel(l)) + .filter((v): v is Version => v instanceof Version), + note: getNoteFromDescription(node.bodyHTML), + }; +} + +/** + * Iterate all of the PRs which have the `version` label + */ +export async function* iterRelevantPullRequests(token: string, version: Version, log: ToolingLog) { + let nextCursor: string | undefined; + let hasNextPage = true; + + while (hasNextPage) { + const resp = await gqlRequest( + token, + gql` + query($cursor: String, $labels: [String!]) { + repository(owner: "elastic", name: "kibana") { + pullRequests(first: 100, after: $cursor, labels: $labels, states: MERGED) { + pageInfo { + hasNextPage + endCursor + } + nodes { + ...PrNode + } + } + } + } + ${PrNodeFragment} + `, + { + cursor: nextCursor, + labels: [version.label], + } + ); + + const pullRequests = resp.data?.repository?.pullRequests; + if (!pullRequests) { + throw new Error(`unexpected github response, unable to fetch PRs: ${inspect(resp)}`); + } + + hasNextPage = pullRequests.pageInfo?.hasNextPage; + nextCursor = pullRequests.pageInfo?.endCursor; + + if (hasNextPage === undefined || (hasNextPage && !nextCursor)) { + throw new Error( + `github response does not include valid pagination information: ${inspect(resp)}` + ); + } + + for (const node of pullRequests.nodes) { + yield parsePullRequestNode(node); + } + } +} + +export async function getPr(token: string, number: number) { + const resp = await gqlRequest( + token, + gql` + query($number: Int!) { + repository(owner: "elastic", name: "kibana") { + pullRequest(number: $number) { + ...PrNode + } + } + } + ${PrNodeFragment} + `, + { + number, + } + ); + + const node = resp.data?.repository?.pullRequest; + if (!node) { + throw new Error(`unexpected github response, unable to fetch PR: ${inspect(resp)}`); + } + + return parsePullRequestNode(node); +} diff --git a/packages/kbn-release-notes/src/lib/streams.ts b/packages/kbn-release-notes/src/lib/streams.ts new file mode 100644 index 00000000000000..f8cb9ec39186ad --- /dev/null +++ b/packages/kbn-release-notes/src/lib/streams.ts @@ -0,0 +1,34 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { promisify } from 'util'; +import { Readable, pipeline } from 'stream'; + +/** + * @types/node still doesn't have this method that was added + * in 10.17.0 https://nodejs.org/api/stream.html#stream_stream_readable_from_iterable_options + */ +export function streamFromIterable( + iter: Iterable | AsyncIterable +): Readable { + // @ts-ignore + return Readable.from(iter); +} + +export const asyncPipeline = promisify(pipeline); diff --git a/packages/kbn-release-notes/src/lib/type_helpers.ts b/packages/kbn-release-notes/src/lib/type_helpers.ts new file mode 100644 index 00000000000000..c9402b3584951b --- /dev/null +++ b/packages/kbn-release-notes/src/lib/type_helpers.ts @@ -0,0 +1,20 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export type ArrayItem = T extends ReadonlyArray ? X : never; diff --git a/packages/kbn-release-notes/src/lib/version.test.ts b/packages/kbn-release-notes/src/lib/version.test.ts new file mode 100644 index 00000000000000..afef2618656977 --- /dev/null +++ b/packages/kbn-release-notes/src/lib/version.test.ts @@ -0,0 +1,146 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Version } from './version'; + +it('parses version labels, returns null on failure', () => { + expect(Version.fromLabel('v1.0.2')).toMatchInlineSnapshot(` + Version { + "label": "v1.0.2", + "major": 1, + "minor": 0, + "patch": 2, + "tag": undefined, + "tagNum": undefined, + "tagOrder": Infinity, + } + `); + expect(Version.fromLabel('v1.0.0')).toMatchInlineSnapshot(` + Version { + "label": "v1.0.0", + "major": 1, + "minor": 0, + "patch": 0, + "tag": undefined, + "tagNum": undefined, + "tagOrder": Infinity, + } + `); + expect(Version.fromLabel('v9.0.2')).toMatchInlineSnapshot(` + Version { + "label": "v9.0.2", + "major": 9, + "minor": 0, + "patch": 2, + "tag": undefined, + "tagNum": undefined, + "tagOrder": Infinity, + } + `); + expect(Version.fromLabel('v9.0.2-alpha0')).toMatchInlineSnapshot(` + Version { + "label": "v9.0.2-alpha0", + "major": 9, + "minor": 0, + "patch": 2, + "tag": "alpha", + "tagNum": 0, + "tagOrder": 1, + } + `); + expect(Version.fromLabel('v9.0.2-beta1')).toMatchInlineSnapshot(` + Version { + "label": "v9.0.2-beta1", + "major": 9, + "minor": 0, + "patch": 2, + "tag": "beta", + "tagNum": 1, + "tagOrder": 2, + } + `); + expect(Version.fromLabel('v9.0')).toMatchInlineSnapshot(`undefined`); + expect(Version.fromLabel('some:area')).toMatchInlineSnapshot(`undefined`); +}); + +it('sorts versions in ascending order', () => { + const versions = [ + 'v1.7.3', + 'v1.7.0', + 'v1.5.0', + 'v2.7.0', + 'v7.0.0-beta2', + 'v7.0.0-alpha1', + 'v2.0.0', + 'v0.0.0', + 'v7.0.0-beta1', + 'v7.0.0', + ].map((l) => Version.fromLabel(l)!); + + const sorted = Version.sort(versions); + + expect(sorted.map((v) => v.label)).toMatchInlineSnapshot(` + Array [ + "v0.0.0", + "v1.5.0", + "v1.7.0", + "v1.7.3", + "v2.0.0", + "v2.7.0", + "v7.0.0-alpha1", + "v7.0.0-beta1", + "v7.0.0-beta2", + "v7.0.0", + ] + `); + + // ensure versions was not mutated + expect(sorted).not.toEqual(versions); +}); + +it('sorts versions in decending order', () => { + const versions = [ + 'v1.7.3', + 'v1.7.0', + 'v1.5.0', + 'v7.0.0-beta1', + 'v2.7.0', + 'v2.0.0', + 'v0.0.0', + 'v7.0.0', + ].map((l) => Version.fromLabel(l)!); + + const sorted = Version.sort(versions, 'desc'); + + expect(sorted.map((v) => v.label)).toMatchInlineSnapshot(` + Array [ + "v7.0.0", + "v7.0.0-beta1", + "v2.7.0", + "v2.0.0", + "v1.7.3", + "v1.7.0", + "v1.5.0", + "v0.0.0", + ] + `); + + // ensure versions was not mutated + expect(sorted).not.toEqual(versions); +}); diff --git a/packages/kbn-release-notes/src/lib/version.ts b/packages/kbn-release-notes/src/lib/version.ts new file mode 100644 index 00000000000000..e0a5c5e177c82a --- /dev/null +++ b/packages/kbn-release-notes/src/lib/version.ts @@ -0,0 +1,123 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +const LABEL_RE = /^v(\d+)\.(\d+)\.(\d+)(?:-(alpha|beta)(\d+))?$/; + +const versionCache = new Map(); + +const multiCompare = (...diffs: number[]) => { + for (const diff of diffs) { + if (diff !== 0) { + return diff; + } + } + return 0; +}; + +export class Version { + static fromFlag(flag: string | string[] | boolean | undefined) { + if (typeof flag !== 'string') { + return; + } + + return Version.fromLabel(flag) || Version.fromLabel(`v${flag}`); + } + + static fromFlags(flag: string | string[] | boolean | undefined) { + const flags = Array.isArray(flag) ? flag : [flag]; + const versions: Version[] = []; + + for (const f of flags) { + const version = Version.fromFlag(f); + if (!version) { + return; + } + versions.push(version); + } + + return versions; + } + + static fromLabel(label: string) { + const match = label.match(LABEL_RE); + if (!match) { + return; + } + + const cached = versionCache.get(label); + if (cached) { + return cached; + } + + const [, major, minor, patch, tag, tagNum] = match; + const version = new Version( + parseInt(major, 10), + parseInt(minor, 10), + parseInt(patch, 10), + tag as 'alpha' | 'beta' | undefined, + tagNum ? parseInt(tagNum, 10) : undefined + ); + + versionCache.set(label, version); + return version; + } + + static sort(versions: Version[], dir: 'asc' | 'desc' = 'asc') { + const order = dir === 'asc' ? 1 : -1; + + return versions.slice().sort((a, b) => a.compare(b) * order); + } + + public readonly label = `v${this.major}.${this.minor}.${this.patch}${ + this.tag ? `-${this.tag}${this.tagNum}` : '' + }`; + private readonly tagOrder: number; + + constructor( + public readonly major: number, + public readonly minor: number, + public readonly patch: number, + public readonly tag: 'alpha' | 'beta' | undefined, + public readonly tagNum: number | undefined + ) { + switch (tag) { + case undefined: + this.tagOrder = Infinity; + break; + case 'alpha': + this.tagOrder = 1; + break; + case 'beta': + this.tagOrder = 2; + break; + default: + throw new Error('unexpected tag'); + } + } + + compare(other: Version) { + return multiCompare( + this.major - other.major, + this.minor - other.minor, + this.patch - other.patch, + this.tagOrder - other.tagOrder, + (this.tagNum ?? 0) - (other.tagNum ?? 0) + ); + } +} diff --git a/packages/kbn-release-notes/src/release_notes_config.ts b/packages/kbn-release-notes/src/release_notes_config.ts new file mode 100644 index 00000000000000..88ab5dfa2fda43 --- /dev/null +++ b/packages/kbn-release-notes/src/release_notes_config.ts @@ -0,0 +1,294 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * Exclude any PR from release notes that has a matching label. String + * labels must match exactly, for more complicated use a RegExp + */ +export const IGNORE_LABELS: Array = [ + 'Team:Docs', + ':KibanaApp/fix-it-week', + 'reverted', + /^test/, + 'non-issue', + 'jenkins', + 'build', + 'chore', + 'backport', + 'release_note:skip', + 'release_note:dev_docs', +]; + +/** + * Define areas that are used to categorize changes in the release notes + * based on the labels a PR has. the `labels` array can contain strings, which + * are matched exactly, or regular expressions. The first area, in definition + * order, which has a `label` which matches and label on a PR is the area + * assigned to that PR. + */ + +export interface Area { + title: string; + labels: Array; +} + +export const AREAS: Area[] = [ + { + title: 'Design', + labels: ['Team:Design', 'Project:Accessibility'], + }, + { + title: 'Logstash', + labels: ['App:Logstash', 'Feature:Logstash Pipelines'], + }, + { + title: 'Management', + labels: [ + 'Feature:license', + 'Feature:Console', + 'Feature:Search Profiler', + 'Feature:watcher', + 'Feature:Index Patterns', + 'Feature:Kibana Management', + 'Feature:Dev Tools', + 'Feature:Inspector', + 'Feature:Index Management', + 'Feature:Snapshot and Restore', + 'Team:Elasticsearch UI', + 'Feature:FieldFormatters', + 'Feature:CCR', + 'Feature:ILM', + 'Feature:Transforms', + ], + }, + { + title: 'Monitoring', + labels: ['Team:Monitoring', 'Feature:Telemetry', 'Feature:Stack Monitoring'], + }, + { + title: 'Operations', + labels: ['Team:Operations', 'Feature:License'], + }, + { + title: 'Kibana UI', + labels: ['Kibana UI', 'Team:Core UI', 'Feature:Header'], + }, + { + title: 'Platform', + labels: [ + 'Team:Platform', + 'Feature:Plugins', + 'Feature:New Platform', + 'Project:i18n', + 'Feature:ExpressionLanguage', + 'Feature:Saved Objects', + 'Team:Stack Services', + 'Feature:NP Migration', + 'Feature:Task Manager', + 'Team:Pulse', + ], + }, + { + title: 'Machine Learning', + labels: [ + ':ml', + 'Feature:Anomaly Detection', + 'Feature:Data Frames', + 'Feature:File Data Viz', + 'Feature:ml-results', + 'Feature:Data Frame Analytics', + ], + }, + { + title: 'Maps', + labels: ['Team:Geo'], + }, + { + title: 'Canvas', + labels: ['Team:Canvas'], + }, + { + title: 'QA', + labels: ['Team:QA'], + }, + { + title: 'Security', + labels: [ + 'Team:Security', + 'Feature:Security/Spaces', + 'Feature:users and roles', + 'Feature:Security/Authentication', + 'Feature:Security/Authorization', + 'Feature:Security/Feature Controls', + ], + }, + { + title: 'Dashboard', + labels: ['Feature:Dashboard', 'Feature:Drilldowns'], + }, + { + title: 'Discover', + labels: ['Feature:Discover'], + }, + { + title: 'Kibana Home & Add Data', + labels: ['Feature:Add Data', 'Feature:Home'], + }, + { + title: 'Querying & Filtering', + labels: [ + 'Feature:Query Bar', + 'Feature:Courier', + 'Feature:Filters', + 'Feature:Timepicker', + 'Feature:Highlight', + 'Feature:KQL', + 'Feature:Rollups', + ], + }, + { + title: 'Reporting', + labels: ['Feature:Reporting', 'Team:Reporting Services'], + }, + { + title: 'Sharing', + labels: ['Feature:Embedding', 'Feature:SharingURLs'], + }, + { + title: 'Visualizations', + labels: [ + 'Feature:Timelion', + 'Feature:TSVB', + 'Feature:Coordinate Map', + 'Feature:Region Map', + 'Feature:Vega', + 'Feature:Gauge Vis', + 'Feature:Tagcloud', + 'Feature:Vis Loader', + 'Feature:Vislib', + 'Feature:Vis Editor', + 'Feature:Aggregations', + 'Feature:Input Control', + 'Feature:Visualizations', + 'Feature:Markdown', + 'Feature:Data Table', + 'Feature:Heatmap', + 'Feature:Pie Chart', + 'Feature:XYAxis', + 'Feature:Graph', + 'Feature:New Feature', + 'Feature:MetricVis', + ], + }, + { + title: 'SIEM', + labels: ['Team:SIEM'], + }, + { + title: 'Code', + labels: ['Team:Code'], + }, + { + title: 'Infrastructure', + labels: ['App:Infrastructure', 'Feature:Infra UI', 'Feature:Service Maps'], + }, + { + title: 'Logs', + labels: ['App:Logs', 'Feature:Logs UI'], + }, + { + title: 'Uptime', + labels: ['App:Uptime', 'Feature:Uptime', 'Team:uptime'], + }, + { + title: 'Beats Management', + labels: ['App:Beats', 'Feature:beats-cm', 'Team:Beats'], + }, + { + title: 'APM', + labels: ['Team:apm', /^apm[:\-]/], + }, + { + title: 'Lens', + labels: ['App:Lens', 'Feature:Lens'], + }, + { + title: 'Alerting', + labels: ['App:Alerting', 'Feature:Alerting', 'Team:Alerting Services', 'Feature:Actions'], + }, + { + title: 'Metrics', + labels: ['App:Metrics', 'Feature:Metrics UI', 'Team:logs-metrics-ui'], + }, + { + title: 'Data ingest', + labels: ['Ingest', 'Feature:Ingest Node Pipelines'], + }, +]; + +export const UNKNOWN_AREA: Area = { + title: 'Unknown', + labels: [], +}; + +/** + * Define the sections that will be assigned to PRs when generating the + * asciidoc formatted report. The order of the sections determines the + * order they will be rendered in the report + */ + +export interface AsciidocSection { + title: string; + labels: Array; + id: string; +} + +export const ASCIIDOC_SECTIONS: AsciidocSection[] = [ + { + id: 'enhancement', + title: 'Enhancements', + labels: ['release_note:enhancement'], + }, + { + id: 'bug', + title: 'Bug fixes', + labels: ['release_note:fix'], + }, + { + id: 'roadmap', + title: 'Roadmap', + labels: ['release_note:roadmap'], + }, + { + id: 'deprecation', + title: 'Deprecations', + labels: ['release_note:deprecation'], + }, + { + id: 'breaking', + title: 'Breaking Changes', + labels: ['release_note:breaking'], + }, +]; + +export const UNKNOWN_ASCIIDOC_SECTION: AsciidocSection = { + id: 'unknown', + title: 'Unknown', + labels: [], +}; diff --git a/packages/kbn-release-notes/tsconfig.json b/packages/kbn-release-notes/tsconfig.json new file mode 100644 index 00000000000000..6ffa64d91fba0f --- /dev/null +++ b/packages/kbn-release-notes/tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "./target", + "declaration": true, + "sourceMap": true, + "target": "ES2019" + }, + "include": [ + "src/**/*" + ] +} diff --git a/packages/kbn-release-notes/yarn.lock b/packages/kbn-release-notes/yarn.lock new file mode 120000 index 00000000000000..3f82ebc9cdbae3 --- /dev/null +++ b/packages/kbn-release-notes/yarn.lock @@ -0,0 +1 @@ +../../yarn.lock \ No newline at end of file diff --git a/scripts/release_notes.js b/scripts/release_notes.js new file mode 100644 index 00000000000000..f46ee5823d70d1 --- /dev/null +++ b/scripts/release_notes.js @@ -0,0 +1,21 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +require('../src/setup_node_env/prebuilt_dev_only_entry'); +require('@kbn/release-notes').runReleaseNotesCli(); diff --git a/yarn.lock b/yarn.lock index 8c795a27e10af5..20089fdb830925 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5355,9 +5355,9 @@ "@types/node" "*" "@types/node@*", "@types/node@8.10.54", "@types/node@>=10.17.17 <10.20.0", "@types/node@>=8.9.0", "@types/node@^12.0.2": - version "10.17.17" - resolved "https://registry.yarnpkg.com/@types/node/-/node-10.17.17.tgz#7a183163a9e6ff720d86502db23ba4aade5999b8" - integrity sha512-gpNnRnZP3VWzzj5k3qrpRC6Rk3H/uclhAVo1aIvwzK5p5cOrs9yEyQ8H/HBsBY0u5rrWxXEiVPQ0dEB6pkjE8Q== + version "10.17.26" + resolved "https://registry.yarnpkg.com/@types/node/-/node-10.17.26.tgz#a8a119960bff16b823be4c617da028570779bcfd" + integrity sha512-myMwkO2Cr82kirHY8uknNRHEVtn0wV3DTQfkrjx17jmkstDRZ24gNUdl8AHXVyVclTYI/bNjgTPTAWvWLqXqkw== "@types/nodemailer@^6.2.1": version "6.2.1" @@ -16148,6 +16148,11 @@ graphql-tag@2.10.1, graphql-tag@^2.9.2: resolved "https://registry.yarnpkg.com/graphql-tag/-/graphql-tag-2.10.1.tgz#10aa41f1cd8fae5373eaf11f1f67260a3cad5e02" integrity sha512-jApXqWBzNXQ8jYa/HLkZJaVw9jgwNqZkywa2zfFn16Iv1Zb7ELNHkJaXHR7Quvd5SIGsy6Ny7SUKATgnu05uEg== +graphql-tag@^2.10.3: + version "2.10.3" + resolved "https://registry.yarnpkg.com/graphql-tag/-/graphql-tag-2.10.3.tgz#ea1baba5eb8fc6339e4c4cf049dabe522b0edf03" + integrity sha512-4FOv3ZKfA4WdOKJeHdz6B3F/vxBLSgmBcGeAFPf4n1F64ltJUvOOerNj0rsJxONQGdhUMynQIvd6LzB+1J5oKA== + graphql-toolkit@0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/graphql-toolkit/-/graphql-toolkit-0.2.0.tgz#91364b69911d51bc915269a37963f4ea2d5f335c" @@ -16194,6 +16199,13 @@ graphql@^0.13.2: dependencies: iterall "^1.2.1" +graphql@^14.0.0: + version "14.6.0" + resolved "https://registry.yarnpkg.com/graphql/-/graphql-14.6.0.tgz#57822297111e874ea12f5cd4419616930cd83e49" + integrity sha512-VKzfvHEKybTKjQVpTFrA5yUq2S9ihcZvfJAtsDBBCuV6wauPu1xl/f9ehgVf0FcEJJs4vz6ysb/ZMkGigQZseg== + dependencies: + iterall "^1.2.2" + graphviz@^0.0.8: version "0.0.8" resolved "https://registry.yarnpkg.com/graphviz/-/graphviz-0.0.8.tgz#e599e40733ef80e1653bfe89a5f031ecf2aa4aaa" @@ -18731,6 +18743,11 @@ iterall@^1.1.3, iterall@^1.2.1: resolved "https://registry.yarnpkg.com/iterall/-/iterall-1.2.2.tgz#92d70deb8028e0c39ff3164fdbf4d8b088130cd7" integrity sha512-yynBb1g+RFUPY64fTrFv7nsjRrENBQJaX2UL+2Szc9REFrSNm1rpSXHGzhmAy7a9uv3vlvgBlXnf9RqmPH1/DA== +iterall@^1.2.2: + version "1.3.0" + resolved "https://registry.yarnpkg.com/iterall/-/iterall-1.3.0.tgz#afcb08492e2915cbd8a0884eb93a8c94d0d72fea" + integrity sha512-QZ9qOMdF+QLHxy1QIpUHUU1D5pS2CG2P69LF6L6CPjPYA/XMOmKV3PZpawHoAjHNyB0swdVTRxdYT4tbBbxqwg== + jest-changed-files@^24.9.0: version "24.9.0" resolved "https://registry.yarnpkg.com/jest-changed-files/-/jest-changed-files-24.9.0.tgz#08d8c15eb79a7fa3fc98269bc14b451ee82f8039" @@ -29202,6 +29219,14 @@ supports-hyperlinks@^1.0.1: has-flag "^2.0.0" supports-color "^5.0.0" +supports-hyperlinks@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/supports-hyperlinks/-/supports-hyperlinks-2.1.0.tgz#f663df252af5f37c5d49bbd7eeefa9e0b9e59e47" + integrity sha512-zoE5/e+dnEijk6ASB6/qrK+oYdm2do1hjoLWrqUC/8WEIW1gbxFcKuBof7sW8ArN6e+AYvsE8HBGiVRWL/F5CA== + dependencies: + has-flag "^4.0.0" + supports-color "^7.0.0" + suricata-sid-db@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/suricata-sid-db/-/suricata-sid-db-1.0.2.tgz#96ceda4db117a9f1282c8f9d785285e5ccf342b1" @@ -29554,6 +29579,14 @@ term-size@^2.1.0: resolved "https://registry.yarnpkg.com/term-size/-/term-size-2.2.0.tgz#1f16adedfe9bdc18800e1776821734086fcc6753" integrity sha512-a6sumDlzyHVJWb8+YofY4TW112G6p2FCPEAFk+59gIYHv3XHRhm9ltVQ9kli4hNWeQBwSpe8cRN25x0ROunMOw== +terminal-link@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/terminal-link/-/terminal-link-2.1.1.tgz#14a64a27ab3c0df933ea546fba55f2d078edc994" + integrity sha512-un0FmiRUQNr5PJqy9kP7c40F5BOfpGlYTrxonDChEZB7pzZxRNp/bt+ymiy9/npwXya9KH99nJ/GXFIiUkYGFQ== + dependencies: + ansi-escapes "^4.2.1" + supports-hyperlinks "^2.0.0" + terser-webpack-plugin@^1.2.4, terser-webpack-plugin@^1.4.3: version "1.4.4" resolved "https://registry.yarnpkg.com/terser-webpack-plugin/-/terser-webpack-plugin-1.4.4.tgz#2c63544347324baafa9a56baaddf1634c8abfc2f" From d110e65624605cba404b4365df602693025b32b6 Mon Sep 17 00:00:00 2001 From: Luke Elmers Date: Mon, 15 Jun 2020 21:54:04 -0600 Subject: [PATCH 07/11] Sort filter fields by items that start with typed characters. (#68585) --- .../public/ui/filter_bar/filter_editor/generic_combo_box.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/src/plugins/data/public/ui/filter_bar/filter_editor/generic_combo_box.tsx b/src/plugins/data/public/ui/filter_bar/filter_editor/generic_combo_box.tsx index 66adbfe9a6fc32..c66518c18a9eab 100644 --- a/src/plugins/data/public/ui/filter_bar/filter_editor/generic_combo_box.tsx +++ b/src/plugins/data/public/ui/filter_bar/filter_editor/generic_combo_box.tsx @@ -59,6 +59,7 @@ export function GenericComboBox(props: GenericComboBoxProps) { options={euiOptions} selectedOptions={selectedEuiOptions} onChange={onComboBoxChange} + sortMatchesBy="startsWith" {...otherProps} /> ); From 7e328417ab891cce72df2b25bfa56b4119a2de3f Mon Sep 17 00:00:00 2001 From: Spencer Date: Mon, 15 Jun 2020 22:15:31 -0700 Subject: [PATCH 08/11] [kbn/optimizer] share all plugin bundles (#68986) Co-authored-by: spalger Co-authored-by: Elastic Machine --- .eslintrc.js | 1 + NOTICE.txt | 5 + ...e-server.pluginmanifest.extrapublicdirs.md | 17 ++ ...ibana-plugin-core-server.pluginmanifest.md | 1 + examples/demo_search/kibana.json | 3 +- examples/embeddable_examples/kibana.json | 3 +- examples/url_generators_examples/kibana.json | 5 +- packages/kbn-optimizer/README.md | 12 ++ packages/kbn-optimizer/package.json | 1 + .../mock_repo/plugins/bar/public/index.ts | 2 +- .../kbn-optimizer/src/common/bundle.test.ts | 10 +- packages/kbn-optimizer/src/common/bundle.ts | 22 ++- .../kbn-optimizer/src/common/bundle_cache.ts | 5 + .../kbn-optimizer/src/common/bundle_refs.ts | 130 ++++++++++++++ packages/kbn-optimizer/src/common/index.ts | 1 + .../basic_optimization.test.ts.snap | 17 +- .../basic_optimization.test.ts | 10 +- .../integration_tests/bundle_cache.test.ts | 53 ++++++ .../watch_bundles_for_changes.test.ts | 4 +- .../assign_bundles_to_workers.test.ts | 2 +- .../src/optimizer/bundle_cache.ts | 33 +++- .../kbn-optimizer/src/optimizer/cache_keys.ts | 12 +- .../src/optimizer/get_plugin_bundles.test.ts | 11 +- .../src/optimizer/get_plugin_bundles.ts | 2 +- .../optimizer/kibana_platform_plugins.test.ts | 4 + .../src/optimizer/kibana_platform_plugins.ts | 16 ++ .../src/optimizer/observe_worker.ts | 8 +- .../src/optimizer/optimizer_config.ts | 24 ++- .../src/worker/bundle_ref_module.ts | 67 +++++++ .../src/worker/bundle_refs_plugin.ts | 165 ++++++++++++++++++ .../src/worker/entry_point_creator.ts | 28 +++ .../kbn-optimizer/src/worker/run_compilers.ts | 71 ++++---- .../kbn-optimizer/src/worker/run_worker.ts | 12 +- .../src/worker/webpack.config.ts | 127 ++++---------- src/core/public/plugins/plugin_reader.test.ts | 33 ++-- src/core/public/plugins/plugin_reader.ts | 14 +- .../discovery/plugin_manifest_parser.ts | 21 ++- src/core/server/plugins/types.ts | 8 + src/core/server/server.api.md | 8 +- .../ui/ui_render/bootstrap/app_bootstrap.js | 4 +- .../bootstrap/kbn_bundles_loader_source.js | 51 ++++++ .../ui/ui_render/bootstrap/template.js.hbs | 8 +- src/legacy/ui/ui_render/ui_render_mixin.js | 12 +- src/plugins/data/kibana.json | 3 +- src/plugins/embeddable/kibana.json | 3 + src/plugins/es_ui_shared/kibana.json | 10 +- src/plugins/expressions/kibana.json | 3 +- src/plugins/inspector/kibana.json | 3 +- src/plugins/kibana_legacy/kibana.json | 3 +- src/plugins/kibana_utils/kibana.json | 7 +- src/plugins/maps_legacy/public/leaflet.js | 6 +- .../saved_objects_management/kibana.json | 3 +- src/plugins/telemetry/kibana.json | 3 + src/plugins/ui_actions/kibana.json | 5 +- .../public/app/components/main.tsx | 2 +- test/scripts/jenkins_build_kibana.sh | 9 +- test/scripts/jenkins_xpack_build_kibana.sh | 8 +- x-pack/plugins/alerts/kibana.json | 3 +- x-pack/plugins/apm/kibana.json | 3 + .../plugins/canvas/common/lib/autocomplete.ts | 2 +- x-pack/plugins/data_enhanced/kibana.json | 1 + x-pack/plugins/features/kibana.json | 3 +- x-pack/plugins/ingest_manager/kibana.json | 3 +- x-pack/plugins/lens/kibana.json | 3 +- x-pack/plugins/license_management/kibana.json | 3 +- x-pack/plugins/maps/kibana.json | 3 +- x-pack/plugins/spaces/kibana.json | 3 +- .../plugins/triggers_actions_ui/kibana.json | 3 +- .../dynamic_actions/dynamic_action_manager.ts | 5 +- 69 files changed, 904 insertions(+), 242 deletions(-) create mode 100644 docs/development/core/server/kibana-plugin-core-server.pluginmanifest.extrapublicdirs.md create mode 100644 packages/kbn-optimizer/src/common/bundle_refs.ts create mode 100644 packages/kbn-optimizer/src/worker/bundle_ref_module.ts create mode 100644 packages/kbn-optimizer/src/worker/bundle_refs_plugin.ts create mode 100644 packages/kbn-optimizer/src/worker/entry_point_creator.ts create mode 100644 src/legacy/ui/ui_render/bootstrap/kbn_bundles_loader_source.js diff --git a/.eslintrc.js b/.eslintrc.js index 9657719f0f526f..8d5b4525d51ba5 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -472,6 +472,7 @@ module.exports = { { files: [ 'test/functional/services/lib/web_element_wrapper/scroll_into_view_if_necessary.js', + 'src/legacy/ui/ui_render/bootstrap/kbn_bundles_loader_source.js', '**/browser_exec_scripts/**/*.js', ], rules: { diff --git a/NOTICE.txt b/NOTICE.txt index 33c1d535d7df32..946b328b8766c1 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -21,6 +21,11 @@ used. Logarithmic ticks are places at powers of ten and at half those values if there are not to many ticks already (e.g. [1, 5, 10, 50, 100]). For details, see https://github.com/flot/flot/pull/1328 +--- +This module was heavily inspired by the externals plugin that ships with webpack@97d58d31 +MIT License http://www.opensource.org/licenses/mit-license.php +Author Tobias Koppers @sokra + --- This product has relied on ASTExplorer that is licensed under MIT. diff --git a/docs/development/core/server/kibana-plugin-core-server.pluginmanifest.extrapublicdirs.md b/docs/development/core/server/kibana-plugin-core-server.pluginmanifest.extrapublicdirs.md new file mode 100644 index 00000000000000..c46e60f2ecf6d2 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.pluginmanifest.extrapublicdirs.md @@ -0,0 +1,17 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [PluginManifest](./kibana-plugin-core-server.pluginmanifest.md) > [extraPublicDirs](./kibana-plugin-core-server.pluginmanifest.extrapublicdirs.md) + +## PluginManifest.extraPublicDirs property + +> Warning: This API is now obsolete. +> +> + +Specifies directory names that can be imported by other ui-plugins built using the same instance of the @kbn/optimizer. A temporary measure we plan to replace with better mechanisms for sharing static code between plugins + +Signature: + +```typescript +readonly extraPublicDirs?: string[]; +``` diff --git a/docs/development/core/server/kibana-plugin-core-server.pluginmanifest.md b/docs/development/core/server/kibana-plugin-core-server.pluginmanifest.md index fe0ca476bbcb27..5edee51d6c523d 100644 --- a/docs/development/core/server/kibana-plugin-core-server.pluginmanifest.md +++ b/docs/development/core/server/kibana-plugin-core-server.pluginmanifest.md @@ -21,6 +21,7 @@ Should never be used in code outside of Core but is exported for documentation p | Property | Type | Description | | --- | --- | --- | | [configPath](./kibana-plugin-core-server.pluginmanifest.configpath.md) | ConfigPath | Root [configuration path](./kibana-plugin-core-server.configpath.md) used by the plugin, defaults to "id" in snake\_case format. | +| [extraPublicDirs](./kibana-plugin-core-server.pluginmanifest.extrapublicdirs.md) | string[] | Specifies directory names that can be imported by other ui-plugins built using the same instance of the @kbn/optimizer. A temporary measure we plan to replace with better mechanisms for sharing static code between plugins | | [id](./kibana-plugin-core-server.pluginmanifest.id.md) | PluginName | Identifier of the plugin. Must be a string in camelCase. Part of a plugin public contract. Other plugins leverage it to access plugin API, navigate to the plugin, etc. | | [kibanaVersion](./kibana-plugin-core-server.pluginmanifest.kibanaversion.md) | string | The version of Kibana the plugin is compatible with, defaults to "version". | | [optionalPlugins](./kibana-plugin-core-server.pluginmanifest.optionalplugins.md) | readonly PluginName[] | An optional list of the other plugins that if installed and enabled \*\*may be\*\* leveraged by this plugin for some additional functionality but otherwise are not required for this plugin to work properly. | diff --git a/examples/demo_search/kibana.json b/examples/demo_search/kibana.json index cdf74121ea2db1..f909ca47fcd55b 100644 --- a/examples/demo_search/kibana.json +++ b/examples/demo_search/kibana.json @@ -5,5 +5,6 @@ "server": true, "ui": true, "requiredPlugins": ["data"], - "optionalPlugins": [] + "optionalPlugins": [], + "extraPublicDirs": ["common"] } diff --git a/examples/embeddable_examples/kibana.json b/examples/embeddable_examples/kibana.json index 489f768552b281..b3ee0de096989e 100644 --- a/examples/embeddable_examples/kibana.json +++ b/examples/embeddable_examples/kibana.json @@ -5,5 +5,6 @@ "server": true, "ui": true, "requiredPlugins": ["embeddable"], - "optionalPlugins": [] + "optionalPlugins": [], + "extraPublicDirs": ["public/todo", "public/hello_world"] } diff --git a/examples/url_generators_examples/kibana.json b/examples/url_generators_examples/kibana.json index cdb2127fdd26f5..9658f5c7300aac 100644 --- a/examples/url_generators_examples/kibana.json +++ b/examples/url_generators_examples/kibana.json @@ -5,5 +5,8 @@ "server": false, "ui": true, "requiredPlugins": ["share"], - "optionalPlugins": [] + "optionalPlugins": [], + "extraPublicDirs": [ + "public/url_generator" + ] } diff --git a/packages/kbn-optimizer/README.md b/packages/kbn-optimizer/README.md index c7f50c6af8dfde..9ff0f563442749 100644 --- a/packages/kbn-optimizer/README.md +++ b/packages/kbn-optimizer/README.md @@ -30,6 +30,18 @@ Bundles built by the the optimizer include a cache file which describes the info When a bundle is determined to be up-to-date a worker is not started for the bundle. If running the optimizer with the `--dev/--watch` flag, then all the files referenced by cached bundles are watched for changes. Once a change is detected in any of the files referenced by the built bundle a worker is started. If a file is changed that is referenced by several bundles then workers will be started for each bundle, combining workers together to respect the worker limit. +## Bundle Refs + +In order to dramatically reduce the size of our bundles, and the time it takes to build them, bundles will "ref" other bundles being built at the same time. When the optimizer starts it creates a list of "refs" that could be had from the list of bundles being built. Each worker uses that list to determine which import statements in a bundle should be replaced with a runtime reference to the output of another bundle. + +At runtime the bundles share a set of entry points via the `__kbnBundles__` global. By default a plugin shares `public` so that other code can use relative imports to access that directory. To expose additional directories they must be listed in the plugin's kibana.json "extraPublicDirs" field. The directories listed there will **also** be exported from the plugins bundle so that any other plugin can import that directory. "common" is commonly in the list of "extraPublicDirs". + +> NOTE: We plan to replace the `extraPublicDirs` functionality soon with better mechanisms for statically sharing code between bundles. + +When a directory is listed in the "extraPublicDirs" it will always be included in the bundle so that other plugins have access to it. The worker building the bundle has no way of knowing whether another plugin is using the directory, so be careful of adding test code or unnecessary directories to that list. + +Any import in a bundle which resolves into another bundles "context" directory, ie `src/plugins/*`, must map explicitly to a "public dir" exported by that plugin. If the resolved import is not in the list of public dirs an error will be thrown and the optimizer will fail to build that bundle until the error is fixed. + ## API To run the optimizer from code, you can import the [`OptimizerConfig`][OptimizerConfig] class and [`runOptimizer`][Optimizer] function. Create an [`OptimizerConfig`][OptimizerConfig] instance by calling it's static `create()` method with some options, then pass it to the [`runOptimizer`][Optimizer] function. `runOptimizer()` returns an observable of update objects, which are summaries of the optimizer state plus an optional `event` property which describes the internal events occuring and may be of use. You can use the [`logOptimizerState()`][LogOptimizerState] helper to write the relevant bits of state to a tooling log or checkout it's implementation to see how the internal events like [`WorkerStdio`][ObserveWorker] and [`WorkerStarted`][ObserveWorker] are used. diff --git a/packages/kbn-optimizer/package.json b/packages/kbn-optimizer/package.json index a372b9e394b9a9..c7bf1dd60985d3 100644 --- a/packages/kbn-optimizer/package.json +++ b/packages/kbn-optimizer/package.json @@ -45,6 +45,7 @@ "terser-webpack-plugin": "^2.1.2", "tinymath": "1.2.1", "url-loader": "^2.2.0", + "val-loader": "^1.1.1", "watchpack": "^1.6.0", "webpack": "^4.41.5", "webpack-merge": "^4.2.2" diff --git a/packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/bar/public/index.ts b/packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/bar/public/index.ts index 7ddd10f4a388fb..c881a15eac5b53 100644 --- a/packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/bar/public/index.ts +++ b/packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/bar/public/index.ts @@ -19,6 +19,6 @@ import './legacy/styles.scss'; import './index.scss'; -import { fooLibFn } from '../../foo/public/index'; +import { fooLibFn } from '../../foo/public'; export * from './lib'; export { fooLibFn }; diff --git a/packages/kbn-optimizer/src/common/bundle.test.ts b/packages/kbn-optimizer/src/common/bundle.test.ts index ec78a1bdf020ec..b209bbca25ac4d 100644 --- a/packages/kbn-optimizer/src/common/bundle.test.ts +++ b/packages/kbn-optimizer/src/common/bundle.test.ts @@ -23,7 +23,7 @@ jest.mock('fs'); const SPEC: BundleSpec = { contextDir: '/foo/bar', - entry: 'entry', + publicDirNames: ['public'], id: 'bar', outputDir: '/foo/bar/target', sourceRoot: '/foo', @@ -49,9 +49,11 @@ it('creates cache keys', () => { }, "spec": Object { "contextDir": "/foo/bar", - "entry": "entry", "id": "bar", "outputDir": "/foo/bar/target", + "publicDirNames": Array [ + "public", + ], "sourceRoot": "/foo", "type": "plugin", }, @@ -82,9 +84,11 @@ it('parses bundles from JSON specs', () => { "state": undefined, }, "contextDir": "/foo/bar", - "entry": "entry", "id": "bar", "outputDir": "/foo/bar/target", + "publicDirNames": Array [ + "public", + ], "sourceRoot": "/foo", "type": "plugin", }, diff --git a/packages/kbn-optimizer/src/common/bundle.ts b/packages/kbn-optimizer/src/common/bundle.ts index 9e2ad186ba40c7..80af94c30f8da4 100644 --- a/packages/kbn-optimizer/src/common/bundle.ts +++ b/packages/kbn-optimizer/src/common/bundle.ts @@ -29,8 +29,8 @@ export interface BundleSpec { readonly type: typeof VALID_BUNDLE_TYPES[0]; /** Unique id for this bundle */ readonly id: string; - /** Webpack entry request for this plugin, relative to the contextDir */ - readonly entry: string; + /** directory names relative to the contextDir that can be imported from */ + readonly publicDirNames: string[]; /** Absolute path to the plugin source directory */ readonly contextDir: string; /** Absolute path to the root of the repository */ @@ -44,8 +44,8 @@ export class Bundle { public readonly type: BundleSpec['type']; /** Unique identifier for this bundle */ public readonly id: BundleSpec['id']; - /** Path, relative to `contextDir`, to the entry file for the Webpack bundle */ - public readonly entry: BundleSpec['entry']; + /** directory names relative to the contextDir that can be imported from */ + public readonly publicDirNames: BundleSpec['publicDirNames']; /** * Absolute path to the root of the bundle context (plugin directory) * where the entry is resolved relative to and the default output paths @@ -62,7 +62,7 @@ export class Bundle { constructor(spec: BundleSpec) { this.type = spec.type; this.id = spec.id; - this.entry = spec.entry; + this.publicDirNames = spec.publicDirNames; this.contextDir = spec.contextDir; this.sourceRoot = spec.sourceRoot; this.outputDir = spec.outputDir; @@ -73,8 +73,6 @@ export class Bundle { /** * Calculate the cache key for this bundle based from current * mtime values. - * - * @param mtimes pre-fetched mtimes (ms || undefined) for all referenced files */ createCacheKey(files: string[], mtimes: Map): unknown { return { @@ -94,7 +92,7 @@ export class Bundle { return { type: this.type, id: this.id, - entry: this.entry, + publicDirNames: this.publicDirNames, contextDir: this.contextDir, sourceRoot: this.sourceRoot, outputDir: this.outputDir, @@ -134,9 +132,9 @@ export function parseBundles(json: string) { throw new Error('`bundles[]` must have a string `id` property'); } - const { entry } = spec; - if (!(typeof entry === 'string')) { - throw new Error('`bundles[]` must have a string `entry` property'); + const { publicDirNames } = spec; + if (!Array.isArray(publicDirNames) || !publicDirNames.every((d) => typeof d === 'string')) { + throw new Error('`bundles[]` must have an array of strings `publicDirNames` property'); } const { contextDir } = spec; @@ -157,7 +155,7 @@ export function parseBundles(json: string) { return new Bundle({ type, id, - entry, + publicDirNames, contextDir, sourceRoot, outputDir, diff --git a/packages/kbn-optimizer/src/common/bundle_cache.ts b/packages/kbn-optimizer/src/common/bundle_cache.ts index 1dbc7f1d1b6b0c..5ae3e4c28a2018 100644 --- a/packages/kbn-optimizer/src/common/bundle_cache.ts +++ b/packages/kbn-optimizer/src/common/bundle_cache.ts @@ -25,6 +25,7 @@ export interface State { cacheKey?: unknown; moduleCount?: number; files?: string[]; + bundleRefExportIds?: string[]; } const DEFAULT_STATE: State = {}; @@ -87,6 +88,10 @@ export class BundleCache { return this.get().files; } + public getBundleRefExportIds() { + return this.get().bundleRefExportIds; + } + public getCacheKey() { return this.get().cacheKey; } diff --git a/packages/kbn-optimizer/src/common/bundle_refs.ts b/packages/kbn-optimizer/src/common/bundle_refs.ts new file mode 100644 index 00000000000000..a5c60f2031c0b6 --- /dev/null +++ b/packages/kbn-optimizer/src/common/bundle_refs.ts @@ -0,0 +1,130 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import Path from 'path'; + +import { Bundle } from './bundle'; +import { UnknownVals } from './ts_helpers'; + +export interface BundleRef { + bundleId: string; + contextDir: string; + contextPrefix: string; + entry: string; + exportId: string; +} + +export class BundleRefs { + static fromBundles(bundles: Bundle[]) { + return new BundleRefs( + bundles.reduce( + (acc: BundleRef[], b) => [ + ...acc, + ...b.publicDirNames.map( + (name): BundleRef => ({ + bundleId: b.id, + contextDir: b.contextDir, + // Path.resolve converts separators and strips the final separator + contextPrefix: Path.resolve(b.contextDir) + Path.sep, + entry: name, + exportId: `${b.type}/${b.id}/${name}`, + }) + ), + ], + [] + ) + ); + } + + static parseSpec(json: unknown) { + if (typeof json !== 'string') { + throw new Error('expected `bundleRefs` spec to be a JSON string'); + } + + let spec; + try { + spec = JSON.parse(json); + } catch (error) { + throw new Error('`bundleRefs` spec must be valid JSON'); + } + + if (!Array.isArray(spec)) { + throw new Error('`bundleRefs` spec must be an array'); + } + + return new BundleRefs( + spec.map( + (refSpec: UnknownVals): BundleRef => { + if (typeof refSpec !== 'object' || !refSpec) { + throw new Error('`bundleRefs[]` must be an object'); + } + + const { bundleId } = refSpec; + if (typeof bundleId !== 'string') { + throw new Error('`bundleRefs[].bundleId` must be a string'); + } + + const { contextDir } = refSpec; + if (typeof contextDir !== 'string' || !Path.isAbsolute(contextDir)) { + throw new Error('`bundleRefs[].contextDir` must be an absolute directory'); + } + + const { contextPrefix } = refSpec; + if (typeof contextPrefix !== 'string' || !Path.isAbsolute(contextPrefix)) { + throw new Error('`bundleRefs[].contextPrefix` must be an absolute directory'); + } + + const { entry } = refSpec; + if (typeof entry !== 'string') { + throw new Error('`bundleRefs[].entry` must be a string'); + } + + const { exportId } = refSpec; + if (typeof exportId !== 'string') { + throw new Error('`bundleRefs[].exportId` must be a string'); + } + + return { + bundleId, + contextDir, + contextPrefix, + entry, + exportId, + }; + } + ) + ); + } + + constructor(private readonly refs: BundleRef[]) {} + + public filterByExportIds(exportIds: string[]) { + return this.refs.filter((r) => exportIds.includes(r.exportId)); + } + + public filterByContextPrefix(bundle: Bundle, absolutePath: string) { + return this.refs.filter( + (ref) => ref.bundleId !== bundle.id && absolutePath.startsWith(ref.contextPrefix) + ); + } + + public toSpecJson() { + return JSON.stringify(this.refs); + } +} diff --git a/packages/kbn-optimizer/src/common/index.ts b/packages/kbn-optimizer/src/common/index.ts index c51905be045653..7d021a5ee78475 100644 --- a/packages/kbn-optimizer/src/common/index.ts +++ b/packages/kbn-optimizer/src/common/index.ts @@ -19,6 +19,7 @@ export * from './bundle'; export * from './bundle_cache'; +export * from './bundle_refs'; export * from './worker_config'; export * from './worker_messages'; export * from './compiler_messages'; diff --git a/packages/kbn-optimizer/src/integration_tests/__snapshots__/basic_optimization.test.ts.snap b/packages/kbn-optimizer/src/integration_tests/__snapshots__/basic_optimization.test.ts.snap index 2814ab32017d29..2265bad9f6afaf 100644 --- a/packages/kbn-optimizer/src/integration_tests/__snapshots__/basic_optimization.test.ts.snap +++ b/packages/kbn-optimizer/src/integration_tests/__snapshots__/basic_optimization.test.ts.snap @@ -9,9 +9,11 @@ OptimizerConfig { "state": undefined, }, "contextDir": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar, - "entry": "./public/index", "id": "bar", "outputDir": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/target/public, + "publicDirNames": Array [ + "public", + ], "sourceRoot": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo, "type": "plugin", }, @@ -21,9 +23,11 @@ OptimizerConfig { "state": undefined, }, "contextDir": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo, - "entry": "./public/index", "id": "foo", "outputDir": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/target/public, + "publicDirNames": Array [ + "public", + ], "sourceRoot": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo, "type": "plugin", }, @@ -35,16 +39,19 @@ OptimizerConfig { "plugins": Array [ Object { "directory": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar, + "extraPublicDirs": Array [], "id": "bar", "isUiPlugin": true, }, Object { "directory": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/baz, + "extraPublicDirs": Array [], "id": "baz", "isUiPlugin": false, }, Object { "directory": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo, + "extraPublicDirs": Array [], "id": "foo", "isUiPlugin": true, }, @@ -55,8 +62,8 @@ OptimizerConfig { } `; -exports[`prepares assets for distribution: bar bundle 1`] = `"var __kbnBundles__=typeof __kbnBundles__===\\"object\\"?__kbnBundles__:{};__kbnBundles__[\\"plugin/bar\\"]=function(modules){function webpackJsonpCallback(data){var chunkIds=data[0];var moreModules=data[1];var moduleId,chunkId,i=0,resolves=[];for(;i { const foo = config.bundles.find((b) => b.id === 'foo')!; expect(foo).toBeTruthy(); foo.cache.refresh(); - expect(foo.cache.getModuleCount()).toBe(5); + expect(foo.cache.getModuleCount()).toBe(6); expect(foo.cache.getReferencedFiles()).toMatchInlineSnapshot(` Array [ /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/async_import.ts, /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/ext.ts, /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/index.ts, /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/lib.ts, + /packages/kbn-optimizer/target/worker/entry_point_creator.js, /packages/kbn-ui-shared-deps/public_path_module_creator.js, ] `); @@ -148,7 +149,7 @@ it('builds expected bundles, saves bundle counts to metadata', async () => { bar.cache.refresh(); expect(bar.cache.getModuleCount()).toBe( // code + styles + style/css-loader runtimes + public path updater - 21 + 18 ); expect(bar.cache.getReferencedFiles()).toMatchInlineSnapshot(` @@ -159,11 +160,8 @@ it('builds expected bundles, saves bundle counts to metadata', async () => { /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/public/index.ts, /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/public/legacy/styles.scss, /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/public/lib.ts, - /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/async_import.ts, - /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/ext.ts, - /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/index.ts, - /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/lib.ts, /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/src/legacy/ui/public/icon.svg, + /packages/kbn-optimizer/target/worker/entry_point_creator.js, /packages/kbn-ui-shared-deps/public_path_module_creator.js, ] `); diff --git a/packages/kbn-optimizer/src/integration_tests/bundle_cache.test.ts b/packages/kbn-optimizer/src/integration_tests/bundle_cache.test.ts index 39064c64062e82..48cab508954a08 100644 --- a/packages/kbn-optimizer/src/integration_tests/bundle_cache.test.ts +++ b/packages/kbn-optimizer/src/integration_tests/bundle_cache.test.ts @@ -75,6 +75,7 @@ it('emits "bundle cached" event when everything is updated', async () => { optimizerCacheKey, files, moduleCount: files.length, + bundleRefExportIds: [], }); const cacheEvents = await getBundleCacheEvent$(config, optimizerCacheKey) @@ -115,6 +116,7 @@ it('emits "bundle not cached" event when cacheKey is up to date but caching is d optimizerCacheKey, files, moduleCount: files.length, + bundleRefExportIds: [], }); const cacheEvents = await getBundleCacheEvent$(config, optimizerCacheKey) @@ -155,6 +157,7 @@ it('emits "bundle not cached" event when optimizerCacheKey is missing', async () optimizerCacheKey: undefined, files, moduleCount: files.length, + bundleRefExportIds: [], }); const cacheEvents = await getBundleCacheEvent$(config, optimizerCacheKey) @@ -195,6 +198,7 @@ it('emits "bundle not cached" event when optimizerCacheKey is outdated, includes optimizerCacheKey: 'old', files, moduleCount: files.length, + bundleRefExportIds: [], }); const cacheEvents = await getBundleCacheEvent$(config, optimizerCacheKey) @@ -217,6 +221,53 @@ it('emits "bundle not cached" event when optimizerCacheKey is outdated, includes `); }); +it('emits "bundle not cached" event when bundleRefExportIds is outdated, includes diff', async () => { + const config = OptimizerConfig.create({ + repoRoot: MOCK_REPO_DIR, + pluginScanDirs: [], + pluginPaths: [Path.resolve(MOCK_REPO_DIR, 'plugins/foo')], + maxWorkerCount: 1, + }); + const [bundle] = config.bundles; + + const optimizerCacheKey = 'optimizerCacheKey'; + const files = [ + Path.resolve(MOCK_REPO_DIR, 'plugins/foo/public/ext.ts'), + Path.resolve(MOCK_REPO_DIR, 'plugins/foo/public/index.ts'), + Path.resolve(MOCK_REPO_DIR, 'plugins/foo/public/lib.ts'), + ]; + const mtimes = await getMtimes(files); + const cacheKey = bundle.createCacheKey(files, mtimes); + + bundle.cache.set({ + cacheKey, + optimizerCacheKey, + files, + moduleCount: files.length, + bundleRefExportIds: ['plugin/bar/public'], + }); + + const cacheEvents = await getBundleCacheEvent$(config, optimizerCacheKey) + .pipe(toArray()) + .toPromise(); + + expect(cacheEvents).toMatchInlineSnapshot(` + Array [ + Object { + "bundle": , + "diff": "- Expected + + Received + +  [ + + \\"plugin/bar/public\\" +  ]", + "reason": "bundle references outdated", + "type": "bundle not cached", + }, + ] + `); +}); + it('emits "bundle not cached" event when cacheKey is missing', async () => { const config = OptimizerConfig.create({ repoRoot: MOCK_REPO_DIR, @@ -238,6 +289,7 @@ it('emits "bundle not cached" event when cacheKey is missing', async () => { optimizerCacheKey, files, moduleCount: files.length, + bundleRefExportIds: [], }); const cacheEvents = await getBundleCacheEvent$(config, optimizerCacheKey) @@ -276,6 +328,7 @@ it('emits "bundle not cached" event when cacheKey is outdated', async () => { optimizerCacheKey, files, moduleCount: files.length, + bundleRefExportIds: [], }); jest.spyOn(bundle, 'createCacheKey').mockImplementation(() => 'new'); diff --git a/packages/kbn-optimizer/src/integration_tests/watch_bundles_for_changes.test.ts b/packages/kbn-optimizer/src/integration_tests/watch_bundles_for_changes.test.ts index 91d0f308e0ef6f..176b17c979da94 100644 --- a/packages/kbn-optimizer/src/integration_tests/watch_bundles_for_changes.test.ts +++ b/packages/kbn-optimizer/src/integration_tests/watch_bundles_for_changes.test.ts @@ -29,14 +29,14 @@ jest.mock('fs'); jest.mock('watchpack'); const MockWatchPack: jest.MockedClass = jest.requireMock('watchpack'); -const bundleEntryPath = (bundle: Bundle) => `${bundle.contextDir}/${bundle.entry}`; +const bundleEntryPath = (bundle: Bundle) => `${bundle.contextDir}/public/index.ts`; const makeTestBundle = (id: string) => { const bundle = new Bundle({ type: 'plugin', id, contextDir: `/repo/plugins/${id}/public`, - entry: 'index.ts', + publicDirNames: ['public'], outputDir: `/repo/plugins/${id}/target/public`, sourceRoot: `/repo`, }); diff --git a/packages/kbn-optimizer/src/optimizer/assign_bundles_to_workers.test.ts b/packages/kbn-optimizer/src/optimizer/assign_bundles_to_workers.test.ts index 4671276797049c..ca50a49e269139 100644 --- a/packages/kbn-optimizer/src/optimizer/assign_bundles_to_workers.test.ts +++ b/packages/kbn-optimizer/src/optimizer/assign_bundles_to_workers.test.ts @@ -57,7 +57,7 @@ const assertReturnVal = (workers: Assignments[]) => { const testBundle = (id: string) => new Bundle({ contextDir: `/repo/plugin/${id}/public`, - entry: 'index.ts', + publicDirNames: ['public'], id, outputDir: `/repo/plugins/${id}/target/public`, sourceRoot: `/repo`, diff --git a/packages/kbn-optimizer/src/optimizer/bundle_cache.ts b/packages/kbn-optimizer/src/optimizer/bundle_cache.ts index 55e8e1d3fd0842..83db8570bd4085 100644 --- a/packages/kbn-optimizer/src/optimizer/bundle_cache.ts +++ b/packages/kbn-optimizer/src/optimizer/bundle_cache.ts @@ -20,7 +20,7 @@ import * as Rx from 'rxjs'; import { mergeAll } from 'rxjs/operators'; -import { Bundle } from '../common'; +import { Bundle, BundleRefs } from '../common'; import { OptimizerConfig } from './optimizer_config'; import { getMtimes } from './get_mtimes'; @@ -35,7 +35,9 @@ export interface BundleNotCachedEvent { | 'optimizer cache key mismatch' | 'missing cache key' | 'cache key mismatch' - | 'cache disabled'; + | 'cache disabled' + | 'bundle references missing' + | 'bundle references outdated'; diff?: string; bundle: Bundle; } @@ -52,6 +54,7 @@ export function getBundleCacheEvent$( return Rx.defer(async () => { const events: BundleCacheEvent[] = []; const eligibleBundles: Bundle[] = []; + const bundleRefs = BundleRefs.fromBundles(config.bundles); for (const bundle of config.bundles) { if (!config.cache) { @@ -93,6 +96,32 @@ export function getBundleCacheEvent$( continue; } + const bundleRefExportIds = bundle.cache.getBundleRefExportIds(); + if (!bundleRefExportIds) { + events.push({ + type: 'bundle not cached', + reason: 'bundle references missing', + bundle, + }); + continue; + } + + const refs = bundleRefs.filterByExportIds(bundleRefExportIds); + + const bundleRefsDiff = diffCacheKey( + refs.map((r) => r.exportId).sort((a, b) => a.localeCompare(b)), + bundleRefExportIds + ); + if (bundleRefsDiff) { + events.push({ + type: 'bundle not cached', + reason: 'bundle references outdated', + diff: bundleRefsDiff, + bundle, + }); + continue; + } + eligibleBundles.push(bundle); } diff --git a/packages/kbn-optimizer/src/optimizer/cache_keys.ts b/packages/kbn-optimizer/src/optimizer/cache_keys.ts index 2766f6d63702b7..d0aaad979485d2 100644 --- a/packages/kbn-optimizer/src/optimizer/cache_keys.ts +++ b/packages/kbn-optimizer/src/optimizer/cache_keys.ts @@ -37,16 +37,6 @@ import { OptimizerConfig } from './optimizer_config'; const OPTIMIZER_DIR = Path.dirname(require.resolve('../../package.json')); const RELATIVE_DIR = Path.relative(REPO_ROOT, OPTIMIZER_DIR); -function omit(obj: T, keys: K[]): Omit { - const result: any = {}; - for (const [key, value] of Object.entries(obj) as any) { - if (!keys.includes(key)) { - result[key] = value; - } - } - return result as Omit; -} - export function diffCacheKey(expected?: unknown, actual?: unknown) { const expectedJson = jsonStable(expected, { space: ' ', @@ -185,7 +175,7 @@ export async function getOptimizerCacheKey(config: OptimizerConfig) { bootstrap, deletedPaths, modifiedTimes: {} as Record, - workerConfig: omit(config.getWorkerConfig('♻'), ['watch', 'profileWebpack', 'cache']), + workerConfig: config.getCacheableWorkerConfig(), }; const mtimes = await getMtimes(modifiedPaths); diff --git a/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.test.ts b/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.test.ts index 2174c488ad6cc4..bbd3ddc11f448d 100644 --- a/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.test.ts +++ b/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.test.ts @@ -31,16 +31,19 @@ it('returns a bundle for core and each plugin', () => { directory: '/repo/plugins/foo', id: 'foo', isUiPlugin: true, + extraPublicDirs: [], }, { directory: '/repo/plugins/bar', id: 'bar', isUiPlugin: false, + extraPublicDirs: [], }, { directory: '/outside/of/repo/plugins/baz', id: 'baz', isUiPlugin: true, + extraPublicDirs: [], }, ], '/repo' @@ -49,17 +52,21 @@ it('returns a bundle for core and each plugin', () => { Array [ Object { "contextDir": /plugins/foo, - "entry": "./public/index", "id": "foo", "outputDir": /plugins/foo/target/public, + "publicDirNames": Array [ + "public", + ], "sourceRoot": , "type": "plugin", }, Object { "contextDir": "/outside/of/repo/plugins/baz", - "entry": "./public/index", "id": "baz", "outputDir": "/outside/of/repo/plugins/baz/target/public", + "publicDirNames": Array [ + "public", + ], "sourceRoot": , "type": "plugin", }, diff --git a/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.ts b/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.ts index b75a8a6edc2647..2635289088725f 100644 --- a/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.ts +++ b/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.ts @@ -31,7 +31,7 @@ export function getPluginBundles(plugins: KibanaPlatformPlugin[], repoRoot: stri new Bundle({ type: 'plugin', id: p.id, - entry: './public/index', + publicDirNames: ['public', ...p.extraPublicDirs], sourceRoot: repoRoot, contextDir: p.directory, outputDir: Path.resolve(p.directory, 'target/public'), diff --git a/packages/kbn-optimizer/src/optimizer/kibana_platform_plugins.test.ts b/packages/kbn-optimizer/src/optimizer/kibana_platform_plugins.test.ts index e047b6d1e44cf4..0961881df461c5 100644 --- a/packages/kbn-optimizer/src/optimizer/kibana_platform_plugins.test.ts +++ b/packages/kbn-optimizer/src/optimizer/kibana_platform_plugins.test.ts @@ -37,21 +37,25 @@ it('parses kibana.json files of plugins found in pluginDirs', () => { Array [ Object { "directory": /packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/bar, + "extraPublicDirs": Array [], "id": "bar", "isUiPlugin": true, }, Object { "directory": /packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/baz, + "extraPublicDirs": Array [], "id": "baz", "isUiPlugin": false, }, Object { "directory": /packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/foo, + "extraPublicDirs": Array [], "id": "foo", "isUiPlugin": true, }, Object { "directory": /packages/kbn-optimizer/src/__fixtures__/mock_repo/test_plugins/test_baz, + "extraPublicDirs": Array [], "id": "test_baz", "isUiPlugin": false, }, diff --git a/packages/kbn-optimizer/src/optimizer/kibana_platform_plugins.ts b/packages/kbn-optimizer/src/optimizer/kibana_platform_plugins.ts index 992feab6cd364c..bfc60a29efa27d 100644 --- a/packages/kbn-optimizer/src/optimizer/kibana_platform_plugins.ts +++ b/packages/kbn-optimizer/src/optimizer/kibana_platform_plugins.ts @@ -26,6 +26,7 @@ export interface KibanaPlatformPlugin { readonly directory: string; readonly id: string; readonly isUiPlugin: boolean; + readonly extraPublicDirs: string[]; } /** @@ -64,9 +65,24 @@ function readKibanaPlatformPlugin(manifestPath: string): KibanaPlatformPlugin { throw new TypeError('expected new platform plugin manifest to have a string id'); } + let extraPublicDirs: string[] | undefined; + if (manifest.extraPublicDirs) { + if ( + !Array.isArray(manifest.extraPublicDirs) || + !manifest.extraPublicDirs.every((p) => typeof p === 'string') + ) { + throw new TypeError( + 'expected new platform plugin manifest to have an array of strings `extraPublicDirs` property' + ); + } + + extraPublicDirs = manifest.extraPublicDirs as string[]; + } + return { directory: Path.dirname(manifestPath), id: manifest.id, isUiPlugin: !!manifest.ui, + extraPublicDirs: extraPublicDirs || [], }; } diff --git a/packages/kbn-optimizer/src/optimizer/observe_worker.ts b/packages/kbn-optimizer/src/optimizer/observe_worker.ts index c929cf62d1bb0a..4527052fa821a4 100644 --- a/packages/kbn-optimizer/src/optimizer/observe_worker.ts +++ b/packages/kbn-optimizer/src/optimizer/observe_worker.ts @@ -24,7 +24,7 @@ import execa from 'execa'; import * as Rx from 'rxjs'; import { map, takeUntil } from 'rxjs/operators'; -import { isWorkerMsg, WorkerConfig, WorkerMsg, Bundle } from '../common'; +import { isWorkerMsg, WorkerConfig, WorkerMsg, Bundle, BundleRefs } from '../common'; import { OptimizerConfig } from './optimizer_config'; @@ -74,7 +74,11 @@ function usingWorkerProc( ) { return Rx.using( (): ProcResource => { - const args = [JSON.stringify(workerConfig), JSON.stringify(bundles.map((b) => b.toSpec()))]; + const args = [ + JSON.stringify(workerConfig), + JSON.stringify(bundles.map((b) => b.toSpec())), + BundleRefs.fromBundles(config.bundles).toSpecJson(), + ]; const proc = execa.node(require.resolve('../worker/run_worker'), args, { nodeOptions: [ diff --git a/packages/kbn-optimizer/src/optimizer/optimizer_config.ts b/packages/kbn-optimizer/src/optimizer/optimizer_config.ts index 37d8a4f5eb8aea..c9e9b3ad01ccc6 100644 --- a/packages/kbn-optimizer/src/optimizer/optimizer_config.ts +++ b/packages/kbn-optimizer/src/optimizer/optimizer_config.ts @@ -20,7 +20,7 @@ import Path from 'path'; import Os from 'os'; -import { Bundle, WorkerConfig } from '../common'; +import { Bundle, WorkerConfig, CacheableWorkerConfig } from '../common'; import { findKibanaPlatformPlugins, KibanaPlatformPlugin } from './kibana_platform_plugins'; import { getPluginBundles } from './get_plugin_bundles'; @@ -34,6 +34,16 @@ function pickMaxWorkerCount(dist: boolean) { return Math.max(maxWorkers, 2); } +function omit(obj: T, keys: K[]): Omit { + const result: any = {}; + for (const [key, value] of Object.entries(obj) as any) { + if (!keys.includes(key)) { + result[key] = value; + } + } + return result as Omit; +} + interface Options { /** absolute path to root of the repo/build */ repoRoot: string; @@ -152,7 +162,7 @@ export class OptimizerConfig { new Bundle({ type: 'entry', id: 'core', - entry: './public/index', + publicDirNames: ['public', 'public/utils'], sourceRoot: options.repoRoot, contextDir: Path.resolve(options.repoRoot, 'src/core'), outputDir: Path.resolve(options.repoRoot, 'src/core/target/public'), @@ -198,4 +208,14 @@ export class OptimizerConfig { browserslistEnv: this.dist ? 'production' : process.env.BROWSERSLIST_ENV || 'dev', }; } + + getCacheableWorkerConfig(): CacheableWorkerConfig { + return omit(this.getWorkerConfig('♻'), [ + // these config options don't change the output of the bundles, so + // should not invalidate caches when they change + 'watch', + 'profileWebpack', + 'cache', + ]); + } } diff --git a/packages/kbn-optimizer/src/worker/bundle_ref_module.ts b/packages/kbn-optimizer/src/worker/bundle_ref_module.ts new file mode 100644 index 00000000000000..cde25564cf5282 --- /dev/null +++ b/packages/kbn-optimizer/src/worker/bundle_ref_module.ts @@ -0,0 +1,67 @@ +/* eslint-disable @kbn/eslint/require-license-header */ + +/** + * @notice + * + * This module was heavily inspired by the externals plugin that ships with webpack@97d58d31 + * MIT License http://www.opensource.org/licenses/mit-license.php + * Author Tobias Koppers @sokra + */ + +// @ts-ignore not typed by @types/webpack +import Module from 'webpack/lib/Module'; + +export class BundleRefModule extends Module { + public built = false; + public buildMeta?: any; + public buildInfo?: any; + public exportsArgument = '__webpack_exports__'; + + constructor(public readonly exportId: string) { + super('kbn/bundleRef', null); + } + + libIdent() { + return this.exportId; + } + + chunkCondition(chunk: any) { + return chunk.hasEntryModule(); + } + + identifier() { + return '@kbn/bundleRef ' + JSON.stringify(this.exportId); + } + + readableIdentifier() { + return this.identifier(); + } + + needRebuild() { + return false; + } + + build(_: any, __: any, ___: any, ____: any, callback: () => void) { + this.built = true; + this.buildMeta = {}; + this.buildInfo = {}; + callback(); + } + + source() { + return ` + __webpack_require__.r(__webpack_exports__); + var ns = __kbnBundles__.get('${this.exportId}'); + Object.defineProperties(__webpack_exports__, Object.getOwnPropertyDescriptors(ns)) + `; + } + + size() { + return 42; + } + + updateHash(hash: any) { + hash.update(this.identifier()); + super.updateHash(hash); + } +} diff --git a/packages/kbn-optimizer/src/worker/bundle_refs_plugin.ts b/packages/kbn-optimizer/src/worker/bundle_refs_plugin.ts new file mode 100644 index 00000000000000..6defcaa787b7d7 --- /dev/null +++ b/packages/kbn-optimizer/src/worker/bundle_refs_plugin.ts @@ -0,0 +1,165 @@ +/* eslint-disable @kbn/eslint/require-license-header */ + +/** + * @notice + * + * This module was heavily inspired by the externals plugin that ships with webpack@97d58d31 + * MIT License http://www.opensource.org/licenses/mit-license.php + * Author Tobias Koppers @sokra + */ + +import Path from 'path'; +import Fs from 'fs'; + +import webpack from 'webpack'; + +import { Bundle, BundleRefs, BundleRef } from '../common'; +import { BundleRefModule } from './bundle_ref_module'; + +const RESOLVE_EXTENSIONS = ['.js', '.ts', '.tsx']; + +function safeStat(path: string): Promise { + return new Promise((resolve, reject) => { + Fs.stat(path, (error, stat) => { + if (error?.code === 'ENOENT') { + resolve(undefined); + } else if (error) { + reject(error); + } else { + resolve(stat); + } + }); + }); +} + +interface RequestData { + context: string; + dependencies: Array<{ request: string }>; +} + +type Callback = (error?: any, result?: T) => void; +type ModuleFactory = (data: RequestData, callback: Callback) => void; + +/** + * Isolate the weired type juggling we have to do to add a hook to the webpack compiler + */ +function hookIntoCompiler( + compiler: webpack.Compiler, + handler: (context: string, request: string) => Promise +) { + compiler.hooks.compile.tap('BundleRefsPlugin', (compilationParams: any) => { + compilationParams.normalModuleFactory.hooks.factory.tap( + 'BundleRefsPlugin/normalModuleFactory/factory', + (wrappedFactory: ModuleFactory): ModuleFactory => (data, callback) => { + const context = data.context; + const dep = data.dependencies[0]; + + handler(context, dep.request).then( + (result) => { + if (!result) { + wrappedFactory(data, callback); + } else { + callback(undefined, result); + } + }, + (error) => callback(error) + ); + } + ); + }); +} + +export class BundleRefsPlugin { + private resolvedRequestCache = new Map>(); + + constructor(private readonly bundle: Bundle, public readonly bundleRefs: BundleRefs) {} + + apply(compiler: webpack.Compiler) { + hookIntoCompiler(compiler, async (context, request) => { + const ref = await this.resolveRef(context, request); + if (ref) { + return new BundleRefModule(ref.exportId); + } + }); + } + + private cachedResolveRequest(context: string, request: string) { + const absoluteRequest = Path.resolve(context, request); + const cached = this.resolvedRequestCache.get(absoluteRequest); + + if (cached) { + return cached; + } + + const promise = this.resolveRequest(absoluteRequest); + this.resolvedRequestCache.set(absoluteRequest, promise); + return promise; + } + + private async resolveRequest(absoluteRequest: string) { + const stats = await safeStat(absoluteRequest); + if (stats && stats.isFile()) { + return absoluteRequest; + } + + if (stats?.isDirectory()) { + for (const ext of RESOLVE_EXTENSIONS) { + const indexPath = Path.resolve(absoluteRequest, `index${ext}`); + const indexStats = await safeStat(indexPath); + if (indexStats?.isFile()) { + return indexPath; + } + } + } + + return; + } + + /** + * Determine if an import request resolves to a bundleRef export id. If the + * request resolves to a bundle ref context but none of the exported directories + * then an error is thrown. If the request does not resolve to a bundleRef then + * undefined is returned. Otherwise it returns the referenced bundleRef. + */ + private async resolveRef(context: string, request: string) { + // ignore imports that have loaders defined or are not relative seeming + if (request.includes('!') || !request.startsWith('.')) { + return; + } + + const requestExt = Path.extname(request); + if (requestExt && !RESOLVE_EXTENSIONS.includes(requestExt)) { + return; + } + + const resolved = await this.cachedResolveRequest(context, request); + if (!resolved) { + return; + } + + const eligibleRefs = this.bundleRefs.filterByContextPrefix(this.bundle, resolved); + if (!eligibleRefs.length) { + // import doesn't match a bundle context + return; + } + + let matchingRef: BundleRef | undefined; + for (const ref of eligibleRefs) { + const resolvedEntry = await this.cachedResolveRequest(ref.contextDir, ref.entry); + if (resolved === resolvedEntry) { + matchingRef = ref; + break; + } + } + + if (!matchingRef) { + const bundleId = Array.from(new Set(eligibleRefs.map((r) => r.bundleId))).join(', '); + const publicDir = eligibleRefs.map((r) => r.entry).join(', '); + throw new Error( + `import [${request}] references a non-public export of the [${bundleId}] bundle and must point to one of the public directories: [${publicDir}]` + ); + } + + return matchingRef; + } +} diff --git a/packages/kbn-optimizer/src/worker/entry_point_creator.ts b/packages/kbn-optimizer/src/worker/entry_point_creator.ts new file mode 100644 index 00000000000000..a613e3e8925a4b --- /dev/null +++ b/packages/kbn-optimizer/src/worker/entry_point_creator.ts @@ -0,0 +1,28 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +module.exports = function ({ entries }: { entries: Array<{ importId: string; relPath: string }> }) { + const lines = entries.map(({ importId, relPath }) => [ + `__kbnBundles__.define('${importId}', __webpack_require__, require.resolve('./${relPath}'))`, + ]); + + return { + code: lines.join('\n'), + }; +}; diff --git a/packages/kbn-optimizer/src/worker/run_compilers.ts b/packages/kbn-optimizer/src/worker/run_compilers.ts index 4ab289d031d721..de5e9372e9e7ac 100644 --- a/packages/kbn-optimizer/src/worker/run_compilers.ts +++ b/packages/kbn-optimizer/src/worker/run_compilers.ts @@ -35,7 +35,9 @@ import { WorkerConfig, ascending, parseFilePath, + BundleRefs, } from '../common'; +import { BundleRefModule } from './bundle_ref_module'; import { getWebpackConfig } from './webpack.config'; import { isFailureStats, failedStatsToErrorMessage } from './webpack_helpers'; import { @@ -43,7 +45,6 @@ import { isNormalModule, isIgnoredModule, isConcatenatedModule, - WebpackNormalModule, getModulePath, } from './webpack_helpers'; @@ -98,40 +99,43 @@ const observeCompiler = ( }); } - const normalModules = stats.compilation.modules.filter( - (module): module is WebpackNormalModule => { - if (isNormalModule(module)) { - return true; - } + const bundleRefExportIds: string[] = []; + const referencedFiles = new Set(); + let normalModuleCount = 0; + + for (const module of stats.compilation.modules) { + if (isNormalModule(module)) { + normalModuleCount += 1; + const path = getModulePath(module); + const parsedPath = parseFilePath(path); - if (isExternalModule(module) || isIgnoredModule(module) || isConcatenatedModule(module)) { - return false; + if (!parsedPath.dirs.includes('node_modules')) { + referencedFiles.add(path); + continue; } - throw new Error(`Unexpected module type: ${inspect(module)}`); + const nmIndex = parsedPath.dirs.lastIndexOf('node_modules'); + const isScoped = parsedPath.dirs[nmIndex + 1].startsWith('@'); + referencedFiles.add( + Path.join( + parsedPath.root, + ...parsedPath.dirs.slice(0, nmIndex + 1 + (isScoped ? 2 : 1)), + 'package.json' + ) + ); + continue; } - ); - - const referencedFiles = new Set(); - for (const module of normalModules) { - const path = getModulePath(module); - const parsedPath = parseFilePath(path); + if (module instanceof BundleRefModule) { + bundleRefExportIds.push(module.exportId); + continue; + } - if (!parsedPath.dirs.includes('node_modules')) { - referencedFiles.add(path); + if (isExternalModule(module) || isIgnoredModule(module) || isConcatenatedModule(module)) { continue; } - const nmIndex = parsedPath.dirs.lastIndexOf('node_modules'); - const isScoped = parsedPath.dirs[nmIndex + 1].startsWith('@'); - referencedFiles.add( - Path.join( - parsedPath.root, - ...parsedPath.dirs.slice(0, nmIndex + 1 + (isScoped ? 2 : 1)), - 'package.json' - ) - ); + throw new Error(`Unexpected module type: ${inspect(module)}`); } const files = Array.from(referencedFiles).sort(ascending((p) => p)); @@ -150,14 +154,15 @@ const observeCompiler = ( ); bundle.cache.set({ + bundleRefExportIds, optimizerCacheKey: workerConfig.optimizerCacheKey, cacheKey: bundle.createCacheKey(files, mtimes), - moduleCount: normalModules.length, + moduleCount: normalModuleCount, files, }); return compilerMsgs.compilerSuccess({ - moduleCount: normalModules.length, + moduleCount: normalModuleCount, }); }) ); @@ -185,8 +190,14 @@ const observeCompiler = ( /** * Run webpack compilers */ -export const runCompilers = (workerConfig: WorkerConfig, bundles: Bundle[]) => { - const multiCompiler = webpack(bundles.map((def) => getWebpackConfig(def, workerConfig))); +export const runCompilers = ( + workerConfig: WorkerConfig, + bundles: Bundle[], + bundleRefs: BundleRefs +) => { + const multiCompiler = webpack( + bundles.map((def) => getWebpackConfig(def, bundleRefs, workerConfig)) + ); return Rx.merge( /** diff --git a/packages/kbn-optimizer/src/worker/run_worker.ts b/packages/kbn-optimizer/src/worker/run_worker.ts index f83c69477f4718..178637d39ab003 100644 --- a/packages/kbn-optimizer/src/worker/run_worker.ts +++ b/packages/kbn-optimizer/src/worker/run_worker.ts @@ -19,7 +19,14 @@ import * as Rx from 'rxjs'; -import { parseBundles, parseWorkerConfig, WorkerMsg, isWorkerMsg, WorkerMsgs } from '../common'; +import { + parseBundles, + parseWorkerConfig, + WorkerMsg, + isWorkerMsg, + WorkerMsgs, + BundleRefs, +} from '../common'; import { runCompilers } from './run_compilers'; @@ -76,11 +83,12 @@ setInterval(() => { Rx.defer(() => { const workerConfig = parseWorkerConfig(process.argv[2]); const bundles = parseBundles(process.argv[3]); + const bundleRefs = BundleRefs.parseSpec(process.argv[4]); // set BROWSERSLIST_ENV so that style/babel loaders see it before running compilers process.env.BROWSERSLIST_ENV = workerConfig.browserslistEnv; - return runCompilers(workerConfig, bundles); + return runCompilers(workerConfig, bundles, bundleRefs); }).subscribe( (msg) => { send(msg); diff --git a/packages/kbn-optimizer/src/worker/webpack.config.ts b/packages/kbn-optimizer/src/worker/webpack.config.ts index d31c098ca1f2eb..3daf21cdc38ccb 100644 --- a/packages/kbn-optimizer/src/worker/webpack.config.ts +++ b/packages/kbn-optimizer/src/worker/webpack.config.ts @@ -17,10 +17,8 @@ * under the License. */ -import Fs from 'fs'; import Path from 'path'; -import normalizePath from 'normalize-path'; import { stringifyRequest } from 'loader-utils'; import webpack from 'webpack'; // @ts-ignore @@ -32,88 +30,22 @@ import { CleanWebpackPlugin } from 'clean-webpack-plugin'; import CompressionPlugin from 'compression-webpack-plugin'; import * as UiSharedDeps from '@kbn/ui-shared-deps'; -import { Bundle, WorkerConfig, parseDirPath, DisallowedSyntaxPlugin } from '../common'; +import { Bundle, BundleRefs, WorkerConfig, parseDirPath, DisallowedSyntaxPlugin } from '../common'; +import { BundleRefsPlugin } from './bundle_refs_plugin'; const IS_CODE_COVERAGE = !!process.env.CODE_COVERAGE; const ISTANBUL_PRESET_PATH = require.resolve('@kbn/babel-preset/istanbul_preset'); const BABEL_PRESET_PATH = require.resolve('@kbn/babel-preset/webpack_preset'); -const SHARED_BUNDLES = [ - { - type: 'entry', - id: 'core', - rootRelativeDir: 'src/core/public', - }, - { - type: 'plugin', - id: 'data', - rootRelativeDir: 'src/plugins/data/public', - }, - { - type: 'plugin', - id: 'kibanaReact', - rootRelativeDir: 'src/plugins/kibana_react/public', - }, - { - type: 'plugin', - id: 'kibanaUtils', - rootRelativeDir: 'src/plugins/kibana_utils/public', - }, - { - type: 'plugin', - id: 'esUiShared', - rootRelativeDir: 'src/plugins/es_ui_shared/public', - }, -]; - -/** - * Determine externals statements for require/import statements by looking - * for requests resolving to the primary public export of the data, kibanaReact, - * amd kibanaUtils plugins. If this module is being imported then rewrite - * the import to access the global `__kbnBundles__` variables and access - * the relavent properties from that global object. - * - * @param bundle - * @param context the directory containing the module which made `request` - * @param request the request for a module from a commonjs require() call or import statement - */ -function dynamicExternals(bundle: Bundle, context: string, request: string) { - // ignore imports that have loaders defined or are not relative seeming - if (request.includes('!') || !request.startsWith('.')) { - return; - } - - // determine the most acurate resolution string we can without running full resolution - const rootRelative = normalizePath( - Path.relative(bundle.sourceRoot, Path.resolve(context, request)) - ); - for (const sharedBundle of SHARED_BUNDLES) { - if ( - rootRelative !== sharedBundle.rootRelativeDir || - `${bundle.type}/${bundle.id}` === `${sharedBundle.type}/${sharedBundle.id}` - ) { - continue; - } - - return `__kbnBundles__['${sharedBundle.type}/${sharedBundle.id}']`; - } - - // import doesn't match a root public import - return undefined; -} - -export function getWebpackConfig(bundle: Bundle, worker: WorkerConfig) { - const extensions = ['.js', '.ts', '.tsx', '.json']; - const entryExtension = extensions.find((ext) => - Fs.existsSync(Path.resolve(bundle.contextDir, bundle.entry) + ext) - ); +export function getWebpackConfig(bundle: Bundle, bundleRefs: BundleRefs, worker: WorkerConfig) { + const ENTRY_CREATOR = require.resolve('./entry_point_creator'); const commonConfig: webpack.Configuration = { node: { fs: 'empty' }, context: bundle.contextDir, cache: true, entry: { - [bundle.id]: `${bundle.entry}${entryExtension}`, + [bundle.id]: ENTRY_CREATOR, }, devtool: worker.dist ? false : '#cheap-source-map', @@ -128,27 +60,19 @@ export function getWebpackConfig(bundle: Bundle, worker: WorkerConfig) { info.absoluteResourcePath )}${info.query}`, jsonpFunction: `${bundle.id}_bundle_jsonpfunction`, - // When the entry point is loaded, assign it's default export - // to a key on the global `__kbnBundles__` object. - library: ['__kbnBundles__', `${bundle.type}/${bundle.id}`], }, optimization: { noEmitOnErrors: true, }, - externals: [ - UiSharedDeps.externals, - function (context, request, cb) { - try { - cb(undefined, dynamicExternals(bundle, context, request)); - } catch (error) { - cb(error, undefined); - } - }, - ], + externals: [UiSharedDeps.externals], - plugins: [new CleanWebpackPlugin(), new DisallowedSyntaxPlugin()], + plugins: [ + new CleanWebpackPlugin(), + new DisallowedSyntaxPlugin(), + new BundleRefsPlugin(bundle, bundleRefs), + ], module: { // no parse rules for a few known large packages which have no require() statements @@ -162,11 +86,28 @@ export function getWebpackConfig(bundle: Bundle, worker: WorkerConfig) { rules: [ { - include: [`${Path.resolve(bundle.contextDir, bundle.entry)}${entryExtension}`], - loader: UiSharedDeps.publicPathLoader, - options: { - key: bundle.id, - }, + include: [ENTRY_CREATOR], + use: [ + { + loader: UiSharedDeps.publicPathLoader, + options: { + key: bundle.id, + }, + }, + { + loader: require.resolve('val-loader'), + options: { + entries: bundle.publicDirNames.map((name) => { + const absolute = Path.resolve(bundle.contextDir, name); + const newContext = Path.dirname(ENTRY_CREATOR); + return { + importId: `${bundle.type}/${bundle.id}/${name}`, + relPath: Path.relative(newContext, absolute), + }; + }), + }, + }, + ], }, { test: /\.css$/, @@ -310,7 +251,7 @@ export function getWebpackConfig(bundle: Bundle, worker: WorkerConfig) { }, resolve: { - extensions, + extensions: ['.js', '.ts', '.tsx', 'json'], mainFields: ['browser', 'main'], alias: { tinymath: require.resolve('tinymath/lib/tinymath.es5.js'), diff --git a/src/core/public/plugins/plugin_reader.test.ts b/src/core/public/plugins/plugin_reader.test.ts index d4324f81de8e64..b3bc84f30daac7 100644 --- a/src/core/public/plugins/plugin_reader.test.ts +++ b/src/core/public/plugins/plugin_reader.test.ts @@ -17,25 +17,37 @@ * under the License. */ -import { CoreWindow, read, UnknownPluginInitializer } from './plugin_reader'; +import { CoreWindow, read } from './plugin_reader'; + +const coreWindow: CoreWindow & { + __kbnBundles__: { stub(key: string, value: any): void }; +} = window as any; -const coreWindow: CoreWindow = window as any; beforeEach(() => { - coreWindow.__kbnBundles__ = {}; + const stubs = new Map(); + coreWindow.__kbnBundles__ = { + get(key) { + return stubs.get(key); + }, + has(key) { + return stubs.has(key); + }, + stub(key, value) { + stubs.set(key, value); + }, + }; }); it('handles undefined plugin exports', () => { - coreWindow.__kbnBundles__['plugin/foo'] = undefined; - expect(() => { read('foo'); }).toThrowError(`Definition of plugin "foo" not found and may have failed to load.`); }); it('handles plugin exports with a "plugin" export that is not a function', () => { - coreWindow.__kbnBundles__['plugin/foo'] = { + coreWindow.__kbnBundles__.stub('plugin/foo/public', { plugin: 1234, - } as any; + }); expect(() => { read('foo'); @@ -43,11 +55,8 @@ it('handles plugin exports with a "plugin" export that is not a function', () => }); it('returns the plugin initializer when the "plugin" named export is a function', () => { - const plugin: UnknownPluginInitializer = () => { - return undefined as any; - }; - - coreWindow.__kbnBundles__['plugin/foo'] = { plugin }; + const plugin = () => {}; + coreWindow.__kbnBundles__.stub('plugin/foo/public', { plugin }); expect(read('foo')).toBe(plugin); }); diff --git a/src/core/public/plugins/plugin_reader.ts b/src/core/public/plugins/plugin_reader.ts index 1907dfa6a3e99c..d80bda7483775a 100644 --- a/src/core/public/plugins/plugin_reader.ts +++ b/src/core/public/plugins/plugin_reader.ts @@ -31,7 +31,8 @@ export type UnknownPluginInitializer = PluginInitializer { optionalPlugins: true, ui: true, server: true, + extraPublicDirs: true, }; return new Set(Object.keys(manifestFields)); @@ -70,7 +71,11 @@ const KNOWN_MANIFEST_FIELDS = (() => { * @param packageInfo Kibana package info. * @internal */ -export async function parseManifest(pluginPath: string, packageInfo: PackageInfo, log: Logger) { +export async function parseManifest( + pluginPath: string, + packageInfo: PackageInfo, + log: Logger +): Promise { const manifestPath = resolve(pluginPath, MANIFEST_FILE_NAME); let manifestContent; @@ -130,6 +135,19 @@ export async function parseManifest(pluginPath: string, packageInfo: PackageInfo ); } + if ( + manifest.extraPublicDirs && + (!Array.isArray(manifest.extraPublicDirs) || + !manifest.extraPublicDirs.every((dir) => typeof dir === 'string')) + ) { + throw PluginDiscoveryError.invalidManifest( + manifestPath, + new Error( + `The "extraPublicDirs" in plugin manifest for "${manifest.id}" should be an array of strings.` + ) + ); + } + const expectedKibanaVersion = typeof manifest.kibanaVersion === 'string' && manifest.kibanaVersion ? manifest.kibanaVersion @@ -175,6 +193,7 @@ export async function parseManifest(pluginPath: string, packageInfo: PackageInfo optionalPlugins: Array.isArray(manifest.optionalPlugins) ? manifest.optionalPlugins : [], ui: includesUiPlugin, server: includesServerPlugin, + extraPublicDirs: manifest.extraPublicDirs, }; } diff --git a/src/core/server/plugins/types.ts b/src/core/server/plugins/types.ts index 4fa4e1780e596c..2ca5c9f6ed3c56 100644 --- a/src/core/server/plugins/types.ts +++ b/src/core/server/plugins/types.ts @@ -153,6 +153,14 @@ export interface PluginManifest { * Specifies whether plugin includes some server-side specific functionality. */ readonly server: boolean; + + /** + * Specifies directory names that can be imported by other ui-plugins built + * using the same instance of the @kbn/optimizer. A temporary measure we plan + * to replace with better mechanisms for sharing static code between plugins + * @deprecated + */ + readonly extraPublicDirs?: string[]; } /** diff --git a/src/core/server/server.api.md b/src/core/server/server.api.md index 78cc02d39e6c43..ecfa09fbd37f39 100644 --- a/src/core/server/server.api.md +++ b/src/core/server/server.api.md @@ -1520,6 +1520,8 @@ export interface PluginInitializerContext { // @public export interface PluginManifest { readonly configPath: ConfigPath; + // @deprecated + readonly extraPublicDirs?: string[]; readonly id: PluginName; readonly kibanaVersion: string; readonly optionalPlugins: readonly PluginName[]; @@ -2549,8 +2551,8 @@ export const validBodyOutput: readonly ["data", "stream"]; // src/core/server/legacy/types.ts:165:3 - (ae-forgotten-export) The symbol "LegacyNavLinkSpec" needs to be exported by the entry point index.d.ts // src/core/server/legacy/types.ts:166:3 - (ae-forgotten-export) The symbol "LegacyAppSpec" needs to be exported by the entry point index.d.ts // src/core/server/legacy/types.ts:167:16 - (ae-forgotten-export) The symbol "LegacyPluginSpec" needs to be exported by the entry point index.d.ts -// src/core/server/plugins/types.ts:230:3 - (ae-forgotten-export) The symbol "KibanaConfigType" needs to be exported by the entry point index.d.ts -// src/core/server/plugins/types.ts:230:3 - (ae-forgotten-export) The symbol "SharedGlobalConfigKeys" needs to be exported by the entry point index.d.ts -// src/core/server/plugins/types.ts:232:3 - (ae-forgotten-export) The symbol "PathConfigType" needs to be exported by the entry point index.d.ts +// src/core/server/plugins/types.ts:238:3 - (ae-forgotten-export) The symbol "KibanaConfigType" needs to be exported by the entry point index.d.ts +// src/core/server/plugins/types.ts:238:3 - (ae-forgotten-export) The symbol "SharedGlobalConfigKeys" needs to be exported by the entry point index.d.ts +// src/core/server/plugins/types.ts:240:3 - (ae-forgotten-export) The symbol "PathConfigType" needs to be exported by the entry point index.d.ts ``` diff --git a/src/legacy/ui/ui_render/bootstrap/app_bootstrap.js b/src/legacy/ui/ui_render/bootstrap/app_bootstrap.js index 0e6936dd64a151..19f75317883d79 100644 --- a/src/legacy/ui/ui_render/bootstrap/app_bootstrap.js +++ b/src/legacy/ui/ui_render/bootstrap/app_bootstrap.js @@ -22,9 +22,11 @@ import { createHash } from 'crypto'; import { readFile } from 'fs'; import { resolve } from 'path'; +import { kbnBundlesLoaderSource } from './kbn_bundles_loader_source'; + export class AppBootstrap { constructor({ templateData }) { - this.templateData = templateData; + this.templateData = { ...templateData, kbnBundlesLoaderSource }; this._rawTemplate = undefined; } diff --git a/src/legacy/ui/ui_render/bootstrap/kbn_bundles_loader_source.js b/src/legacy/ui/ui_render/bootstrap/kbn_bundles_loader_source.js new file mode 100644 index 00000000000000..cb5488118cc772 --- /dev/null +++ b/src/legacy/ui/ui_render/bootstrap/kbn_bundles_loader_source.js @@ -0,0 +1,51 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +module.exports = { + kbnBundlesLoaderSource: `(${kbnBundlesLoader.toString()})();`, +}; + +function kbnBundlesLoader() { + var modules = {}; + + function has(prop) { + return Object.prototype.hasOwnProperty.call(modules, prop); + } + + function define(key, bundleRequire, bundleModuleKey) { + if (has(key)) { + throw new Error('__kbnBundles__ already has a module defined for "' + key + '"'); + } + + modules[key] = { + bundleRequire, + bundleModuleKey, + }; + } + + function get(key) { + if (!has(key)) { + throw new Error('__kbnBundles__ does not have a module defined for "' + key + '"'); + } + + return modules[key].bundleRequire(modules[key].bundleModuleKey); + } + + return { has: has, define: define, get: get }; +} diff --git a/src/legacy/ui/ui_render/bootstrap/template.js.hbs b/src/legacy/ui/ui_render/bootstrap/template.js.hbs index e8f05b46f70611..ca2e944489a734 100644 --- a/src/legacy/ui/ui_render/bootstrap/template.js.hbs +++ b/src/legacy/ui/ui_render/bootstrap/template.js.hbs @@ -3,6 +3,7 @@ window.__kbnStrictCsp__ = kbnCsp.strictCsp; window.__kbnDarkMode__ = {{darkMode}}; window.__kbnThemeVersion__ = "{{themeVersion}}"; window.__kbnPublicPath__ = {{publicPathMap}}; +window.__kbnBundles__ = {{kbnBundlesLoaderSource}} if (window.__kbnStrictCsp__ && window.__kbnCspNotEnforced__) { var legacyBrowserError = document.getElementById('kbn_legacy_browser_error'); @@ -78,12 +79,7 @@ if (window.__kbnStrictCsp__ && window.__kbnCspNotEnforced__) { {{/each}} ], function () { {{#unless legacyBundlePath}} - if (!__kbnBundles__ || !__kbnBundles__['entry/core'] || typeof __kbnBundles__['entry/core'].__kbnBootstrap__ !== 'function') { - console.error('entry/core bundle did not load correctly'); - failure(); - } else { - __kbnBundles__['entry/core'].__kbnBootstrap__() - } + __kbnBundles__.get('entry/core/public').__kbnBootstrap__(); {{/unless}} load([ diff --git a/src/legacy/ui/ui_render/ui_render_mixin.js b/src/legacy/ui/ui_render/ui_render_mixin.js index b09d4861b343ba..673e19155879a3 100644 --- a/src/legacy/ui/ui_render/ui_render_mixin.js +++ b/src/legacy/ui/ui_render/ui_render_mixin.js @@ -26,8 +26,6 @@ import { AppBootstrap } from './bootstrap'; import { getApmConfig } from '../apm'; import { DllCompiler } from '../../../optimize/dynamic_dll_plugin'; -const uniq = (...items) => Array.from(new Set(items)); - /** * @typedef {import('../../server/kbn_server').default} KbnServer * @typedef {import('../../server/kbn_server').ResponseToolkit} ResponseToolkit @@ -150,15 +148,7 @@ export function uiRenderMixin(kbnServer, server, config) { ]), ]; - const kpPluginIds = uniq( - // load these plugins first, they are "shared" and other bundles access their - // public/index exports without considering topographic sorting by plugin deps (for now) - 'kibanaUtils', - 'kibanaReact', - 'data', - 'esUiShared', - ...kbnServer.newPlatform.__internals.uiPlugins.public.keys() - ); + const kpPluginIds = Array.from(kbnServer.newPlatform.__internals.uiPlugins.public.keys()); const jsDependencyPaths = [ ...UiSharedDeps.jsDepFilenames.map( diff --git a/src/plugins/data/kibana.json b/src/plugins/data/kibana.json index f5df747f17e1ea..3e5d96a4bc47bf 100644 --- a/src/plugins/data/kibana.json +++ b/src/plugins/data/kibana.json @@ -7,5 +7,6 @@ "expressions", "uiActions" ], - "optionalPlugins": ["usageCollection"] + "optionalPlugins": ["usageCollection"], + "extraPublicDirs": ["common", "common/utils/abort_utils"] } diff --git a/src/plugins/embeddable/kibana.json b/src/plugins/embeddable/kibana.json index 535527b4d09db1..06b0e88da334fa 100644 --- a/src/plugins/embeddable/kibana.json +++ b/src/plugins/embeddable/kibana.json @@ -6,5 +6,8 @@ "requiredPlugins": [ "inspector", "uiActions" + ], + "extraPublicDirs": [ + "public/lib/test_samples" ] } diff --git a/src/plugins/es_ui_shared/kibana.json b/src/plugins/es_ui_shared/kibana.json index 5f772b3c83da95..980f43ea46a68f 100644 --- a/src/plugins/es_ui_shared/kibana.json +++ b/src/plugins/es_ui_shared/kibana.json @@ -2,5 +2,13 @@ "id": "esUiShared", "version": "kibana", "ui": true, - "server": true + "server": true, + "extraPublicDirs": [ + "static/ace_x_json/hooks", + "static/validators/string", + "static/forms/hook_form_lib", + "static/forms/helpers", + "static/forms/components", + "static/forms/helpers/field_validators/types" + ] } diff --git a/src/plugins/expressions/kibana.json b/src/plugins/expressions/kibana.json index 5d2112103e94d3..4774c69cc29ffd 100644 --- a/src/plugins/expressions/kibana.json +++ b/src/plugins/expressions/kibana.json @@ -5,5 +5,6 @@ "ui": true, "requiredPlugins": [ "bfetch" - ] + ], + "extraPublicDirs": ["common", "common/fonts"] } diff --git a/src/plugins/inspector/kibana.json b/src/plugins/inspector/kibana.json index 39d3ff65eed531..99a38d2928df69 100644 --- a/src/plugins/inspector/kibana.json +++ b/src/plugins/inspector/kibana.json @@ -2,5 +2,6 @@ "id": "inspector", "version": "kibana", "server": false, - "ui": true + "ui": true, + "extraPublicDirs": ["common", "common/adapters/request"] } diff --git a/src/plugins/kibana_legacy/kibana.json b/src/plugins/kibana_legacy/kibana.json index e96b4859a36d0b..606acd8b88b050 100644 --- a/src/plugins/kibana_legacy/kibana.json +++ b/src/plugins/kibana_legacy/kibana.json @@ -2,5 +2,6 @@ "id": "kibanaLegacy", "version": "kibana", "server": true, - "ui": true + "ui": true, + "extraPublicDirs": ["common/kbn_base_url"] } diff --git a/src/plugins/kibana_utils/kibana.json b/src/plugins/kibana_utils/kibana.json index 6fa39d82d10215..7e2127c27548e0 100644 --- a/src/plugins/kibana_utils/kibana.json +++ b/src/plugins/kibana_utils/kibana.json @@ -1,5 +1,10 @@ { "id": "kibanaUtils", "version": "kibana", - "ui": true + "ui": true, + "extraPublicDirs": [ + "common", + "demos/state_containers/todomvc", + "common/state_containers" + ] } diff --git a/src/plugins/maps_legacy/public/leaflet.js b/src/plugins/maps_legacy/public/leaflet.js index e36da2c52b8c51..bee75021c76ade 100644 --- a/src/plugins/maps_legacy/public/leaflet.js +++ b/src/plugins/maps_legacy/public/leaflet.js @@ -17,8 +17,6 @@ * under the License. */ -export let L; - if (!window.hasOwnProperty('L')) { require('leaflet/dist/leaflet.css'); window.L = require('leaflet/dist/leaflet.js'); @@ -31,6 +29,6 @@ if (!window.hasOwnProperty('L')) { require('leaflet-draw/dist/leaflet.draw.js'); require('leaflet-responsive-popup/leaflet.responsive.popup.css'); require('leaflet-responsive-popup/leaflet.responsive.popup.js'); -} else { - L = window.L; } + +export const L = window.L; diff --git a/src/plugins/saved_objects_management/kibana.json b/src/plugins/saved_objects_management/kibana.json index 22135ce4558ae4..6184d890c415cf 100644 --- a/src/plugins/saved_objects_management/kibana.json +++ b/src/plugins/saved_objects_management/kibana.json @@ -4,5 +4,6 @@ "server": true, "ui": true, "requiredPlugins": ["home", "management", "data"], - "optionalPlugins": ["dashboard", "visualizations", "discover"] + "optionalPlugins": ["dashboard", "visualizations", "discover"], + "extraPublicDirs": ["public/lib"] } diff --git a/src/plugins/telemetry/kibana.json b/src/plugins/telemetry/kibana.json index f623f4f2a565dc..a4975977625207 100644 --- a/src/plugins/telemetry/kibana.json +++ b/src/plugins/telemetry/kibana.json @@ -6,5 +6,8 @@ "requiredPlugins": [ "telemetryCollectionManager", "usageCollection" + ], + "extraPublicDirs": [ + "common/constants" ] } diff --git a/src/plugins/ui_actions/kibana.json b/src/plugins/ui_actions/kibana.json index 44ecbbfa684087..907cbabbdf9c97 100644 --- a/src/plugins/ui_actions/kibana.json +++ b/src/plugins/ui_actions/kibana.json @@ -2,5 +2,8 @@ "id": "uiActions", "version": "kibana", "server": false, - "ui": true + "ui": true, + "extraPublicDirs": [ + "public/tests/test_samples" + ] } diff --git a/test/interpreter_functional/plugins/kbn_tp_run_pipeline/public/app/components/main.tsx b/test/interpreter_functional/plugins/kbn_tp_run_pipeline/public/app/components/main.tsx index ace2af2b4f0cfe..b4f9634b23d299 100644 --- a/test/interpreter_functional/plugins/kbn_tp_run_pipeline/public/app/components/main.tsx +++ b/test/interpreter_functional/plugins/kbn_tp_run_pipeline/public/app/components/main.tsx @@ -21,7 +21,7 @@ import React from 'react'; import { EuiPage, EuiPageBody, EuiPageContent, EuiPageContentHeader } from '@elastic/eui'; import { first } from 'rxjs/operators'; import { IInterpreterRenderHandlers, ExpressionValue } from 'src/plugins/expressions'; -import { RequestAdapter, DataAdapter } from '../../../../../../../src/plugins/inspector'; +import { RequestAdapter, DataAdapter } from '../../../../../../../src/plugins/inspector/public'; import { Adapters, ExpressionRenderHandler } from '../../types'; import { getExpressions } from '../../services'; diff --git a/test/scripts/jenkins_build_kibana.sh b/test/scripts/jenkins_build_kibana.sh index e3f46e7a6ada41..3e49edc8e6ae5f 100755 --- a/test/scripts/jenkins_build_kibana.sh +++ b/test/scripts/jenkins_build_kibana.sh @@ -2,9 +2,16 @@ source src/dev/ci_setup/setup_env.sh -echo " -> building kibana platform plugins" +echo " -> building examples separate from test plugins" node scripts/build_kibana_platform_plugins \ --oss \ + --examples \ + --verbose; + +echo " -> building test plugins" +node scripts/build_kibana_platform_plugins \ + --oss \ + --no-examples \ --scan-dir "$KIBANA_DIR/test/plugin_functional/plugins" \ --scan-dir "$KIBANA_DIR/test/interpreter_functional/plugins" \ --verbose; diff --git a/test/scripts/jenkins_xpack_build_kibana.sh b/test/scripts/jenkins_xpack_build_kibana.sh index c962b962b1e5e6..58ef6a42d3fe4c 100755 --- a/test/scripts/jenkins_xpack_build_kibana.sh +++ b/test/scripts/jenkins_xpack_build_kibana.sh @@ -3,8 +3,14 @@ cd "$KIBANA_DIR" source src/dev/ci_setup/setup_env.sh -echo " -> building kibana platform plugins" +echo " -> building examples separate from test plugins" node scripts/build_kibana_platform_plugins \ + --examples \ + --verbose; + +echo " -> building test plugins" +node scripts/build_kibana_platform_plugins \ + --no-examples \ --scan-dir "$KIBANA_DIR/test/plugin_functional/plugins" \ --scan-dir "$XPACK_DIR/test/plugin_functional/plugins" \ --scan-dir "$XPACK_DIR/test/functional_with_es_ssl/fixtures/plugins" \ diff --git a/x-pack/plugins/alerts/kibana.json b/x-pack/plugins/alerts/kibana.json index 3509f79dbbe4d3..eef61ff4b3d53a 100644 --- a/x-pack/plugins/alerts/kibana.json +++ b/x-pack/plugins/alerts/kibana.json @@ -6,5 +6,6 @@ "kibanaVersion": "kibana", "configPath": ["xpack", "alerts"], "requiredPlugins": ["licensing", "taskManager", "encryptedSavedObjects", "actions", "eventLog"], - "optionalPlugins": ["usageCollection", "spaces", "security"] + "optionalPlugins": ["usageCollection", "spaces", "security"], + "extraPublicDirs": ["common", "common/parse_duration"] } diff --git a/x-pack/plugins/apm/kibana.json b/x-pack/plugins/apm/kibana.json index 1b8e7c4dc54318..56a9e226b65284 100644 --- a/x-pack/plugins/apm/kibana.json +++ b/x-pack/plugins/apm/kibana.json @@ -25,5 +25,8 @@ "configPath": [ "xpack", "apm" + ], + "extraPublicDirs": [ + "public/style/variables" ] } diff --git a/x-pack/plugins/canvas/common/lib/autocomplete.ts b/x-pack/plugins/canvas/common/lib/autocomplete.ts index 0ab549bd14e832..c97879de2137e6 100644 --- a/x-pack/plugins/canvas/common/lib/autocomplete.ts +++ b/x-pack/plugins/canvas/common/lib/autocomplete.ts @@ -14,7 +14,7 @@ import { ExpressionFunction, ExpressionFunctionParameter, getByAlias, -} from '../../../../../src/plugins/expressions'; +} from '../../../../../src/plugins/expressions/common'; const MARKER = 'CANVAS_SUGGESTION_MARKER'; diff --git a/x-pack/plugins/data_enhanced/kibana.json b/x-pack/plugins/data_enhanced/kibana.json index 443bb63a277997..1be55d2b7a6352 100644 --- a/x-pack/plugins/data_enhanced/kibana.json +++ b/x-pack/plugins/data_enhanced/kibana.json @@ -8,6 +8,7 @@ "requiredPlugins": [ "data" ], + "optionalPlugins": ["kibanaReact", "kibanaUtils"], "server": true, "ui": true } diff --git a/x-pack/plugins/features/kibana.json b/x-pack/plugins/features/kibana.json index 1cab1821b1bf59..92fdd08e934784 100644 --- a/x-pack/plugins/features/kibana.json +++ b/x-pack/plugins/features/kibana.json @@ -6,5 +6,6 @@ "optionalPlugins": ["visTypeTimelion"], "configPath": ["xpack", "features"], "server": true, - "ui": true + "ui": true, + "extraPublicDirs": ["common"] } diff --git a/x-pack/plugins/ingest_manager/kibana.json b/x-pack/plugins/ingest_manager/kibana.json index 382ea0444093d4..35447139607a6b 100644 --- a/x-pack/plugins/ingest_manager/kibana.json +++ b/x-pack/plugins/ingest_manager/kibana.json @@ -5,5 +5,6 @@ "ui": true, "configPath": ["xpack", "ingestManager"], "requiredPlugins": ["licensing", "data", "encryptedSavedObjects"], - "optionalPlugins": ["security", "features", "cloud"] + "optionalPlugins": ["security", "features", "cloud"], + "extraPublicDirs": ["common"] } diff --git a/x-pack/plugins/lens/kibana.json b/x-pack/plugins/lens/kibana.json index a8b22b3e22750f..346a5a24c269f4 100644 --- a/x-pack/plugins/lens/kibana.json +++ b/x-pack/plugins/lens/kibana.json @@ -13,5 +13,6 @@ "dashboard" ], "optionalPlugins": ["embeddable", "usageCollection", "taskManager", "uiActions"], - "configPath": ["xpack", "lens"] + "configPath": ["xpack", "lens"], + "extraPublicDirs": ["common/constants"] } diff --git a/x-pack/plugins/license_management/kibana.json b/x-pack/plugins/license_management/kibana.json index be28c8e978d8aa..6da923c5cff5a3 100644 --- a/x-pack/plugins/license_management/kibana.json +++ b/x-pack/plugins/license_management/kibana.json @@ -5,5 +5,6 @@ "ui": true, "requiredPlugins": ["home", "licensing", "management"], "optionalPlugins": ["telemetry"], - "configPath": ["xpack", "license_management"] + "configPath": ["xpack", "license_management"], + "extraPublicDirs": ["common/constants"] } diff --git a/x-pack/plugins/maps/kibana.json b/x-pack/plugins/maps/kibana.json index 67520321de7612..f8a30b8d0337e4 100644 --- a/x-pack/plugins/maps/kibana.json +++ b/x-pack/plugins/maps/kibana.json @@ -18,5 +18,6 @@ "usageCollection" ], "ui": true, - "server": true + "server": true, + "extraPublicDirs": ["common/constants"] } diff --git a/x-pack/plugins/spaces/kibana.json b/x-pack/plugins/spaces/kibana.json index 7e6d0425f2ae07..9483cb67392c43 100644 --- a/x-pack/plugins/spaces/kibana.json +++ b/x-pack/plugins/spaces/kibana.json @@ -13,5 +13,6 @@ "savedObjectsManagement" ], "server": true, - "ui": true + "ui": true, + "extraPublicDirs": ["common"] } diff --git a/x-pack/plugins/triggers_actions_ui/kibana.json b/x-pack/plugins/triggers_actions_ui/kibana.json index d8f50553688310..158cfa100d5463 100644 --- a/x-pack/plugins/triggers_actions_ui/kibana.json +++ b/x-pack/plugins/triggers_actions_ui/kibana.json @@ -5,5 +5,6 @@ "ui": true, "optionalPlugins": ["alerts", "alertingBuiltins"], "requiredPlugins": ["management", "charts", "data"], - "configPath": ["xpack", "trigger_actions_ui"] + "configPath": ["xpack", "trigger_actions_ui"], + "extraPublicDirs": ["public/common", "public/common/constants"] } diff --git a/x-pack/plugins/ui_actions_enhanced/public/dynamic_actions/dynamic_action_manager.ts b/x-pack/plugins/ui_actions_enhanced/public/dynamic_actions/dynamic_action_manager.ts index 3e37bac0a17af4..58344026079e79 100644 --- a/x-pack/plugins/ui_actions_enhanced/public/dynamic_actions/dynamic_action_manager.ts +++ b/x-pack/plugins/ui_actions_enhanced/public/dynamic_actions/dynamic_action_manager.ts @@ -12,7 +12,10 @@ import { UiActionsActionDefinition as ActionDefinition, } from '../../../../../src/plugins/ui_actions/public'; import { defaultState, transitions, selectors, State } from './dynamic_action_manager_state'; -import { StateContainer, createStateContainer } from '../../../../../src/plugins/kibana_utils'; +import { + StateContainer, + createStateContainer, +} from '../../../../../src/plugins/kibana_utils/common'; import { StartContract } from '../plugin'; import { SerializedAction, SerializedEvent } from './types'; From 88eed9819c9888986f148d742377a8a92ecefd8e Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Tue, 16 Jun 2020 07:23:10 +0100 Subject: [PATCH 09/11] [CI] GitHub check details link to test reports and errorlevel (#69157) --- .ci/end2end.groovy | 12 ++++++++---- x-pack/plugins/apm/e2e/run-e2e.sh | 7 +++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/.ci/end2end.groovy b/.ci/end2end.groovy index 0cd64dcfd41fd2..ed642f22cfeb4c 100644 --- a/.ci/end2end.groovy +++ b/.ci/end2end.groovy @@ -76,7 +76,7 @@ pipeline { } } steps{ - notifyStatus('Running smoke tests', 'PENDING') + notifyTestStatus('Running smoke tests', 'PENDING') dir("${BASE_DIR}"){ sh "${E2E_DIR}/ci/run-e2e.sh" } @@ -95,10 +95,10 @@ pipeline { } } unsuccessful { - notifyStatus('Test failures', 'FAILURE') + notifyTestStatus('Test failures', 'FAILURE') } success { - notifyStatus('Tests passed', 'SUCCESS') + notifyTestStatus('Tests passed', 'SUCCESS') } } } @@ -113,5 +113,9 @@ pipeline { } def notifyStatus(String description, String status) { - withGithubNotify.notify('end2end-for-apm-ui', description, status, getBlueoceanDisplayURL()) + withGithubNotify.notify('end2end-for-apm-ui', description, status, getBlueoceanTabURL('pipeline')) +} + +def notifyTestStatus(String description, String status) { + withGithubNotify.notify('end2end-for-apm-ui', description, status, getBlueoceanTabURL('tests')) } diff --git a/x-pack/plugins/apm/e2e/run-e2e.sh b/x-pack/plugins/apm/e2e/run-e2e.sh index 4bebab8e0c6a80..aa7c0e21425ad8 100755 --- a/x-pack/plugins/apm/e2e/run-e2e.sh +++ b/x-pack/plugins/apm/e2e/run-e2e.sh @@ -164,6 +164,7 @@ echo "✅ Setup completed successfully. Running tests..." # run cypress tests ################################################## yarn cypress run --config pageLoadTimeout=100000,watchForFileChanges=true +e2e_status=$? # # Run interactively @@ -171,3 +172,9 @@ yarn cypress run --config pageLoadTimeout=100000,watchForFileChanges=true echo "${bold}If you want to run the test interactively, run:${normal}" echo "" # newline echo "cd ${E2E_DIR} && yarn cypress open --config pageLoadTimeout=100000,watchForFileChanges=true" + +# Report the e2e status at the very end +if [ $e2e_status -ne 0 ]; then + echo "⚠️ Running tests failed." + exit 1 +fi From 833ce082591f99baf740292edb298598130100a7 Mon Sep 17 00:00:00 2001 From: Gidi Meir Morris Date: Tue, 16 Jun 2020 09:33:10 +0100 Subject: [PATCH 10/11] [Alerting] Adds API for executing actions immediately (#68897) This PR adds an API to the AlertsClient which allows users to execute actions immediately, rather than via TaskManager. This also moves the TaskManager based execution to a new api on ActionsClient along side the immediate execution. --- x-pack/plugins/actions/README.md | 52 +++++- .../actions/server/actions_client.mock.ts | 2 + .../actions/server/actions_client.test.ts | 58 +++++++ .../plugins/actions/server/actions_client.ts | 39 ++++- .../server/create_execute_function.test.ts | 152 ++---------------- .../actions/server/create_execute_function.ts | 49 ++---- x-pack/plugins/actions/server/mocks.ts | 1 - x-pack/plugins/actions/server/plugin.ts | 39 +++-- .../actions/server/routes/execute.test.ts | 89 +++++----- .../plugins/actions/server/routes/execute.ts | 16 +- .../create_execution_handler.test.ts | 35 ++-- .../task_runner/create_execution_handler.ts | 7 +- .../server/task_runner/task_runner.test.ts | 132 ++++++++++++++- .../alerts/server/task_runner/task_runner.ts | 11 +- 14 files changed, 401 insertions(+), 281 deletions(-) diff --git a/x-pack/plugins/actions/README.md b/x-pack/plugins/actions/README.md index 96d5f04ac088f6..605676cee363d1 100644 --- a/x-pack/plugins/actions/README.md +++ b/x-pack/plugins/actions/README.md @@ -234,9 +234,24 @@ Payload: ## Firing actions -The plugin exposes an execute function that you can use to run actions. +Running actions is possible by using the ActionsClient which is provided by the `getActionsClientWithRequest` function part of the plugin's Start Contract. +By providing the user's Request you'll receive an instance of the ActionsClient which is tailered to the current user and is scoped to the resources the user is authorized to access. -**server.plugins.actions.execute(options)** +## Accessing a scoped ActionsClient + +``` +const actionsClient = server.plugins.actions.getActionsClientWithRequest(request); +``` + +Once you have a scoped ActionsClient you can execute an action by caling either the `enqueueExecution` which will schedule the action to run later or the `execute` apis which will run it immediately and return the result respectively. + +### actionsClient.enqueueExecution(options) + +This api schedules a task which will run the action using the current user scope at the soonest opportunity. + +Running the action by scheduling a task means that we will no longer have a user request by which to ascertain the action's privileges and so you might need to provide these yourself: +- The **SpaceId** in which the user's action is expected to run +- When security is enabled you'll also need to provide an **apiKey** which allows us to mimic the user and their privileges. The following table describes the properties of the `options` object. @@ -251,8 +266,9 @@ The following table describes the properties of the `options` object. This example makes action `3c5b2bd4-5424-4e4b-8cf5-c0a58c762cc5` send an email. The action plugin will load the saved object and find what action type to call with `params`. -``` -server.plugins.actions.execute({ +```typescript +const actionsClient = await server.plugins.actions.getActionsClientWithRequest(request); +await actionsClient.enqueueExecution({ id: '3c5b2bd4-5424-4e4b-8cf5-c0a58c762cc5', spaceId: 'default', // The spaceId of the action params: { @@ -264,6 +280,34 @@ server.plugins.actions.execute({ }); ``` +### actionsClient.execute(options) + +This api runs the action and asynchronously returns the result of running the action. + +The following table describes the properties of the `options` object. + +| Property | Description | Type | +| -------- | ------------------------------------------------------------------------------------------------------ | ------ | +| id | The id of the action you want to execute. | string | +| params | The `params` value to give the action type executor. | object | + +## Example + +As with the previous example, we'll use the action `3c5b2bd4-5424-4e4b-8cf5-c0a58c762cc5` to send an email. + +```typescript +const actionsClient = await server.plugins.actions.getActionsClientWithRequest(request); +const result = await actionsClient.execute({ + id: '3c5b2bd4-5424-4e4b-8cf5-c0a58c762cc5', + params: { + from: 'example@elastic.co', + to: ['destination@elastic.co'], + subject: 'My email subject', + body: 'My email body', + }, +}); +``` + # Built-in Action Types Kibana ships with a set of built-in action types: diff --git a/x-pack/plugins/actions/server/actions_client.mock.ts b/x-pack/plugins/actions/server/actions_client.mock.ts index a2b64e49f76e34..efd044c7e2493a 100644 --- a/x-pack/plugins/actions/server/actions_client.mock.ts +++ b/x-pack/plugins/actions/server/actions_client.mock.ts @@ -17,6 +17,8 @@ const createActionsClientMock = () => { update: jest.fn(), getAll: jest.fn(), getBulk: jest.fn(), + execute: jest.fn(), + enqueueExecution: jest.fn(), }; return mocked; }; diff --git a/x-pack/plugins/actions/server/actions_client.test.ts b/x-pack/plugins/actions/server/actions_client.test.ts index bf55a1c18d169c..2865bbbe1d944a 100644 --- a/x-pack/plugins/actions/server/actions_client.test.ts +++ b/x-pack/plugins/actions/server/actions_client.test.ts @@ -19,10 +19,16 @@ import { elasticsearchServiceMock, savedObjectsClientMock, } from '../../../../src/core/server/mocks'; +import { actionExecutorMock } from './lib/action_executor.mock'; +import uuid from 'uuid'; +import { KibanaRequest } from 'kibana/server'; const defaultKibanaIndex = '.kibana'; const savedObjectsClient = savedObjectsClientMock.create(); const scopedClusterClient = elasticsearchServiceMock.createScopedClusterClient(); +const actionExecutor = actionExecutorMock.create(); +const executionEnqueuer = jest.fn(); +const request = {} as KibanaRequest; const mockTaskManager = taskManagerMock.setup(); @@ -53,6 +59,9 @@ beforeEach(() => { scopedClusterClient, defaultKibanaIndex, preconfiguredActions: [], + actionExecutor, + executionEnqueuer, + request, }); }); @@ -232,6 +241,9 @@ describe('create()', () => { scopedClusterClient, defaultKibanaIndex, preconfiguredActions: [], + actionExecutor, + executionEnqueuer, + request, }); const savedObjectCreateResult = { @@ -328,6 +340,9 @@ describe('get()', () => { savedObjectsClient, scopedClusterClient, defaultKibanaIndex, + actionExecutor, + executionEnqueuer, + request, preconfiguredActions: [ { id: 'testPreconfigured', @@ -388,6 +403,9 @@ describe('getAll()', () => { savedObjectsClient, scopedClusterClient, defaultKibanaIndex, + actionExecutor, + executionEnqueuer, + request, preconfiguredActions: [ { id: 'testPreconfigured', @@ -453,6 +471,9 @@ describe('getBulk()', () => { savedObjectsClient, scopedClusterClient, defaultKibanaIndex, + actionExecutor, + executionEnqueuer, + request, preconfiguredActions: [ { id: 'testPreconfigured', @@ -718,3 +739,40 @@ describe('update()', () => { ).rejects.toThrowErrorMatchingInlineSnapshot(`"Fail"`); }); }); + +describe('execute()', () => { + test('calls the actionExecutor with the appropriate parameters', async () => { + const actionId = uuid.v4(); + actionExecutor.execute.mockResolvedValue({ status: 'ok', actionId }); + await expect( + actionsClient.execute({ + actionId, + params: { + name: 'my name', + }, + }) + ).resolves.toMatchObject({ status: 'ok', actionId }); + + expect(actionExecutor.execute).toHaveBeenCalledWith({ + actionId, + request, + params: { + name: 'my name', + }, + }); + }); +}); + +describe('enqueueExecution()', () => { + test('calls the executionEnqueuer with the appropriate parameters', async () => { + const opts = { + id: uuid.v4(), + params: { baz: false }, + spaceId: 'default', + apiKey: Buffer.from('123:abc').toString('base64'), + }; + await expect(actionsClient.enqueueExecution(opts)).resolves.toMatchInlineSnapshot(`undefined`); + + expect(executionEnqueuer).toHaveBeenCalledWith(savedObjectsClient, opts); + }); +}); diff --git a/x-pack/plugins/actions/server/actions_client.ts b/x-pack/plugins/actions/server/actions_client.ts index 48703f01f55091..a2feac83cba9f5 100644 --- a/x-pack/plugins/actions/server/actions_client.ts +++ b/x-pack/plugins/actions/server/actions_client.ts @@ -9,13 +9,25 @@ import { SavedObjectsClientContract, SavedObjectAttributes, SavedObject, + KibanaRequest, } from 'src/core/server'; import { i18n } from '@kbn/i18n'; import { ActionTypeRegistry } from './action_type_registry'; -import { validateConfig, validateSecrets } from './lib'; -import { ActionResult, FindActionResult, RawAction, PreConfiguredAction } from './types'; +import { validateConfig, validateSecrets, ActionExecutorContract } from './lib'; +import { + ActionResult, + FindActionResult, + RawAction, + PreConfiguredAction, + ActionTypeExecutorResult, +} from './types'; import { PreconfiguredActionDisabledModificationError } from './lib/errors/preconfigured_action_disabled_modification'; +import { ExecuteOptions } from './lib/action_executor'; +import { + ExecutionEnqueuer, + ExecuteOptions as EnqueueExecutionOptions, +} from './create_execute_function'; // We are assuming there won't be many actions. This is why we will load // all the actions in advance and assume the total count to not go over 10000. @@ -42,6 +54,9 @@ interface ConstructorOptions { actionTypeRegistry: ActionTypeRegistry; savedObjectsClient: SavedObjectsClientContract; preconfiguredActions: PreConfiguredAction[]; + actionExecutor: ActionExecutorContract; + executionEnqueuer: ExecutionEnqueuer; + request: KibanaRequest; } interface UpdateOptions { @@ -55,6 +70,9 @@ export class ActionsClient { private readonly savedObjectsClient: SavedObjectsClientContract; private readonly actionTypeRegistry: ActionTypeRegistry; private readonly preconfiguredActions: PreConfiguredAction[]; + private readonly actionExecutor: ActionExecutorContract; + private readonly request: KibanaRequest; + private readonly executionEnqueuer: ExecutionEnqueuer; constructor({ actionTypeRegistry, @@ -62,12 +80,18 @@ export class ActionsClient { scopedClusterClient, savedObjectsClient, preconfiguredActions, + actionExecutor, + executionEnqueuer, + request, }: ConstructorOptions) { this.actionTypeRegistry = actionTypeRegistry; this.savedObjectsClient = savedObjectsClient; this.scopedClusterClient = scopedClusterClient; this.defaultKibanaIndex = defaultKibanaIndex; this.preconfiguredActions = preconfiguredActions; + this.actionExecutor = actionExecutor; + this.executionEnqueuer = executionEnqueuer; + this.request = request; } /** @@ -251,6 +275,17 @@ export class ActionsClient { } return await this.savedObjectsClient.delete('action', id); } + + public async execute({ + actionId, + params, + }: Omit): Promise { + return this.actionExecutor.execute({ actionId, params, request: this.request }); + } + + public async enqueueExecution(options: EnqueueExecutionOptions): Promise { + return this.executionEnqueuer(this.savedObjectsClient, options); + } } function actionFromSavedObject(savedObject: SavedObject): ActionResult { diff --git a/x-pack/plugins/actions/server/create_execute_function.test.ts b/x-pack/plugins/actions/server/create_execute_function.test.ts index 1b7752588e3d39..04d4d92945cdb3 100644 --- a/x-pack/plugins/actions/server/create_execute_function.test.ts +++ b/x-pack/plugins/actions/server/create_execute_function.test.ts @@ -5,23 +5,20 @@ */ import { taskManagerMock } from '../../task_manager/server/task_manager.mock'; -import { createExecuteFunction } from './create_execute_function'; +import { createExecutionEnqueuerFunction } from './create_execute_function'; import { savedObjectsClientMock } from '../../../../src/core/server/mocks'; import { actionTypeRegistryMock } from './action_type_registry.mock'; const mockTaskManager = taskManagerMock.start(); const savedObjectsClient = savedObjectsClientMock.create(); -const getBasePath = jest.fn(); beforeEach(() => jest.resetAllMocks()); describe('execute()', () => { test('schedules the action with all given parameters', async () => { - const executeFn = createExecuteFunction({ - getBasePath, + const executeFn = createExecutionEnqueuerFunction({ taskManager: mockTaskManager, actionTypeRegistry: actionTypeRegistryMock.create(), - getScopedSavedObjectsClient: jest.fn().mockReturnValueOnce(savedObjectsClient), isESOUsingEphemeralEncryptionKey: false, preconfiguredActions: [], }); @@ -39,7 +36,7 @@ describe('execute()', () => { attributes: {}, references: [], }); - await executeFn({ + await executeFn(savedObjectsClient, { id: '123', params: { baz: false }, spaceId: 'default', @@ -70,11 +67,9 @@ describe('execute()', () => { }); test('schedules the action with all given parameters with a preconfigured action', async () => { - const executeFn = createExecuteFunction({ - getBasePath, + const executeFn = createExecutionEnqueuerFunction({ taskManager: mockTaskManager, actionTypeRegistry: actionTypeRegistryMock.create(), - getScopedSavedObjectsClient: jest.fn().mockReturnValueOnce(savedObjectsClient), isESOUsingEphemeralEncryptionKey: false, preconfiguredActions: [ { @@ -101,7 +96,7 @@ describe('execute()', () => { attributes: {}, references: [], }); - await executeFn({ + await executeFn(savedObjectsClient, { id: '123', params: { baz: false }, spaceId: 'default', @@ -131,115 +126,15 @@ describe('execute()', () => { }); }); - test('uses API key when provided', async () => { - const getScopedSavedObjectsClient = jest.fn().mockReturnValueOnce(savedObjectsClient); - const executeFn = createExecuteFunction({ - getBasePath, - taskManager: mockTaskManager, - getScopedSavedObjectsClient, - isESOUsingEphemeralEncryptionKey: false, - actionTypeRegistry: actionTypeRegistryMock.create(), - preconfiguredActions: [], - }); - savedObjectsClient.get.mockResolvedValueOnce({ - id: '123', - type: 'action', - attributes: { - actionTypeId: 'mock-action', - }, - references: [], - }); - savedObjectsClient.create.mockResolvedValueOnce({ - id: '234', - type: 'action_task_params', - attributes: {}, - references: [], - }); - - await executeFn({ - id: '123', - params: { baz: false }, - spaceId: 'default', - apiKey: Buffer.from('123:abc').toString('base64'), - }); - expect(getScopedSavedObjectsClient).toHaveBeenCalledWith({ - getBasePath: expect.anything(), - headers: { - // base64 encoded "123:abc" - authorization: 'ApiKey MTIzOmFiYw==', - }, - path: '/', - route: { settings: {} }, - url: { - href: '/', - }, - raw: { - req: { - url: '/', - }, - }, - }); - }); - - test(`doesn't use API keys when not provided`, async () => { - const getScopedSavedObjectsClient = jest.fn().mockReturnValueOnce(savedObjectsClient); - const executeFn = createExecuteFunction({ - getBasePath, - taskManager: mockTaskManager, - getScopedSavedObjectsClient, - isESOUsingEphemeralEncryptionKey: false, - actionTypeRegistry: actionTypeRegistryMock.create(), - preconfiguredActions: [], - }); - savedObjectsClient.get.mockResolvedValueOnce({ - id: '123', - type: 'action', - attributes: { - actionTypeId: 'mock-action', - }, - references: [], - }); - savedObjectsClient.create.mockResolvedValueOnce({ - id: '234', - type: 'action_task_params', - attributes: {}, - references: [], - }); - - await executeFn({ - id: '123', - params: { baz: false }, - spaceId: 'default', - apiKey: null, - }); - expect(getScopedSavedObjectsClient).toHaveBeenCalledWith({ - getBasePath: expect.anything(), - headers: {}, - path: '/', - route: { settings: {} }, - url: { - href: '/', - }, - raw: { - req: { - url: '/', - }, - }, - }); - }); - test('throws when passing isESOUsingEphemeralEncryptionKey with true as a value', async () => { - const getScopedSavedObjectsClient = jest.fn().mockReturnValueOnce(savedObjectsClient); - const executeFn = createExecuteFunction({ - getBasePath, + const executeFn = createExecutionEnqueuerFunction({ taskManager: mockTaskManager, - getScopedSavedObjectsClient, isESOUsingEphemeralEncryptionKey: true, actionTypeRegistry: actionTypeRegistryMock.create(), preconfiguredActions: [], }); await expect( - executeFn({ + executeFn(savedObjectsClient, { id: '123', params: { baz: false }, spaceId: 'default', @@ -252,11 +147,8 @@ describe('execute()', () => { test('should ensure action type is enabled', async () => { const mockedActionTypeRegistry = actionTypeRegistryMock.create(); - const getScopedSavedObjectsClient = jest.fn().mockReturnValueOnce(savedObjectsClient); - const executeFn = createExecuteFunction({ - getBasePath, + const executeFn = createExecutionEnqueuerFunction({ taskManager: mockTaskManager, - getScopedSavedObjectsClient, isESOUsingEphemeralEncryptionKey: false, actionTypeRegistry: mockedActionTypeRegistry, preconfiguredActions: [], @@ -274,7 +166,7 @@ describe('execute()', () => { }); await expect( - executeFn({ + executeFn(savedObjectsClient, { id: '123', params: { baz: false }, spaceId: 'default', @@ -285,11 +177,8 @@ describe('execute()', () => { test('should skip ensure action type if action type is preconfigured and license is valid', async () => { const mockedActionTypeRegistry = actionTypeRegistryMock.create(); - const getScopedSavedObjectsClient = jest.fn().mockReturnValueOnce(savedObjectsClient); - const executeFn = createExecuteFunction({ - getBasePath, + const executeFn = createExecutionEnqueuerFunction({ taskManager: mockTaskManager, - getScopedSavedObjectsClient, isESOUsingEphemeralEncryptionKey: false, actionTypeRegistry: mockedActionTypeRegistry, preconfiguredActions: [ @@ -303,9 +192,6 @@ describe('execute()', () => { }, ], }); - mockedActionTypeRegistry.ensureActionTypeEnabled.mockImplementation(() => { - throw new Error('Fail'); - }); mockedActionTypeRegistry.isActionExecutable.mockImplementation(() => true); savedObjectsClient.get.mockResolvedValueOnce({ id: '123', @@ -322,25 +208,13 @@ describe('execute()', () => { references: [], }); - await executeFn({ + await executeFn(savedObjectsClient, { id: '123', params: { baz: false }, spaceId: 'default', apiKey: null, }); - expect(getScopedSavedObjectsClient).toHaveBeenCalledWith({ - getBasePath: expect.anything(), - headers: {}, - path: '/', - route: { settings: {} }, - url: { - href: '/', - }, - raw: { - req: { - url: '/', - }, - }, - }); + + expect(mockedActionTypeRegistry.ensureActionTypeEnabled).not.toHaveBeenCalled(); }); }); diff --git a/x-pack/plugins/actions/server/create_execute_function.ts b/x-pack/plugins/actions/server/create_execute_function.ts index a16b2f4295072d..2bad33d56f228e 100644 --- a/x-pack/plugins/actions/server/create_execute_function.ts +++ b/x-pack/plugins/actions/server/create_execute_function.ts @@ -4,19 +4,12 @@ * you may not use this file except in compliance with the Elastic License. */ -import { SavedObjectsClientContract, KibanaRequest } from '../../../../src/core/server'; +import { SavedObjectsClientContract } from '../../../../src/core/server'; import { TaskManagerStartContract } from '../../task_manager/server'; -import { - GetBasePathFunction, - RawAction, - ActionTypeRegistryContract, - PreConfiguredAction, -} from './types'; +import { RawAction, ActionTypeRegistryContract, PreConfiguredAction } from './types'; interface CreateExecuteFunctionOptions { taskManager: TaskManagerStartContract; - getScopedSavedObjectsClient: (request: KibanaRequest) => SavedObjectsClientContract; - getBasePath: GetBasePathFunction; isESOUsingEphemeralEncryptionKey: boolean; actionTypeRegistry: ActionTypeRegistryContract; preconfiguredActions: PreConfiguredAction[]; @@ -29,45 +22,27 @@ export interface ExecuteOptions { apiKey: string | null; } -export function createExecuteFunction({ - getBasePath, +export type ExecutionEnqueuer = ( + savedObjectsClient: SavedObjectsClientContract, + options: ExecuteOptions +) => Promise; + +export function createExecutionEnqueuerFunction({ taskManager, actionTypeRegistry, - getScopedSavedObjectsClient, isESOUsingEphemeralEncryptionKey, preconfiguredActions, }: CreateExecuteFunctionOptions) { - return async function execute({ id, params, spaceId, apiKey }: ExecuteOptions) { + return async function execute( + savedObjectsClient: SavedObjectsClientContract, + { id, params, spaceId, apiKey }: ExecuteOptions + ) { if (isESOUsingEphemeralEncryptionKey === true) { throw new Error( `Unable to execute action due to the Encrypted Saved Objects plugin using an ephemeral encryption key. Please set xpack.encryptedSavedObjects.encryptionKey in kibana.yml` ); } - const requestHeaders: Record = {}; - - if (apiKey) { - requestHeaders.authorization = `ApiKey ${apiKey}`; - } - - // Since we're using API keys and accessing elasticsearch can only be done - // via a request, we're faking one with the proper authorization headers. - const fakeRequest: unknown = { - headers: requestHeaders, - getBasePath: () => getBasePath(spaceId), - path: '/', - route: { settings: {} }, - url: { - href: '/', - }, - raw: { - req: { - url: '/', - }, - }, - }; - - const savedObjectsClient = getScopedSavedObjectsClient(fakeRequest as KibanaRequest); const actionTypeId = await getActionTypeId(id); if (!actionTypeRegistry.isActionExecutable(id, actionTypeId)) { diff --git a/x-pack/plugins/actions/server/mocks.ts b/x-pack/plugins/actions/server/mocks.ts index 09bc137da736f5..1763d275c6fb07 100644 --- a/x-pack/plugins/actions/server/mocks.ts +++ b/x-pack/plugins/actions/server/mocks.ts @@ -23,7 +23,6 @@ const createSetupMock = () => { const createStartMock = () => { const mock: jest.Mocked = { - execute: jest.fn(), isActionTypeEnabled: jest.fn(), isActionExecutable: jest.fn(), getActionsClientWithRequest: jest.fn().mockResolvedValue(actionsClientMock.create()), diff --git a/x-pack/plugins/actions/server/plugin.ts b/x-pack/plugins/actions/server/plugin.ts index 8c1b21e600ae17..ad3fa97ee0c366 100644 --- a/x-pack/plugins/actions/server/plugin.ts +++ b/x-pack/plugins/actions/server/plugin.ts @@ -35,8 +35,7 @@ import { Services, ActionType, PreConfiguredAction } from './types'; import { ActionExecutor, TaskRunnerFactory, LicenseState, ILicenseState } from './lib'; import { ActionsClient } from './actions_client'; import { ActionTypeRegistry } from './action_type_registry'; -import { ExecuteOptions } from './create_execute_function'; -import { createExecuteFunction } from './create_execute_function'; +import { createExecutionEnqueuerFunction } from './create_execute_function'; import { registerBuiltInActionTypes } from './builtin_action_types'; import { registerActionsUsageCollector } from './usage'; @@ -68,7 +67,6 @@ export interface PluginSetupContract { export interface PluginStartContract { isActionTypeEnabled(id: string): boolean; isActionExecutable(actionId: string, actionTypeId: string): boolean; - execute(options: ExecuteOptions): Promise; getActionsClientWithRequest(request: KibanaRequest): Promise>; preconfiguredActions: PreConfiguredAction[]; } @@ -203,7 +201,7 @@ export class ActionsPlugin implements Plugin, Plugi getAllActionRoute(router, this.licenseState); updateActionRoute(router, this.licenseState); listActionTypesRoute(router, this.licenseState); - executeActionRoute(router, this.licenseState, actionExecutor); + executeActionRoute(router, this.licenseState); return { registerType: (actionType: ActionType) => { @@ -269,14 +267,6 @@ export class ActionsPlugin implements Plugin, Plugi scheduleActionsTelemetry(this.telemetryLogger, plugins.taskManager); return { - execute: createExecuteFunction({ - taskManager: plugins.taskManager, - actionTypeRegistry: actionTypeRegistry!, - getScopedSavedObjectsClient, - getBasePath: this.getBasePath, - isESOUsingEphemeralEncryptionKey: isESOUsingEphemeralEncryptionKey!, - preconfiguredActions, - }), isActionTypeEnabled: (id) => { return this.actionTypeRegistry!.isActionTypeEnabled(id); }, @@ -296,6 +286,14 @@ export class ActionsPlugin implements Plugin, Plugi defaultKibanaIndex: await kibanaIndex, scopedClusterClient: core.elasticsearch.legacy.client.asScoped(request), preconfiguredActions, + request, + actionExecutor: actionExecutor!, + executionEnqueuer: createExecutionEnqueuerFunction({ + taskManager: plugins.taskManager, + actionTypeRegistry: actionTypeRegistry!, + isESOUsingEphemeralEncryptionKey: isESOUsingEphemeralEncryptionKey!, + preconfiguredActions, + }), }); }, preconfiguredActions, @@ -319,10 +317,15 @@ export class ActionsPlugin implements Plugin, Plugi core: CoreSetup, defaultKibanaIndex: string ): IContextProvider, 'actions'> => { - const { actionTypeRegistry, isESOUsingEphemeralEncryptionKey, preconfiguredActions } = this; + const { + actionTypeRegistry, + isESOUsingEphemeralEncryptionKey, + preconfiguredActions, + actionExecutor, + } = this; return async function actionsRouteHandlerContext(context, request) { - const [{ savedObjects }] = await core.getStartServices(); + const [{ savedObjects }, { taskManager }] = await core.getStartServices(); return { getActionsClient: () => { if (isESOUsingEphemeralEncryptionKey === true) { @@ -336,6 +339,14 @@ export class ActionsPlugin implements Plugin, Plugi defaultKibanaIndex, scopedClusterClient: context.core.elasticsearch.legacy.client, preconfiguredActions, + request, + actionExecutor: actionExecutor!, + executionEnqueuer: createExecutionEnqueuerFunction({ + taskManager, + actionTypeRegistry: actionTypeRegistry!, + isESOUsingEphemeralEncryptionKey: isESOUsingEphemeralEncryptionKey!, + preconfiguredActions, + }), }); }, listTypes: actionTypeRegistry!.list.bind(actionTypeRegistry!), diff --git a/x-pack/plugins/actions/server/routes/execute.test.ts b/x-pack/plugins/actions/server/routes/execute.test.ts index 4467ddf372b21d..6e8ebbf6f91cd7 100644 --- a/x-pack/plugins/actions/server/routes/execute.test.ts +++ b/x-pack/plugins/actions/server/routes/execute.test.ts @@ -8,7 +8,9 @@ import { executeActionRoute } from './execute'; import { httpServiceMock } from 'src/core/server/mocks'; import { licenseStateMock } from '../lib/license_state.mock'; import { mockHandlerArguments } from './_mock_handler_arguments'; -import { ActionExecutorContract, verifyApiAccess, ActionTypeDisabledError } from '../lib'; +import { verifyApiAccess, ActionTypeDisabledError } from '../lib'; +import { actionsClientMock } from '../actions_client.mock'; +import { ActionTypeExecutorResult } from '../types'; jest.mock('../lib/verify_api_access.ts', () => ({ verifyApiAccess: jest.fn(), @@ -23,8 +25,11 @@ describe('executeActionRoute', () => { const licenseState = licenseStateMock.create(); const router = httpServiceMock.createRouter(); + const actionsClient = actionsClientMock.create(); + actionsClient.execute.mockResolvedValueOnce({ status: 'ok', actionId: '1' }); + const [context, req, res] = mockHandlerArguments( - {}, + { actionsClient }, { body: { params: { @@ -42,14 +47,8 @@ describe('executeActionRoute', () => { actionId: '1', status: 'ok', }; - const actionExecutor = { - initialize: jest.fn(), - execute: jest.fn(async ({ params, request, actionId }) => { - return executeResult; - }), - } as jest.Mocked; - executeActionRoute(router, licenseState, actionExecutor); + executeActionRoute(router, licenseState); const [config, handler] = router.post.mock.calls[0]; @@ -64,12 +63,11 @@ describe('executeActionRoute', () => { expect(await handler(context, req, res)).toEqual({ body: executeResult }); - expect(actionExecutor.execute).toHaveBeenCalledWith({ + expect(actionsClient.execute).toHaveBeenCalledWith({ actionId: '1', params: { someData: 'data', }, - request: req, }); expect(res.ok).toHaveBeenCalled(); @@ -79,8 +77,11 @@ describe('executeActionRoute', () => { const licenseState = licenseStateMock.create(); const router = httpServiceMock.createRouter(); + const actionsClient = actionsClientMock.create(); + actionsClient.execute.mockResolvedValueOnce((null as unknown) as ActionTypeExecutorResult); + const [context, req, res] = mockHandlerArguments( - {}, + { actionsClient }, { body: { params: {}, @@ -92,21 +93,15 @@ describe('executeActionRoute', () => { ['noContent'] ); - const actionExecutor = { - initialize: jest.fn(), - execute: jest.fn(), - } as jest.Mocked; - - executeActionRoute(router, licenseState, actionExecutor); + executeActionRoute(router, licenseState); const [, handler] = router.post.mock.calls[0]; expect(await handler(context, req, res)).toEqual(undefined); - expect(actionExecutor.execute).toHaveBeenCalledWith({ + expect(actionsClient.execute).toHaveBeenCalledWith({ actionId: '1', params: {}, - request: req, }); expect(res.ok).not.toHaveBeenCalled(); @@ -117,8 +112,14 @@ describe('executeActionRoute', () => { const licenseState = licenseStateMock.create(); const router = httpServiceMock.createRouter(); + const actionsClient = actionsClientMock.create(); + actionsClient.execute.mockResolvedValue({ + actionId: '1', + status: 'ok', + }); + const [context, req, res] = mockHandlerArguments( - {}, + { actionsClient }, { body: {}, params: {}, @@ -126,17 +127,7 @@ describe('executeActionRoute', () => { ['ok'] ); - const actionExecutor = { - initialize: jest.fn(), - execute: jest.fn(async ({ params, request, actionId }) => { - return { - actionId: '1', - status: 'ok', - }; - }), - } as jest.Mocked; - - executeActionRoute(router, licenseState, actionExecutor); + executeActionRoute(router, licenseState); const [, handler] = router.post.mock.calls[0]; @@ -149,12 +140,18 @@ describe('executeActionRoute', () => { const licenseState = licenseStateMock.create(); const router = httpServiceMock.createRouter(); + const actionsClient = actionsClientMock.create(); + actionsClient.execute.mockResolvedValue({ + actionId: '1', + status: 'ok', + }); + (verifyApiAccess as jest.Mock).mockImplementation(() => { throw new Error('OMG'); }); const [context, req, res] = mockHandlerArguments( - {}, + { actionsClient }, { body: {}, params: {}, @@ -162,17 +159,7 @@ describe('executeActionRoute', () => { ['ok'] ); - const actionExecutor = { - initialize: jest.fn(), - execute: jest.fn(async ({ params, request, actionId }) => { - return { - actionId: '1', - status: 'ok', - }; - }), - } as jest.Mocked; - - executeActionRoute(router, licenseState, actionExecutor); + executeActionRoute(router, licenseState); const [, handler] = router.post.mock.calls[0]; @@ -185,8 +172,11 @@ describe('executeActionRoute', () => { const licenseState = licenseStateMock.create(); const router = httpServiceMock.createRouter(); + const actionsClient = actionsClientMock.create(); + actionsClient.execute.mockRejectedValue(new ActionTypeDisabledError('Fail', 'license_invalid')); + const [context, req, res] = mockHandlerArguments( - {}, + { actionsClient }, { body: {}, params: {}, @@ -194,14 +184,7 @@ describe('executeActionRoute', () => { ['ok', 'forbidden'] ); - const actionExecutor = { - initialize: jest.fn(), - execute: jest.fn().mockImplementation(() => { - throw new ActionTypeDisabledError('Fail', 'license_invalid'); - }), - } as jest.Mocked; - - executeActionRoute(router, licenseState, actionExecutor); + executeActionRoute(router, licenseState); const [, handler] = router.post.mock.calls[0]; diff --git a/x-pack/plugins/actions/server/routes/execute.ts b/x-pack/plugins/actions/server/routes/execute.ts index 820bc191993d1e..28e6a54f5e92d9 100644 --- a/x-pack/plugins/actions/server/routes/execute.ts +++ b/x-pack/plugins/actions/server/routes/execute.ts @@ -13,7 +13,6 @@ import { } from 'kibana/server'; import { ILicenseState, verifyApiAccess, isErrorThatHandlesItsOwnResponse } from '../lib'; -import { ActionExecutorContract } from '../lib'; import { ActionTypeExecutorResult } from '../types'; import { BASE_ACTION_API_PATH } from '../../common'; @@ -25,11 +24,7 @@ const bodySchema = schema.object({ params: schema.recordOf(schema.string(), schema.any()), }); -export const executeActionRoute = ( - router: IRouter, - licenseState: ILicenseState, - actionExecutor: ActionExecutorContract -) => { +export const executeActionRoute = (router: IRouter, licenseState: ILicenseState) => { router.post( { path: `${BASE_ACTION_API_PATH}/action/{id}/_execute`, @@ -47,12 +42,17 @@ export const executeActionRoute = ( res: KibanaResponseFactory ): Promise { verifyApiAccess(licenseState); + + if (!context.actions) { + return res.badRequest({ body: 'RouteHandlerContext is not registered for actions' }); + } + + const actionsClient = context.actions.getActionsClient(); const { params } = req.body; const { id } = req.params; try { - const body: ActionTypeExecutorResult = await actionExecutor.execute({ + const body: ActionTypeExecutorResult = await actionsClient.execute({ params, - request: req, actionId: id, }); return body diff --git a/x-pack/plugins/alerts/server/task_runner/create_execution_handler.test.ts b/x-pack/plugins/alerts/server/task_runner/create_execution_handler.test.ts index ceae9f04fe5ae6..dd5a9f531bd58b 100644 --- a/x-pack/plugins/alerts/server/task_runner/create_execution_handler.test.ts +++ b/x-pack/plugins/alerts/server/task_runner/create_execution_handler.test.ts @@ -7,8 +7,9 @@ import { AlertType } from '../types'; import { createExecutionHandler } from './create_execution_handler'; import { loggingServiceMock } from '../../../../../src/core/server/mocks'; -import { actionsMock } from '../../../actions/server/mocks'; +import { actionsMock, actionsClientMock } from '../../../actions/server/mocks'; import { eventLoggerMock } from '../../../event_log/server/event_logger.mock'; +import { KibanaRequest } from 'kibana/server'; const alertType: AlertType = { id: 'test', @@ -22,6 +23,7 @@ const alertType: AlertType = { producer: 'alerting', }; +const actionsClient = actionsClientMock.create(); const createExecutionHandlerParams = { actionsPlugin: actionsMock.createStart(), spaceId: 'default', @@ -47,15 +49,19 @@ const createExecutionHandlerParams = { }, }, ], + request: {} as KibanaRequest, }; beforeEach(() => { jest.resetAllMocks(); createExecutionHandlerParams.actionsPlugin.isActionTypeEnabled.mockReturnValue(true); createExecutionHandlerParams.actionsPlugin.isActionExecutable.mockReturnValue(true); + createExecutionHandlerParams.actionsPlugin.getActionsClientWithRequest.mockResolvedValue( + actionsClient + ); }); -test('calls actionsPlugin.execute per selected action', async () => { +test('enqueues execution per selected action', async () => { const executionHandler = createExecutionHandler(createExecutionHandlerParams); await executionHandler({ actionGroup: 'default', @@ -63,8 +69,11 @@ test('calls actionsPlugin.execute per selected action', async () => { context: {}, alertInstanceId: '2', }); - expect(createExecutionHandlerParams.actionsPlugin.execute).toHaveBeenCalledTimes(1); - expect(createExecutionHandlerParams.actionsPlugin.execute.mock.calls[0]).toMatchInlineSnapshot(` + expect( + createExecutionHandlerParams.actionsPlugin.getActionsClientWithRequest + ).toHaveBeenCalledWith(createExecutionHandlerParams.request); + expect(actionsClient.enqueueExecution).toHaveBeenCalledTimes(1); + expect(actionsClient.enqueueExecution.mock.calls[0]).toMatchInlineSnapshot(` Array [ Object { "apiKey": "MTIzOmFiYw==", @@ -139,8 +148,8 @@ test(`doesn't call actionsPlugin.execute for disabled actionTypes`, async () => context: {}, alertInstanceId: '2', }); - expect(createExecutionHandlerParams.actionsPlugin.execute).toHaveBeenCalledTimes(1); - expect(createExecutionHandlerParams.actionsPlugin.execute).toHaveBeenCalledWith({ + expect(actionsClient.enqueueExecution).toHaveBeenCalledTimes(1); + expect(actionsClient.enqueueExecution).toHaveBeenCalledWith({ id: '2', params: { foo: true, @@ -180,7 +189,7 @@ test('trow error error message when action type is disabled', async () => { alertInstanceId: '2', }); - expect(createExecutionHandlerParams.actionsPlugin.execute).toHaveBeenCalledTimes(0); + expect(actionsClient.enqueueExecution).toHaveBeenCalledTimes(0); createExecutionHandlerParams.actionsPlugin.isActionExecutable.mockImplementation(() => true); const executionHandlerForPreconfiguredAction = createExecutionHandler({ @@ -193,7 +202,7 @@ test('trow error error message when action type is disabled', async () => { context: {}, alertInstanceId: '2', }); - expect(createExecutionHandlerParams.actionsPlugin.execute).toHaveBeenCalledTimes(1); + expect(actionsClient.enqueueExecution).toHaveBeenCalledTimes(1); }); test('limits actionsPlugin.execute per action group', async () => { @@ -204,7 +213,7 @@ test('limits actionsPlugin.execute per action group', async () => { context: {}, alertInstanceId: '2', }); - expect(createExecutionHandlerParams.actionsPlugin.execute).not.toHaveBeenCalled(); + expect(actionsClient.enqueueExecution).not.toHaveBeenCalled(); }); test('context attribute gets parameterized', async () => { @@ -215,8 +224,8 @@ test('context attribute gets parameterized', async () => { state: {}, alertInstanceId: '2', }); - expect(createExecutionHandlerParams.actionsPlugin.execute).toHaveBeenCalledTimes(1); - expect(createExecutionHandlerParams.actionsPlugin.execute.mock.calls[0]).toMatchInlineSnapshot(` + expect(actionsClient.enqueueExecution).toHaveBeenCalledTimes(1); + expect(actionsClient.enqueueExecution.mock.calls[0]).toMatchInlineSnapshot(` Array [ Object { "apiKey": "MTIzOmFiYw==", @@ -241,8 +250,8 @@ test('state attribute gets parameterized', async () => { state: { value: 'state-val' }, alertInstanceId: '2', }); - expect(createExecutionHandlerParams.actionsPlugin.execute).toHaveBeenCalledTimes(1); - expect(createExecutionHandlerParams.actionsPlugin.execute.mock.calls[0]).toMatchInlineSnapshot(` + expect(actionsClient.enqueueExecution).toHaveBeenCalledTimes(1); + expect(actionsClient.enqueueExecution.mock.calls[0]).toMatchInlineSnapshot(` Array [ Object { "apiKey": "MTIzOmFiYw==", diff --git a/x-pack/plugins/alerts/server/task_runner/create_execution_handler.ts b/x-pack/plugins/alerts/server/task_runner/create_execution_handler.ts index 3c58c6d9ba2883..8d859a570ba91f 100644 --- a/x-pack/plugins/alerts/server/task_runner/create_execution_handler.ts +++ b/x-pack/plugins/alerts/server/task_runner/create_execution_handler.ts @@ -6,7 +6,7 @@ import { pluck } from 'lodash'; import { AlertAction, State, Context, AlertType } from '../types'; -import { Logger } from '../../../../../src/core/server'; +import { Logger, KibanaRequest } from '../../../../../src/core/server'; import { transformActionParams } from './transform_action_params'; import { PluginStartContract as ActionsPluginStartContract } from '../../../actions/server'; import { IEventLogger, IEvent, SAVED_OBJECT_REL_PRIMARY } from '../../../event_log/server'; @@ -23,6 +23,7 @@ interface CreateExecutionHandlerOptions { alertType: AlertType; logger: Logger; eventLogger: IEventLogger; + request: KibanaRequest; } interface ExecutionHandlerOptions { @@ -43,6 +44,7 @@ export function createExecutionHandler({ apiKey, alertType, eventLogger, + request, }: CreateExecutionHandlerOptions) { const alertTypeActionGroups = new Set(pluck(alertType.actionGroups, 'id')); return async ({ actionGroup, context, state, alertInstanceId }: ExecutionHandlerOptions) => { @@ -80,7 +82,8 @@ export function createExecutionHandler({ // TODO would be nice to add the action name here, but it's not available const actionLabel = `${action.actionTypeId}:${action.id}`; - await actionsPlugin.execute({ + const actionsClient = await actionsPlugin.getActionsClientWithRequest(request); + await actionsClient.enqueueExecution({ id: action.id, params: action.params, spaceId, diff --git a/x-pack/plugins/alerts/server/task_runner/task_runner.test.ts b/x-pack/plugins/alerts/server/task_runner/task_runner.test.ts index 983dff86d5602b..690971bc870062 100644 --- a/x-pack/plugins/alerts/server/task_runner/task_runner.test.ts +++ b/x-pack/plugins/alerts/server/task_runner/task_runner.test.ts @@ -13,7 +13,7 @@ import { TaskRunner } from './task_runner'; import { encryptedSavedObjectsMock } from '../../../encrypted_saved_objects/server/mocks'; import { loggingServiceMock } from '../../../../../src/core/server/mocks'; import { PluginStartContract as ActionsPluginStart } from '../../../actions/server'; -import { actionsMock } from '../../../actions/server/mocks'; +import { actionsMock, actionsClientMock } from '../../../actions/server/mocks'; import { alertsMock } from '../mocks'; import { eventLoggerMock } from '../../../event_log/server/event_logger.mock'; import { IEventLogger } from '../../../event_log/server'; @@ -57,6 +57,7 @@ describe('Task Runner', () => { const encryptedSavedObjectsClient = encryptedSavedObjectsMock.createClient(); const services = alertsMock.createAlertServices(); const savedObjectsClient = services.savedObjectsClient; + const actionsClient = actionsClientMock.create(); const taskRunnerFactoryInitializerParams: jest.Mocked & { actionsPlugin: jest.Mocked; @@ -108,6 +109,9 @@ describe('Task Runner', () => { beforeEach(() => { jest.resetAllMocks(); taskRunnerFactoryInitializerParams.getServices.mockReturnValue(services); + taskRunnerFactoryInitializerParams.actionsPlugin.getActionsClientWithRequest.mockResolvedValue( + actionsClient + ); }); test('successfully executes the task', async () => { @@ -206,9 +210,129 @@ describe('Task Runner', () => { references: [], }); await taskRunner.run(); - expect(taskRunnerFactoryInitializerParams.actionsPlugin.execute).toHaveBeenCalledTimes(1); - expect(taskRunnerFactoryInitializerParams.actionsPlugin.execute.mock.calls[0]) - .toMatchInlineSnapshot(` + expect(actionsClient.enqueueExecution).toHaveBeenCalledTimes(1); + expect(actionsClient.enqueueExecution.mock.calls[0]).toMatchInlineSnapshot(` + Array [ + Object { + "apiKey": "MTIzOmFiYw==", + "id": "1", + "params": Object { + "foo": true, + }, + "spaceId": undefined, + }, + ] + `); + + const eventLogger = taskRunnerFactoryInitializerParams.eventLogger; + expect(eventLogger.logEvent).toHaveBeenCalledTimes(3); + expect(eventLogger.logEvent).toHaveBeenCalledWith({ + event: { + action: 'execute', + outcome: 'success', + }, + kibana: { + saved_objects: [ + { + id: '1', + namespace: undefined, + rel: 'primary', + type: 'alert', + }, + ], + }, + message: "alert executed: test:1: 'alert-name'", + }); + expect(eventLogger.logEvent).toHaveBeenCalledWith({ + event: { + action: 'new-instance', + }, + kibana: { + alerting: { + instance_id: '1', + }, + saved_objects: [ + { + id: '1', + namespace: undefined, + rel: 'primary', + type: 'alert', + }, + ], + }, + message: "test:1: 'alert-name' created new instance: '1'", + }); + expect(eventLogger.logEvent).toHaveBeenCalledWith({ + event: { + action: 'execute-action', + }, + kibana: { + alerting: { + instance_id: '1', + }, + saved_objects: [ + { + id: '1', + namespace: undefined, + rel: 'primary', + type: 'alert', + }, + { + id: '1', + namespace: undefined, + type: 'action', + }, + ], + }, + message: + "alert: test:1: 'alert-name' instanceId: '1' scheduled actionGroup: 'default' action: undefined:1", + }); + }); + + test('includes the apiKey in the request used to initialize the actionsClient', async () => { + taskRunnerFactoryInitializerParams.actionsPlugin.isActionTypeEnabled.mockReturnValue(true); + taskRunnerFactoryInitializerParams.actionsPlugin.isActionExecutable.mockReturnValue(true); + alertType.executor.mockImplementation( + ({ services: executorServices }: AlertExecutorOptions) => { + executorServices.alertInstanceFactory('1').scheduleActions('default'); + } + ); + const taskRunner = new TaskRunner( + alertType, + mockedTaskInstance, + taskRunnerFactoryInitializerParams + ); + savedObjectsClient.get.mockResolvedValueOnce(mockedAlertTypeSavedObject); + encryptedSavedObjectsClient.getDecryptedAsInternalUser.mockResolvedValueOnce({ + id: '1', + type: 'alert', + attributes: { + apiKey: Buffer.from('123:abc').toString('base64'), + }, + references: [], + }); + await taskRunner.run(); + expect( + taskRunnerFactoryInitializerParams.actionsPlugin.getActionsClientWithRequest + ).toHaveBeenCalledWith({ + getBasePath: expect.anything(), + headers: { + // base64 encoded "123:abc" + authorization: 'ApiKey MTIzOmFiYw==', + }, + path: '/', + route: { settings: {} }, + url: { + href: '/', + }, + raw: { + req: { + url: '/', + }, + }, + }); + expect(actionsClient.enqueueExecution).toHaveBeenCalledTimes(1); + expect(actionsClient.enqueueExecution.mock.calls[0]).toMatchInlineSnapshot(` Array [ Object { "apiKey": "MTIzOmFiYw==", diff --git a/x-pack/plugins/alerts/server/task_runner/task_runner.ts b/x-pack/plugins/alerts/server/task_runner/task_runner.ts index be399893088e33..3512ab16a37125 100644 --- a/x-pack/plugins/alerts/server/task_runner/task_runner.ts +++ b/x-pack/plugins/alerts/server/task_runner/task_runner.ts @@ -71,14 +71,14 @@ export class TaskRunner { return apiKey; } - async getServicesWithSpaceLevelPermissions(spaceId: string, apiKey: string | null) { + private getFakeKibanaRequest(spaceId: string, apiKey: string | null) { const requestHeaders: Record = {}; if (apiKey) { requestHeaders.authorization = `ApiKey ${apiKey}`; } - const fakeRequest = { + return ({ headers: requestHeaders, getBasePath: () => this.context.getBasePath(spaceId), path: '/', @@ -91,9 +91,11 @@ export class TaskRunner { url: '/', }, }, - }; + } as unknown) as KibanaRequest; + } - return this.context.getServices((fakeRequest as unknown) as KibanaRequest); + async getServicesWithSpaceLevelPermissions(spaceId: string, apiKey: string | null) { + return this.context.getServices(this.getFakeKibanaRequest(spaceId, apiKey)); } private getExecutionHandler( @@ -128,6 +130,7 @@ export class TaskRunner { spaceId, alertType: this.alertType, eventLogger: this.context.eventLogger, + request: this.getFakeKibanaRequest(spaceId, apiKey), }); } From 721125772f32d4780409000516e5eddbf856069d Mon Sep 17 00:00:00 2001 From: Anton Dosov Date: Tue, 16 Jun 2020 10:42:37 +0200 Subject: [PATCH 11/11] stabilize flaky embeddables/adding_children take 2 (#68873) * wait for loading indicator to be deleted * improve with retry * Update test/examples/embeddables/adding_children.ts Co-authored-by: Spencer Co-authored-by: Spencer Co-authored-by: Elastic Machine --- test/examples/embeddables/adding_children.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/test/examples/embeddables/adding_children.ts b/test/examples/embeddables/adding_children.ts index 4e3d9d30b01aa2..9c088f0a5d3135 100644 --- a/test/examples/embeddables/adding_children.ts +++ b/test/examples/embeddables/adding_children.ts @@ -24,6 +24,7 @@ import { PluginFunctionalProviderContext } from 'test/plugin_functional/services export default function ({ getService }: PluginFunctionalProviderContext) { const testSubjects = getService('testSubjects'); const flyout = getService('flyout'); + const retry = getService('retry'); describe('creating and adding children', () => { before(async () => { @@ -33,8 +34,15 @@ export default function ({ getService }: PluginFunctionalProviderContext) { it('Can create a new child', async () => { await testSubjects.click('embeddablePanelToggleMenuIcon'); await testSubjects.click('embeddablePanelAction-ACTION_ADD_PANEL'); - await testSubjects.click('createNew'); + + // this seem like an overkill, but clicking this button which opens context menu was flaky + await testSubjects.waitForEnabled('createNew'); + await retry.waitFor('createNew popover opened', async () => { + await testSubjects.click('createNew'); + return await testSubjects.exists('createNew-TODO_EMBEDDABLE'); + }); await testSubjects.click('createNew-TODO_EMBEDDABLE'); + await testSubjects.setValue('taskInputField', 'new task'); await testSubjects.click('createTodoEmbeddable'); const tasks = await testSubjects.getVisibleTextAll('todoEmbeddableTask'); @@ -44,6 +52,7 @@ export default function ({ getService }: PluginFunctionalProviderContext) { it('Can add a child backed off a saved object', async () => { await testSubjects.click('embeddablePanelToggleMenuIcon'); await testSubjects.click('embeddablePanelAction-ACTION_ADD_PANEL'); + await testSubjects.waitForDeleted('savedObjectFinderLoadingIndicator'); await testSubjects.click('savedObjectTitleGarbage'); await testSubjects.moveMouseTo('euiFlyoutCloseButton'); await flyout.ensureClosed('dashboardAddPanel');