[kbn/optimizer] share all plugin bundles

.eslintrc.js

@@ -472,6 +472,7 @@ module.exports = {
     {
       files: [
         'test/functional/services/lib/web_element_wrapper/scroll_into_view_if_necessary.js',
+        'src/legacy/ui/ui_render/bootstrap/kbn_bundles_loader_source.js',
         '**/browser_exec_scripts/**/*.js',
       ],
       rules: {

NOTICE.txt

@@ -21,6 +21,11 @@ used. Logarithmic ticks are places at powers of ten and at half those
 values if there are not to many ticks already (e.g. [1, 5, 10, 50, 100]).
 For details,  see https://github.com/flot/flot/pull/1328
 
+---
+This module was heavily inspired by the externals plugin that ships with webpack@97d58d31
+MIT License http://www.opensource.org/licenses/mit-license.php
+Author Tobias Koppers @sokra
+
 ---
 This product has relied on ASTExplorer that is licensed under MIT.
 

docs/development/core/server/kibana-plugin-core-server.pluginmanifest.extrapublicdirs.md

@@ -0,0 +1,17 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-core-server](./kibana-plugin-core-server.md) &gt; [PluginManifest](./kibana-plugin-core-server.pluginmanifest.md) &gt; [extraPublicDirs](./kibana-plugin-core-server.pluginmanifest.extrapublicdirs.md)
+
+## PluginManifest.extraPublicDirs property
+
+> Warning: This API is now obsolete.
+> 
+> 
+
+Specifies directory names that can be imported by other ui-plugins built using the same instance of the @<!-- -->kbn/optimizer. A temporary measure we plan to replace with better mechanisms for sharing static code between plugins
+
+<b>Signature:</b>
+
+```typescript
+readonly extraPublicDirs?: string[];
+```

docs/development/core/server/kibana-plugin-core-server.pluginmanifest.md

@@ -21,6 +21,7 @@ Should never be used in code outside of Core but is exported for documentation p
 |  Property | Type | Description |
 |  --- | --- | --- |
 |  [configPath](./kibana-plugin-core-server.pluginmanifest.configpath.md) | <code>ConfigPath</code> | Root [configuration path](./kibana-plugin-core-server.configpath.md) used by the plugin, defaults to "id" in snake\_case format. |
+|  [extraPublicDirs](./kibana-plugin-core-server.pluginmanifest.extrapublicdirs.md) | <code>string[]</code> | Specifies directory names that can be imported by other ui-plugins built using the same instance of the @<!-- -->kbn/optimizer. A temporary measure we plan to replace with better mechanisms for sharing static code between plugins |
 |  [id](./kibana-plugin-core-server.pluginmanifest.id.md) | <code>PluginName</code> | Identifier of the plugin. Must be a string in camelCase. Part of a plugin public contract. Other plugins leverage it to access plugin API, navigate to the plugin, etc. |
 |  [kibanaVersion](./kibana-plugin-core-server.pluginmanifest.kibanaversion.md) | <code>string</code> | The version of Kibana the plugin is compatible with, defaults to "version". |
 |  [optionalPlugins](./kibana-plugin-core-server.pluginmanifest.optionalplugins.md) | <code>readonly PluginName[]</code> | An optional list of the other plugins that if installed and enabled \*\*may be\*\* leveraged by this plugin for some additional functionality but otherwise are not required for this plugin to work properly. |

examples/demo_search/kibana.json

@@ -5,5 +5,6 @@
   "server": true,
   "ui": true,
   "requiredPlugins": ["data"],
-  "optionalPlugins": []
+  "optionalPlugins": [],
+  "extraPublicDirs": ["common"]
 }

examples/embeddable_examples/kibana.json

@@ -5,5 +5,6 @@
   "server": true,
   "ui": true,
   "requiredPlugins": ["embeddable"],
-  "optionalPlugins": []
+  "optionalPlugins": [],
+  "extraPublicDirs": ["public/todo", "public/hello_world"]
 }

examples/url_generators_examples/kibana.json

@@ -5,5 +5,8 @@
   "server": false,
   "ui": true,
   "requiredPlugins": ["share"],
-  "optionalPlugins": []
+  "optionalPlugins": [],
+  "extraPublicDirs": [
+    "public/url_generator"
+  ]
 }

packages/kbn-optimizer/README.md

@@ -30,6 +30,18 @@ Bundles built by the the optimizer include a cache file which describes the info
 
 When a bundle is determined to be up-to-date a worker is not started for the bundle. If running the optimizer with the `--dev/--watch` flag, then all the files referenced by cached bundles are watched for changes. Once a change is detected in any of the files referenced by the built bundle a worker is started. If a file is changed that is referenced by several bundles then workers will be started for each bundle, combining workers together to respect the worker limit.
 
+## Bundle Refs
+
+In order to dramatically reduce the size of our bundles, and the time it takes to build them, bundles will "ref" other bundles being built at the same time. When the optimizer starts it creates a list of "refs" that could be had from the list of bundles being built. Each worker uses that list to determine which import statements in a bundle should be replaced with a runtime reference to the output of another bundle.
+
+At runtime the bundles share a set of entry points via the `__kbnBundles__` global. By default a plugin shares `public` so that other code can use relative imports to access that directory. To expose additional directories they must be listed in the plugin's kibana.json "extraPublicDirs" field. The directories listed there will **also** be exported from the plugins bundle so that any other plugin can import that directory. "common" is commonly in the list of "extraPublicDirs".
+
+> NOTE: We plan to replace the `extraPublicDirs` functionality soon with better mechanisms for statically sharing code between bundles.
+
+When a directory is listed in the "extraPublicDirs" it will always be included in the bundle so that other plugins have access to it. The worker building the bundle has no way of knowing whether another plugin is using the directory, so be careful of adding test code or unnecessary directories to that list.
+
+Any import in a bundle which resolves into another bundles "context" directory, ie `src/plugins/*`, must map explicitly to a "public dir" exported by that plugin. If the resolved import is not in the list of public dirs an error will be thrown and the optimizer will fail to build that bundle until the error is fixed.
+
 ## API
 
 To run the optimizer from code, you can import the [`OptimizerConfig`][OptimizerConfig] class and [`runOptimizer`][Optimizer] function. Create an [`OptimizerConfig`][OptimizerConfig] instance by calling it's static `create()` method with some options, then pass it to the [`runOptimizer`][Optimizer] function. `runOptimizer()` returns an observable of update objects, which are summaries of the optimizer state plus an optional `event` property which describes the internal events occuring and may be of use. You can use the [`logOptimizerState()`][LogOptimizerState] helper to write the relevant bits of state to a tooling log or checkout it's implementation to see how the internal events like [`WorkerStdio`][ObserveWorker] and [`WorkerStarted`][ObserveWorker] are used.

packages/kbn-optimizer/package.json

@@ -45,6 +45,7 @@
     "terser-webpack-plugin": "^2.1.2",
     "tinymath": "1.2.1",
     "url-loader": "^2.2.0",
+    "val-loader": "^1.1.1",
     "watchpack": "^1.6.0",
     "webpack": "^4.41.5",
     "webpack-merge": "^4.2.2"

packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/bar/public/index.ts

@@ -19,6 +19,6 @@
 
 import './legacy/styles.scss';
 import './index.scss';
-import { fooLibFn } from '../../foo/public/index';
+import { fooLibFn } from '../../foo/public';
 export * from './lib';
 export { fooLibFn };

packages/kbn-optimizer/src/common/bundle.test.ts

@@ -23,7 +23,7 @@ jest.mock('fs');
 
 const SPEC: BundleSpec = {
   contextDir: '/foo/bar',
-  entry: 'entry',
+  publicDirNames: ['public'],
   id: 'bar',
   outputDir: '/foo/bar/target',
   sourceRoot: '/foo',
@@ -49,9 +49,11 @@ it('creates cache keys', () => {
       },
       "spec": Object {
         "contextDir": "/foo/bar",
-        "entry": "entry",
         "id": "bar",
         "outputDir": "/foo/bar/target",
+        "publicDirNames": Array [
+          "public",
+        ],
         "sourceRoot": "/foo",
         "type": "plugin",
       },
@@ -82,9 +84,11 @@ it('parses bundles from JSON specs', () => {
           "state": undefined,
         },
         "contextDir": "/foo/bar",
-        "entry": "entry",
         "id": "bar",
         "outputDir": "/foo/bar/target",
+        "publicDirNames": Array [
+          "public",
+        ],
         "sourceRoot": "/foo",
         "type": "plugin",
       },

packages/kbn-optimizer/src/common/bundle.ts

@@ -29,8 +29,8 @@ export interface BundleSpec {
   readonly type: typeof VALID_BUNDLE_TYPES[0];
   /** Unique id for this bundle */
   readonly id: string;
-  /** Webpack entry request for this plugin, relative to the contextDir */
-  readonly entry: string;
+  /** directory names relative to the contextDir that can be imported from */
+  readonly publicDirNames: string[];
   /** Absolute path to the plugin source directory */
   readonly contextDir: string;
   /** Absolute path to the root of the repository */
@@ -44,8 +44,8 @@ export class Bundle {
   public readonly type: BundleSpec['type'];
   /** Unique identifier for this bundle */
   public readonly id: BundleSpec['id'];
-  /** Path, relative to `contextDir`, to the entry file for the Webpack bundle */
-  public readonly entry: BundleSpec['entry'];
+  /** directory names relative to the contextDir that can be imported from */
+  public readonly publicDirNames: BundleSpec['publicDirNames'];
   /**
    * Absolute path to the root of the bundle context (plugin directory)
    * where the entry is resolved relative to and the default output paths
@@ -62,7 +62,7 @@ export class Bundle {
   constructor(spec: BundleSpec) {
     this.type = spec.type;
     this.id = spec.id;
-    this.entry = spec.entry;
+    this.publicDirNames = spec.publicDirNames;
     this.contextDir = spec.contextDir;
     this.sourceRoot = spec.sourceRoot;
     this.outputDir = spec.outputDir;
@@ -73,8 +73,6 @@ export class Bundle {
   /**
    * Calculate the cache key for this bundle based from current
    * mtime values.
-   *
-   * @param mtimes pre-fetched mtimes (ms || undefined) for all referenced files
    */
   createCacheKey(files: string[], mtimes: Map<string, number | undefined>): unknown {
     return {
@@ -94,7 +92,7 @@ export class Bundle {
     return {
       type: this.type,
       id: this.id,
-      entry: this.entry,
+      publicDirNames: this.publicDirNames,
       contextDir: this.contextDir,
       sourceRoot: this.sourceRoot,
       outputDir: this.outputDir,
@@ -134,9 +132,9 @@ export function parseBundles(json: string) {
           throw new Error('`bundles[]` must have a string `id` property');
         }
 
-        const { entry } = spec;
-        if (!(typeof entry === 'string')) {
-          throw new Error('`bundles[]` must have a string `entry` property');
+        const { publicDirNames } = spec;
+        if (!Array.isArray(publicDirNames) || !publicDirNames.every((d) => typeof d === 'string')) {
+          throw new Error('`bundles[]` must have an array of strings `publicDirNames` property');
         }
 
         const { contextDir } = spec;
@@ -157,7 +155,7 @@ export function parseBundles(json: string) {
         return new Bundle({
           type,
           id,
-          entry,
+          publicDirNames,
           contextDir,
           sourceRoot,
           outputDir,

packages/kbn-optimizer/src/common/bundle_cache.ts

@@ -25,6 +25,7 @@ export interface State {
   cacheKey?: unknown;
   moduleCount?: number;
   files?: string[];
+  bundleRefExportIds?: string[];
 }
 
 const DEFAULT_STATE: State = {};
@@ -87,6 +88,10 @@ export class BundleCache {
     return this.get().files;
   }
 
+  public getBundleRefExportIds() {
+    return this.get().bundleRefExportIds;
+  }
+
   public getCacheKey() {
     return this.get().cacheKey;
   }

packages/kbn-optimizer/src/common/bundle_refs.ts

@@ -0,0 +1,130 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import Path from 'path';
+
+import { Bundle } from './bundle';
+import { UnknownVals } from './ts_helpers';
+
+export interface BundleRef {
+  bundleId: string;
+  contextDir: string;
+  contextPrefix: string;
+  entry: string;
+  exportId: string;
+}
+
+export class BundleRefs {
+  static fromBundles(bundles: Bundle[]) {
+    return new BundleRefs(
+      bundles.reduce(
+        (acc: BundleRef[], b) => [
+          ...acc,
+          ...b.publicDirNames.map(
+            (name): BundleRef => ({
+              bundleId: b.id,
+              contextDir: b.contextDir,
+              // Path.resolve converts separators and strips the final separator
+              contextPrefix: Path.resolve(b.contextDir) + Path.sep,
+              entry: name,
+              exportId: `${b.type}/${b.id}/${name}`,
+            })
+          ),
+        ],
+        []
+      )
+    );
+  }
+
+  static parseSpec(json: unknown) {
+    if (typeof json !== 'string') {
+      throw new Error('expected `bundleRefs` spec to be a JSON string');
+    }
+
+    let spec;
+    try {
+      spec = JSON.parse(json);
+    } catch (error) {
+      throw new Error('`bundleRefs` spec must be valid JSON');
+    }
+
+    if (!Array.isArray(spec)) {
+      throw new Error('`bundleRefs` spec must be an array');
+    }
+
+    return new BundleRefs(
+      spec.map(
+        (refSpec: UnknownVals<BundleRef>): BundleRef => {
+          if (typeof refSpec !== 'object' || !refSpec) {
+            throw new Error('`bundleRefs[]` must be an object');
+          }
+
+          const { bundleId } = refSpec;
+          if (typeof bundleId !== 'string') {
+            throw new Error('`bundleRefs[].bundleId` must be a string');
+          }
+
+          const { contextDir } = refSpec;
+          if (typeof contextDir !== 'string' || !Path.isAbsolute(contextDir)) {
+            throw new Error('`bundleRefs[].contextDir` must be an absolute directory');
+          }
+
+          const { contextPrefix } = refSpec;
+          if (typeof contextPrefix !== 'string' || !Path.isAbsolute(contextPrefix)) {
+            throw new Error('`bundleRefs[].contextPrefix` must be an absolute directory');
+          }
+
+          const { entry } = refSpec;
+          if (typeof entry !== 'string') {
+            throw new Error('`bundleRefs[].entry` must be a string');
+          }
+
+          const { exportId } = refSpec;
+          if (typeof exportId !== 'string') {
+            throw new Error('`bundleRefs[].exportId` must be a string');
+          }
+
+          return {
+            bundleId,
+            contextDir,
+            contextPrefix,
+            entry,
+            exportId,
+          };
+        }
+      )
+    );
+  }
+
+  constructor(private readonly refs: BundleRef[]) {}
+
+  public filterByExportIds(exportIds: string[]) {
+    return this.refs.filter((r) => exportIds.includes(r.exportId));
+  }
+
+  public filterByContextPrefix(bundle: Bundle, absolutePath: string) {
+    return this.refs.filter(
+      (ref) => ref.bundleId !== bundle.id && absolutePath.startsWith(ref.contextPrefix)
+    );
+  }
+
+  public toSpecJson() {
+    return JSON.stringify(this.refs);
+  }
+}

packages/kbn-optimizer/src/common/index.ts

@@ -19,6 +19,7 @@
 
 export * from './bundle';
 export * from './bundle_cache';
+export * from './bundle_refs';
 export * from './worker_config';
 export * from './worker_messages';
 export * from './compiler_messages';