Functional test setup with kbn-test package (elastic#18568) (elastic#…

…19174)

Restructure testing with kbn-test package

 - Run with multiple configs, move cli options to config
 - Package-ify kbn-test
 - Eventually we'll have functional_test_runner live in a package
of its own, and then this kbn-test will use that as a dependency,
probably still as a devDependency.
 - Implement functional_tests_server
 - Collapse single and multiple config apis into one command

Use kbn-es

Replace es_test_cluster + es_test_config with kbn/test utils

Implement new createEsTestCluster

Improve scripts, jsdocs, cli top-level tools

Lift error handling to the top level

.eslintrc.js

@@ -31,6 +31,7 @@ module.exports = {
         'packages/kbn-es/**/*',
         'packages/kbn-datemath/**/*.js',
         'packages/kbn-plugin-generator/**/*',
+        'packages/kbn-test/**/*',
         'packages/kbn-eslint-import-resolver-kibana/**/*',
         'x-pack/plugins/apm/**/*',
       ],

CONTRIBUTING.md

@@ -355,6 +355,7 @@ yarn test:browser --dev # remove the --dev flag to run them once and close
 
 [Read about the `FunctionalTestRunner`](https://www.elastic.co/guide/en/kibana/current/development-functional-tests.html) to learn more about how you can run and develop functional tests for Kibana core and plugins.
 
+You can also look into the [Scripts README.md](./scripts/README.md) to learn more about using the node scripts we provide for building Kibana, running integration tests, and starting up Kibana and Elasticsearch while you develop.
 
 ### Building OS packages
 

package.json

@@ -224,6 +224,7 @@
     "@kbn/eslint-import-resolver-kibana": "link:packages/kbn-eslint-import-resolver-kibana",
     "@kbn/eslint-plugin-license-header": "link:packages/kbn-eslint-plugin-license-header",
     "@kbn/plugin-generator": "link:packages/kbn-plugin-generator",
+    "@kbn/test": "link:packages/kbn-test",
     "angular-mocks": "1.4.7",
     "babel-eslint": "8.1.2",
     "babel-jest": "^22.4.3",

packages/kbn-dev-utils/src/proc_runner/proc.js

@@ -78,7 +78,7 @@ export function createProc(name, { cmd, args, cwd, env, stdin, log }) {
         .map(code => {
           // JVM exits with 143 on SIGTERM and 130 on SIGINT, dont' treat then as errors
           if (code > 0 && !(code === 143 || code === 130)) {
-            throw createCliError(`[${name}] exitted with code ${code}`);
+            throw createCliError(`[${name}] exited with code ${code}`);
           }
 
           return code;

packages/kbn-dev-utils/src/proc_runner/proc_runner.js

@@ -76,7 +76,9 @@ export class ProcRunner {
           .first()
           .catch(err => {
             if (err.name !== 'EmptyError') {
-              throw createCliError(`[${name}] exitted without matching pattern: ${wait}`);
+              throw createCliError(
+                `[${name}] exited without matching pattern: ${wait}`
+              );
             } else {
               throw err;
             }
@@ -171,7 +173,12 @@ export class ProcRunner {
     proc.outcome$.subscribe({
       next: (code) => {
         const duration = moment.duration(Date.now() - startMs);
-        this._log.info('[%s] exitted with %s after %s', name, code, duration.humanize());
+        this._log.info(
+          '[%s] exited with %s after %s',
+          name,
+          code,
+          duration.humanize()
+        );
       },
       complete: () => {
         remove();

packages/kbn-test/.babelrc

@@ -0,0 +1,3 @@
+{
+  "presets": ["@kbn/babel-preset/node_preset"]
+}

packages/kbn-test/README.md

@@ -0,0 +1,39 @@
+Kibana Testing Library
+======================
+
+The @kbn/test package provides ways to run tests. Currently only functional testing is provided by this library, with unit and other testing possibly added here.
+
+Functional Testing
+-------------------
+
+### Dependencies
+
+Functional testing methods exist in the `src/functional_tests` directory. They depend on the Functional Test Runner, which is found in [`{KIBANA_ROOT}/src/functional_test_runner`](../../src/functional_test_runner). Ideally libraries provided by kibana packages such as this one should not depend on kibana source code that lives in [`{KIBANA_ROOT}/src`](../../src). The goal is to start pulling test and development utilities out into packages so they can be used across Kibana and plugins. Accordingly the Functional Test Runner itself will be pulled out into a package (or part of a package), and this package's dependence on it will not be an issue.
+
+### Exposed methods
+
+#### runTests(configPaths: Array<string>)
+For each config file specified in configPaths, starts Elasticsearch and Kibana once, runs tests specified in that config file, and shuts down Elasticsearch and Kibana once completed. (Repeats for every config file.)
+
+`configPaths`: array of strings, each an absolute path to a config file that looks like [this](../../test/functional/config.js), following the config schema specified [here](../../src/functional_test_runner/lib/config/schema.js).
+
+Internally the method that starts Elasticsearch comes from [kbn-es](../../packages/kbn-es).
+
+#### startServers(configPath: string)
+Starts Elasticsearch and Kibana servers given a specified config.
+
+`configPath`: absolute path to a config file that looks like [this](../../test/functional/config.js), following the config schema specified [here](../../src/functional_test_runner/lib/config/schema.js).
+
+Allows users to start another process to run just the tests while keeping the servers running with this method. Start servers _and_ run tests using the same config file ([see how](../../scripts/README.md)).
+
+## Rationale
+
+### Single config per setup
+
+We think it makes sense to specify the tests to run along with the particular server configuration for Elasticsearch and Kibana servers, because the tests expect a particular configuration. For example, saml api integration tests expect certain xml files to exist in Elasticsearch's config directory, and certain saml specific options to be passed in via the command line (or alternatively via the `.yml` config file) to both Elasticsearch and Kibana. It makes sense to keep all these config options together with the list of test files.
+
+### Multiple configs running in succession
+
+We also think it makes sense to have a test runner intelligently (but simply) start servers, run tests, tear down servers, and repeat for each config, uninterrupted. There's nothing special about each kind of config that specifies running some set of functional tests against some kind of Elasticsearch/Kibana servers. There doesn't need to be a separate job to run each kind of setup/test/teardown. These can all be orchestrated sequentially via the current `runTests` implementation. This is how we envision tests to run on CI.
+
+This inherently means that grouping test files in configs matters, such that a group of test files that depends on a particular server config appears together in that config's `testFiles` list. Given how quickly and easily we can start servers using [@kbn/es](../../packages/kbn-es), it should not impact performance to logically group tests by domain even if multiple groups of tests share the same server config. We can think about how to group test files together across domains when that time comes.

packages/kbn-test/package.json

@@ -0,0 +1,27 @@
+{
+  "name": "@kbn/test",
+  "main": "./target/index.js",
+  "version": "1.0.0",
+  "license": "Apache-2.0",
+  "private": true,
+  "scripts": {
+    "build": "babel src --out-dir target",
+    "kbn:bootstrap": "yarn build",
+    "kbn:watch": "yarn build --watch"
+  },
+  "devDependencies": {
+    "@kbn/babel-preset": "link:../kbn-babel-preset",
+    "@kbn/dev-utils": "link:../kbn-dev-utils",
+    "babel-cli": "^6.26.0"
+  },
+  "dependencies": {
+    "chalk": "^2.4.1",
+    "dedent": "^0.7.0",
+    "getopts": "^2.0.6",
+    "glob": "^7.1.2",
+    "rxjs": "^5.4.3",
+    "tar-fs": "^1.16.2",
+    "tmp": "^0.0.33",
+    "zlib": "^1.0.5"
+  }
+}

src/test_utils/es/es_test_cluster.js → packages/kbn-test/src/es/es_test_cluster.js

@@ -1,37 +1,39 @@
 import { resolve } from 'path';
-import { get } from 'lodash';
 import { format } from 'url';
-import elasticsearch from 'elasticsearch';
+import { get } from 'lodash';
 import toPath from 'lodash/internal/toPath';
 import { Cluster } from '@kbn/es';
 import { esTestConfig } from './es_test_config';
 import { rmrfSync } from './rmrf_sync';
+import { KIBANA_ROOT } from '../';
+import elasticsearch from 'elasticsearch';
 
-export function createTestCluster(options = {}) {
+export function createEsTestCluster(options = {}) {
   const {
     port = esTestConfig.getPort(),
     password = 'changeme',
     license = 'oss',
     log,
-    basePath = resolve(__dirname, '../../../.es'),
+    basePath = resolve(KIBANA_ROOT, '.es'),
+    // Use source when running on CI
+    from = esTestConfig.getBuildFrom(),
   } = options;
 
-  const randomHash = Math.random().toString(36).substring(2);
+  const randomHash = Math.random()
+    .toString(36)
+    .substring(2);
   const clusterName = `test-${randomHash}`;
   const config = {
     version: esTestConfig.getVersion(),
     installPath: resolve(basePath, clusterName),
-    sourcePath: resolve(__dirname, '../../../../elasticsearch'),
+    sourcePath: resolve(KIBANA_ROOT, '../elasticsearch'),
     password,
     license,
     basePath,
   };
 
   const cluster = new Cluster(log);
 
-  // Use source when running on CI
-  const from = options.from || esTestConfig.getBuildFrom();
-
   return new class EsTestCluster {
     getStartTimeout() {
       const second = 1000;
@@ -51,18 +53,20 @@ export function createTestCluster(options = {}) {
           `cluster.name=${clusterName}`,
           `http.port=${port}`,
           `discovery.zen.ping.unicast.hosts=localhost:${port}`,
-          ...esArgs
+          ...esArgs,
         ],
       });
     }
 
     async stop() {
       await cluster.stop();
+      log.info('[es] stopped');
     }
 
     async cleanup() {
       await this.stop();
       rmrfSync(config.installPath);
+      log.info('[es] cleanup complete');
     }
 
     /**
@@ -84,7 +88,7 @@ export function createTestCluster(options = {}) {
 
       return format(parts);
     }
-  };
+  }();
 }
 
 /**

src/test_utils/es/es_test_config.js → packages/kbn-test/src/es/es_test_config.js

@@ -1,6 +1,6 @@
-import url, {  format as formatUrl } from 'url';
-import pkg from '../../../package.json';
-import { admin } from '../../../test/shield';
+import url, { format as formatUrl } from 'url';
+import pkg from '../../../../package.json';
+import { adminTestUser } from '../kbn';
 
 export const esTestConfig = new class EsTestConfig {
   getVersion() {
@@ -30,22 +30,21 @@ export const esTestConfig = new class EsTestConfig {
         port: parseInt(testEsUrl.port, 10),
         username: testEsUrl.auth.split(':')[0],
         password: testEsUrl.auth.split(':')[1],
-        auth: testEsUrl.auth
+        auth: testEsUrl.auth,
       };
     }
 
-    const username = process.env.TEST_KIBANA_USERNAME || admin.username;
-    const password = process.env.TEST_KIBANA_PASSWORD || admin.password;
+    const username = process.env.TEST_KIBANA_USERNAME || adminTestUser.username;
+    const password = process.env.TEST_KIBANA_PASSWORD || adminTestUser.password;
     return {
       // Allow setting any individual component(s) of the URL,
-      // or use default values (username and password from shield.js)
+      // or use default values (username and password from ../kbn/users.js)
       protocol: process.env.TEST_ES_PROTOCOL || 'http',
       hostname: process.env.TEST_ES_HOSTNAME || 'localhost',
       port: parseInt(process.env.TEST_ES_PORT, 10) || 9220,
       auth: `${username}:${password}`,
       username: username,
       password: password,
     };
-
   }
-};
+}();

packages/kbn-test/src/es/index.js

@@ -0,0 +1,2 @@
+export { createEsTestCluster } from './es_test_cluster.js';
+export { esTestConfig } from './es_test_config';

src/test_utils/es/rmrf_sync.js → packages/kbn-test/src/es/rmrf_sync.js

@@ -18,4 +18,4 @@ export function rmrfSync(path) {
     });
     fs.rmdirSync(path);
   }
-}
+}

packages/kbn-test/src/functional_tests/cli/index.js

@@ -0,0 +1,2 @@
+export { runTestsCli } from './run_tests_cli';
+export { startServersCli } from './start_servers_cli';

packages/kbn-test/src/functional_tests/cli/run_tests_cli.js

@@ -0,0 +1,78 @@
+import dedent from 'dedent';
+import getopts from 'getopts';
+import { createToolingLog, pickLevelFromFlags } from '@kbn/dev-utils';
+import { runTests } from '../../';
+
+/**
+ * Run servers and tests for each config
+ * Only cares about --config option. Other options
+ * are passed directly to functional_test_runner, such as
+ * --bail, --verbose, etc.
+ * @param {string[]} defaultConfigPaths  Array of paths to configs to use
+ *                                       if no config option is passed
+ */
+export async function runTestsCli(defaultConfigPaths) {
+  const { configs, help, bail, log } = processArgs(defaultConfigPaths);
+
+  if (help) return displayHelp();
+
+  if (!configs || configs.length === 0) {
+    log.error(
+      `Run Tests requires at least one path to a config. Leave blank to use defaults.`
+    );
+    process.exit(9);
+  }
+
+  try {
+    await runTests(configs, { bail, log });
+  } catch (err) {
+    log.error('FATAL ERROR');
+    log.error(err);
+    process.exit(1);
+  }
+}
+
+function processArgs(defaultConfigPaths) {
+  // If no args are passed, use {}
+  const options = getopts(process.argv.slice(2)) || {};
+
+  // If --config is passed without paths, it's "true", so use default
+  const configs =
+    typeof options.config === 'string' || Array.isArray(options.config)
+      ? [].concat(options.config)
+      : defaultConfigPaths;
+
+  const log = createToolingLog(pickLevelFromFlags(options));
+  log.pipe(process.stdout);
+
+  return {
+    configs,
+    log,
+    help: options.help,
+    bail: options.bail,
+    rest: options._,
+  };
+}
+
+function displayHelp() {
+  console.log(
+    dedent(`
+    Run Functional Tests
+
+    Usage:  node scripts/functional_tests [options]
+
+    --config      Option to pass in a config
+                  Can pass in multiple configs with
+                  --config file1 --config file2 --config file3
+    --bail        Stop the test run at the first failure
+    --help        Display this menu and exit
+
+    Log level options:
+
+    --verbose
+    --debug
+    --quiet       Log errors
+    --silent
+    `)
+  );
+}