[FTR] Add test suite metrics tracking/output

packages/kbn-test/src/functional_test_runner/functional_test_runner.ts

@@ -30,11 +30,13 @@ import {
   setupMocha,
   runTests,
   Config,
+  SuiteTracker,
 } from './lib';
 
 export class FunctionalTestRunner {
   public readonly lifecycle = new Lifecycle();
   public readonly failureMetadata = new FailureMetadata(this.lifecycle);
+  public readonly suiteTracker = new SuiteTracker(this.lifecycle);
   private closed = false;
 
   constructor(

packages/kbn-test/src/functional_test_runner/lib/config/config.ts

@@ -35,6 +35,7 @@ interface Options {
 
 export class Config {
   private [$values]: Record<string, any>;
+  public path: string;
 
   constructor(options: Options) {
     const { settings = {}, primary = false, path = null } = options || {};
@@ -43,6 +44,8 @@ export class Config {
       throw new TypeError('path is a required option');
     }
 
+    this.path = path;
+
     const { error, value } = schema.validate(settings, {
       abortEarly: false,
       context: {

packages/kbn-test/src/functional_test_runner/lib/index.ts

@@ -23,3 +23,4 @@ export { readConfigFile, Config } from './config';
 export { readProviderSpec, ProviderCollection, Provider } from './providers';
 export { runTests, setupMocha } from './mocha';
 export { FailureMetadata } from './failure_metadata';
+export { SuiteTracker } from './suite_tracker';

packages/kbn-test/src/functional_test_runner/lib/mocha/decorate_mocha_ui.js

@@ -22,7 +22,7 @@ import { createAssignmentProxy } from './assignment_proxy';
 import { wrapFunction } from './wrap_function';
 import { wrapRunnableArgs } from './wrap_runnable_args';
 
-export function decorateMochaUi(lifecycle, context) {
+export function decorateMochaUi(lifecycle, context, config) {
   // incremented at the start of each suite, decremented after
   // so that in each non-suite call we can know if we are within
   // a suite, or that when a suite is defined it is within a suite
@@ -70,6 +70,8 @@ export function decorateMochaUi(lifecycle, context) {
           this.tags(relativeFilePath);
           this.suiteTag = relativeFilePath; // The tag that uniquely targets this suite/file
 
+          this.ftrConfig = config;
+
           provider.call(this);
 
           after(async () => {

packages/kbn-test/src/functional_test_runner/lib/mocha/load_test_files.js

@@ -31,7 +31,15 @@ import { decorateMochaUi } from './decorate_mocha_ui';
  *  @param  {String} path
  *  @return {undefined} - mutates mocha, no return value
  */
-export const loadTestFiles = ({ mocha, log, lifecycle, providers, paths, updateBaselines }) => {
+export const loadTestFiles = ({
+  config,
+  mocha,
+  log,
+  lifecycle,
+  providers,
+  paths,
+  updateBaselines,
+}) => {
   const innerLoadTestFile = path => {
     if (typeof path !== 'string' || !isAbsolute(path)) {
       throw new TypeError('loadTestFile() only accepts absolute paths');
@@ -55,7 +63,7 @@ export const loadTestFiles = ({ mocha, log, lifecycle, providers, paths, updateB
     loadTracer(provider, `testProvider[${path}]`, () => {
       // mocha.suite hocus-pocus comes from: https://git.io/vDnXO
 
-      const context = decorateMochaUi(lifecycle, global);
+      const context = decorateMochaUi(lifecycle, global, config);
       mocha.suite.emit('pre-require', context, path, mocha);
 
       const returnVal = provider({

packages/kbn-test/src/functional_test_runner/lib/mocha/setup_mocha.js

@@ -51,6 +51,7 @@ export async function setupMocha(lifecycle, log, config, providers) {
     log,
     lifecycle,
     providers,
+    config,
     paths: config.get('testFiles'),
     updateBaselines: config.get('updateBaselines'),
   });

packages/kbn-test/src/functional_test_runner/lib/suite_tracker.test.ts

@@ -0,0 +1,197 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import fs from 'fs';
+import { join, resolve } from 'path';
+
+jest.mock('fs');
+jest.mock('@kbn/dev-utils', () => {
+  return { REPO_ROOT: '/dev/null/root' };
+});
+
+import { REPO_ROOT } from '@kbn/dev-utils';
+import { Lifecycle } from './lifecycle';
+import { SuiteTracker } from './suite_tracker';
+
+const DEFAULT_TEST_METADATA_PATH = join(REPO_ROOT, 'target', 'test_metadata.json');
+const MOCK_CONFIG_PATH = join('test', 'config.js');
+const MOCK_TEST_PATH = join('test', 'apps', 'test.js');
+const ENVS_TO_RESET = ['TEST_METADATA_PATH'];
+
+describe('SuiteTracker', () => {
+  const originalEnvs: Record<string, string> = {};
+
+  beforeEach(() => {
+    for (const env of ENVS_TO_RESET) {
+      if (env in process.env) {
+        originalEnvs[env] = process.env[env] || '';
+        delete process.env[env];
+      }
+    }
+  });
+
+  afterEach(() => {
+    for (const env of ENVS_TO_RESET) {
+      delete process.env[env];
+    }
+
+    for (const env of Object.keys(originalEnvs)) {
+      process.env[env] = originalEnvs[env];
+    }
+
+    jest.resetAllMocks();
+  });
+
+  let MOCKS: Record<string, object>;
+
+  const createMock = (overrides = {}) => {
+    return {
+      ftrConfig: {
+        path: resolve(REPO_ROOT, MOCK_CONFIG_PATH),
+      },
+      file: resolve(REPO_ROOT, MOCK_TEST_PATH),
+      title: 'A Test',
+      suiteTag: MOCK_TEST_PATH,
+      ...overrides,
+    };
+  };
+
+  const runLifecycleWithMocks = async (mocks: object[], fn: (objs: any) => any = () => {}) => {
+    const lifecycle = new Lifecycle();
+    const suiteTracker = new SuiteTracker(lifecycle);
+
+    const ret = { lifecycle, suiteTracker };
+
+    for (const mock of mocks) {
+      await lifecycle.beforeTestSuite.trigger(mock);
+    }
+
+    if (fn) {
+      fn(ret);
+    }
+
+    for (const mock of mocks.reverse()) {
+      await lifecycle.afterTestSuite.trigger(mock);
+    }
+
+    return ret;
+  };
+
+  beforeEach(() => {
+    MOCKS = {
+      WITH_TESTS: createMock({ tests: [{}] }), // i.e. a describe with tests in it
+      WITHOUT_TESTS: createMock(), // i.e. a describe with only other describes in it
+    };
+  });
+
+  it('collects metadata for a single suite with multiple describe()s', async () => {
+    const { suiteTracker } = await runLifecycleWithMocks([MOCKS.WITHOUT_TESTS, MOCKS.WITH_TESTS]);
+
+    const suites = suiteTracker.getAllFinishedSuites();
+    expect(suites.length).toBe(1);
+    const suite = suites[0];
+
+    expect(suite).toMatchObject({
+      config: MOCK_CONFIG_PATH,
+      file: MOCK_TEST_PATH,
+      tag: MOCK_TEST_PATH,
+      leafSuite: true,
+      success: true,
+    });
+  });
+
+  it('writes metadata to a file when cleanup is triggered', async () => {
+    const { lifecycle, suiteTracker } = await runLifecycleWithMocks([MOCKS.WITH_TESTS]);
+    await lifecycle.cleanup.trigger();
+
+    const suites = suiteTracker.getAllFinishedSuites();
+
+    const call = (fs.writeFileSync as jest.Mock).mock.calls[0];
+    expect(call[0]).toEqual(DEFAULT_TEST_METADATA_PATH);
+    expect(call[1]).toEqual(JSON.stringify(suites, null, 2));
+  });
+
+  it('respects TEST_METADATA_PATH env var for metadata target override', async () => {
+    process.env.TEST_METADATA_PATH = '/dev/null/fake-test-path';
+    const { lifecycle } = await runLifecycleWithMocks([MOCKS.WITH_TESTS]);
+    await lifecycle.cleanup.trigger();
+
+    expect((fs.writeFileSync as jest.Mock).mock.calls[0][0]).toEqual(
+      process.env.TEST_METADATA_PATH
+    );
+  });
+
+  it('identifies suites with tests as leaf suites', async () => {
+    const root = createMock({ title: 'root', file: join(REPO_ROOT, 'root.js') });
+    const parent = createMock({ parent: root });
+    const withTests = createMock({ parent, tests: [{}] });
+
+    const { suiteTracker } = await runLifecycleWithMocks([root, parent, withTests]);
+    const suites = suiteTracker.getAllFinishedSuites();
+
+    const finishedRoot = suites.find(s => s.title === 'root');
+    const finishedWithTests = suites.find(s => s.title !== 'root');
+
+    expect(finishedRoot).toBeTruthy();
+    expect(finishedRoot?.leafSuite).toBeFalsy();
+    expect(finishedWithTests?.leafSuite).toBe(true);
+  });
+
+  describe('with a failing suite', () => {
+    let root: any;
+    let parent: any;
+    let failed: any;
+
+    beforeEach(() => {
+      root = createMock({ file: join(REPO_ROOT, 'root.js') });
+      parent = createMock({ parent: root });
+      failed = createMock({ parent, tests: [{}] });
+    });
+
+    it('marks parent suites as not successful when a test fails', async () => {
+      const { suiteTracker } = await runLifecycleWithMocks(
+        [root, parent, failed],
+        async ({ lifecycle }) => {
+          await lifecycle.testFailure.trigger(Error('test'), { parent: failed });
+        }
+      );
+
+      const suites = suiteTracker.getAllFinishedSuites();
+      expect(suites.length).toBe(2);
+      for (const suite of suites) {
+        expect(suite.success).toBeFalsy();
+      }
+    });
+
+    it('marks parent suites as not successful when a test hook fails', async () => {
+      const { suiteTracker } = await runLifecycleWithMocks(
+        [root, parent, failed],
+        async ({ lifecycle }) => {
+          await lifecycle.testHookFailure.trigger(Error('test'), { parent: failed });
+        }
+      );
+
+      const suites = suiteTracker.getAllFinishedSuites();
+      expect(suites.length).toBe(2);
+      for (const suite of suites) {
+        expect(suite.success).toBeFalsy();
+      }
+    });
+  });
+});