Move all reporting tests into their own folder and add chromium tests (…

…elastic#19330)

* Move all reporting tests into their own folder to allow for multiple kibana.yml configuration tests, including chromium

* Add debugging and try to skip other tests an jenkins to speed things up

* More debug output

* more logging (remove other line which failed on jenkins)

* Remove no sandbox flag, it doesn't help

* Add fix for socket hangup and clean up tests

* fix path to logstash_functional

* Extend timeout for chromium, add verbose logging, add better comment, conditionally output curl command

* fix path... again

* Ah, other functional tests still need access to reporting page object, put it back

* fix sp err

* Add debug logs for screenshot stitching for png.bitblt error

* Fix tests that don't pass logger to screenshotStitcher

* Fix logger being undefined

* Add more debug output

* png has data, too much info to spit out

* Add comment with link to issue for extra debug messages so they can be left in since so many passes

* Dont use spawnSync with curl, use http.request instead, more support for it

* Comment out chromium tests for now to avoid flakiness

* Wait... lets at least make sure the other fix worked (the http.request instead of spawnsync and curl)

* New http.request code doesn't seem to work on jenkins, timing out after 10 secs maybe

* go back to spawnsync to see if it's an issue with the rxjs code or the http.request code

* I think I figured it out...

* Comment out tests to avoid flaky png error in the screenshot stitcher

* Use a const for OSS archive path

* use path.resolve correctly.

x-pack/plugins/reporting/server/browsers/chromium/driver/index.js

@@ -101,8 +101,9 @@ export class HeadlessChromiumDriver {
           scale: 1
         }
       });
+      this._logger.debug(`captured screenshot clip ${JSON.stringify(screenshotClip)}`);
       return data;
-    });
+    }, this._logger);
   }
 
   async _writeData(writePath, base64EncodedData) {

x-pack/plugins/reporting/server/browsers/chromium/driver/screenshot_stitcher/combine.js

@@ -16,10 +16,11 @@ const canUseFirstScreenshot = (screenshots, outputDimensions) => {
   }
 
   const firstScreenshot = screenshots[0];
-  return firstScreenshot.dimensions.width === outputDimensions.width && firstScreenshot.dimensions.height === outputDimensions.height;
+  return firstScreenshot.dimensions.width === outputDimensions.width &&
+    firstScreenshot.dimensions.height === outputDimensions.height;
 };
 
-export function $combine(screenshots, outputDimensions) {
+export function $combine(screenshots, outputDimensions, logger) {
   if (screenshots.length === 0) {
     return Observable.throw('Unable to combine 0 screenshots');
   }
@@ -42,6 +43,11 @@ export function $combine(screenshots, outputDimensions) {
   const output$ = pngs$
     .reduce(
       (output, { dimensions, png }) => {
+        // Spitting out a lot of output to help debug https://github.com/elastic/kibana/issues/19563. Once that is
+        // fixed, this should probably get pared down.
+        logger.debug(`Output dimensions is ${JSON.stringify(outputDimensions)}`);
+        logger.debug(`Input png w: ${png.width} and h: ${png.height}`);
+        logger.debug(`Creating output png with ${JSON.stringify(dimensions)}`);
         png.bitblt(output, 0, 0, dimensions.width, dimensions.height, dimensions.x, dimensions.y);
         return output;
       },

x-pack/plugins/reporting/server/browsers/chromium/driver/screenshot_stitcher/index.js

@@ -16,7 +16,7 @@ const scaleRect = (rect, scale) => {
   };
 };
 
-export async function screenshotStitcher(outputClip, zoom, max, captureScreenshotFn) {
+export async function screenshotStitcher(outputClip, zoom, max, captureScreenshotFn, logger) {
   // We have to divide the max by the zoom because we want to be limiting the resolution
   // of the output screenshots, which is implicitly multiplied by the zoom, but we don't
   // want the zoom to affect the clipping rects that we use
@@ -44,6 +44,6 @@ export async function screenshotStitcher(outputClip, zoom, max, captureScreensho
 
   return screenshotPngDimensions$
     .toArray()
-    .switchMap(screenshots => $combine(screenshots, scaleRect(outputClip, zoom)))
+    .switchMap(screenshots => $combine(screenshots, scaleRect(outputClip, zoom), logger))
     .toPromise();
-}
+}

x-pack/plugins/reporting/server/browsers/chromium/driver/screenshot_stitcher/index.test.js

@@ -10,6 +10,10 @@ import { promisify } from 'bluebird';
 
 import { screenshotStitcher } from './index';
 
+const loggerMock = {
+  debug: () => {}
+};
+
 const fsp = {
   readFile: promisify(fs.readFile)
 };
@@ -53,7 +57,7 @@ test(`single screenshot`, async () => {
 
   const fn = jest.fn();
   fn.mockReturnValueOnce(getSingleWhitePixel());
-  const data = await screenshotStitcher(clip, 1, 1, fn);
+  const data = await screenshotStitcher(clip, 1, 1, fn, loggerMock);
 
   expect(fn.mock.calls.length).toBe(1);
   expect(fn.mock.calls[0][0]).toEqual({ x: 0, y: 0, width: 1, height: 1 });
@@ -72,7 +76,7 @@ test(`single screenshot, when zoom creates partial pixel we round up`, async ()
 
   const fn = jest.fn();
   fn.mockReturnValueOnce(get2x2White());
-  const data = await screenshotStitcher(clip, 2, 1, fn);
+  const data = await screenshotStitcher(clip, 2, 1, fn, loggerMock);
 
   expect(fn.mock.calls.length).toBe(1);
   expect(fn.mock.calls[0][0]).toEqual({ x: 0, y: 0, width: 1, height: 1 });
@@ -92,7 +96,7 @@ test(`two screenshots, no zoom`, async () => {
   const fn = jest.fn();
   fn.mockReturnValueOnce(getSingleWhitePixel());
   fn.mockReturnValueOnce(getSingleBlackPixel());
-  const data = await screenshotStitcher(clip, 1, 1, fn);
+  const data = await screenshotStitcher(clip, 1, 1, fn, loggerMock);
 
   expect(fn.mock.calls.length).toBe(2);
   expect(fn.mock.calls[0][0]).toEqual({ x: 0, y: 0, width: 1, height: 1 });
@@ -116,7 +120,7 @@ test(`four screenshots, zoom`, async () => {
   fn.mockReturnValueOnce(get2x2Black());
   fn.mockReturnValueOnce(get2x2White());
 
-  const data = await screenshotStitcher(clip, 2, 1, fn);
+  const data = await screenshotStitcher(clip, 2, 1, fn, loggerMock);
 
   expect(fn.mock.calls.length).toBe(4);
   expect(fn.mock.calls[0][0]).toEqual({ x: 0, y: 0, width: 1, height: 1 });
@@ -143,7 +147,7 @@ test(`four screenshots, zoom and offset`, async () => {
   fn.mockReturnValueOnce(get2x2Black());
   fn.mockReturnValueOnce(get2x2White());
 
-  const data = await screenshotStitcher(clip, 2, 1, fn);
+  const data = await screenshotStitcher(clip, 2, 1, fn, loggerMock);
 
   expect(fn.mock.calls.length).toBe(4);
   expect(fn.mock.calls[0][0]).toEqual({ x: 1, y: 1, width: 1, height: 1 });

x-pack/plugins/reporting/server/browsers/chromium/driver_factory/args.js

@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-export const args = ({ userDataDir, bridgePort, viewport, disableSandbox, proxyConfig }) => {
+export const args = ({ userDataDir, bridgePort, viewport, disableSandbox, proxyConfig, verboseLogging }) => {
   const flags = [
     // Disable built-in Google Translate service
     '--disable-translate',
@@ -44,6 +44,11 @@ export const args = ({ userDataDir, bridgePort, viewport, disableSandbox, proxyC
     flags.push('--no-sandbox');
   }
 
+  if (verboseLogging) {
+    flags.push('--enable-logging');
+    flags.push('--v=1');
+  }
+
   if (process.platform === 'linux') {
     flags.push('--disable-setuid-sandbox');
   }

x-pack/plugins/reporting/server/browsers/chromium/driver_factory/ensure_chromium_is_listening.js

@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import http from 'http';
+
+
+// See https://github.com/elastic/kibana/issues/19351 for why this is necessary. Long story short, on certain
+// linux platforms (fwiw, we have only experienced this on jenkins agents) the first bootup of chromium takes
+// a long time doing something with fontconfig packages loading up a cache. The cdp command will timeout
+// if we don't wait for this manually. Note that this may still timeout based on the value of
+// xpack.reporting.queue.timeout. Subsequent runs should be fast because the cache will already be
+// initialized.
+/**
+ *
+ * @param {string} port
+ * @param {Object} logger
+ * @return {Promise}
+ */
+export async function ensureChromiumIsListening(port, logger) {
+  const options = {
+    port,
+    hostname: '127.0.0.1',
+    timeout: 120000,
+    path: '/json',
+  };
+
+  return new Promise((resolve, reject) => {
+    http.get(
+      options,
+      res => {
+        res.on('data', function (chunk) {
+          logger.debug(`Response from chromium: ${chunk}`);
+        });
+        res.on('end', function () {
+          logger.debug(`Chromium response complete`);
+          resolve();
+        });
+      })
+      .on('error', e => {
+        logger.error(`Ensure chromium is listening failed with error ${e.message}`);
+        reject(e);
+      });
+  });
+}

x-pack/plugins/reporting/server/browsers/chromium/driver_factory/index.js

@@ -3,7 +3,6 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-
 import fs from 'fs';
 import os from 'os';
 import path from 'path';
@@ -13,7 +12,9 @@ import Rx from 'rxjs/Rx';
 import cdp from 'chrome-remote-interface';
 import { HeadlessChromiumDriver } from '../driver';
 import { args } from './args';
+
 import { safeChildProcess } from '../../safe_child_process';
+import { ensureChromiumIsListening } from './ensure_chromium_is_listening';
 
 const compactWhitespace = (str) => {
   return str.replace(/\s+/, ' ');
@@ -35,6 +36,7 @@ export class HeadlessChromiumDriverFactory {
         userDataDir,
         bridgePort,
         viewport,
+        verboseLogging: this.logger.isVerbose,
         disableSandbox: this.browserConfig.disableSandbox,
         proxyConfig: this.browserConfig.proxy,
       });
@@ -56,7 +58,9 @@ export class HeadlessChromiumDriverFactory {
 
       const driver$ = message$
         .first(line => line.indexOf(`DevTools listening on ws://127.0.0.1:${bridgePort}`) >= 0)
-        .do(() => this.logger.debug('Connecting chrome remote inspector'))
+        .do(() => this.logger.debug('Ensure chromium is running and listening'))
+        .mergeMap(() => ensureChromiumIsListening(bridgePort, this.logger))
+        .do(() => this.logger.debug('Connecting chrome remote interface'))
         .mergeMap(() => cdp({ port: bridgePort, local: true }))
         .do(() => this.logger.debug('Initializing chromium driver'))
         .map(client => new HeadlessChromiumDriver(client, {
@@ -106,6 +110,5 @@ export class HeadlessChromiumDriverFactory {
         });
       };
     });
-
   }
 }

x-pack/plugins/reporting/server/lib/level_logger.js

@@ -8,12 +8,16 @@
 export class LevelLogger {
 
   static createForServer(server, tags) {
-    return new LevelLogger((tags, msg) => server.log(tags, msg), tags);
+    return new LevelLogger(
+      (tags, msg) => server.log(tags, msg),
+      tags,
+      server.config().get('logging.verbose'));
   }
 
-  constructor(logger, tags) {
+  constructor(logger, tags, isVerbose) {
     this._logger = logger;
     this._tags = tags;
+    this.isVerbose = isVerbose;
   }
 
   error(msg, tags = []) {
@@ -33,6 +37,6 @@ export class LevelLogger {
   }
 
   clone(tags) {
-    return new LevelLogger(this._logger, [...this._tags, ...tags]);
+    return new LevelLogger(this._logger, [...this._tags, ...tags], this.isVerbose);
   }
 }

x-pack/scripts/functional_tests.js

@@ -6,6 +6,11 @@
 
 require('@kbn/plugin-helpers').babelRegister();
 require('@kbn/test').runTestsCli([
+  // Uncomment when https://github.com/elastic/kibana/issues/19563 is resolved.
+  // require.resolve('../test/reporting/configs/chromium_api.js'),
+  // require.resolve('../test/reporting/configs/chromium_functional.js'),
+  require.resolve('../test/reporting/configs/phantom_api.js'),
+  require.resolve('../test/reporting/configs/phantom_functional.js'),
   require.resolve('../test/functional/config.js'),
   require.resolve('../test/api_integration/config.js'),
   require.resolve('../test/saml_api_integration/config.js'),

x-pack/test/api_integration/apis/index.js

@@ -9,7 +9,6 @@ export default function ({ loadTestFile }) {
     loadTestFile(require.resolve('./security'));
     loadTestFile(require.resolve('./monitoring'));
     loadTestFile(require.resolve('./xpack_main'));
-    loadTestFile(require.resolve('./reporting'));
     loadTestFile(require.resolve('./logstash'));
   });
 }

x-pack/test/api_integration/config.js

@@ -6,7 +6,6 @@
 
 import {
   SupertestWithoutAuthProvider,
-  ReportingAPIProvider,
   UsageAPIProvider,
 } from './services';
 
@@ -26,7 +25,6 @@ export default async function ({ readConfigFile }) {
       supertestWithoutAuth: SupertestWithoutAuthProvider,
       es: kibanaCommonConfig.get('services.es'),
       esArchiver: kibanaCommonConfig.get('services.esArchiver'),
-      reportingAPI: ReportingAPIProvider,
       usageAPI: UsageAPIProvider,
       kibanaServer: kibanaCommonConfig.get('services.kibanaServer'),
     },
@@ -35,7 +33,13 @@ export default async function ({ readConfigFile }) {
       reportName: 'X-Pack API Integration Tests',
     },
     env: xPackFunctionalTestsConfig.get('env'),
-    kbnTestServer: xPackFunctionalTestsConfig.get('kbnTestServer'),
+    kbnTestServer: {
+      ...xPackFunctionalTestsConfig.get('kbnTestServer'),
+      serverArgs: [
+        ...xPackFunctionalTestsConfig.get('kbnTestServer.serverArgs'),
+        '--optimize.enabled=false',
+      ],
+    },
     esTestCluster: xPackFunctionalTestsConfig.get('esTestCluster'),
   };
 }

x-pack/test/api_integration/services/index.js

@@ -5,5 +5,4 @@
  */
 
 export { SupertestWithoutAuthProvider } from './supertest_without_auth';
-export { ReportingAPIProvider } from './reporting_api';
 export { UsageAPIProvider } from './usage_api';

x-pack/test/functional/config.js

@@ -11,12 +11,12 @@ import { format as formatUrl } from 'url';
 
 import {
   SecurityPageProvider,
-  ReportingPageProvider,
   MonitoringPageProvider,
   LogstashPageProvider,
   GraphPageProvider,
   GrokDebuggerPageProvider,
   WatcherPageProvider,
+  ReportingPageProvider,
 } from './page_objects';
 
 import {
@@ -92,7 +92,6 @@ export default async function ({ readConfigFile }) {
       resolve(__dirname, './apps/watcher'),
       resolve(__dirname, './apps/dashboard_mode'),
       resolve(__dirname, './apps/security'),
-      resolve(__dirname, './apps/reporting'),
       resolve(__dirname, './apps/logstash'),
       resolve(__dirname, './apps/grok_debugger'),
     ],
@@ -136,12 +135,12 @@ export default async function ({ readConfigFile }) {
     pageObjects: {
       ...kibanaFunctionalConfig.get('pageObjects'),
       security: SecurityPageProvider,
-      reporting: ReportingPageProvider,
       monitoring: MonitoringPageProvider,
       logstash: LogstashPageProvider,
       graph: GraphPageProvider,
       grokDebugger: GrokDebuggerPageProvider,
       watcher: WatcherPageProvider,
+      reporting: ReportingPageProvider,
     },
 
     servers,

x-pack/test/functional/page_objects/index.js

@@ -5,9 +5,9 @@
  */
 
 export { SecurityPageProvider } from './security_page';
-export { ReportingPageProvider } from './reporting_page';
 export { MonitoringPageProvider } from './monitoring_page';
 export { LogstashPageProvider } from './logstash_page';
 export { GraphPageProvider } from './graph_page';
 export { GrokDebuggerPageProvider } from './grok_debugger_page';
 export { WatcherPageProvider } from './watcher_page';
+export { ReportingPageProvider } from './reporting_page';

x-pack/test/functional/page_objects/reporting_page.js

@@ -21,8 +21,8 @@ export function ReportingPageProvider({ getService, getPageObjects }) {
     async initTests() {
       log.debug('ReportingPage:initTests');
       await PageObjects.settings.navigateTo();
-      await esArchiver.loadIfNeeded('logstash_functional');
-      await esArchiver.load('reporting/historic');
+      await esArchiver.loadIfNeeded('../../functional/es_archives/logstash_functional');
+      await esArchiver.load('historic');
       await kibanaServer.uiSettings.replace({
         'dateFormat:tz': 'UTC',
         'defaultIndex': 'logstash-*'

x-pack/test/api_integration/apis/reporting/bwc_existing_indexes.js → x-pack/test/reporting/api/bwc_existing_indexes.js

@@ -27,7 +27,7 @@ export default function ({ getService }) {
     describe('existing 6_2 index', () => {
       before('load data and add index alias', async () => {
         await reportingAPI.deleteAllReportingIndexes();
-        await esArchiver.load('reporting/bwc/6_2');
+        await esArchiver.load('bwc/6_2');
 
         // The index name in the 6_2 archive.
         const ARCHIVED_REPORTING_INDEX = '.reporting-2018.03.11';