[7.x] refactor failed_tests_reporter to use TS, no octokit (#4… (elas…

…tic#47631)

* refactor failed_tests_reporter to use TS, no octokit

* update renovate config

* ensure that all kbn-test files are in ts project

* fix some type errors

* add some more tests

* [kbn-test/githubapi] cleanup and document

* collect log messages as strings instead of message objects

* ensure issue is open when updating body

* improve readability of getKibanaIssues

* expose axios helpers from dev-utils

* fix request params for fetching github issues and validate locally

* include a README for failed_tests_reporter

* improve axios error helpers

# Conflicts:
#	packages/kbn-test/src/index.ts
#	renovate.json5

Jenkinsfile

@@ -294,6 +294,6 @@ def buildXpack() {
 def runErrorReporter() {
   bash """
     source src/dev/ci_setup/setup_env.sh
-    node src/dev/failed_tests/cli
+    node scripts/report_failed_tests
   """
 }

package.json

@@ -280,7 +280,6 @@
     "@kbn/utility-types": "1.0.0",
     "@microsoft/api-documenter": "7.4.3",
     "@microsoft/api-extractor": "7.4.2",
-    "@octokit/rest": "^15.10.0",
     "@percy/agent": "^0.11.0",
     "@types/angular": "^1.6.56",
     "@types/angular-mocks": "^1.7.0",

packages/kbn-dev-utils/src/kbn_client/errors.ts → packages/kbn-dev-utils/src/axios/errors.ts

@@ -28,15 +28,9 @@ export interface AxiosResponseError<T> extends AxiosError {
 }
 
 export const isAxiosRequestError = (error: any): error is AxiosRequestError => {
-  return error && error.code === undefined && error.response === undefined;
+  return error && error.config && error.response === undefined;
 };
 
-export const isAxiosResponseError = (error: any): error is AxiosResponseError<any> => {
-  return error && error.code !== undefined && error.response !== undefined;
-};
-
-export const isConcliftOnGetError = (error: any) => {
-  return (
-    isAxiosResponseError(error) && error.config.method === 'GET' && error.response.status === 409
-  );
+export const isAxiosResponseError = <T = any>(error: any): error is AxiosResponseError<T> => {
+  return error && error.response && error.response.status !== undefined;
 };

packages/kbn-dev-utils/src/axios/index.ts

@@ -0,0 +1,20 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export * from './errors';

packages/kbn-dev-utils/src/index.ts

@@ -18,9 +18,15 @@
  */
 
 export { withProcRunner } from './proc_runner';
-export { ToolingLog, ToolingLogTextWriter, pickLevelFromFlags } from './tooling_log';
+export {
+  ToolingLog,
+  ToolingLogTextWriter,
+  pickLevelFromFlags,
+  ToolingLogCollectingWriter,
+} from './tooling_log';
 export { createAbsolutePathSerializer } from './serializers';
 export { CA_CERT_PATH, ES_KEY_PATH, ES_CERT_PATH } from './certs';
 export { run, createFailError, createFlagError, combineErrors, isFailError, Flags } from './run';
 export { REPO_ROOT } from './constants';
 export { KbnClient } from './kbn_client';
+export * from './axios';

packages/kbn-dev-utils/src/kbn_client/kbn_client_requester.ts

@@ -21,9 +21,15 @@ import Url from 'url';
 
 import Axios from 'axios';
 
-import { isAxiosRequestError, isConcliftOnGetError } from './errors';
+import { isAxiosRequestError, isAxiosResponseError } from '../axios';
 import { ToolingLog } from '../tooling_log';
 
+const isConcliftOnGetError = (error: any) => {
+  return (
+    isAxiosResponseError(error) && error.config.method === 'GET' && error.response.status === 409
+  );
+};
+
 export const uriencode = (
   strings: TemplateStringsArray,
   ...values: Array<string | number | boolean>

packages/kbn-dev-utils/src/tooling_log/index.ts

@@ -20,3 +20,4 @@
 export { ToolingLog } from './tooling_log';
 export { ToolingLogTextWriter, ToolingLogTextWriterConfig } from './tooling_log_text_writer';
 export { pickLevelFromFlags, LogLevel } from './log_levels';
+export { ToolingLogCollectingWriter } from './tooling_log_collecting_writer';

src/dev/github_utils/index.js → packages/kbn-dev-utils/src/tooling_log/tooling_log_collecting_writer.ts

@@ -17,27 +17,20 @@
  * under the License.
  */
 
-import Octokit from '@octokit/rest';
-import { markdownMetadata } from './metadata';
+import { ToolingLogTextWriter } from './tooling_log_text_writer';
 
-export { markdownMetadata };
+export class ToolingLogCollectingWriter extends ToolingLogTextWriter {
+  messages: string[] = [];
 
-export function getGithubClient() {
-  const client = new Octokit();
-  client.authenticate({
-    type: 'token',
-    token: process.env.GITHUB_TOKEN
-  });
-
-  return client;
-}
-
-export async function paginate(client, promise) {
-  let response = await promise;
-  let { data } = response;
-  while (client.hasNextPage(response)) {
-    response = await client.getNextPage(response);
-    data = data.concat(response.data);
+  constructor() {
+    super({
+      level: 'verbose',
+      writeTo: {
+        write: msg => {
+          // trim trailing new line
+          this.messages.push(msg.slice(0, -1));
+        },
+      },
+    });
   }
-  return data;
 }

packages/kbn-dev-utils/tsconfig.json

@@ -6,5 +6,5 @@
   },
   "include": [
     "src/**/*"
-  ],
+  ]
 }

packages/kbn-test/package.json

@@ -12,17 +12,23 @@
   "devDependencies": {
     "@babel/cli": "^7.5.5",
     "@kbn/babel-preset": "1.0.0",
-    "@kbn/dev-utils": "1.0.0"
+    "@kbn/dev-utils": "1.0.0",
+    "@types/parse-link-header": "^1.0.0",
+    "@types/strip-ansi": "^5.2.1",
+    "@types/xml2js": "^0.4.5"
   },
   "dependencies": {
     "chalk": "^2.4.2",
     "dedent": "^0.7.0",
     "del": "^4.1.1",
     "getopts": "^2.2.4",
     "glob": "^7.1.2",
+    "parse-link-header": "^1.0.1",
+    "strip-ansi": "^5.2.0",
     "rxjs": "^6.2.1",
     "tar-fs": "^1.16.3",
     "tmp": "^0.1.0",
+    "xml2js": "^0.4.22",
     "zlib": "^1.0.5"
   }
 }

packages/kbn-test/src/failed_tests_reporter/README.md

@@ -0,0 +1,21 @@
+# failed tests reporter
+
+A little CLI that runs in CI to find the failed tests in the JUnit reports, then create/update github issues for each failure.
+
+## Test this script locally
+
+To fetch some JUnit reports from a recent build on CI, visit its `Google Cloud Storage Upload Report` and execute the following in the JS Console:
+
+```js
+copy(`wget "${Array.from($$('a[href$=".xml"]')).filter(a => a.innerText === 'Download').map(a => a.href.replace('https://storage.cloud.google.com/', 'https://storage.googleapis.com/')).join('" "')}"`)
+```
+
+This copies a script to download the reporets, which can be executed in the `test/junit` directory.
+
+Next, run the CLI in `--dry-run` mode so that it doesn't actually communicate with Github.
+
+```sh
+node scripts/report_failed_tests.js --verbose --dry-run --build-url foo
+```
+
+If you specify the `GITHUB_TOKEN` environment variable then `--dry-run` will execute read operations but still won't execute write operations.

packages/kbn-test/src/failed_tests_reporter/get_failures.test.ts

@@ -0,0 +1,91 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { ToolingLog } from '@kbn/dev-utils';
+
+import { getFailures } from './get_failures';
+
+const log = new ToolingLog();
+
+it('discovers failures in ftr report', async () => {
+  const failures = await getFailures(log, require.resolve('./__fixtures__/ftr_report.xml'));
+  expect(failures).toMatchInlineSnapshot(`
+    Array [
+      Object {
+        "classname": "Chrome X-Pack UI Functional Tests.x-pack/test/functional/apps/maps/sample_data·js",
+        "failure": "
+            Error: retry.try timeout: TimeoutError: Waiting for element to be located By(css selector, [data-test-subj~=\\"layerTocActionsPanelToggleButtonRoad_Map_-_Bright\\"])
+    Wait timed out after 10055ms
+        at /var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/node_modules/selenium-webdriver/lib/webdriver.js:834:17
+        at process._tickCallback (internal/process/next_tick.js:68:7)
+        at lastError (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/test/common/services/retry/retry_for_success.ts:28:9)
+        at onFailure (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/test/common/services/retry/retry_for_success.ts:68:13)
+          ",
+        "name": "maps app  maps loaded from sample data ecommerce \\"before all\\" hook",
+        "time": "154.378",
+      },
+    ]
+  `);
+});
+
+it('discovers failures in jest report', async () => {
+  const failures = await getFailures(log, require.resolve('./__fixtures__/jest_report.xml'));
+  expect(failures).toMatchInlineSnapshot(`
+    Array [
+      Object {
+        "classname": "X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp",
+        "failure": "
+            TypeError: Cannot read property '0' of undefined
+        at Object.<anonymous>.test (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-intake/node/immutable/kibana/x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts:166:10)
+          ",
+        "name": "launcher can reconnect if process died",
+        "time": "7.060",
+      },
+    ]
+  `);
+});
+
+it('discovers failures in karma report', async () => {
+  const failures = await getFailures(log, require.resolve('./__fixtures__/karma_report.xml'));
+  expect(failures).toMatchInlineSnapshot(`
+    Array [
+      Object {
+        "classname": "Browser Unit Tests.CoordinateMapsVisualizationTest",
+        "failure": "Error: expected 7069 to be below 64
+        at Assertion.__kbnBundles__.tests../packages/kbn-expect/expect.js.Assertion.assert (http://localhost:5610/bundles/tests.bundle.js?shards=4&shard_num=1:13671:11)
+        at Assertion.__kbnBundles__.tests../packages/kbn-expect/expect.js.Assertion.lessThan.Assertion.below (http://localhost:5610/bundles/tests.bundle.js?shards=4&shard_num=1:13891:8)
+        at Function.lessThan (http://localhost:5610/bundles/tests.bundle.js?shards=4&shard_num=1:14078:15)
+        at _callee3$ (http://localhost:5610/bundles/tests.bundle.js?shards=4&shard_num=1:158985:60)
+        at tryCatch (webpack://%5Bname%5D/./node_modules/regenerator-runtime/runtime.js?:62:40)
+        at Generator.invoke [as _invoke] (webpack://%5Bname%5D/./node_modules/regenerator-runtime/runtime.js?:288:22)
+        at Generator.prototype.<computed> [as next] (webpack://%5Bname%5D/./node_modules/regenerator-runtime/runtime.js?:114:21)
+        at asyncGeneratorStep (http://localhost:5610/bundles/tests.bundle.js?shards=4&shard_num=1:158772:103)
+        at _next (http://localhost:5610/bundles/tests.bundle.js?shards=4&shard_num=1:158774:194)
+    ",
+        "name": "CoordinateMapsVisualizationTest CoordinateMapsVisualization - basics should initialize OK",
+        "time": "0.265",
+      },
+    ]
+  `);
+});
+
+it('discovers failures in mocha report', async () => {
+  const failures = await getFailures(log, require.resolve('./__fixtures__/mocha_report.xml'));
+  expect(failures).toMatchInlineSnapshot(`Array []`);
+});