Upgrade screenshotting Chromium and Puppeteer to v15

package.json

@@ -555,7 +555,7 @@
     "prop-types": "^15.8.1",
     "proxy-from-env": "1.0.0",
     "puid": "1.0.7",
-    "puppeteer": "^10.2.0",
+    "puppeteer": "15.5.0",
     "query-string": "^6.13.2",
     "rbush": "^3.0.1",
     "re-resizable": "^6.1.1",

src/dev/chromium_version.ts

@@ -25,6 +25,9 @@ const forkCompatibilityMap: Record<string, PuppeteerRelease> = {
   '5.4.1-patch.1': '5.4.1',
 };
 
+/**
+ * Get the currently installed version of puppeteer
+ */
 async function getPuppeteerRelease(log: ToolingLog): Promise<PuppeteerRelease> {
   // open node_modules/puppeteer/package.json
   const puppeteerPackageJson = JSON.parse(
@@ -42,6 +45,9 @@ async function getPuppeteerRelease(log: ToolingLog): Promise<PuppeteerRelease> {
   return puppeteerRelease;
 }
 
+/**
+ * Get the chromium revision for a puppeteer version
+ */
 async function getChromiumRevision(
   kibanaPuppeteerVersion: PuppeteerRelease,
   log: ToolingLog
@@ -80,6 +86,9 @@ async function getChromiumRevision(
   return revision;
 }
 
+/**
+ * Get the git commit tied to the chromium revision
+ */
 async function getChromiumCommit(
   revision: ChromiumRevision,
   log: ToolingLog
@@ -103,10 +112,29 @@ async function getChromiumCommit(
     throw new Error(`Could not find a Chromium commit! Check ${url} in a browser.`);
   }
 
-  log.info(`Found Chromium commit ${commit} from revision ${revision}.`);
+  log.debug(`Found Chromium commit ${commit} from revision ${revision}.`);
   return commit;
 }
 
+/**
+ * Get the Chrome release from git commit
+ */
+async function getChromiumRelease(commit: ChromiumCommit, log: ToolingLog) {
+  const url =
+    `https://storage.googleapis.com/chromium-find-releases-static/` +
+    `${commit.substring(0, 3)}.html#${commit}`;
+
+  log.info(`Fetching ${url}`);
+  const response = await fetch(url);
+  const pageText = await response.text();
+  const commitIndex = pageText.indexOf(commit);
+  if (commitIndex > 0) {
+    const substr = pageText.substring(commitIndex, commitIndex + 100);
+    const releaseVersion = substr.replace(/^.*"([\d\.]+)".*$/, '$1');
+    log.info(`Release version: ${releaseVersion}` + `\nCommit:          ${commit}`);
+  }
+}
+
 run(
   async ({
     log,
@@ -123,7 +151,8 @@ run(
       }
 
       const chromiumRevision = await getChromiumRevision(puppeteerVersion, log);
-      await getChromiumCommit(chromiumRevision, log);
+      const commit = await getChromiumCommit(chromiumRevision, log);
+      await getChromiumRelease(commit, log);
     } catch (err) {
       log.error(err);
     }
@@ -134,8 +163,6 @@ run(
 
       -  node scripts/chromium_version 5.5.0  {dim # gets the Chromium commit for Puppeteer v5.5.0}
       -  node scripts/chromium_version       {dim  # gets the Chromium commit for the Kibana dependency version of Puppeteer}
-
-      You can use https://omahaproxy.appspot.com/ to look up the Chromium release that first shipped with that commit.
     `,
   }
 );

x-pack/build_chromium/README.md

@@ -24,6 +24,11 @@ location](https://commondatastorage.googleapis.com/chromium-browser-snapshots/in
 
 ## Build Script Usage
 
+The system OS requires a few setup steps:
+1. Required packages: `bzip2`, `git`, `lsb_release`, `python3`
+2. The `python` command needs to launch Python 3.
+3. Recommended: `tmux`, as your ssh session may get interrupted
+
 These commands show how to set up an environment to build:
 ```sh
 # Allow our scripts to use depot_tools commands
@@ -36,7 +41,7 @@ mkdir ~/chromium && cd ~/chromium
 gsutil cp -r gs://headless_shell_staging/build_chromium .
 
 # Install the OS packages, configure the environment, download the chromium source (25GB)
-python ./build_chromium/init.py [arch_name]
+python ./build_chromium/init.py
 
 # Run the build script with the path to the chromium src directory, the git commit hash
 python ./build_chromium/build.py 70f5d88ea95298a18a85c33c98ea00e02358ad75 x64
@@ -67,7 +72,7 @@ A good how-to on building Chromium from source is
 We have an `linux/args.gn` file that is automatically copied to the build target directory.
 
 To get a list of the build arguments that are enabled, install `depot_tools` and run
-`gn args out/headless --list`. It prints out all of the flags and their
+`gn args out/headless --list` from the `chromium/src` directory. It prints out all of the flags and their
 settings, including the defaults. Some build flags are documented
 [here](https://www.chromium.org/developers/gn-build-configuration).
 
@@ -86,10 +91,6 @@ are created in x64 using cross-compiling. CentOS is not supported for building C
    - 8 CPU
    - 30GB memory
    - 80GB free space on disk (Try `ncdu /home` to see where space is used.)
-   - git
-   - python2 (`python` must link to `python2`)
-   - lsb_release
-   - tmux is recommended in case your ssh session is interrupted
    - "Cloud API access scopes": must have **read / write** scope for the Storage API
 4. Install [Google Cloud SDK](https://cloud.google.com/sdk) locally to ssh into the GCP instance
 

x-pack/build_chromium/build.py

@@ -66,12 +66,8 @@
 os.environ['PATH'] = full_path
 
 # configure environment: build dependencies
-if platform.system() == 'Linux':
-  if arch_name:
-    print('Running sysroot install script...')
-    runcmd(src_path + '/build/linux/sysroot_scripts/install-sysroot.py --arch=' + arch_name)
-  print('Running install-build-deps...')
-  runcmd(src_path + '/build/install-build-deps.sh')
+print('Running sysroot install script...')
+runcmd(src_path + '/build/linux/sysroot_scripts/install-sysroot.py --arch=' + arch_name)
 
 print('Updating all modules')
 runcmd('gclient sync -D')
@@ -95,14 +91,14 @@
 
 # Optimize the output on Linux x64 by stripping inessentials from the binary
 # ARM must be cross-compiled from Linux and can not read the ARM binary in order to strip
-if platform.system() != 'Windows' and arch_name != 'arm64':
+if arch_name != 'arm64':
   print('Optimizing headless_shell')
   shutil.move('out/headless/headless_shell', 'out/headless/headless_shell_raw')
   runcmd('strip -o out/headless/headless_shell out/headless/headless_shell_raw')
 
 # Create the zip and generate the md5 hash using filenames like:
 # chromium-4747cc2-linux_x64.zip
-base_filename = 'out/headless/chromium-' + base_version + '-' + platform.system().lower() + '_' + arch_name
+base_filename = 'out/headless/chromium-' + base_version + '-locales-' + platform.system().lower() + '_' + arch_name
 zip_filename = base_filename + '.zip'
 md5_filename = base_filename + '.md5'
 
@@ -112,9 +108,14 @@
 path_prefix = 'headless_shell-' + platform.system().lower() + '_' + arch_name
 
 # Add dependencies that must be bundled with the Chromium executable.
+# Refer to https://chromium.googlesource.com/chromium/src/+/[commit_hash]/chrome/installer/linux/common/installer.include#220
 archive.write('out/headless/headless_shell', path.join(path_prefix, 'headless_shell'))
 archive.write('out/headless/libEGL.so', path.join(path_prefix, 'libEGL.so'))
 archive.write('out/headless/libGLESv2.so', path.join(path_prefix, 'libGLESv2.so'))
+archive.write('out/headless/libvulkan.so.1', path.join(path_prefix, 'libvulkan.so.1'))
+archive.write('out/headless/libvk_swiftshader.so', path.join(path_prefix, 'libvk_swiftshader.so'))
+# Install the ICD json file to point ANGLE to libvk_swiftshader.so
+archive.write('out/headless/vk_swiftshader_icd.json', path.join(path_prefix, 'vk_swiftshader_icd.json'))
 archive.write(en_us_locale_file_path, path.join(path_prefix, 'locales', en_us_locale_pak_file_name))
 
 archive.close()

x-pack/build_chromium/init.py

@@ -6,13 +6,9 @@
 # once per environment.
 
 # Set to "arm" to build for ARM
-arch_name = sys.argv[1] if len(sys.argv) >= 2 else 'undefined'
 build_path = path.abspath(os.curdir)
 src_path = path.abspath(path.join(build_path, 'chromium', 'src'))
 
-if arch_name != 'x64' and arch_name != 'arm64':
-  raise Exception('Unexpected architecture: ' + arch_name + '. `x64` and `arm64` are supported.')
-
 # Configure git
 print('Configuring git globals...')
 runcmd('git config --global core.autocrlf false')
@@ -29,8 +25,8 @@
   print('Updating depot_tools...')
   original_dir = os.curdir
   os.chdir(path.join(build_path, 'depot_tools'))
-  runcmd('git checkout master')
-  runcmd('git pull origin master')
+  runcmd('git checkout main')
+  runcmd('git pull origin main')
   os.chdir(original_dir)
 
 # Fetch the Chromium source code

x-pack/build_chromium/linux/args.gn

@@ -1,9 +1,16 @@
+# Build args reference: https://www.chromium.org/developers/gn-build-configuration/
+
 import("//build/args/headless.gn")
+
 is_debug = false
+
 symbol_level = 0
+
 is_component_build = false
+
 enable_nacl = false
+
 # Please, consult @elastic/kibana-security before changing/removing this option.
 use_kerberos = false
 
-# target_cpu is appended before build: "x64" or "arm64"
+# target_cpu is added at build time

x-pack/plugins/screenshotting/server/browsers/chromium/driver.ts

@@ -12,7 +12,7 @@ import {
 } from '@kbn/screenshot-mode-plugin/server';
 import { truncate } from 'lodash';
 import open from 'opn';
-import puppeteer, { ElementHandle, EvaluateFn, Page, SerializableOrJSHandle } from 'puppeteer';
+import puppeteer, { ElementHandle, Page, EvaluateFunc } from 'puppeteer';
 import { Subject } from 'rxjs';
 import { parse as parseUrl } from 'url';
 import { getDisallowedOutgoingUrlError } from '.';
@@ -23,6 +23,16 @@ import { allowRequest } from '../network_policy';
 import { stripUnsafeHeaders } from './strip_unsafe_headers';
 import { getFooterTemplate, getHeaderTemplate } from './templates';
 
+declare module 'puppeteer' {
+  interface Page {
+    _client(): CDPSession;
+  }
+
+  interface Target {
+    _targetId: string;
+  }
+}
+
 export type Context = Record<string, unknown>;
 
 export interface ElementPosition {
@@ -51,8 +61,8 @@ interface WaitForSelectorOpts {
 }
 
 interface EvaluateOpts {
-  fn: EvaluateFn;
-  args: SerializableOrJSHandle[];
+  fn: EvaluateFunc<any>;
+  args: unknown[];
 }
 
 interface EvaluateMetaOpts {
@@ -312,8 +322,8 @@ export class HeadlessChromiumDriver {
     args,
     timeout,
   }: {
-    fn: EvaluateFn;
-    args: SerializableOrJSHandle[];
+    fn: EvaluateFunc<any>;
+    args: unknown[];
     timeout: number;
   }): Promise<void> {
     await this.page.waitForFunction(fn, { timeout, polling: WAIT_FOR_DELAY_MS }, ...args);
@@ -345,8 +355,7 @@ export class HeadlessChromiumDriver {
       return;
     }
 
-    // FIXME: retrieve the client in open() and  pass in the client
-    const client = this.page.client();
+    const client = this.page._client();
 
     // We have to reach into the Chrome Devtools Protocol to apply headers as using
     // puppeteer's API will cause map tile requests to hang indefinitely:
@@ -437,12 +446,12 @@ export class HeadlessChromiumDriver {
     // In order to get the inspector running, we have to know the page's internal ID (again, private)
     // in order to construct the final debugging URL.
 
+    const client = this.page._client();
     const target = this.page.target();
-    const client = await target.createCDPSession();
+    const targetId = target._targetId;
 
     await client.send('Debugger.enable');
     await client.send('Debugger.pause');
-    const targetId = target._targetId;
     const wsEndpoint = this.page.browser().wsEndpoint();
     const { port } = parseUrl(wsEndpoint);
 

x-pack/plugins/screenshotting/server/browsers/chromium/driver_factory/index.ts

@@ -293,16 +293,14 @@ export class HeadlessChromiumDriverFactory {
    */
   private async getErrorMessage(message: ConsoleMessage): Promise<undefined | string> {
     for (const arg of message.args()) {
-      const errorMessage = await arg
-        .executionContext()
-        .evaluate<undefined | string>((_arg: unknown) => {
-          /* !! We are now in the browser context !! */
-          if (_arg instanceof Error) {
-            return _arg.message;
-          }
-          return undefined;
-          /* !! End of browser context !! */
-        }, arg);
+      const errorMessage = await arg.executionContext().evaluate((_arg: unknown) => {
+        /* !! We are now in the browser context !! */
+        if (_arg instanceof Error) {
+          return _arg.message;
+        }
+        return undefined;
+        /* !! End of browser context !! */
+      }, arg);
       if (errorMessage) {
         return errorMessage;
       }