From fff74900fb90c0a06cce85ee3ab8766fe22af36f Mon Sep 17 00:00:00 2001 From: Cacie Prins Date: Thu, 10 Aug 2023 13:49:13 -0400 Subject: [PATCH] feat: protocol terminal output (#27402) * uploading initiation & legit no upload messaging * report errors * makes upload reporting uniform across artifact types * retrieve capture meta from correct cloud endpoint * moves skipped artifact insertion to more reasonable point * rm unneccessary paren around Promise.all on upload * improve zipped filesize determination for protocol uploads, clean up get db signature in protocol manager * changelog * add url onto protocol failure report * rm unused err.cause consts * ensure artifact PUT server mock resolves in system tests * extract terminal output logic from upload flow, mask filepaths and filesizes in system tests * update return shape for postRun when test replay is enabled * pad beginning of liine for upload reports * update upload messaging snapshots for record spec * improve trailing whitespace for artifact upload terminal output * since we are now waiting for artifact report, must include urls in test assertion * respect quiet mode * address correct index of reqs for api reordering specs test * updates snapshots & adds missing artifacts PUT req for api skips specs not in parallel * updates tests for skipping specs in parallel * update snapshot for no upload when video disabled test * update snapshot for update instance 500 * updates snapshot for postInstanceTests 500 * update instance stdout 500 snapshot update * improve message format when error on uploading, update snapshots * snapshot for api retry on error * update snapshot for sendPreflight system tests * update snapshots for api interaction private tests limit warning * update snapshots when over tests limit * updates snapshots for protocol retrieval, bypass stub verification in test mode * set gzip header on stubbed capture code server endpoint so client can verify * accept BROWSER env var to reduce screenshot dimension difference from local to ci * adds artifacts PUT to manifest for stdout 500 system test * fixes snapshot browser workaround; updates url manifest for record system tests * fix whitespace between filesize and path in upload manifest * manually update snapshots for video_compression * adds system tests for disabled message from server, file size exceeded * additional tests, bugfixes * add logging to determine source of ci error with db files * ensure protocol tmp dir is created before tests * rm test env force return of failed sig check on protocol runtime * code review comments * fix priority nums on artifact readout * rm commented code from protocol stub --- cli/CHANGELOG.md | 5 + packages/server/lib/cloud/api.ts | 24 +- packages/server/lib/cloud/protocol.ts | 96 ++- packages/server/lib/modes/record.js | 359 ++++++--- packages/server/lib/util/print-run.ts | 79 ++ packages/server/package.json | 1 + packages/types/src/protocol.ts | 8 +- system-tests/README.md | 2 + system-tests/__snapshots__/record_spec.js | 705 ++++++++++++++++-- .../__snapshots__/video_compression_spec.js | 20 +- system-tests/lib/protocolStub.ts | 19 - system-tests/lib/protocolStubResponse.ts | 38 +- .../lib/protocolStubWithRuntimeErrors.ts | 39 + system-tests/lib/serverStub.ts | 109 ++- system-tests/lib/system-tests.ts | 10 +- system-tests/test/record_spec.js | 209 +++++- 16 files changed, 1453 insertions(+), 270 deletions(-) create mode 100644 system-tests/lib/protocolStubWithRuntimeErrors.ts diff --git a/cli/CHANGELOG.md b/cli/CHANGELOG.md index 227b4125b66a..bdb757a47cdd 100644 --- a/cli/CHANGELOG.md +++ b/cli/CHANGELOG.md @@ -11,6 +11,11 @@ _Released 08/1/2023 (PENDING)_ - The [`videoUploadOnPasses`](https://docs.cypress.io/guides/references/configuration#Videos) configuration option has been removed. Please see our [screenshots & videos guide](https://docs.cypress.io/guides/guides/screenshots-and-videos#Delete-videos-for-specs-without-failing-or-retried-tests) on how to accomplish similar functionality. Addresses [#26899](https://github.com/cypress-io/cypress/issues/26899). - The deprecated configuration option, `nodeVersion` has been removed. Addresses [#27016](https://github.com/cypress-io/cypress/issues/27016). - The properties and values returned by the [Module API](https://docs.cypress.io/guides/guides/module-api) and included in the arguments of handlers for the [`after:run`](https://docs.cypress.io/api/plugins/after-run-api) and [`after:spec`](https://docs.cypress.io/api/plugins/after-spec-api) have been changed to be more consistent. Addresses [#23805](https://github.com/cypress-io/cypress/issues/23805). + +**Features:** + +- Consolidates and improves terminal output when uploading test artifacts to Cypress Cloud. Addressed in [#27402](https://github.com/cypress-io/cypress/pull/27402) + ## 12.17.4 _Released 08/15/2023 (PENDING)_ diff --git a/packages/server/lib/cloud/api.ts b/packages/server/lib/cloud/api.ts index f6ca1b1c543b..27f046b9e62b 100644 --- a/packages/server/lib/cloud/api.ts +++ b/packages/server/lib/cloud/api.ts @@ -266,6 +266,10 @@ type CreateRunResponse = { name: string })[] captureProtocolUrl?: string | undefined + capture?: { + url?: string + tags: string[] | null + } | undefined } type UpdateInstanceArtifactsOptions = { @@ -382,9 +386,12 @@ module.exports = { .then(async (result: CreateRunResponse) => { let protocolManager = new ProtocolManager() + const captureProtocolUrl = result.capture?.url || result.captureProtocolUrl + + debugProtocol({ captureProtocolUrl }) try { - if (result.captureProtocolUrl || process.env.CYPRESS_LOCAL_PROTOCOL_PATH) { - const script = await this.getCaptureProtocolScript(result.captureProtocolUrl || process.env.CYPRESS_LOCAL_PROTOCOL_PATH) + if (captureProtocolUrl || process.env.CYPRESS_LOCAL_PROTOCOL_PATH) { + const script = await this.getCaptureProtocolScript(captureProtocolUrl || process.env.CYPRESS_LOCAL_PROTOCOL_PATH) if (script) { options.project.protocolManager = protocolManager @@ -633,6 +640,17 @@ module.exports = { return fs.promises.readFile(process.env.CYPRESS_LOCAL_PROTOCOL_PATH, 'utf8') } + debugProtocol({ + url, + headers: { + 'x-route-version': '1', + 'x-cypress-signature': PUBLIC_KEY_VERSION, + }, + agent, + encrypt: 'signed', + resolveWithFullResponse: true, + }) + return retryWithBackoff(async (attemptIndex) => { return rp.get({ url, @@ -649,8 +667,6 @@ module.exports = { const verified = enc.verifySignature(res.body, res.headers['x-cypress-signature']) if (!verified) { - debugProtocol(`Unable to verify protocol signature %s`, url) - return null } diff --git a/packages/server/lib/cloud/protocol.ts b/packages/server/lib/cloud/protocol.ts index bd64fab21915..51753c0a18c3 100644 --- a/packages/server/lib/cloud/protocol.ts +++ b/packages/server/lib/cloud/protocol.ts @@ -1,12 +1,13 @@ import fs from 'fs-extra' import Debug from 'debug' -import type { ProtocolManagerShape, AppCaptureProtocolInterface, CDPClient, ProtocolError } from '@packages/types' +import type { ProtocolManagerShape, AppCaptureProtocolInterface, CDPClient, ProtocolError, CaptureArtifact } from '@packages/types' import Database from 'better-sqlite3' import path from 'path' import os from 'os' import { createGzip } from 'zlib' import fetch from 'cross-fetch' import Module from 'module' +import env from '../util/env' const routes = require('./routes') const pkg = require('@packages/root') @@ -20,6 +21,13 @@ const DELETE_DB = !process.env.CYPRESS_LOCAL_PROTOCOL_PATH // Timeout for upload const TWO_MINUTES = 120000 +const DB_SIZE_LIMIT = 5000000000 + +const dbSizeLimit = () => { + return env.get('CYPRESS_INTERNAL_SYSTEM_TESTS') === '1' ? + 200 : DB_SIZE_LIMIT +} + /** * requireScript, does just that, requires the passed in script as if it was a module. * @param script - string @@ -64,6 +72,7 @@ export class ProtocolManager implements ProtocolManagerShape { this._protocol = new AppCaptureProtocol() } } catch (error) { + debug(error) if (CAPTURE_ERRORS) { this._errors.push({ error, @@ -173,39 +182,53 @@ export class ProtocolManager implements ProtocolManagerShape { this.invokeSync('resetTest', testId) } - async uploadCaptureArtifact ({ uploadUrl, timeout }) { - const dbPath = this._dbPath + canUpload (): boolean { + return !!this._protocol && !!this._dbPath && !!this._db + } - if (!this._protocol || !dbPath || !this._db) { - if (this._errors.length) { - await this.sendErrors() - } + hasErrors (): boolean { + return !!this._errors.length + } + async getZippedDb (): Promise { + const dbPath = this._dbPath + + debug('reading db from', dbPath) + if (!dbPath) { return } - debug(`uploading %s to %s`, dbPath, uploadUrl) + return new Promise((resolve, reject) => { + const gzip = createGzip() + const buffers: Buffer[] = [] - let zippedFileSize = 0 + gzip.on('data', (args) => { + buffers.push(args) + }) - try { - const body = await new Promise((resolve, reject) => { - const gzip = createGzip() - const buffers: Buffer[] = [] + gzip.on('end', () => { + resolve(Buffer.concat(buffers)) + }) - gzip.on('data', (args) => { - zippedFileSize += args.length - buffers.push(args) - }) + gzip.on('error', reject) - gzip.on('end', () => { - resolve(Buffer.concat(buffers)) - }) + fs.createReadStream(dbPath).pipe(gzip, { end: true }) + }) + } - gzip.on('error', reject) + async uploadCaptureArtifact ({ uploadUrl, payload, fileSize }: CaptureArtifact, timeout) { + const dbPath = this._dbPath - fs.createReadStream(dbPath).pipe(gzip, { end: true }) - }) + if (!this._protocol || !dbPath || !this._db) { + return + } + + debug(`uploading %s to %s`, dbPath, uploadUrl, fileSize) + + try { + if (fileSize > dbSizeLimit()) { + throw new Error(`Spec recording too large: db is ${fileSize} bytes, limit is ${dbSizeLimit()} bytes`) + } const controller = new AbortController() @@ -214,30 +237,30 @@ export class ProtocolManager implements ProtocolManagerShape { }, timeout ?? TWO_MINUTES) const res = await fetch(uploadUrl, { + // @ts-expect-error - this is supported agent, method: 'PUT', - // @ts-expect-error - this is supported - body, + body: payload, headers: { 'Content-Encoding': 'gzip', 'Content-Type': 'binary/octet-stream', - 'Content-Length': `${zippedFileSize}`, + 'Content-Length': `${fileSize}`, }, signal: controller.signal, }) if (res.ok) { return { - fileSize: zippedFileSize, + fileSize, success: true, } } - const err = await res.text() + const errorMessage = await res.text() - debug(`error response text: %s`, err) + debug(`error response text: %s`, errorMessage) - throw new Error(err) + throw new Error(errorMessage) } catch (e) { if (CAPTURE_ERRORS) { this._errors.push({ @@ -248,19 +271,16 @@ export class ProtocolManager implements ProtocolManagerShape { throw e } finally { - await Promise.all([ - this.sendErrors(), + await ( DELETE_DB ? fs.unlink(dbPath).catch((e) => { debug(`Error unlinking db %o`, e) - }) : Promise.resolve(), - ]) - - // Reset errors after they have been sent - this._errors = [] + }) : Promise.resolve() + ) } } async sendErrors (protocolErrors: ProtocolError[] = this._errors) { + debug('invoke: sendErrors for protocol %O', protocolErrors) if (protocolErrors.length === 0) { return } @@ -295,6 +315,8 @@ export class ProtocolManager implements ProtocolManagerShape { } catch (e) { debug(`Error calling ProtocolManager.sendErrors: %o, original errors %o`, e, protocolErrors) } + + this._errors = [] } /** diff --git a/packages/server/lib/modes/record.js b/packages/server/lib/modes/record.js index 187eba5fe291..3e78658c417f 100644 --- a/packages/server/lib/modes/record.js +++ b/packages/server/lib/modes/record.js @@ -21,6 +21,7 @@ const Config = require('../config') const env = require('../util/env') const terminal = require('../util/terminal') const ciProvider = require('../util/ci_provider') +const { printPendingArtifactUpload, printCompletedArtifactUpload } = require('../util/print-run') const testsUtils = require('../util/tests_utils') const specWriter = require('../util/spec_writer') const { fs } = require('../util/fs') @@ -108,145 +109,300 @@ const getSpecRelativePath = (spec) => { return _.get(spec, 'relative', null) } -const uploadArtifacts = (options = {}) => { - const { protocolManager, video, screenshots, videoUploadUrl, captureUploadUrl, screenshotUploadUrls, quiet } = options - - const uploads = [] - const uploadReport = { - protocol: undefined, - screenshots: [], - video: undefined, +/* +artifacts : [ + { + reportKey: 'protocol' | 'screenshots' | 'video', + uploadUrl: string, + filePath?: string, + url: string, + fileSize?: number | bigint, + payload?: Buffer, + message?: string, + } +] + +returns: +[ + { + success: boolean, + error?: string, + url: artifact.uploadUrl, + pathToFile: artifact.filePath, + fileSize: artifact.fileSize, + key: artifact.reportKey, + }, + ... +] +*/ + +const uploadArtifactBatch = async (artifacts, protocolManager, quiet) => { + const priority = { + 'video': 0, + 'screenshots': 1, + 'protocol': 2, + } + const labels = { + 'video': 'Video', + 'screenshots': 'Screenshot', + 'protocol': 'Test Replay', } - const attachMetadataToUploadReport = async (key, pathToFile, statFile, initialUploadMetadata) => { - const uploadMetadata = { - ...initialUploadMetadata, + artifacts.sort((a, b) => { + return priority[a.reportKey] - priority[b.reportKey] + }) + + const preparedArtifacts = await Promise.all(artifacts.map(async (artifact) => { + if (artifact.skip) { + return artifact } - if (statFile) { + if (artifact.reportKey === 'protocol') { try { - const { size } = await fs.statAsync(pathToFile) + if (protocolManager.hasErrors()) { + return { + ...artifact, + skip: true, + error: true, + } + } - uploadMetadata.fileSize = size + const zippedDb = await protocolManager.getZippedDb() + + if (zippedDb === undefined) { + return { + ...artifact, + skip: true, + error: true, + message: 'No test data recorded', + } + } + + return { + ...artifact, + fileSize: Buffer.byteLength(zippedDb), + payload: zippedDb, + } + } catch (err) { + debug('failed to zip protocol db', { + stack: err.stack, + }) + } + } + + if (artifact.filePath) { + try { + const { size } = await fs.statAsync(artifact.filePath) + + return { + ...artifact, + fileSize: size, + } } catch (err) { debug('failed to get stats for upload artifact %o', { - file: pathToFile, + file: artifact.filePath, stack: err.stack, }) } } - uploadReport[key] = Array.isArray(uploadReport[key]) ? - [...uploadReport[key], uploadMetadata] : uploadMetadata - } + return artifact + })) - let count = 0 + if (!quiet) { + // eslint-disable-next-line no-console + console.log('') - const nums = () => { - count += 1 + terminal.header('Uploading Cloud Artifacts', { + color: ['blue'], + }) - return chalk.gray(`(${count}/${uploads.length})`) + // eslint-disable-next-line no-console + console.log('') } - const success = (pathToFile, url, uploadReportOptions) => { - const { statFile, key } = uploadReportOptions + preparedArtifacts.forEach((artifact) => { + debug('preparing to upload artifact %O', artifact) + if (!quiet) { + printPendingArtifactUpload(artifact, labels) + } + }) - return async (res) => { - await attachMetadataToUploadReport(key, pathToFile, statFile, { - success: true, - url, - ...res, - }) + const uploadResults = await Promise.all( + preparedArtifacts.map(async (artifact) => { + if (artifact.skip) { + debug('nothing to upload for artifact %O', artifact) - if (!quiet) { - // eslint-disable-next-line no-console - return console.log(` - Done Uploading ${nums()}`, chalk.blue(pathToFile)) + return { + key: artifact.reportKey, + skipped: true, + error: artifact.message, + } } - } - } - const fail = (pathToFile, url, uploadReportOptions) => { - const { statFile, key } = uploadReportOptions + debug('uploading artifact %O', artifact) + try { + if (artifact.reportKey === 'protocol') { + const res = await protocolManager.uploadCaptureArtifact(artifact) + + return { + ...res, + pathToFile: 'Test Replay', + url: artifact.uploadUrl, + fileSize: artifact.fileSize, + key: artifact.reportKey, + } + } - return async (err) => { - await attachMetadataToUploadReport(key, pathToFile, statFile, { - success: false, - url, - error: err.message, - }) + const res = await upload.send(artifact.filePath, artifact.uploadUrl) - debug('failed to upload artifact %o', { - file: pathToFile, - url, - stack: err.stack, - }) + return { + ...res, + success: true, + url: artifact.uploadUrl, + pathToFile: artifact.filePath, + fileSize: artifact.fileSize, + key: artifact.reportKey, + } + } catch (err) { + debug('failed to upload artifact %o', { + file: artifact.filePath, + url: artifact.uploadUrl, + stack: err.stack, + }) - if (!quiet) { - // eslint-disable-next-line no-console - return console.log(` - Failed Uploading ${nums()}`, chalk.red(pathToFile)) + return { + key: artifact.reportKey, + success: false, + error: err.message, + url: artifact.uploadUrl, + pathToFile: artifact.filePath, + } } - } - } + }), + ) - const send = (pathToFile, url, reportKey) => { - return uploads.push( - upload.send(pathToFile, url) - .then(success(pathToFile, url, { key: reportKey, statFile: true })) - .catch(fail(pathToFile, url, { key: reportKey, statFile: true })), - ) + const attemptedUploadResults = uploadResults.filter(({ skipped }) => { + return !skipped + }) + + if (!quiet && attemptedUploadResults.length) { + // eslint-disable-next-line no-console + console.log('') + + terminal.header('Uploaded Cloud Artifacts', { + color: ['blue'], + }) + + // eslint-disable-next-line no-console + console.log('') } + return attemptedUploadResults.reduce((acc, { key, skipped, ...report }, i, { length }) => { + if (!quiet) { + printCompletedArtifactUpload({ key, ...report }, labels, chalk.grey(`${i + 1}/${length}`)) + } + + return skipped ? acc : { + ...acc, + [key]: (key === 'screenshots') ? [...acc.screenshots, report] : report, + } + }, { + video: undefined, + screenshots: [], + protocol: undefined, + }) +} + +const uploadArtifacts = async (options = {}) => { + const { protocolManager, video, screenshots, videoUploadUrl, captureUploadUrl, protocolCaptureMeta, screenshotUploadUrls, quiet, runId, instanceId } = options + + const artifacts = [] + if (videoUploadUrl) { - send(video, videoUploadUrl, 'video') + artifacts.push({ + reportKey: 'video', + uploadUrl: videoUploadUrl, + filePath: video, + }) + } else { + artifacts.push({ + reportKey: 'video', + skip: true, + }) } - if (screenshotUploadUrls) { - screenshotUploadUrls.forEach((obj) => { - const screenshot = _.find(screenshots, { screenshotId: obj.screenshotId }) + if (screenshotUploadUrls.length) { + screenshotUploadUrls.map(({ screenshotId, uploadUrl }) => { + const screenshot = _.find(screenshots, { screenshotId }) + + debug('screenshot: %o', screenshot) - return send(screenshot.path, obj.uploadUrl, 'screenshots') + return { + reportKey: 'screenshots', + uploadUrl, + filePath: screenshot.path, + } + }).forEach((screenshotArtifact) => { + artifacts.push(screenshotArtifact) + }) + } else { + artifacts.push({ + reportKey: 'screenshots', + skip: true, }) } - if (captureUploadUrl && protocolManager) { - uploads.push( - protocolManager.uploadCaptureArtifact({ uploadUrl: captureUploadUrl }) - .then(success('Test Replay', captureUploadUrl, { key: 'protocol', statFile: false })) - .catch(fail('Test Replay', captureUploadUrl, { key: 'protocol', statFile: false })), - ) + debug('capture manifest: %O', { captureUploadUrl, protocolCaptureMeta, protocolManager }) + if ((captureUploadUrl || (protocolCaptureMeta && protocolCaptureMeta.url)) && protocolManager) { + artifacts.push({ + reportKey: 'protocol', + uploadUrl: captureUploadUrl || protocolCaptureMeta.url, + }) + } else if (protocolCaptureMeta && protocolCaptureMeta.disabledMessage) { + artifacts.push({ + reportKey: 'protocol', + message: protocolCaptureMeta.disabledMessage, + skip: true, + }) } - if (!uploads.length && !quiet) { - // eslint-disable-next-line no-console - console.log(' - Nothing to Upload') - } + let uploadReport - return Promise - .all(uploads) - .catch((err) => { + try { + uploadReport = await uploadArtifactBatch(artifacts, protocolManager, quiet) + } catch (err) { errors.warning('CLOUD_CANNOT_UPLOAD_ARTIFACTS', err) return exception.create(err) - }) - .finally(() => { - api.updateInstanceArtifacts({ - runId: options.runId, - instanceId: options.instanceId, - ...uploadReport, - }) - .catch((err) => { - debug('failed updating artifact status %o', { - stack: err.stack, - }) + } - errors.warning('CLOUD_CANNOT_UPLOAD_ARTIFACTS_PROTOCOL', err) + debug('checking for protocol errors', protocolManager?.hasErrors()) + if (protocolManager && protocolManager.hasErrors()) { + try { + await protocolManager.sendErrors() + } catch (err) { + debug('Failed to send protocol errors %O', err) + } + } - // don't log exceptions if we have a 503 status code - if (err.statusCode !== 503) { - return exception.create(err) - } + try { + const res = await api.updateInstanceArtifacts({ + runId, instanceId, ...uploadReport, }) - }) + + return res + } catch (err) { + debug('failed updating artifact status %o', { + stack: err.stack, + }) + + errors.warning('CLOUD_CANNOT_UPLOAD_ARTIFACTS_PROTOCOL', err) + + if (err.statusCode !== 503) { + return exception.create(err) + } + } } const updateInstanceStdout = (options = {}) => { @@ -722,6 +878,7 @@ const createRunAndRecordSpecs = (options = {}) => { } const { runUrl, runId, machineId, groupId } = resp + const protocolCaptureMeta = resp.capture || {} let captured = null let instanceId = null @@ -766,18 +923,6 @@ const createRunAndRecordSpecs = (options = {}) => { debug('after spec run %o', { spec }) - if (!quiet) { - // eslint-disable-next-line no-console - console.log('') - - terminal.header('Uploading Screenshots & Videos', { - color: ['blue'], - }) - - // eslint-disable-next-line no-console - console.log('') - } - return specWriter.countStudioUsage(spec.absolute) .then((metadata) => { return postInstanceResults({ @@ -795,6 +940,7 @@ const createRunAndRecordSpecs = (options = {}) => { return } + debug('postInstanceResults resp %O', resp) const { video, screenshots } = results const { videoUploadUrl, captureUploadUrl, screenshotUploadUrls } = resp @@ -805,6 +951,7 @@ const createRunAndRecordSpecs = (options = {}) => { screenshots, videoUploadUrl, captureUploadUrl, + protocolCaptureMeta, protocolManager: project.protocolManager, screenshotUploadUrls, quiet, diff --git a/packages/server/lib/util/print-run.ts b/packages/server/lib/util/print-run.ts index 2a17bedaa148..c105bb99c1fc 100644 --- a/packages/server/lib/util/print-run.ts +++ b/packages/server/lib/util/print-run.ts @@ -3,6 +3,7 @@ import _ from 'lodash' import logSymbols from 'log-symbols' import chalk from 'chalk' import human from 'human-interval' +import prettyBytes from 'pretty-bytes' import pkg from '@packages/root' import humanTime from './human_time' import duration from './duration' @@ -560,3 +561,81 @@ export const printVideoPath = (videoName?: string) => { console.log('') } } + +const formatFileSize = (bytes: number) => { + // in test environments, mask the value as it may differ from environment + // to environment + if (env.get('CYPRESS_INTERNAL_ENV') === 'test') { + return prettyBytes(1000) + } + + return prettyBytes(bytes) +} + +type ArtifactLike = { + reportKey: 'protocol' | 'screenshots' | 'video' + filePath?: string + fileSize?: number | BigInt + message?: string + skip?: boolean + error: boolean +} + +export const printPendingArtifactUpload = (artifact: T, labels: Record<'protocol' | 'screenshots' | 'video', string>): void => { + process.stdout.write(` - ${labels[artifact.reportKey]} `) + + if (artifact.skip) { + if (artifact.reportKey === 'protocol' && artifact.error) { + process.stdout.write('- Failed Capturing ') + } else { + process.stdout.write('- Nothing to upload ') + } + } + + if (artifact.reportKey === 'protocol' && artifact.message) { + process.stdout.write(`- ${artifact.message}`) + } + + if (artifact.fileSize) { + process.stdout.write(`- ${formatFileSize(Number(artifact.fileSize))}`) + } + + if (artifact.filePath) { + process.stdout.write(` ${formatPath(artifact.filePath, undefined, 'cyan')}`) + } + + process.stdout.write('\n') +} + +type ArtifactUploadResultLike = { + pathToFile?: string + key: string + fileSize?: number | BigInt + success: boolean + error?: string + skipped?: boolean +} + +export const printCompletedArtifactUpload = (artifactUploadResult: T, labels: Record<'protocol' | 'screenshots' | 'video', string>, num: string): void => { + const { pathToFile, key, fileSize, success, error, skipped } = artifactUploadResult + + process.stdout.write(` - ${labels[key]} `) + + if (success) { + process.stdout.write(`- Done Uploading ${formatFileSize(Number(fileSize))} ${num}`) + } else if (skipped) { + process.stdout.write(`- Nothing to Upload ${num}`) + } else { + process.stdout.write(`- Failed Uploading ${num}`) + } + + if (pathToFile && key !== 'protocol') { + process.stdout.write(` ${formatPath(pathToFile, undefined, 'cyan')}`) + } + + if (error) { + process.stdout.write(` - ${error}`) + } + + process.stdout.write('\n') +} diff --git a/packages/server/package.json b/packages/server/package.json index 38872bd671f8..372068f21a88 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -105,6 +105,7 @@ "p-queue": "6.1.0", "pidusage": "3.0.2", "pluralize": "8.0.0", + "pretty-bytes": "^5.6.0", "randomstring": "1.1.5", "recast": "0.20.4", "resolve": "1.17.0", diff --git a/packages/types/src/protocol.ts b/packages/types/src/protocol.ts index 6c573e85a9bf..aeda511cf754 100644 --- a/packages/types/src/protocol.ts +++ b/packages/types/src/protocol.ts @@ -37,10 +37,16 @@ export interface ProtocolError { captureMethod: keyof AppCaptureProtocolInterface | 'setupProtocol' | 'uploadCaptureArtifact' | 'getCaptureProtocolScript' | 'cdpClient.on' } +export type CaptureArtifact = { + uploadUrl: string + fileSize: number + payload: Buffer +} + export interface ProtocolManagerShape extends AppCaptureProtocolCommon { protocolEnabled: boolean setupProtocol(script: string, runId: string): Promise beforeSpec (spec: { instanceId: string}): void sendErrors (errors: ProtocolError[]): Promise - uploadCaptureArtifact(options: { uploadUrl: string, timeout: number }): Promise<{ fileSize: number, success: boolean, error?: string } | void> + uploadCaptureArtifact(artifact: CaptureArtifact, timeout?: number): Promise<{ fileSize: number, success: boolean, error?: string } | void> } diff --git a/system-tests/README.md b/system-tests/README.md index 57f9788574fe..1d9d5b278cc0 100644 --- a/system-tests/README.md +++ b/system-tests/README.md @@ -114,6 +114,8 @@ Prepend `SNAPSHOT_UPDATE=1` to any test command. See [`snap-shot-it` instruction SNAPSHOT_UPDATE=1 yarn test go_spec ``` +If you are on a Retina device, you may get mismatching screenshot dimensions when updating snapshots. To resolve this, you can set the `SNAPSHOT_BROWSER` environment variable to `chrome` when you update the snapshots. + ### Test Projects Every folder in [`./projects`](./lib/projects) represents a self-contained Cypress project. When you pass the `project` property to `systemTests.it` or `systemTests.exec`, Cypress launches using this project. diff --git a/system-tests/__snapshots__/record_spec.js b/system-tests/__snapshots__/record_spec.js index deccc2dc98df..10a4565c3ce2 100644 --- a/system-tests/__snapshots__/record_spec.js +++ b/system-tests/__snapshots__/record_spec.js @@ -73,9 +73,15 @@ Fix the error in your code and re-run your tests. - Video output: /XXX/XXX/XXX/cypress/videos/record_error.cy.js.mp4 - (Uploading Screenshots & Videos) + (Uploading Cloud Artifacts) - - Done Uploading (1/1) /foo/bar/.projects/e2e/cypress/videos/record_error.cy.js.mp4 + - Video - 1 kB /XXX/XXX/XXX/cypress/videos/record_error.cy.js.mp4 + - Screenshot - Nothing to upload + - Test Replay - Nothing to upload - Test Replay is disabled for this project. Enable Test Replay in Cloud project settings + + (Uploaded Cloud Artifacts) + + - Video - Done Uploading 1 kB 1/1 /XXX/XXX/XXX/cypress/videos/record_error.cy.js.mp4 ──────────────────────────────────────────────────────────────────────────────────────────────────── @@ -130,10 +136,16 @@ Because this error occurred during a \`before each\` hook we are skipping the re - Video output: /XXX/XXX/XXX/cypress/videos/record_fail.cy.js.mp4 - (Uploading Screenshots & Videos) + (Uploading Cloud Artifacts) + + - Video - 1 kB /XXX/XXX/XXX/cypress/videos/record_fail.cy.js.mp4 + - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_fail.cy.js/record fails -- fails 1 -- before each hook (failed).png + - Test Replay - Nothing to upload - Test Replay is disabled for this project. Enable Test Replay in Cloud project settings + + (Uploaded Cloud Artifacts) - - Done Uploading (*/2) /foo/bar/.projects/e2e/cypress/screenshots/record_fail.cy.js/record fails -- fails 1 -- before each hook (failed).png - - Done Uploading (*/2) /foo/bar/.projects/e2e/cypress/videos/record_fail.cy.js.mp4 + - Video - Done Uploading 1 kB 1/2 /XXX/XXX/XXX/cypress/videos/record_fail.cy.js.mp4 + - Screenshot - Done Uploading 1 kB 2/2 /XXX/XXX/XXX/cypress/screenshots/record_fail.cy.js/record fails -- fails 1 -- before each hook (failed).png ──────────────────────────────────────────────────────────────────────────────────────────────────── @@ -172,9 +184,15 @@ plugin stdout - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) - (Uploading Screenshots & Videos) + (Uploading Cloud Artifacts) - - Done Uploading (1/1) /foo/bar/.projects/e2e/cypress/screenshots/record_pass.cy.js/yay it passes.png + - Video - Nothing to upload + - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png + - Test Replay - Nothing to upload - Test Replay is disabled for this project. Enable Test Replay in Cloud project settings + + (Uploaded Cloud Artifacts) + + - Screenshot - Done Uploading 1 kB 1/1 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png ──────────────────────────────────────────────────────────────────────────────────────────────────── @@ -232,10 +250,16 @@ We dynamically generated a new test to display this failure. - Video output: /XXX/XXX/XXX/cypress/videos/record_uncaught.cy.js.mp4 - (Uploading Screenshots & Videos) + (Uploading Cloud Artifacts) + + - Video - 1 kB /XXX/XXX/XXX/cypress/videos/record_uncaught.cy.js.mp4 + - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_uncaught.cy.js/An uncaught error was detected outside of a test (failed).png + - Test Replay - Nothing to upload - Test Replay is disabled for this project. Enable Test Replay in Cloud project settings + + (Uploaded Cloud Artifacts) - - Done Uploading (*/2) /foo/bar/.projects/e2e/cypress/screenshots/record_uncaught.cy.js/An uncaught error was detected outside of a test (failed).png - - Done Uploading (*/2) /foo/bar/.projects/e2e/cypress/videos/record_uncaught.cy.js.mp4 + - Video - Done Uploading 1 kB 1/2 /XXX/XXX/XXX/cypress/videos/record_uncaught.cy.js.mp4 + - Screenshot - Done Uploading 1 kB 2/2 /XXX/XXX/XXX/cypress/screenshots/record_uncaught.cy.js/An uncaught error was detected outside of a test (failed).png ==================================================================================================== @@ -329,9 +353,15 @@ exports['e2e record api interaction errors update instance stdout warns but proc - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) - (Uploading Screenshots & Videos) + (Uploading Cloud Artifacts) - - Done Uploading (1/1) /foo/bar/.projects/e2e/cypress/screenshots/record_pass.cy.js/yay it passes.png + - Video - Nothing to upload + - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png + - Test Replay - Nothing to upload - Test Replay is disabled for this project. Enable Test Replay in Cloud project settings + + (Uploaded Cloud Artifacts) + + - Screenshot - Done Uploading 1 kB 1/1 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png Warning: We encountered an error communicating with our servers. This run will proceed, but will not be recorded. @@ -455,10 +485,16 @@ exports['e2e record api interaction errors uploading assets warns but proceeds 1 - Video output: /XXX/XXX/XXX/cypress/videos/record_pass.cy.js.mp4 - (Uploading Screenshots & Videos) + (Uploading Cloud Artifacts) + + - Video - 1 kB /XXX/XXX/XXX/cypress/videos/record_pass.cy.js.mp4 + - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png + - Test Replay - Nothing to upload - Test Replay is disabled for this project. Enable Test Replay in Cloud project settings + + (Uploaded Cloud Artifacts) - - Failed Uploading (*/2) /foo/bar/.projects/e2e/cypress/screenshots/record_pass.cy.js/yay it passes.png - - Failed Uploading (*/2) /foo/bar/.projects/e2e/cypress/videos/record_pass.cy.js.mp4 + - Video - Failed Uploading 1/2 /XXX/XXX/XXX/cypress/videos/record_pass.cy.js.mp4 - 500 - "Internal Server Error" + - Screenshot - Failed Uploading 2/2 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png - 500 - "Internal Server Error" ==================================================================================================== @@ -678,9 +714,6 @@ exports['e2e record api interaction errors update instance 500 does not proceed - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) - - (Uploading Screenshots & Videos) - We encountered an unexpected error communicating with our servers. StatusCodeError: 500 - "Internal Server Error" @@ -768,9 +801,15 @@ We will retry 3 more times in X second(s)... - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) - (Uploading Screenshots & Videos) + (Uploading Cloud Artifacts) + + - Video - Nothing to upload + - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png + - Test Replay - Nothing to upload - Test Replay is disabled for this project. Enable Test Replay in Cloud project settings + + (Uploaded Cloud Artifacts) - - Done Uploading (1/1) /foo/bar/.projects/e2e/cypress/screenshots/record_pass.cy.js/yay it passes.png + - Screenshot - Done Uploading 1 kB 1/1 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png ==================================================================================================== @@ -968,9 +1007,14 @@ https://on.cypress.io/dashboard/organizations/org-id-1234/billing - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) - (Uploading Screenshots & Videos) + (Uploading Cloud Artifacts) - - Done Uploading (1/1) /foo/bar/.projects/e2e/cypress/screenshots/record_pass.cy.js/yay it passes.png + - Video - Nothing to upload + - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png + + (Uploaded Cloud Artifacts) + + - Screenshot - Done Uploading 1 kB 1/1 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png ==================================================================================================== @@ -1048,9 +1092,14 @@ https://on.cypress.io/dashboard/organizations/org-id-1234/billing - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) - (Uploading Screenshots & Videos) + (Uploading Cloud Artifacts) + + - Video - Nothing to upload + - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png + + (Uploaded Cloud Artifacts) - - Done Uploading (1/1) /foo/bar/.projects/e2e/cypress/screenshots/record_pass.cy.js/yay it passes.png + - Screenshot - Done Uploading 1 kB 1/1 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png ==================================================================================================== @@ -1133,9 +1182,15 @@ Details: - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) - (Uploading Screenshots & Videos) + (Uploading Cloud Artifacts) - - Done Uploading (1/1) /foo/bar/.projects/e2e/cypress/screenshots/record_pass.cy.js/yay it passes.png + - Video - Nothing to upload + - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png + - Test Replay - Nothing to upload - Test Replay is disabled for this project. Enable Test Replay in Cloud project settings + + (Uploaded Cloud Artifacts) + + - Screenshot - Done Uploading 1 kB 1/1 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png ==================================================================================================== @@ -1469,9 +1524,14 @@ https://on.cypress.io/dashboard/organizations/org-id-1234/billing - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) - (Uploading Screenshots & Videos) + (Uploading Cloud Artifacts) + + - Video - Nothing to upload + - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png + + (Uploaded Cloud Artifacts) - - Done Uploading (1/1) /foo/bar/.projects/e2e/cypress/screenshots/record_pass.cy.js/yay it passes.png + - Screenshot - Done Uploading 1 kB 1/1 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png ==================================================================================================== @@ -1551,9 +1611,14 @@ https://on.cypress.io/dashboard/organizations/org-id-1234/billing - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) - (Uploading Screenshots & Videos) + (Uploading Cloud Artifacts) - - Done Uploading (1/1) /foo/bar/.projects/e2e/cypress/screenshots/record_pass.cy.js/yay it passes.png + - Video - Nothing to upload + - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png + + (Uploaded Cloud Artifacts) + + - Screenshot - Done Uploading 1 kB 1/1 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png ==================================================================================================== @@ -1631,9 +1696,14 @@ https://on.cypress.io/dashboard/organizations/org-id-1234/billing - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) - (Uploading Screenshots & Videos) + (Uploading Cloud Artifacts) + + - Video - Nothing to upload + - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png + + (Uploaded Cloud Artifacts) - - Done Uploading (1/1) /foo/bar/.projects/e2e/cypress/screenshots/record_pass.cy.js/yay it passes.png + - Screenshot - Done Uploading 1 kB 1/1 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png ==================================================================================================== @@ -1711,9 +1781,14 @@ https://on.cypress.io/dashboard/organizations/org-id-1234/billing - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) - (Uploading Screenshots & Videos) + (Uploading Cloud Artifacts) - - Done Uploading (1/1) /foo/bar/.projects/e2e/cypress/screenshots/record_pass.cy.js/yay it passes.png + - Video - Nothing to upload + - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png + + (Uploaded Cloud Artifacts) + + - Screenshot - Done Uploading 1 kB 1/1 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png ==================================================================================================== @@ -1791,9 +1866,14 @@ https://on.cypress.io/dashboard/organizations/org-id-1234/billing - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) - (Uploading Screenshots & Videos) + (Uploading Cloud Artifacts) + + - Video - Nothing to upload + - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png - - Done Uploading (1/1) /foo/bar/.projects/e2e/cypress/screenshots/record_pass.cy.js/yay it passes.png + (Uploaded Cloud Artifacts) + + - Screenshot - Done Uploading 1 kB 1/1 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png ==================================================================================================== @@ -1984,9 +2064,6 @@ exports['e2e record api interaction errors postInstanceResults errors and exits - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) - - (Uploading Screenshots & Videos) - We encountered an unexpected error communicating with our servers. StatusCodeError: 500 - "Internal Server Error" @@ -2045,9 +2122,11 @@ exports['e2e record api skips specs records tests and exits without executing 1' └────────────────────────────────────────────────────────────────────────────────────────────────┘ - (Uploading Screenshots & Videos) + (Uploading Cloud Artifacts) - - Nothing to Upload + - Video - Nothing to upload + - Screenshot - Nothing to upload + - Test Replay - Nothing to upload - Test Replay is disabled for this project. Enable Test Replay in Cloud project settings ==================================================================================================== @@ -2124,9 +2203,11 @@ exports['e2e record api skips specs records tests and exits without executing in └────────────────────────────────────────────────────────────────────────────────────────────────┘ - (Uploading Screenshots & Videos) + (Uploading Cloud Artifacts) - - Nothing to Upload + - Video - Nothing to upload + - Screenshot - Nothing to upload + - Test Replay - Nothing to upload - Test Replay is disabled for this project. Enable Test Replay in Cloud project settings ==================================================================================================== @@ -2192,9 +2273,11 @@ exports['e2e record empty specs succeeds when empty spec file 1'] = ` └────────────────────────────────────────────────────────────────────────────────────────────────┘ - (Uploading Screenshots & Videos) + (Uploading Cloud Artifacts) - - Nothing to Upload + - Video - Nothing to upload + - Screenshot - Nothing to upload + - Test Replay - Nothing to upload - Test Replay is disabled for this project. Enable Test Replay in Cloud project settings ──────────────────────────────────────────────────────────────────────────────────────────────────── @@ -2221,9 +2304,11 @@ exports['e2e record empty specs succeeds when empty spec file 1'] = ` └────────────────────────────────────────────────────────────────────────────────────────────────┘ - (Uploading Screenshots & Videos) + (Uploading Cloud Artifacts) - - Nothing to Upload + - Video - Nothing to upload + - Screenshot - Nothing to upload + - Test Replay - Nothing to upload - Test Replay is disabled for this project. Enable Test Replay in Cloud project settings ==================================================================================================== @@ -2354,9 +2439,15 @@ plugin stdout - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) - (Uploading Screenshots & Videos) + (Uploading Cloud Artifacts) + + - Video - Nothing to upload + - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png + - Test Replay - Nothing to upload - Test Replay is disabled for this project. Enable Test Replay in Cloud project settings - - Done Uploading (1/1) /foo/bar/.projects/e2e/cypress/screenshots/record_pass.cy.js/yay it passes.png + (Uploaded Cloud Artifacts) + + - Screenshot - Done Uploading 1 kB 1/1 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png ==================================================================================================== @@ -2561,9 +2652,14 @@ https://on.cypress.io/dashboard/organizations/org-id-1234/billing - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) - (Uploading Screenshots & Videos) + (Uploading Cloud Artifacts) + + - Video - Nothing to upload + - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png - - Done Uploading (1/1) /foo/bar/.projects/e2e/cypress/screenshots/record_pass.cy.js/yay it passes.png + (Uploaded Cloud Artifacts) + + - Screenshot - Done Uploading 1 kB 1/1 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png ==================================================================================================== @@ -2645,7 +2741,7 @@ https://on.cypress.io/run-group-name-not-unique ` -exports['e2e record capture-protocol passing retrieves the capture protocol 1'] = ` +exports['e2e record capture-protocol disabled messaging displays disabled message but continues 1'] = ` ==================================================================================================== @@ -2697,9 +2793,510 @@ exports['e2e record capture-protocol passing retrieves the capture protocol 1'] - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) - (Uploading Screenshots & Videos) + (Uploading Cloud Artifacts) + + - Video - Nothing to upload + - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png + - Test Replay - Nothing to upload - Test Replay is only supported in Chromium browsers + + (Uploaded Cloud Artifacts) + + - Screenshot - Done Uploading 1 kB 1/1 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png + +==================================================================================================== + + (Run Finished) + + + Spec Tests Passing Failing Pending Skipped + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ ✔ record_pass.cy.js XX:XX 2 1 - 1 - │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + ✔ All specs passed! XX:XX 2 1 - 1 - + + +─────────────────────────────────────────────────────────────────────────────────────────────────────── + + Recorded Run: https://dashboard.cypress.io/projects/cjvoj7/runs/12 + + +` + +exports['e2e record capture-protocol enabled passing retrieves the capture protocol and uploads the db 1'] = ` + +==================================================================================================== + + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 1 found (record_pass.cy.js) │ + │ Searched: cypress/e2e/record_pass* │ + │ Params: Tag: false, Group: false, Parallel: false │ + │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: record_pass.cy.js (1 of 1) + Estimated: X second(s) + + + record pass + ✓ passes + - is pending + + + 1 passing + 1 pending + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 2 │ + │ Passing: 1 │ + │ Failing: 0 │ + │ Pending: 1 │ + │ Skipped: 0 │ + │ Screenshots: 1 │ + │ Video: false │ + │ Duration: X seconds │ + │ Estimated: X second(s) │ + │ Spec Ran: record_pass.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + + (Screenshots) + + - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) + + + (Uploading Cloud Artifacts) + + - Video - Nothing to upload + - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png + - Test Replay - 1 kB + + (Uploaded Cloud Artifacts) + + - Screenshot - Done Uploading 1 kB 1/2 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png + - Test Replay - Done Uploading 1 kB 2/2 + +==================================================================================================== + + (Run Finished) + + + Spec Tests Passing Failing Pending Skipped + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ ✔ record_pass.cy.js XX:XX 2 1 - 1 - │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + ✔ All specs passed! XX:XX 2 1 - 1 - + + +─────────────────────────────────────────────────────────────────────────────────────────────────────── + + Recorded Run: https://dashboard.cypress.io/projects/cjvoj7/runs/12 + + +` + +exports['e2e record capture-protocol enabled protocol errors db size too large displays error and does not upload if db size is too large 1'] = ` + +==================================================================================================== + + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 1 found (record_pass.cy.js) │ + │ Searched: cypress/e2e/record_pass* │ + │ Params: Tag: false, Group: false, Parallel: false │ + │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: record_pass.cy.js (1 of 1) + Estimated: X second(s) + + + record pass + ✓ passes + - is pending + + + 1 passing + 1 pending + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 2 │ + │ Passing: 1 │ + │ Failing: 0 │ + │ Pending: 1 │ + │ Skipped: 0 │ + │ Screenshots: 1 │ + │ Video: false │ + │ Duration: X seconds │ + │ Estimated: X second(s) │ + │ Spec Ran: record_pass.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + + (Screenshots) + + - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) + + + (Uploading Cloud Artifacts) + + - Video - Nothing to upload + - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png + - Test Replay - 1 kB + + (Uploaded Cloud Artifacts) + + - Screenshot - Done Uploading 1 kB 1/2 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png + - Test Replay - Failed Uploading 2/2 - Spec recording too large: db is 1047 bytes, limit is 200 bytes + +==================================================================================================== + + (Run Finished) + + + Spec Tests Passing Failing Pending Skipped + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ ✔ record_pass.cy.js XX:XX 2 1 - 1 - │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + ✔ All specs passed! XX:XX 2 1 - 1 - + + +─────────────────────────────────────────────────────────────────────────────────────────────────────── + + Recorded Run: https://dashboard.cypress.io/projects/cjvoj7/runs/12 + + +` + +exports['e2e record capture-protocol enabled protocol errors error initializing protocol displays the error and reports to cloud 1'] = ` + +==================================================================================================== + + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 1 found (record_pass.cy.js) │ + │ Searched: cypress/e2e/record_pass* │ + │ Params: Tag: false, Group: false, Parallel: false │ + │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: record_pass.cy.js (1 of 1) + Estimated: X second(s) + + + record pass + ✓ passes + - is pending + + + 1 passing + 1 pending + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 2 │ + │ Passing: 1 │ + │ Failing: 0 │ + │ Pending: 1 │ + │ Skipped: 0 │ + │ Screenshots: 1 │ + │ Video: false │ + │ Duration: X seconds │ + │ Estimated: X second(s) │ + │ Spec Ran: record_pass.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + + (Screenshots) + + - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) + + + (Uploading Cloud Artifacts) + + - Video - Nothing to upload + - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png + - Test Replay - Failed Capturing + + (Uploaded Cloud Artifacts) + + - Screenshot - Done Uploading 1 kB 1/1 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png + +==================================================================================================== + + (Run Finished) + + + Spec Tests Passing Failing Pending Skipped + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ ✔ record_pass.cy.js XX:XX 2 1 - 1 - │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + ✔ All specs passed! XX:XX 2 1 - 1 - + + +─────────────────────────────────────────────────────────────────────────────────────────────────────── + + Recorded Run: https://dashboard.cypress.io/projects/cjvoj7/runs/12 + + +` + +exports['capture-protocol api errors upload 500 continues 1'] = ` + +==================================================================================================== + + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 1 found (record_pass.cy.js) │ + │ Searched: cypress/e2e/record_pass* │ + │ Params: Tag: false, Group: false, Parallel: false │ + │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: record_pass.cy.js (1 of 1) + Estimated: X second(s) + + + record pass + ✓ passes + - is pending + + + 1 passing + 1 pending + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 2 │ + │ Passing: 1 │ + │ Failing: 0 │ + │ Pending: 1 │ + │ Skipped: 0 │ + │ Screenshots: 1 │ + │ Video: false │ + │ Duration: X seconds │ + │ Estimated: X second(s) │ + │ Spec Ran: record_pass.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + + (Screenshots) + + - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) + + + (Uploading Cloud Artifacts) + + - Video - Nothing to upload + - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png + - Test Replay - 1 kB + + (Uploaded Cloud Artifacts) + + - Screenshot - Done Uploading 1 kB 1/2 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png + - Test Replay - Failed Uploading 2/2 + +==================================================================================================== + + (Run Finished) + + + Spec Tests Passing Failing Pending Skipped + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ ✔ record_pass.cy.js XX:XX 2 1 - 1 - │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + ✔ All specs passed! XX:XX 2 1 - 1 - + + +─────────────────────────────────────────────────────────────────────────────────────────────────────── + + Recorded Run: https://dashboard.cypress.io/projects/cjvoj7/runs/12 + + +` + +exports['capture-protocol api errors fetch script 500 continues 1'] = ` +We encountered an unexpected error communicating with our servers. + +StatusCodeError: 500 - "" + +We will retry 1 more time in X second(s)... + + +==================================================================================================== + + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 1 found (record_pass.cy.js) │ + │ Searched: cypress/e2e/record_pass* │ + │ Params: Tag: false, Group: false, Parallel: false │ + │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: record_pass.cy.js (1 of 1) + Estimated: X second(s) + + + record pass + ✓ passes + - is pending + + + 1 passing + 1 pending + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 2 │ + │ Passing: 1 │ + │ Failing: 0 │ + │ Pending: 1 │ + │ Skipped: 0 │ + │ Screenshots: 1 │ + │ Video: false │ + │ Duration: X seconds │ + │ Estimated: X second(s) │ + │ Spec Ran: record_pass.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + + (Screenshots) + + - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) + + + (Uploading Cloud Artifacts) + + - Video - Nothing to upload + - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png + + (Uploaded Cloud Artifacts) + + - Screenshot - Done Uploading 1 kB 1/1 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png + +==================================================================================================== + + (Run Finished) + + + Spec Tests Passing Failing Pending Skipped + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ ✔ record_pass.cy.js XX:XX 2 1 - 1 - │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + ✔ All specs passed! XX:XX 2 1 - 1 - + + +─────────────────────────────────────────────────────────────────────────────────────────────────────── + + Recorded Run: https://dashboard.cypress.io/projects/cjvoj7/runs/12 + + +` + +exports['capture-protocol api errors error report 500 continues 1'] = ` + +==================================================================================================== + + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 1 found (record_pass.cy.js) │ + │ Searched: cypress/e2e/record_pass* │ + │ Params: Tag: false, Group: false, Parallel: false │ + │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: record_pass.cy.js (1 of 1) + Estimated: X second(s) + + + record pass + ✓ passes + - is pending + + + 1 passing + 1 pending + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 2 │ + │ Passing: 1 │ + │ Failing: 0 │ + │ Pending: 1 │ + │ Skipped: 0 │ + │ Screenshots: 1 │ + │ Video: false │ + │ Duration: X seconds │ + │ Estimated: X second(s) │ + │ Spec Ran: record_pass.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + + (Screenshots) + + - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) + + + (Uploading Cloud Artifacts) + + - Video - Nothing to upload + - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png + - Test Replay - 1 kB + + (Uploaded Cloud Artifacts) - - Done Uploading (1/1) /foo/bar/.projects/e2e/cypress/screenshots/record_pass.cy.js/yay it passes.png + - Screenshot - Done Uploading 1 kB 1/2 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png + - Test Replay - Done Uploading 1 kB 2/2 ==================================================================================================== diff --git a/system-tests/__snapshots__/video_compression_spec.js b/system-tests/__snapshots__/video_compression_spec.js index 24516a90ca40..1f97f2ec0987 100644 --- a/system-tests/__snapshots__/video_compression_spec.js +++ b/system-tests/__snapshots__/video_compression_spec.js @@ -103,9 +103,15 @@ exports['video compression true / compresses to 32 CRF'] = ` - Video output: /XXX/XXX/XXX/cypress/videos/video_compression.cy.js.mp4 - (Uploading Screenshots & Videos) + (Uploading Cloud Artifacts) - - Done Uploading (1/1) /foo/bar/.projects/e2e/cypress/videos/video_compression.cy.js.mp4 + - Video - 1 kB /XXX/XXX/XXX/cypress/videos/video_compression.cy.js.mp4 + - Screenshot - Nothing to upload + - Test Replay - Nothing to upload - Test Replay is disabled for this project. Enable Test Replay in Cloud project settings + + (Uploaded Cloud Artifacts) + + - Video - Done Uploading 1 kB 1/1 /XXX/XXX/XXX/cypress/videos/video_compression.cy.js.mp4 ==================================================================================================== @@ -175,9 +181,15 @@ exports['video compression true / coerces true to 32 CRF'] = ` - Video output: /XXX/XXX/XXX/cypress/videos/video_compression.cy.js.mp4 - (Uploading Screenshots & Videos) + (Uploading Cloud Artifacts) + + - Video - 1 kB /XXX/XXX/XXX/cypress/videos/video_compression.cy.js.mp4 + - Screenshot - Nothing to upload + - Test Replay - Nothing to upload - Test Replay is disabled for this project. Enable Test Replay in Cloud project settings + + (Uploaded Cloud Artifacts) - - Done Uploading (1/1) /foo/bar/.projects/e2e/cypress/videos/video_compression.cy.js.mp4 + - Video - Done Uploading 1 kB 1/1 /XXX/XXX/XXX/cypress/videos/video_compression.cy.js.mp4 ==================================================================================================== diff --git a/system-tests/lib/protocolStub.ts b/system-tests/lib/protocolStub.ts index c530f478b34d..62ba6a485c80 100644 --- a/system-tests/lib/protocolStub.ts +++ b/system-tests/lib/protocolStub.ts @@ -1,25 +1,6 @@ import type { ProtocolManagerShape } from '@packages/types' -declare const Debug: (namespace) => import('debug').IDebugger -declare const performance: { - now(): number - timeOrigin: number -} -declare const createHash: { - (text: string): string -} - export class AppCaptureProtocol implements ProtocolManagerShape { - private Debug: typeof Debug - private performance: typeof performance - private createHash: typeof createHash - - constructor () { - this.Debug = Debug - this.performance = performance - this.createHash = createHash - } - protocolEnabled: boolean setupProtocol = (script, runId) => { diff --git a/system-tests/lib/protocolStubResponse.ts b/system-tests/lib/protocolStubResponse.ts index a2ca44e83363..f35de829bfb2 100644 --- a/system-tests/lib/protocolStubResponse.ts +++ b/system-tests/lib/protocolStubResponse.ts @@ -8,17 +8,37 @@ export const SYSTEM_TESTS_PRIVATE = 'LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV export const TEST_PRIVATE = crypto.createPrivateKey(Buffer.from(SYSTEM_TESTS_PRIVATE, 'base64').toString('utf8')) -const { outputFiles: [{ contents: stubProtocolRaw }] } = esbuild.buildSync({ - entryPoints: [path.join(__dirname, 'protocolStub.ts')], - bundle: true, - format: 'cjs', - write: false, -}) +const buildStub = (filename: string): string => { + const { outputFiles: [{ contents }] } = esbuild.buildSync({ + entryPoints: [path.join(__dirname, filename)], + bundle: true, + format: 'cjs', + write: false, + }) -export const CYPRESS_LOCAL_PROTOCOL_STUB = new TextDecoder('utf-8').decode(stubProtocolRaw) + return new TextDecoder('utf-8').decode(contents) +} + +const hash = (stub: string): string => { + return base64Url.fromBase64(crypto.createHash('SHA256').update(stub).digest('base64')) +} + +const sign = (stub: string): string => { + return base64Url.fromBase64(crypto.createSign('SHA256').update(stub).sign(TEST_PRIVATE, 'base64')) +} + +export const CYPRESS_LOCAL_PROTOCOL_STUB = buildStub('protocolStub.ts') export const CYPRESS_LOCAL_PROTOCOL_STUB_COMPRESSED = gzipSync(CYPRESS_LOCAL_PROTOCOL_STUB) -export const CYPRESS_LOCAL_PROTOCOL_STUB_HASH = base64Url.fromBase64(crypto.createHash('SHA256').update(CYPRESS_LOCAL_PROTOCOL_STUB).digest('base64')) +export const CYPRESS_LOCAL_PROTOCOL_STUB_HASH = hash(CYPRESS_LOCAL_PROTOCOL_STUB) + +export const CYPRESS_LOCAL_PROTOCOL_STUB_SIGN = sign(CYPRESS_LOCAL_PROTOCOL_STUB) + +export const CYPRESS_LOCAL_FAULTY_PROTOCOL_STUB = buildStub('protocolStubWithRuntimeErrors.ts') + +export const CYPRESS_LOCAL_FAULTY_PROTOCOL_STUB_COMPRESSED = gzipSync(CYPRESS_LOCAL_FAULTY_PROTOCOL_STUB) + +export const CYPRESS_LOCAL_FAULTY_PROTOCOL_STUB_HASH = hash(CYPRESS_LOCAL_FAULTY_PROTOCOL_STUB) -export const CYPRESS_LOCAL_PROTOCOL_STUB_SIGN = base64Url.fromBase64(crypto.createSign('SHA256').update(CYPRESS_LOCAL_PROTOCOL_STUB).sign(TEST_PRIVATE, 'base64')) +export const CYPRESS_LOCAL_FAULTY_PROTOCOL_STUB_SIGN = sign(CYPRESS_LOCAL_FAULTY_PROTOCOL_STUB) diff --git a/system-tests/lib/protocolStubWithRuntimeErrors.ts b/system-tests/lib/protocolStubWithRuntimeErrors.ts new file mode 100644 index 000000000000..1e5c54fe9a99 --- /dev/null +++ b/system-tests/lib/protocolStubWithRuntimeErrors.ts @@ -0,0 +1,39 @@ +import type { ProtocolManagerShape } from '@packages/types' + +export class AppCaptureProtocol implements ProtocolManagerShape { + constructor () { + throw new Error() + } + + protocolEnabled: boolean + + setupProtocol = (script, runId) => { + return Promise.resolve() + } + connectToBrowser = (cdpClient) => { + return Promise.resolve() + } + addRunnables = (runnables) => {} + beforeSpec = (spec) => {} + afterSpec = () => { + return Promise.resolve() + } + beforeTest = (test) => { + return Promise.resolve() + } + commandLogAdded = (log) => {} + commandLogChanged = (log) => {} + viewportChanged = (input) => {} + urlChanged = (input) => {} + pageLoading = (input) => {} + resetTest (testId) {} + sendErrors (errors) { + return Promise.resolve() + } + uploadCaptureArtifact ({ uploadUrl }) { + return Promise.resolve() + } + afterTest = (test) => { + return Promise.resolve() + } +} diff --git a/system-tests/lib/serverStub.ts b/system-tests/lib/serverStub.ts index c27492afa21b..403dbd617b49 100644 --- a/system-tests/lib/serverStub.ts +++ b/system-tests/lib/serverStub.ts @@ -2,6 +2,7 @@ import crypto from 'crypto' import _ from 'lodash' import Bluebird from 'bluebird' import bodyParser from 'body-parser' +import Debug from 'debug' import type { RequestHandler } from 'express' import { getExample, assertSchema, RecordSchemaVersions } from './validations/cloudValidations' @@ -12,14 +13,21 @@ import base64Url from 'base64url' import systemTests from './system-tests' let CAPTURE_PROTOCOL_ENABLED = false +let CAPTURE_PROTOCOL_MESSAGE: string | undefined +let FAULTY_CAPTURE_PROTOCOL_ENABLED = false import { TEST_PRIVATE, CYPRESS_LOCAL_PROTOCOL_STUB_COMPRESSED, CYPRESS_LOCAL_PROTOCOL_STUB_HASH, CYPRESS_LOCAL_PROTOCOL_STUB_SIGN, + CYPRESS_LOCAL_FAULTY_PROTOCOL_STUB_SIGN, + CYPRESS_LOCAL_FAULTY_PROTOCOL_STUB_COMPRESSED, + CYPRESS_LOCAL_FAULTY_PROTOCOL_STUB_HASH, } from './protocolStubResponse' +const debug = Debug('cypress:system-tests:server-stub') + export const postRunResponseWithWarnings = getExample('createRun', 4, 'res') export const postRunInstanceResponse = getExample('createInstance', 5, 'res') @@ -29,6 +37,38 @@ export const postInstanceTestsResponse = getExample('postInstanceTests', 1, 'res postInstanceTestsResponse.actions = [] export const postRunResponse = _.assign({}, postRunResponseWithWarnings, { warnings: [] }) +// mocked here rather than attempting to intercept and mock an s3 req +export const CAPTURE_PROTOCOL_UPLOAD_URL = '/capture-protocol/upload/' + +export const postRunResponseWithProtocolEnabled = () => { + const hash = FAULTY_CAPTURE_PROTOCOL_ENABLED ? CYPRESS_LOCAL_FAULTY_PROTOCOL_STUB_HASH : CYPRESS_LOCAL_PROTOCOL_STUB_HASH + + debug('building postRunResponse', { + hash, + FAULTY_CAPTURE_PROTOCOL_ENABLED, + }) + + return { + ...postRunResponse, + captureProtocolUrl: `http://localhost:1234/capture-protocol/script/${hash}.js`, + capture: { + url: `http://localhost:1234/capture-protocol/script/${hash}.js`, + }, + } +} + +export const postRunResponseWithProtocolDisabled = () => { + return { + ...postRunResponse, + captureProtocolUrl: '', + + capture: { + url: '', + disabledMessage: CAPTURE_PROTOCOL_MESSAGE || postRunResponse.capture?.disabledMessage, + }, + } +} + type DeepPartial = { [P in keyof T]?: DeepPartial; }; @@ -54,6 +94,10 @@ const sendUploadUrls = function (req, res) { json.screenshotUploadUrls = screenshotUploadUrls + if (CAPTURE_PROTOCOL_ENABLED) { + json.captureUploadUrl = `http://localhost:1234${CAPTURE_PROTOCOL_UPLOAD_URL}` + } + return res.json(json) } const mockServerState = { @@ -111,16 +155,12 @@ export const routeHandlers: Record = { } mockServerState.setSpecs(req) - if (CAPTURE_PROTOCOL_ENABLED && req.body.runnerCapabilities.protocolMountVersion === 1) { - res.json({ - ...postRunResponse, - captureProtocolUrl: `http://localhost:1234/capture-protocol/script/${CYPRESS_LOCAL_PROTOCOL_STUB_HASH}.js`, - }) - return - } + const postRunResponseReturnVal = (CAPTURE_PROTOCOL_ENABLED && req.body.runnerCapabilities.protocolMountVersion === 1) ? + (postRunResponseWithProtocolEnabled()) : + (postRunResponseWithProtocolDisabled()) - return res.json(postRunResponse) + return res.json(postRunResponseReturnVal) }, }, postRunInstance: { @@ -158,7 +198,7 @@ export const routeHandlers: Record = { url: '/instances/:id/artifacts', // reqSchema: TODO(protocol): export this as part of manifest from cloud res: async (req, res) => { - res.status(200) + return res.sendStatus(200) }, }, putInstanceStdout: { @@ -190,8 +230,30 @@ export const routeHandlers: Record = { method: 'get', url: '/capture-protocol/script/*', res: async (req, res) => { - res.header('x-cypress-signature', CYPRESS_LOCAL_PROTOCOL_STUB_SIGN) - res.status(200).send(CYPRESS_LOCAL_PROTOCOL_STUB_COMPRESSED) + res.header('Content-Encoding', 'gzip') + if (FAULTY_CAPTURE_PROTOCOL_ENABLED) { + res.header('x-cypress-signature', CYPRESS_LOCAL_FAULTY_PROTOCOL_STUB_SIGN) + res.status(200).send(CYPRESS_LOCAL_FAULTY_PROTOCOL_STUB_COMPRESSED) + } else { + res.header('x-cypress-signature', CYPRESS_LOCAL_PROTOCOL_STUB_SIGN) + res.status(200).send(CYPRESS_LOCAL_PROTOCOL_STUB_COMPRESSED) + } + }, + }, + putCaptureProtocolUpload: { + method: 'put', + url: '/capture-protocol/upload', + res: (req, res) => { + return res.status(200).json({ + ok: true, + }) + }, + }, + postCaptureProtocolErrors: { + method: 'post', + url: '/capture-protocol/errors', + res: (req, res) => { + return res.status(200).send('') }, }, } @@ -372,10 +434,35 @@ const onServer = (routes) => { export const enableCaptureProtocol = () => { beforeEach(() => { CAPTURE_PROTOCOL_ENABLED = true + CAPTURE_PROTOCOL_MESSAGE = undefined }) afterEach(() => { CAPTURE_PROTOCOL_ENABLED = false + CAPTURE_PROTOCOL_MESSAGE = undefined + }) +} + +export const useFaultyCaptureProtocol = () => { + debug('setting tests to use faulty protocol stub') + beforeEach(() => { + debug('using faulty capture protocol') + FAULTY_CAPTURE_PROTOCOL_ENABLED = true + }) + + afterEach(() => { + FAULTY_CAPTURE_PROTOCOL_ENABLED = false + }) +} + +export const disableCaptureProtocolWithMessage = (message: string) => { + beforeEach(() => { + CAPTURE_PROTOCOL_ENABLED = false + CAPTURE_PROTOCOL_MESSAGE = message + }) + + afterEach(() => { + CAPTURE_PROTOCOL_MESSAGE = undefined }) } diff --git a/system-tests/lib/system-tests.ts b/system-tests/lib/system-tests.ts index 4ab0d4ee2327..108867438fca 100644 --- a/system-tests/lib/system-tests.ts +++ b/system-tests/lib/system-tests.ts @@ -629,7 +629,7 @@ const systemTests = { } _.defaults(options, { - browser: 'electron', + browser: process.env.SNAPSHOT_BROWSER || 'electron', headed: process.env.HEADED || false, project: 'e2e', timeout: 120000, @@ -809,10 +809,18 @@ const systemTests = { const specifiedBrowser = process.env.BROWSER const projectPath = Fixtures.projectPath(options.project) + if (process.env.SNAPSHOT_BROWSER) { + debug('setting browser to ', process.env.SNAPSHOT_BROWSER) + options.browser = options.browser || process.env.SNAPSHOT_BROWSER as BrowserName + debug(options.browser) + } + if (specifiedBrowser && (![].concat(options.browser).includes(specifiedBrowser))) { ctx.skip() } + debug(process.env.SNAPSHOT_BROWSER, options.browser) + if (!options.skipScaffold) { // symlinks won't work via docker options.dockerImage || await DepInstaller.scaffoldCommonNodeModules() diff --git a/system-tests/test/record_spec.js b/system-tests/test/record_spec.js index 26e7d41f8f9b..bfc980bc0a79 100644 --- a/system-tests/test/record_spec.js +++ b/system-tests/test/record_spec.js @@ -6,6 +6,9 @@ const dedent = require('dedent') const systemTests = require('../lib/system-tests').default const { fs } = require('@packages/server/lib/util/fs') +const { promises: fsPromise } = require('fs') +const os = require('os') + const Fixtures = require('../lib/fixtures') const { assertSchema } = require('../lib/validations/cloudValidations') const { @@ -19,9 +22,14 @@ const { postRunInstanceResponse, postInstanceTestsResponse, encryptBody, + disableCaptureProtocolWithMessage, + useFaultyCaptureProtocol, + CAPTURE_PROTOCOL_UPLOAD_URL, + postRunResponseWithProtocolDisabled, } = require('../lib/serverStub') const { expectRunsToHaveCorrectTimings } = require('../lib/resultsUtils') - +const { randomBytes } = require('crypto') +const debug = require('debug')('cypress:system-tests:record_spec') const e2ePath = Fixtures.projectPath('e2e') const outputPath = path.join(e2ePath, 'output.json') @@ -57,7 +65,7 @@ describe('e2e record', () => { const urls = getRequestUrls() const requests = getRequests() - const instanceReqs = urls.slice(0, 22) + const instanceReqs = urls.slice(0, 26) expect(instanceReqs).to.deep.eq([ // first create run request @@ -68,6 +76,7 @@ describe('e2e record', () => { // no instances/:id/tests because spec failed during eval `POST /instances/${instanceId}/results`, 'PUT /videos/video.mp4', + `PUT /instances/${instanceId}/artifacts`, `PUT /instances/${instanceId}/stdout`, // spec 2 @@ -76,6 +85,7 @@ describe('e2e record', () => { `POST /instances/${instanceId}/results`, 'PUT /videos/video.mp4', 'PUT /screenshots/1.png', + `PUT /instances/${instanceId}/artifacts`, `PUT /instances/${instanceId}/stdout`, // spec 3 @@ -84,6 +94,7 @@ describe('e2e record', () => { `POST /instances/${instanceId}/results`, // no video because no tests failed 'PUT /screenshots/1.png', + `PUT /instances/${instanceId}/artifacts`, `PUT /instances/${instanceId}/stdout`, // spec 4 @@ -92,6 +103,7 @@ describe('e2e record', () => { `POST /instances/${instanceId}/results`, 'PUT /videos/video.mp4', 'PUT /screenshots/1.png', + `PUT /instances/${instanceId}/artifacts`, `PUT /instances/${instanceId}/stdout`, ]) @@ -126,23 +138,23 @@ describe('e2e record', () => { expect(firstInstancePostResults.body.stats.failures).to.eq(1) expect(firstInstancePostResults.body.stats.passes).to.eq(0) - const firstInstanceStdout = requests[4] + const firstInstanceStdout = requests[5] expect(firstInstanceStdout.body.stdout).to.include('record_error.cy.js') - const secondInstance = requests[5] + const secondInstance = requests[6] expect(secondInstance.body.groupId).to.eq(groupId) expect(secondInstance.body.machineId).to.eq(machineId) expect(secondInstance.body.spec).to.eq(null) - const secondInstancePostTests = requests[6].body + const secondInstancePostTests = requests[7].body expect(secondInstancePostTests.tests).length(2) expect(secondInstancePostTests.hooks).length(1) expect(secondInstancePostTests.config).is.an('object') - const secondInstancePostResults = requests[7] + const secondInstancePostResults = requests[8] expect(secondInstancePostResults.body.exception).to.be.null expect(secondInstancePostResults.body.tests).to.have.length(2) @@ -154,25 +166,25 @@ describe('e2e record', () => { expect(secondInstancePostResults.body.hooks).not.exist expect(secondInstancePostResults.body.cypressConfig).not.exist - const secondInstanceStdout = requests[10] + const secondInstanceStdout = requests[12] expect(secondInstanceStdout.body.stdout).to.include('record_fail.cy.js') expect(secondInstanceStdout.body.stdout).not.to.include('record_error.cy.js') - const thirdInstance = requests[11] + const thirdInstance = requests[13] expect(thirdInstance.body.groupId).to.eq(groupId) expect(thirdInstance.body.machineId).to.eq(machineId) expect(thirdInstance.body.spec).to.eq(null) - const thirdInstancePostTests = requests[12].body + const thirdInstancePostTests = requests[14].body expect(thirdInstancePostTests.tests[0].config.env.foo).eq(true) expect(thirdInstancePostTests.tests).length(2) expect(thirdInstancePostTests.hooks).length(0) expect(thirdInstancePostTests.config).is.an('object') - const thirdInstancePostResults = requests[13] + const thirdInstancePostResults = requests[15] expect(thirdInstancePostResults.body.exception).to.be.null expect(thirdInstancePostResults.body.tests).to.have.length(2) @@ -182,7 +194,7 @@ describe('e2e record', () => { expect(thirdInstancePostResults.body.stats.failures).to.eq(0) expect(thirdInstancePostResults.body.stats.pending).to.eq(1) - const thirdInstanceStdout = requests[15] + const thirdInstanceStdout = requests[18] console.log('13') @@ -192,7 +204,7 @@ describe('e2e record', () => { expect(thirdInstanceStdout.body.stdout).to.include('plugin stdout') expect(thirdInstanceStdout.body.stdout).to.not.include('plugin stderr') - const fourthInstance = requests[16] + const fourthInstance = requests[19] console.log('14') @@ -200,7 +212,7 @@ describe('e2e record', () => { expect(fourthInstance.body.machineId).to.eq(machineId) expect(fourthInstance.body.spec).to.eq(null) - const fourthInstancePostResults = requests[18] + const fourthInstancePostResults = requests[21] console.log('15') @@ -211,7 +223,7 @@ describe('e2e record', () => { expect(fourthInstancePostResults.body.stats.failures).to.eq(1) expect(fourthInstancePostResults.body.stats.passes).to.eq(0) - const forthInstanceStdout = requests[21] + const forthInstanceStdout = requests[25] console.log('18') @@ -467,13 +479,13 @@ describe('e2e record', () => { `POST /runs/${runId}/instances`, `POST /instances/${instanceId}/tests`, `POST /instances/${instanceId}/results`, + `PUT /instances/${instanceId}/artifacts`, `PUT /instances/${instanceId}/stdout`, - `POST /runs/${runId}/instances`, `POST /instances/${instanceId}/tests`, `POST /instances/${instanceId}/results`, + `PUT /instances/${instanceId}/artifacts`, `PUT /instances/${instanceId}/stdout`, - `POST /runs/${runId}/instances`, ]) }) @@ -638,7 +650,7 @@ describe('e2e record', () => { mockServerState.specs = req.body.specs.slice().reverse() console.log(mockServerState.specs) mockServerState.allSpecs = req.body.specs - res.json(postRunResponse) + res.json(postRunResponseWithProtocolDisabled()) }, }, })) @@ -656,7 +668,7 @@ describe('e2e record', () => { // specs were reordered expect(requests[2].body.tests[0].title[1]).eq('b test') - expect(requests[6].body.tests[0].title[1]).eq('a test') + expect(requests[7].body.tests[0].title[1]).eq('a test') }) }) }) @@ -707,6 +719,7 @@ describe('e2e record', () => { 'POST /runs/00748421-e035-4a3d-8604-8468cc48bdb5/instances', 'POST /instances/e9e81b5e-cc58-4026-b2ff-8ae3161435a6/tests', 'POST /instances/e9e81b5e-cc58-4026-b2ff-8ae3161435a6/results', + 'PUT /instances/e9e81b5e-cc58-4026-b2ff-8ae3161435a6/artifacts', 'PUT /instances/e9e81b5e-cc58-4026-b2ff-8ae3161435a6/stdout', 'POST /runs/00748421-e035-4a3d-8604-8468cc48bdb5/instances', ]) @@ -739,6 +752,7 @@ describe('e2e record', () => { 'POST /runs/00748421-e035-4a3d-8604-8468cc48bdb5/instances', 'POST /instances/e9e81b5e-cc58-4026-b2ff-8ae3161435a6/tests', 'POST /instances/e9e81b5e-cc58-4026-b2ff-8ae3161435a6/results', + 'PUT /instances/e9e81b5e-cc58-4026-b2ff-8ae3161435a6/artifacts', 'PUT /instances/e9e81b5e-cc58-4026-b2ff-8ae3161435a6/stdout', 'POST /runs/00748421-e035-4a3d-8604-8468cc48bdb5/instances', ]) @@ -1451,6 +1465,7 @@ describe('e2e record', () => { `POST /instances/${instanceId}/tests`, `POST /instances/${instanceId}/results`, 'PUT /screenshots/1.png', + `PUT /instances/${instanceId}/artifacts`, `PUT /instances/${instanceId}/stdout`, `POST /runs/${runId}/instances`, ]) @@ -1492,14 +1507,16 @@ describe('e2e record', () => { .then(() => { const urls = getRequestUrls() - expect(urls).to.have.members([ + expect(urls).to.deep.eq([ 'POST /runs', `POST /runs/${runId}/instances`, `POST /instances/${instanceId}/tests`, `POST /instances/${instanceId}/results`, - 'PUT /videos/video.mp4', 'PUT /screenshots/1.png', + 'PUT /videos/video.mp4', + `PUT /instances/${instanceId}/artifacts`, `PUT /instances/${instanceId}/stdout`, + `POST /runs/${runId}/instances`, ]) }) }) @@ -1514,7 +1531,7 @@ describe('e2e record', () => { count += 1 if (count === 4) { - return res.json(postRunResponse) + return res.json(postRunResponseWithProtocolDisabled()) } return res.sendStatus(500) @@ -1579,6 +1596,7 @@ describe('e2e record', () => { 'POST /instances/e9e81b5e-cc58-4026-b2ff-8ae3161435a6/tests', 'POST /instances/e9e81b5e-cc58-4026-b2ff-8ae3161435a6/results', 'PUT /screenshots/1.png', + 'PUT /instances/e9e81b5e-cc58-4026-b2ff-8ae3161435a6/artifacts', 'PUT /instances/e9e81b5e-cc58-4026-b2ff-8ae3161435a6/stdout', 'POST /runs/00748421-e035-4a3d-8604-8468cc48bdb5/instances', ]) @@ -2234,10 +2252,11 @@ describe('e2e record', () => { describe('capture-protocol', () => { setupStubbedServer(createRoutes()) - enableCaptureProtocol() - describe('passing', () => { - it('retrieves the capture protocol', function () { + describe('disabled messaging', () => { + disableCaptureProtocolWithMessage('Test Replay is only supported in Chromium browsers') + + it('displays disabled message but continues', function () { return systemTests.exec(this, { key: 'f858a2bc-b469-4e48-be67-0876339ee7e1', configFile: 'cypress-with-project-id.config.js', @@ -2247,5 +2266,147 @@ describe('e2e record', () => { }) }) }) + + describe('enabled', () => { + let dbFile = '' + + beforeEach(async () => { + const dbPath = path.join(os.tmpdir(), 'cypress', 'protocol') + + dbFile = path.join(dbPath, `${instanceId}.db`) + + await fsPromise.mkdir(dbPath, { recursive: true }) + + debug('writing db to', dbFile) + + return fsPromise.writeFile(dbFile, randomBytes(128)) + }) + + describe('passing', () => { + enableCaptureProtocol() + it('retrieves the capture protocol and uploads the db', function () { + return systemTests.exec(this, { + key: 'f858a2bc-b469-4e48-be67-0876339ee7e1', + configFile: 'cypress-with-project-id.config.js', + spec: 'record_pass*', + record: true, + snapshot: true, + }).then((ret) => { + const urls = getRequestUrls() + + expect(urls).to.include.members([`PUT ${CAPTURE_PROTOCOL_UPLOAD_URL}`]) + }) + }) + }) + + describe('protocol errors', () => { + enableCaptureProtocol() + describe('db size too large', () => { + beforeEach(() => { + return fsPromise.writeFile(dbFile, randomBytes(1024)) + }) + + afterEach(async () => { + if (fs.existsSync(dbFile)) { + return fsPromise.rm(dbFile) + } + }) + + it('displays error and does not upload if db size is too large', function () { + // have to write the db to fs here instead of in the t + return systemTests.exec(this, { + key: 'f858a2bc-b469-4e48-be67-0876339ee7e1', + configFile: 'cypress-with-project-id.config.js', + spec: 'record_pass*', + record: true, + snapshot: true, + }).then(() => { + const urls = getRequestUrls() + + expect(urls).not.to.include.members([`PUT ${CAPTURE_PROTOCOL_UPLOAD_URL}`]) + }) + }) + }) + + describe('error initializing protocol', () => { + useFaultyCaptureProtocol() + + it('displays the error and reports to cloud', function () { + return systemTests.exec(this, { + key: 'f858a2bc-b469-4e48-be67-0876339ee7e1', + configFile: 'cypress-with-project-id.config.js', + spec: 'record_pass*', + record: true, + snapshot: true, + }).then(() => { + const urls = getRequestUrls() + + debug(urls) + expect(urls).to.include.members(['POST /capture-protocol/errors']) + expect(urls).not.to.include.members([`PUT ${CAPTURE_PROTOCOL_UPLOAD_URL}`]) + }) + }) + }) + }) + }) + }) +}) + +describe('capture-protocol api errors', () => { + enableCaptureProtocol() + + const stubbedServerWithErrorOn = (endpoint) => { + return setupStubbedServer(createRoutes({ + [endpoint]: { + res: (req, res) => { + res.status(500).send() + }, + }, + })) + } + + describe('upload 500', () => { + stubbedServerWithErrorOn('putCaptureProtocolUpload') + it('continues', function () { + process.env.API_RETRY_INTERVALS = '1000' + + return systemTests.exec(this, { + key: 'f858a2bc-b469-4e48-be67-0876339ee7e1', + configFile: 'cypress-with-project-id.config.js', + spec: 'record_pass*', + record: true, + snapshot: true, + }) + }) + }) + + describe('fetch script 500', () => { + stubbedServerWithErrorOn('getCaptureScript') + it('continues', function () { + process.env.API_RETRY_INTERVALS = '1000' + + return systemTests.exec(this, { + key: 'f858a2bc-b469-4e48-be67-0876339ee7e1', + configFile: 'cypress-with-project-id.config.js', + spec: 'record_pass*', + record: true, + snapshot: true, + }) + }) + }) + + describe('error report 500', () => { + stubbedServerWithErrorOn('postCaptureProtocolErrors') + it('continues', function () { + process.env.API_RETRY_INTERVALS = '1000' + + return systemTests.exec(this, { + key: 'f858a2bc-b469-4e48-be67-0876339ee7e1', + configFile: 'cypress-with-project-id.config.js', + spec: 'record_pass*', + record: true, + snapshot: true, + }) + }) }) })