diff --git a/.buildkite/ftr_configs.yml b/.buildkite/ftr_configs.yml index b890e8d45cc9c7..ca2dcd500029f9 100644 --- a/.buildkite/ftr_configs.yml +++ b/.buildkite/ftr_configs.yml @@ -29,6 +29,9 @@ disabled: - x-pack/test/defend_workflows_cypress/config.ts - x-pack/test/defend_workflows_cypress/endpoint_config.ts - x-pack/test/defend_workflows_cypress/visual_config.ts + - x-pack/plugins/observability_onboarding/e2e/ftr_config_open.ts + - x-pack/plugins/observability_onboarding/e2e/ftr_config_runner.ts + - x-pack/plugins/observability_onboarding/e2e/ftr_config.ts - x-pack/test/osquery_cypress/cli_config.ts - x-pack/test/osquery_cypress/config.ts - x-pack/test/osquery_cypress/visual_config.ts diff --git a/.buildkite/pipelines/pull_request/observability_onboarding_cypress.yml b/.buildkite/pipelines/pull_request/observability_onboarding_cypress.yml new file mode 100644 index 00000000000000..e5a1b6a445fd28 --- /dev/null +++ b/.buildkite/pipelines/pull_request/observability_onboarding_cypress.yml @@ -0,0 +1,14 @@ +steps: + - command: .buildkite/scripts/steps/functional/observability_onboarding_cypress.sh + label: 'Observability onboarding Cypress Tests' + agents: + queue: n2-4-spot + depends_on: build + timeout_in_minutes: 120 + parallelism: 2 + retry: + automatic: + - exit_status: '-1' + limit: 3 + - exit_status: '*' + limit: 1 diff --git a/.buildkite/scripts/steps/functional/observability_onboarding_cypress.sh b/.buildkite/scripts/steps/functional/observability_onboarding_cypress.sh new file mode 100644 index 00000000000000..1155cfdae07f7a --- /dev/null +++ b/.buildkite/scripts/steps/functional/observability_onboarding_cypress.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/scripts/common/util.sh + +.buildkite/scripts/bootstrap.sh +.buildkite/scripts/download_build_artifacts.sh + +export JOB=kibana-observability-onboarding-cypress + +echo "--- Observability onboarding Cypress Tests" + +cd "$XPACK_DIR" + +node plugins/observability_onboarding/scripts/test/e2e.js \ + --kibana-install-dir "$KIBANA_BUILD_LOCATION" \ \ No newline at end of file diff --git a/x-pack/plugins/observability_onboarding/e2e/README.md b/x-pack/plugins/observability_onboarding/e2e/README.md new file mode 100644 index 00000000000000..4d3e15318c64fb --- /dev/null +++ b/x-pack/plugins/observability_onboarding/e2e/README.md @@ -0,0 +1,51 @@ +# Observability onboarding E2E tests + +Observability onboarding uses [FTR](../../../../packages/kbn-test/README.mdx) (functional test runner) and [Cypress](https://www.cypress.io/) to run the e2e tests. The tests are located at `kibana/x-pack/plugins/observability_onboarding/e2e/cypress/e2e`. + +## E2E Tests (Cypress) + +The E2E tests are located in [`x-pack/plugins/observability_onboarding/e2e`](./cypress/e2e). + +Tests run on buildkite PR pipeline are parallelized (2 parallel jobs) and are orchestrated by the Cypress dashboard service. It can be configured in [.buildkite/pipelines/pull_request/observability_onboarding_cypress.yml](https://github.com/elastic/kibana/blob/main/.buildkite/pipelines/pull_request/observability_onboarding_cypress.yml) with the property `parallelism`. + +```yml + ... + depends_on: build + parallelism: 2 + ... +``` + +## Running it locally + +### Start test server + +``` +node x-pack/plugins/observability_onboarding/scripts/test/e2e --server +``` + +### Run tests +Runs all tests in the terminal + +``` +node x-pack/plugins/observability_onboarding/scripts/test/e2e --runner +``` + +### Open cypress dashboard +Opens cypress dashboard, there it's possible to select what test you want to run. + +``` +node x-pack/plugins/observability_onboarding/scripts/test/e2e --open +``` +### Arguments + +| Option | Description | +| ------------ | ----------------------------------------------- | +| --server | Only start ES and Kibana | +| --runner | Only run tests | +| --spec | Specify the specs to run | +| --times | Repeat the test n number of times | +| --bail | stop tests after the first failure | + +``` +node x-pack/plugins/observability_onboarding/scripts/test/e2e.js --runner --spec cypress/e2e/home.cy.ts --times 2 +``` \ No newline at end of file diff --git a/x-pack/plugins/observability_onboarding/e2e/cypress.config.ts b/x-pack/plugins/observability_onboarding/e2e/cypress.config.ts new file mode 100644 index 00000000000000..93a2a543528c3f --- /dev/null +++ b/x-pack/plugins/observability_onboarding/e2e/cypress.config.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { defineCypressConfig } from '@kbn/cypress-config'; + +export default defineCypressConfig({ + fileServerFolder: './cypress', + fixturesFolder: './cypress/fixtures', + screenshotsFolder: './cypress/screenshots', + videosFolder: './cypress/videos', + requestTimeout: 10000, + responseTimeout: 40000, + defaultCommandTimeout: 30000, + execTimeout: 120000, + pageLoadTimeout: 120000, + viewportHeight: 1800, + viewportWidth: 1440, + video: false, + videoUploadOnPasses: false, + screenshotOnRunFailure: false, + retries: { + runMode: 1, + }, + e2e: { + baseUrl: 'http://localhost:5601', + supportFile: './cypress/support/e2e.ts', + specPattern: './cypress/e2e/**/*.cy.{js,jsx,ts,tsx}', + }, +}); diff --git a/x-pack/plugins/observability_onboarding/e2e/cypress/e2e/home.cy.ts b/x-pack/plugins/observability_onboarding/e2e/cypress/e2e/home.cy.ts new file mode 100644 index 00000000000000..8f65ec2e721f5a --- /dev/null +++ b/x-pack/plugins/observability_onboarding/e2e/cypress/e2e/home.cy.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +describe('[Observability onboarding] Landing page', () => { + beforeEach(() => { + cy.loginAsElastic(); + }); + + it('shows landing page', () => { + cy.visitKibana('/app/observabilityOnboarding'); + cy.contains('Get started with Observability'); + }); +}); diff --git a/x-pack/plugins/observability_onboarding/e2e/cypress/support/commands.ts b/x-pack/plugins/observability_onboarding/e2e/cypress/support/commands.ts new file mode 100644 index 00000000000000..299729f1d8fa7b --- /dev/null +++ b/x-pack/plugins/observability_onboarding/e2e/cypress/support/commands.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import URL from 'url'; + +Cypress.Commands.add( + 'loginAs', + ({ username, password }: { username: string; password: string }) => { + const kibanaUrl = Cypress.env('KIBANA_URL'); + cy.log(`Logging in as ${username} on ${kibanaUrl}`); + cy.visit('/'); + cy.request({ + log: true, + method: 'POST', + url: `${kibanaUrl}/internal/security/login`, + body: { + providerType: 'basic', + providerName: 'basic', + currentURL: `${kibanaUrl}/login`, + params: { username, password }, + }, + headers: { + 'kbn-xsrf': 'e2e_test', + }, + }); + cy.visit('/'); + } +); + +Cypress.Commands.add('loginAsElastic', () => { + return cy.loginAs({ + username: 'elastic', + password: 'changeme', + }); +}); + +Cypress.Commands.add('getByTestSubj', (selector: string) => { + return cy.get(`[data-test-subj="${selector}"]`); +}); + +Cypress.Commands.add( + 'visitKibana', + (url: string, rangeFrom?: string, rangeTo?: string) => { + const urlPath = URL.format({ + pathname: url, + query: { rangeFrom, rangeTo }, + }); + + cy.visit(urlPath); + cy.getByTestSubj('kbnLoadingMessage').should('exist'); + cy.getByTestSubj('kbnLoadingMessage').should('not.exist', { + timeout: 50000, + }); + } +); diff --git a/x-pack/plugins/observability_onboarding/e2e/cypress/support/e2e.ts b/x-pack/plugins/observability_onboarding/e2e/cypress/support/e2e.ts new file mode 100644 index 00000000000000..5f5d1eb3b36146 --- /dev/null +++ b/x-pack/plugins/observability_onboarding/e2e/cypress/support/e2e.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +Cypress.on('uncaught:exception', (err, runnable) => { + return false; +}); + +import './commands'; diff --git a/x-pack/plugins/observability_onboarding/e2e/cypress/support/types.d.ts b/x-pack/plugins/observability_onboarding/e2e/cypress/support/types.d.ts new file mode 100644 index 00000000000000..8db14ad70561ad --- /dev/null +++ b/x-pack/plugins/observability_onboarding/e2e/cypress/support/types.d.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +declare namespace Cypress { + interface Chainable { + loginAs(params: { + username: string; + password: string; + }): Cypress.Chainable>; + loginAsElastic(): Cypress.Chainable>; + getByTestSubj(selector: string): Chainable>; + visitKibana(url: string, rangeFrom?: string, rangeTo?: string): void; + } +} diff --git a/x-pack/plugins/observability_onboarding/e2e/cypress_test_runner.ts b/x-pack/plugins/observability_onboarding/e2e/cypress_test_runner.ts new file mode 100644 index 00000000000000..1d056897a6d026 --- /dev/null +++ b/x-pack/plugins/observability_onboarding/e2e/cypress_test_runner.ts @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import cypress from 'cypress'; +import path from 'path'; +import Url from 'url'; +import { FtrProviderContext } from './ftr_provider_context'; + +export async function cypressTestRunner({ + ftrProviderContext: { getService }, + cypressExecution, +}: { + ftrProviderContext: FtrProviderContext; + cypressExecution: typeof cypress.run | typeof cypress.open; +}) { + const config = getService('config'); + + const username = config.get('servers.elasticsearch.username'); + const password = config.get('servers.elasticsearch.password'); + + const esNode = Url.format({ + protocol: config.get('servers.elasticsearch.protocol'), + port: config.get('servers.elasticsearch.port'), + hostname: config.get('servers.elasticsearch.hostname'), + auth: `${username}:${password}`, + }); + + const esRequestTimeout = config.get('timeouts.esRequestTimeout'); + + const kibanaUrlWithoutAuth = Url.format({ + protocol: config.get('servers.kibana.protocol'), + hostname: config.get('servers.kibana.hostname'), + port: config.get('servers.kibana.port'), + }); + + const cypressProjectPath = path.join(__dirname); + const { open, ...cypressCliArgs } = getCypressCliArgs(); + + const res = await cypressExecution({ + ...cypressCliArgs, + project: cypressProjectPath, + config: { + e2e: { + baseUrl: kibanaUrlWithoutAuth, + }, + }, + env: { + KIBANA_URL: kibanaUrlWithoutAuth, + ES_NODE: esNode, + ES_REQUEST_TIMEOUT: esRequestTimeout, + TEST_CLOUD: process.env.TEST_CLOUD, + }, + }); + + return res; +} + +function getCypressCliArgs(): Record { + if (!process.env.CYPRESS_CLI_ARGS) { + return {}; + } + + const { $0, _, ...cypressCliArgs } = JSON.parse( + process.env.CYPRESS_CLI_ARGS + ) as Record; + + const spec = + typeof cypressCliArgs.spec === 'string' && + !cypressCliArgs.spec.includes('**') + ? `**/${cypressCliArgs.spec}*` + : cypressCliArgs.spec; + + return { ...cypressCliArgs, spec }; +} diff --git a/x-pack/plugins/observability_onboarding/e2e/ftr_config.ts b/x-pack/plugins/observability_onboarding/e2e/ftr_config.ts new file mode 100644 index 00000000000000..c042c085b37d03 --- /dev/null +++ b/x-pack/plugins/observability_onboarding/e2e/ftr_config.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrConfigProviderContext } from '@kbn/test'; +import { CA_CERT_PATH } from '@kbn/dev-utils'; +import path from 'path'; + +const kibanaYamlFilePath = path.join(__dirname, './ftr_kibana.yml'); + +async function ftrConfig({ readConfigFile }: FtrConfigProviderContext) { + const kibanaCommonTestsConfig = await readConfigFile( + require.resolve('../../../../test/common/config.js') + ); + const xpackFunctionalTestsConfig = await readConfigFile( + require.resolve('../../../test/functional/config.base.js') + ); + + return { + ...kibanaCommonTestsConfig.getAll(), + + esTestCluster: { + ...xpackFunctionalTestsConfig.get('esTestCluster'), + serverArgs: [ + ...xpackFunctionalTestsConfig.get('esTestCluster.serverArgs'), + // define custom es server here + // API Keys is enabled at the top level + 'xpack.security.enabled=true', + ], + }, + + kbnTestServer: { + ...xpackFunctionalTestsConfig.get('kbnTestServer'), + serverArgs: [ + ...xpackFunctionalTestsConfig.get('kbnTestServer.serverArgs'), + '--home.disableWelcomeScreen=true', + '--csp.strict=false', + '--csp.warnLegacyBrowsers=false', + // define custom kibana server args here + `--elasticsearch.ssl.certificateAuthorities=${CA_CERT_PATH}`, + `--config=${kibanaYamlFilePath}`, + ], + }, + }; +} + +// eslint-disable-next-line import/no-default-export +export default ftrConfig; diff --git a/x-pack/plugins/observability_onboarding/e2e/ftr_config_open.ts b/x-pack/plugins/observability_onboarding/e2e/ftr_config_open.ts new file mode 100644 index 00000000000000..18508abf27f0cc --- /dev/null +++ b/x-pack/plugins/observability_onboarding/e2e/ftr_config_open.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrConfigProviderContext } from '@kbn/test'; +import cypress from 'cypress'; +import { FtrProviderContext } from './ftr_provider_context'; +import { cypressTestRunner } from './cypress_test_runner'; + +async function ftrConfigOpen({ readConfigFile }: FtrConfigProviderContext) { + const kibanaConfig = await readConfigFile(require.resolve('./ftr_config.ts')); + return { + ...kibanaConfig.getAll(), + testRunner, + }; +} + +export async function testRunner(ftrProviderContext: FtrProviderContext) { + await cypressTestRunner({ + ftrProviderContext, + cypressExecution: cypress.open, + }); +} + +// eslint-disable-next-line import/no-default-export +export default ftrConfigOpen; diff --git a/x-pack/plugins/observability_onboarding/e2e/ftr_config_runner.ts b/x-pack/plugins/observability_onboarding/e2e/ftr_config_runner.ts new file mode 100644 index 00000000000000..9ce0abaeed231f --- /dev/null +++ b/x-pack/plugins/observability_onboarding/e2e/ftr_config_runner.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrConfigProviderContext } from '@kbn/test'; +import cypress from 'cypress'; +import { cypressTestRunner } from './cypress_test_runner'; +import { FtrProviderContext } from './ftr_provider_context'; + +async function ftrConfigRun({ readConfigFile }: FtrConfigProviderContext) { + const kibanaConfig = await readConfigFile(require.resolve('./ftr_config.ts')); + + return { + ...kibanaConfig.getAll(), + testRunner, + }; +} + +async function testRunner(ftrProviderContext: FtrProviderContext) { + const result = await cypressTestRunner({ + ftrProviderContext, + cypressExecution: cypress.run, + }); + + if (result && (result.status === 'failed' || result.totalFailed > 0)) { + process.exit(1); + } +} + +// eslint-disable-next-line import/no-default-export +export default ftrConfigRun; diff --git a/x-pack/plugins/observability_onboarding/e2e/ftr_kibana.yml b/x-pack/plugins/observability_onboarding/e2e/ftr_kibana.yml new file mode 100644 index 00000000000000..640dca6564d009 --- /dev/null +++ b/x-pack/plugins/observability_onboarding/e2e/ftr_kibana.yml @@ -0,0 +1,3 @@ +xpack.observability_onboarding.ui.enabled: true + +xpack.cloud.id: 'foo' diff --git a/x-pack/plugins/observability_onboarding/e2e/ftr_provider_context.d.ts b/x-pack/plugins/observability_onboarding/e2e/ftr_provider_context.d.ts new file mode 100644 index 00000000000000..b87f35adcccf2f --- /dev/null +++ b/x-pack/plugins/observability_onboarding/e2e/ftr_provider_context.d.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { GenericFtrProviderContext } from '@kbn/test'; + +export type FtrProviderContext = GenericFtrProviderContext<{}, {}>; diff --git a/x-pack/plugins/observability_onboarding/e2e/tsconfig.json b/x-pack/plugins/observability_onboarding/e2e/tsconfig.json new file mode 100644 index 00000000000000..c4de3fff85cfd9 --- /dev/null +++ b/x-pack/plugins/observability_onboarding/e2e/tsconfig.json @@ -0,0 +1,26 @@ +{ + "extends": "../../../../tsconfig.base.json", + "include": [ + "**/*" + ], + "exclude": [ + "tmp", + "target/**/*" + ], + "compilerOptions": { + "outDir": "target/types", + "types": [ + "cypress", + "node", + "cypress-real-events" + ], + "isolatedModules": false + }, + "kbn_references": [ + { "path": "../../../test/tsconfig.json" }, + { "path": "../../../../test/tsconfig.json" }, + "@kbn/test", + "@kbn/dev-utils", + "@kbn/cypress-config", + ] +} diff --git a/x-pack/plugins/observability_onboarding/scripts/test/e2e.js b/x-pack/plugins/observability_onboarding/scripts/test/e2e.js new file mode 100644 index 00000000000000..d5cd56175d2239 --- /dev/null +++ b/x-pack/plugins/observability_onboarding/scripts/test/e2e.js @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/* eslint-disable no-console */ +const { times } = require('lodash'); +const path = require('path'); +const yargs = require('yargs'); +const childProcess = require('child_process'); + +const { argv } = yargs(process.argv.slice(2)) + .parserConfiguration({ 'unknown-options-as-args': true }) + .option('kibana-install-dir', { + default: '', + type: 'string', + description: 'Path to the Kibana install directory', + }) + .option('server', { + default: false, + type: 'boolean', + description: 'Start Elasticsearch and Kibana', + }) + .option('runner', { + default: false, + type: 'boolean', + description: + 'Run all tests (an instance of Elasticsearch and kibana are needs to be available)', + }) + .option('open', { + default: false, + type: 'boolean', + description: + 'Open cypress dashboard (an instance of Elasticsearch and kibana are needs to be available)', + }) + .option('times', { + type: 'number', + description: 'Repeat the test n number of times', + }) + .option('bail', { + default: false, + type: 'boolean', + description: 'stop tests after the first failure', + }) + .help(); + +const e2eDir = path.join(__dirname, '../../e2e'); + +let ftrScript = 'functional_tests.js'; +if (argv.server) { + ftrScript = 'functional_tests_server.js'; +} else if (argv.runner || argv.open) { + ftrScript = 'functional_test_runner.js'; +} + +const cypressCliArgs = yargs(argv._).parserConfiguration({ + 'boolean-negation': false, +}).argv; + +if (cypressCliArgs.grep) { + throw new Error('--grep is not supported. Please use --spec instead'); +} + +const ftrConfig = argv.open ? './ftr_config_open.ts' : './ftr_config_runner.ts'; +const spawnArgs = [ + `../../../../scripts/${ftrScript}`, + `--config=${ftrConfig}`, + `--kibana-install-dir=${argv.kibanaInstallDir}`, + ...(argv.bail ? [`--bail`] : []), +]; + +function runTests() { + console.log(`Running e2e tests: "node ${spawnArgs.join(' ')}"`); + + return childProcess.spawnSync('node', spawnArgs, { + cwd: e2eDir, + env: { ...process.env, CYPRESS_CLI_ARGS: JSON.stringify(cypressCliArgs) }, + encoding: 'utf8', + stdio: 'inherit', + }); +} + +const runCounter = { succeeded: 0, failed: 0, remaining: argv.times }; +let exitStatus = 0; +times(argv.times ?? 1, () => { + const child = runTests(); + if (child.status === 0) { + runCounter.succeeded++; + } else { + exitStatus = child.status; + runCounter.failed++; + } + + runCounter.remaining--; + + if (argv.times > 1) { + console.log(runCounter); + } +}); + +process.exitCode = exitStatus; +console.log(`Quitting with exit code ${exitStatus}`);