From d6f76faee00d5c53182918ed03ac0a03e32cb022 Mon Sep 17 00:00:00 2001 From: spalger Date: Tue, 1 Mar 2022 00:40:56 +0000 Subject: [PATCH] [ftr] implement support for accessing ES through CCS --- packages/kbn-test/BUILD.bazel | 1 + .../kbn-test/src/es/es_client_for_testing.ts | 21 ++++ packages/kbn-test/src/es/index.ts | 6 +- packages/kbn-test/src/es/test_es_cluster.ts | 11 +- .../lib/config/schema.ts | 8 +- .../functional_tests/lib/run_elasticsearch.ts | 109 +++++++++++++++--- .../kbn-test/src/functional_tests/tasks.ts | 12 +- packages/kbn-test/src/index.ts | 1 + scripts/functional_tests.js | 2 +- test/common/services/es_archiver.ts | 5 +- test/common/services/security/test_user.ts | 22 ++++ .../kbn_archiver/date_nested_ccs.json | 4 +- .../fixtures/kbn_archiver/discover_ccs.json | 6 +- .../{_data_view_ccs.ts => data_view_ccs.ts} | 10 +- test/functional_ccs/apps/discover/index.ts | 30 ++--- ...ed_queries_ccs.ts => saved_queries_ccs.ts} | 6 +- test/functional_ccs/config.js | 25 ---- test/functional_ccs/config.ts | 48 ++++++++ ...r_context.d.ts => ftr_provider_context.ts} | 4 +- test/functional_ccs/services/ccs_remote_es.ts | 24 ++++ test/functional_ccs/services/index.ts | 15 +++ 21 files changed, 279 insertions(+), 91 deletions(-) rename test/functional_ccs/apps/discover/{_data_view_ccs.ts => data_view_ccs.ts} (91%) rename test/functional_ccs/apps/discover/{_saved_queries_ccs.ts => saved_queries_ccs.ts} (97%) delete mode 100644 test/functional_ccs/config.js create mode 100644 test/functional_ccs/config.ts rename test/functional_ccs/{apps/discover/ftr_provider_context.d.ts => ftr_provider_context.ts} (80%) create mode 100644 test/functional_ccs/services/ccs_remote_es.ts create mode 100644 test/functional_ccs/services/index.ts diff --git a/packages/kbn-test/BUILD.bazel b/packages/kbn-test/BUILD.bazel index 6732b08d8bc72af..4e9458823dd3c22 100644 --- a/packages/kbn-test/BUILD.bazel +++ b/packages/kbn-test/BUILD.bazel @@ -53,6 +53,7 @@ RUNTIME_DEPS = [ "@npm//execa", "@npm//exit-hook", "@npm//form-data", + "@npm//get-port", "@npm//getopts", "@npm//globby", "@npm//he", diff --git a/packages/kbn-test/src/es/es_client_for_testing.ts b/packages/kbn-test/src/es/es_client_for_testing.ts index 084cb8d77eac578..d0ecfe008c965e6 100644 --- a/packages/kbn-test/src/es/es_client_for_testing.ts +++ b/packages/kbn-test/src/es/es_client_for_testing.ts @@ -27,6 +27,27 @@ export interface EsClientForTestingOptions extends Omit +) { + const ccsConfig = config.get('esTestCluster.ccs'); + if (!ccsConfig) { + throw new Error('FTR config is missing esTestCluster.ccs'); + } + + const esUrl = Url.format({ + ...config.get('servers.elasticsearch'), + port: ccsConfig.remoteClusterPort, + }); + + return createEsClientForTesting({ + esUrl, + requestTimeout: config.get('timeouts.esRequestTimeout'), + ...overrides, + }); +} + export function createEsClientForFtrConfig( config: Config, overrides?: Omit diff --git a/packages/kbn-test/src/es/index.ts b/packages/kbn-test/src/es/index.ts index 641253acc3647e5..bdc338894582795 100644 --- a/packages/kbn-test/src/es/index.ts +++ b/packages/kbn-test/src/es/index.ts @@ -9,5 +9,9 @@ export { createTestEsCluster } from './test_es_cluster'; export type { CreateTestEsClusterOptions, EsTestCluster, ICluster } from './test_es_cluster'; export { esTestConfig } from './es_test_config'; -export { createEsClientForTesting, createEsClientForFtrConfig } from './es_client_for_testing'; +export { + createEsClientForTesting, + createEsClientForFtrConfig, + createRemoteEsClientForFtrConfig, +} from './es_client_for_testing'; export type { EsClientForTestingOptions } from './es_client_for_testing'; diff --git a/packages/kbn-test/src/es/test_es_cluster.ts b/packages/kbn-test/src/es/test_es_cluster.ts index 6e4fc2fb14628ff..27f29ce6995a7d1 100644 --- a/packages/kbn-test/src/es/test_es_cluster.ts +++ b/packages/kbn-test/src/es/test_es_cluster.ts @@ -136,7 +136,15 @@ export interface CreateTestEsClusterOptions { * } */ port?: number; + /** + * Should this ES cluster use SSL? + */ ssl?: boolean; + /** + * Explicit transport port for a single node to run on, or a string port range to use eg. '9300-9400' + * defaults to the transport port from `packages/kbn-test/src/es/es_test_config.ts` + */ + transportPort?: number | string; } export function createTestEsCluster< @@ -155,13 +163,14 @@ export function createTestEsCluster< esJavaOpts, clusterName: customClusterName = 'es-test-cluster', ssl, + transportPort, } = options; const clusterName = `${CI_PARALLEL_PROCESS_PREFIX}${customClusterName}`; const defaultEsArgs = [ `cluster.name=${clusterName}`, - `transport.port=${esTestConfig.getTransportPort()}`, + `transport.port=${transportPort ?? esTestConfig.getTransportPort()}`, // For multi-node clusters, we make all nodes master-eligible by default. ...(nodes.length > 1 ? ['discovery.type=zen', `cluster.initial_master_nodes=${nodes.map((n) => n.name).join(',')}`] diff --git a/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts b/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts index f65cb3c41f42182..c7a16c93689b18b 100644 --- a/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts +++ b/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts @@ -176,12 +176,15 @@ export const schema = Joi.object() esTestCluster: Joi.object() .keys({ - license: Joi.string().default('basic'), + license: Joi.valid('basic', 'trial', 'gold').default('basic'), from: Joi.string().default('snapshot'), - serverArgs: Joi.array(), + serverArgs: Joi.array().items(Joi.string()), esJavaOpts: Joi.string(), dataArchive: Joi.string(), ssl: Joi.boolean().default(false), + ccs: Joi.object().keys({ + remoteClusterPort: Joi.number(), + }), }) .default(), @@ -274,6 +277,7 @@ export const schema = Joi.object() security: Joi.object() .keys({ roles: Joi.object().default(), + remoteEsRoles: Joi.object(), defaultRoles: Joi.array() .items(Joi.string()) .when('$primary', { diff --git a/packages/kbn-test/src/functional_tests/lib/run_elasticsearch.ts b/packages/kbn-test/src/functional_tests/lib/run_elasticsearch.ts index 68e7a4992fcfc87..0cfaff8642039b2 100644 --- a/packages/kbn-test/src/functional_tests/lib/run_elasticsearch.ts +++ b/packages/kbn-test/src/functional_tests/lib/run_elasticsearch.ts @@ -8,6 +8,7 @@ import { resolve } from 'path'; import type { ToolingLog } from '@kbn/dev-utils'; +import getPort from 'get-port'; import { KIBANA_ROOT } from './paths'; import type { Config } from '../../functional_test_runner/'; import { createTestEsCluster } from '../../es'; @@ -15,32 +16,102 @@ import { createTestEsCluster } from '../../es'; interface RunElasticsearchOptions { log: ToolingLog; esFrom?: string; + config: Config; +} + +interface CcsConfig { + remoteClusterPort: number; } -export async function runElasticsearch({ + +type EsConfig = ReturnType; + +function getEsConfig({ config, - options, -}: { - config: Config; - options: RunElasticsearchOptions; -}) { - const { log, esFrom } = options; - const ssl = config.get('esTestCluster.ssl'); - const license = config.get('esTestCluster.license'); - const esArgs = config.get('esTestCluster.serverArgs'); - const esJavaOpts = config.get('esTestCluster.esJavaOpts'); + esFrom = config.get('esTestCluster.from'), +}: RunElasticsearchOptions) { + const ssl = !!config.get('esTestCluster.ssl'); + const license: 'basic' | 'trial' | 'gold' = config.get('esTestCluster.license'); + const esArgs: string[] = config.get('esTestCluster.serverArgs') ?? []; + const esJavaOpts: string | undefined = config.get('esTestCluster.esJavaOpts'); const isSecurityEnabled = esArgs.includes('xpack.security.enabled=true'); - const cluster = createTestEsCluster({ - port: config.get('servers.elasticsearch.port'), - password: isSecurityEnabled ? 'changeme' : config.get('servers.elasticsearch.password'), + const port: number | undefined = config.get('servers.elasticsearch.port'); + const ccsConfig: CcsConfig | undefined = config.get('esTestCluster.ccs'); + + const password: string | undefined = isSecurityEnabled + ? 'changeme' + : config.get('servers.elasticsearch.password'); + + const dataArchive: string | undefined = config.get('esTestCluster.dataArchive'); + + return { + ssl, license, - log, - basePath: resolve(KIBANA_ROOT, '.es'), - esFrom: esFrom || config.get('esTestCluster.from'), - dataArchive: config.get('esTestCluster.dataArchive'), esArgs, esJavaOpts, - ssl, + isSecurityEnabled, + esFrom, + port, + password, + dataArchive, + ccsConfig, + }; +} + +export async function runElasticsearch( + options: RunElasticsearchOptions +): Promise<() => Promise> { + const { log } = options; + const config = getEsConfig(options); + + if (!config.ccsConfig) { + const node = await startEsNode(log, 'ftr', config); + return async () => { + await node.cleanup(); + }; + } + + const remotePort = await getPort(); + const remoteNode = await startEsNode(log, 'ftr-remote', { + ...config, + port: config.ccsConfig.remoteClusterPort, + transportPort: remotePort, + }); + + const localNode = await startEsNode(log, 'ftr-local', { + ...config, + esArgs: [...config.esArgs, `cluster.remote.ftr-remote.seeds=localhost:${remotePort}`], + }); + + return async () => { + await localNode.cleanup(); + await remoteNode.cleanup(); + }; +} + +async function startEsNode( + log: ToolingLog, + name: string, + config: EsConfig & { transportPort?: number } +) { + const cluster = createTestEsCluster({ + clusterName: `cluster-${name}`, + esArgs: config.esArgs, + esFrom: config.esFrom, + esJavaOpts: config.esJavaOpts, + license: config.license, + password: config.password, + port: config.port, + ssl: config.ssl, + log, + basePath: resolve(KIBANA_ROOT, '.es'), + nodes: [ + { + name, + dataArchive: config.dataArchive, + }, + ], + transportPort: config.transportPort, }); await cluster.start(); diff --git a/packages/kbn-test/src/functional_tests/tasks.ts b/packages/kbn-test/src/functional_tests/tasks.ts index 5906193ca145c77..b1213ceef2905e3 100644 --- a/packages/kbn-test/src/functional_tests/tasks.ts +++ b/packages/kbn-test/src/functional_tests/tasks.ts @@ -108,10 +108,10 @@ export async function runTests(options: RunTestsParams) { await withProcRunner(log, async (procs) => { const config = await readConfigFile(log, options.esVersion, configPath); - let es; + let shutdownEs; try { if (process.env.TEST_ES_DISABLE_STARTUP !== 'true') { - es = await runElasticsearch({ config, options: { ...options, log } }); + shutdownEs = await runElasticsearch({ ...options, log, config }); } await runKibanaServer({ procs, config, options }); await runFtr({ configPath, options: { ...options, log } }); @@ -125,8 +125,8 @@ export async function runTests(options: RunTestsParams) { await procs.stop('kibana'); } finally { - if (es) { - await es.cleanup(); + if (shutdownEs) { + await shutdownEs(); } } } @@ -166,7 +166,7 @@ export async function startServers({ ...options }: StartServerOptions) { await withProcRunner(log, async (procs) => { const config = await readConfigFile(log, options.esVersion, options.config); - const es = await runElasticsearch({ config, options: opts }); + const shutdownEs = await runElasticsearch({ ...opts, config }); await runKibanaServer({ procs, config, @@ -190,7 +190,7 @@ export async function startServers({ ...options }: StartServerOptions) { log.success(makeSuccessMessage(options)); await procs.waitForAllToStop(); - await es.cleanup(); + await shutdownEs(); }); } diff --git a/packages/kbn-test/src/index.ts b/packages/kbn-test/src/index.ts index 26d40f70edb7808..c9f0e67c558f19c 100644 --- a/packages/kbn-test/src/index.ts +++ b/packages/kbn-test/src/index.ts @@ -36,6 +36,7 @@ export { createTestEsCluster, createEsClientForTesting, createEsClientForFtrConfig, + createRemoteEsClientForFtrConfig, } from './es'; export { diff --git a/scripts/functional_tests.js b/scripts/functional_tests.js index 972c682ab0d9f83..b185bcb0ea5d0c2 100644 --- a/scripts/functional_tests.js +++ b/scripts/functional_tests.js @@ -9,7 +9,7 @@ require('../src/setup_node_env'); require('@kbn/test').runTestsCli([ require.resolve('../test/functional/config.js'), - require.resolve('../test/functional_ccs/config.js'), + require.resolve('../test/functional_ccs/config.ts'), require.resolve('../test/plugin_functional/config.ts'), require.resolve('../test/ui_capabilities/newsfeed_err/config.ts'), require.resolve('../test/new_visualize_flow/config.ts'), diff --git a/test/common/services/es_archiver.ts b/test/common/services/es_archiver.ts index 2ea4b6ce3a43421..b03d262fe264631 100644 --- a/test/common/services/es_archiver.ts +++ b/test/common/services/es_archiver.ts @@ -12,7 +12,10 @@ import * as KibanaServer from './kibana_server'; export function EsArchiverProvider({ getService }: FtrProviderContext): EsArchiver { const config = getService('config'); - const client = getService('es'); + const client = !!config.get('esTestCluster.ccs') + ? getService('ccsRemoteEs' as 'es') + : getService('es'); + const log = getService('log'); const kibanaServer = getService('kibanaServer'); const retry = getService('retry'); diff --git a/test/common/services/security/test_user.ts b/test/common/services/security/test_user.ts index 1161e7b493f4191..dc61dae74df3cc6 100644 --- a/test/common/services/security/test_user.ts +++ b/test/common/services/security/test_user.ts @@ -86,6 +86,28 @@ export async function createTestUserService(ctx: FtrProviderContext, role: Role, await role.create(name, definition); } + // when configured to setup remote roles, load the remote es service and set them up directly via es + const remoteEsRoles: undefined | Record = config.get('security.remoteEsRoles'); + if (remoteEsRoles) { + let ccsRemoteEs; + try { + ccsRemoteEs = ctx.getService('ccsRemoteEs' as 'es'); + } catch (error) { + throw new Error( + 'unable to load `ccsRemoteEs` cluster, which should provide an ES client configured to talk to the remote cluster. Include that service from another FTR config or fix the error it is throwing on creation: ' + + error.message + ); + } + + for (const [name, body] of Object.entries(remoteEsRoles)) { + log.info(`creating ${name} role on remote cluster`); + await ccsRemoteEs.security.putRole({ + name, + ...body, + }); + } + } + // delete the test_user if present (will it error if the user doesn't exist?) try { await user.delete(TEST_USER_NAME); diff --git a/test/functional/fixtures/kbn_archiver/date_nested_ccs.json b/test/functional/fixtures/kbn_archiver/date_nested_ccs.json index 933b21d920c00b9..9a411ba96705a94 100644 --- a/test/functional/fixtures/kbn_archiver/date_nested_ccs.json +++ b/test/functional/fixtures/kbn_archiver/date_nested_ccs.json @@ -2,10 +2,10 @@ "attributes": { "fields": "[]", "timeFieldName": "nested.timestamp", - "title": "remote:date-nested" + "title": "ftr-remote:date-nested" }, "coreMigrationVersion": "8.2.0", - "id": "remote:date-nested", + "id": "ftr-remote:date-nested", "migrationVersion": { "index-pattern": "8.0.0" }, diff --git a/test/functional/fixtures/kbn_archiver/discover_ccs.json b/test/functional/fixtures/kbn_archiver/discover_ccs.json index d53aa1bc759afbd..4c1143ed4e798f3 100644 --- a/test/functional/fixtures/kbn_archiver/discover_ccs.json +++ b/test/functional/fixtures/kbn_archiver/discover_ccs.json @@ -3,10 +3,10 @@ "fieldAttrs": "{\"referer\":{\"customLabel\":\"Referer custom\"}}", "fields": "[{\"name\":\"@message\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"@message.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"@message\"}}},{\"name\":\"@tags\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"@tags.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"@tags\"}}},{\"name\":\"@timestamp\",\"type\":\"date\",\"esTypes\":[\"date\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"agent\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"agent.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"agent\"}}},{\"name\":\"bytes\",\"type\":\"number\",\"esTypes\":[\"long\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"clientip\",\"type\":\"ip\",\"esTypes\":[\"ip\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"extension\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"extension.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"extension\"}}},{\"name\":\"geo.coordinates\",\"type\":\"geo_point\",\"esTypes\":[\"geo_point\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"geo.dest\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"geo.src\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"geo.srcdest\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"headings\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"headings.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"headings\"}}},{\"name\":\"host\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"host.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"host\"}}},{\"name\":\"id\",\"type\":\"number\",\"esTypes\":[\"integer\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"index\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"index.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"index\"}}},{\"name\":\"ip\",\"type\":\"ip\",\"esTypes\":[\"ip\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"links\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"links.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"links\"}}},{\"name\":\"machine.os\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"machine.os.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"machine.os\"}}},{\"name\":\"machine.ram\",\"type\":\"number\",\"esTypes\":[\"long\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"memory\",\"type\":\"number\",\"esTypes\":[\"double\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"meta.char\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"meta.related\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"meta.user.firstname\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"meta.user.lastname\",\"type\":\"number\",\"esTypes\":[\"integer\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"nestedField.child\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"nested\":{\"path\":\"nestedField\"}}},{\"name\":\"phpmemory\",\"type\":\"number\",\"esTypes\":[\"long\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"referer\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"relatedContent.article:modified_time\",\"type\":\"date\",\"esTypes\":[\"date\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"relatedContent.article:published_time\",\"type\":\"date\",\"esTypes\":[\"date\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"relatedContent.article:section\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.article:section.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.article:section\"}}},{\"name\":\"relatedContent.article:tag\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.article:tag.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.article:tag\"}}},{\"name\":\"relatedContent.og:description\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:description.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:description\"}}},{\"name\":\"relatedContent.og:image\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:image.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:image\"}}},{\"name\":\"relatedContent.og:image:height\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:image:height.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:image:height\"}}},{\"name\":\"relatedContent.og:image:width\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:image:width.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:image:width\"}}},{\"name\":\"relatedContent.og:site_name\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:site_name.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:site_name\"}}},{\"name\":\"relatedContent.og:title\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:title.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:title\"}}},{\"name\":\"relatedContent.og:type\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:type.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:type\"}}},{\"name\":\"relatedContent.og:url\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.og:url.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.og:url\"}}},{\"name\":\"relatedContent.twitter:card\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:card.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.twitter:card\"}}},{\"name\":\"relatedContent.twitter:description\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:description.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.twitter:description\"}}},{\"name\":\"relatedContent.twitter:image\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:image.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.twitter:image\"}}},{\"name\":\"relatedContent.twitter:site\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:site.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.twitter:site\"}}},{\"name\":\"relatedContent.twitter:title\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.twitter:title.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.twitter:title\"}}},{\"name\":\"relatedContent.url\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"relatedContent.url.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"relatedContent.url\"}}},{\"name\":\"request\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"request.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"request\"}}},{\"name\":\"response\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"response.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"response\"}}},{\"name\":\"spaces\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"spaces.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"spaces\"}}},{\"name\":\"type\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"url\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"url.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"url\"}}},{\"name\":\"utc_time\",\"type\":\"date\",\"esTypes\":[\"date\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"xss\",\"type\":\"string\",\"esTypes\":[\"text\"],\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"xss.raw\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"xss\"}}}]", "timeFieldName": "@timestamp", - "title": "remote:logstash-*" + "title": "ftr-remote:logstash-*" }, "coreMigrationVersion": "8.0.0", - "id": "remote:logstash-*", + "id": "ftr-remote:logstash-*", "migrationVersion": { "index-pattern": "7.11.0" }, @@ -41,7 +41,7 @@ }, "references": [ { - "id": "remote:logstash-*", + "id": "ftr-remote:logstash-*", "name": "kibanaSavedObjectMeta.searchSourceJSON.index", "type": "index-pattern" } diff --git a/test/functional_ccs/apps/discover/_data_view_ccs.ts b/test/functional_ccs/apps/discover/data_view_ccs.ts similarity index 91% rename from test/functional_ccs/apps/discover/_data_view_ccs.ts rename to test/functional_ccs/apps/discover/data_view_ccs.ts index 91d9cb2faf68168..21190e1c5cdff86 100644 --- a/test/functional_ccs/apps/discover/_data_view_ccs.ts +++ b/test/functional_ccs/apps/discover/data_view_ccs.ts @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -import { FtrProviderContext } from './ftr_provider_context'; +import { FtrProviderContext } from '../../ftr_provider_context'; export default function ({ getService, getPageObjects }: FtrProviderContext) { const retry = getService('retry'); @@ -29,7 +29,11 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { describe('discover integration with data view editor', function describeIndexTests() { before(async function () { - await security.testUser.setRoles(['kibana_admin', 'test_logstash_reader']); + await security.testUser.setRoles([ + 'kibana_admin', + 'test_logstash_reader', + 'ccs_remote_search', + ]); await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/logstash_functional'); await kibanaServer.savedObjects.clean({ types: ['saved-search', 'index-pattern'] }); await kibanaServer.importExport.load('test/functional/fixtures/kbn_archiver/discover'); @@ -44,7 +48,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); it('use ccs to create a new data view', async function () { - const dataViewToCreate = 'remote:logstash'; + const dataViewToCreate = 'ftr-remote:logstash'; await createDataView(dataViewToCreate); await PageObjects.header.waitUntilLoadingHasFinished(); await retry.waitForWithTimeout( diff --git a/test/functional_ccs/apps/discover/index.ts b/test/functional_ccs/apps/discover/index.ts index 629423b1b75aa4f..2e9d428f44c6012 100644 --- a/test/functional_ccs/apps/discover/index.ts +++ b/test/functional_ccs/apps/discover/index.ts @@ -6,38 +6,24 @@ * Side Public License, v 1. */ -import { FtrProviderContext } from './ftr_provider_context'; +import { FtrProviderContext } from '../../ftr_provider_context'; export default function ({ getService, loadTestFile }: FtrProviderContext) { const esArchiver = getService('esArchiver'); const browser = getService('browser'); - const esClient = getService('es'); describe('discover app css', function () { this.tags('ciGroup6'); - before(async function () { - await esClient.cluster.putSettings({ - persistent: { - cluster: { - remote: { - remote: { - skip_unavailable: 'true', - seeds: ['localhost:9300'], - }, - }, - }, - }, - }); - return browser.setWindowSize(1300, 800); + before(async () => { + await browser.setWindowSize(1300, 800); }); - after(function unloadMakelogs() { - // Make sure to clean up the cluster setting from the before above. - return esArchiver.unload('test/functional/fixtures/es_archiver/logstash_functional'); - }); + loadTestFile(require.resolve('./data_view_ccs')); + loadTestFile(require.resolve('./saved_queries_ccs')); - loadTestFile(require.resolve('./_data_view_ccs')); - loadTestFile(require.resolve('./_saved_queries_ccs')); + after(async () => { + await esArchiver.unload('test/functional/fixtures/es_archiver/logstash_functional'); + }); }); } diff --git a/test/functional_ccs/apps/discover/_saved_queries_ccs.ts b/test/functional_ccs/apps/discover/saved_queries_ccs.ts similarity index 97% rename from test/functional_ccs/apps/discover/_saved_queries_ccs.ts rename to test/functional_ccs/apps/discover/saved_queries_ccs.ts index 325f279ff28ab7c..35c70d4b18029fd 100644 --- a/test/functional_ccs/apps/discover/_saved_queries_ccs.ts +++ b/test/functional_ccs/apps/discover/saved_queries_ccs.ts @@ -8,7 +8,7 @@ import expect from '@kbn/expect'; -import { FtrProviderContext } from './ftr_provider_context'; +import { FtrProviderContext } from '../../ftr_provider_context'; export default function ({ getService, getPageObjects }: FtrProviderContext) { const retry = getService('retry'); @@ -87,12 +87,12 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { expect(await filterBar.hasFilter('extension.raw', 'jpg')).to.be(false); expect(await queryBar.getQueryString()).to.eql(''); - await PageObjects.discover.selectIndexPattern('remote:date-nested'); + await PageObjects.discover.selectIndexPattern('ftr-remote:date-nested'); expect(await filterBar.hasFilter('extension.raw', 'jpg')).to.be(false); expect(await queryBar.getQueryString()).to.eql(''); - await PageObjects.discover.selectIndexPattern('remote:logstash-*'); + await PageObjects.discover.selectIndexPattern('ftr-remote:logstash-*'); expect(await filterBar.hasFilter('extension.raw', 'jpg')).to.be(false); expect(await queryBar.getQueryString()).to.eql(''); diff --git a/test/functional_ccs/config.js b/test/functional_ccs/config.js deleted file mode 100644 index 4cd887579837202..000000000000000 --- a/test/functional_ccs/config.js +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { services } from '../functional/services'; - -export default async function ({ readConfigFile }) { - const functionalConfig = await readConfigFile(require.resolve('../functional/config')); - - return { - ...functionalConfig.getAll(), - - testFiles: [require.resolve('./apps/discover')], - - services, - - junit: { - reportName: 'Kibana CCS Tests', - }, - }; -} diff --git a/test/functional_ccs/config.ts b/test/functional_ccs/config.ts new file mode 100644 index 000000000000000..146fed63436b7c5 --- /dev/null +++ b/test/functional_ccs/config.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { FtrConfigProviderContext } from '@kbn/test'; +import { services } from './services'; + +export default async function ({ readConfigFile }: FtrConfigProviderContext) { + const functionalConfig = await readConfigFile(require.resolve('../functional/config')); + + return { + ...functionalConfig.getAll(), + + testFiles: [require.resolve('./apps/discover')], + + services, + + junit: { + reportName: 'Kibana CCS Tests', + }, + + security: { + ...functionalConfig.get('security'), + remoteEsRoles: { + ccs_remote_search: { + indices: [ + { + names: ['*'], + privileges: ['read', 'read_cross_cluster'], + }, + ], + }, + }, + defaultRoles: [...(functionalConfig.get('security.defaultRoles') ?? []), 'ccs_remote_search'], + }, + + esTestCluster: { + ...functionalConfig.get('esTestCluster'), + ccs: { + remoteClusterPort: functionalConfig.get('servers.elasticsearch.port') + 1, + }, + }, + }; +} diff --git a/test/functional_ccs/apps/discover/ftr_provider_context.d.ts b/test/functional_ccs/ftr_provider_context.ts similarity index 80% rename from test/functional_ccs/apps/discover/ftr_provider_context.d.ts rename to test/functional_ccs/ftr_provider_context.ts index ea232d23463e657..8fa82b46ac40631 100644 --- a/test/functional_ccs/apps/discover/ftr_provider_context.d.ts +++ b/test/functional_ccs/ftr_provider_context.ts @@ -7,7 +7,7 @@ */ import { GenericFtrProviderContext } from '@kbn/test'; -import { services } from '../../../functional/services'; -import { pageObjects } from '../../../functional/page_objects'; +import { services } from './services'; +import { pageObjects } from '../functional/page_objects'; export type FtrProviderContext = GenericFtrProviderContext; diff --git a/test/functional_ccs/services/ccs_remote_es.ts b/test/functional_ccs/services/ccs_remote_es.ts new file mode 100644 index 000000000000000..44f8cfc67d671bf --- /dev/null +++ b/test/functional_ccs/services/ccs_remote_es.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { Client } from '@elastic/elasticsearch'; + +import { systemIndicesSuperuser, createRemoteEsClientForFtrConfig } from '@kbn/test'; +import { FtrProviderContext } from '../ftr_provider_context'; + +/** + * Kibana-specific @elastic/elasticsearch client instance. + */ +export function CcsRemoteEsProvider({ getService }: FtrProviderContext): Client { + const config = getService('config'); + + return createRemoteEsClientForFtrConfig(config, { + // Use system indices user so tests can write to system indices + authOverride: systemIndicesSuperuser, + }); +} diff --git a/test/functional_ccs/services/index.ts b/test/functional_ccs/services/index.ts new file mode 100644 index 000000000000000..2d0536ce231de66 --- /dev/null +++ b/test/functional_ccs/services/index.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { services as functionalServices } from '../../functional/services'; +import { CcsRemoteEsProvider } from './ccs_remote_es'; + +export const services = { + ...functionalServices, + ccsRemoteEs: CcsRemoteEsProvider, +};