From a23a07239e6b30facfd33b2a82c96efbf1f2e21d Mon Sep 17 00:00:00 2001 From: Spencer Date: Wed, 9 Jun 2021 17:13:00 -0700 Subject: [PATCH] [kbnArchiver] convert archive names to root-relative paths (#101839) * [kbnArchiver] convert archive names to root-relative paths * ensure that newly multiline hooks are explicitly async * missed a newly multiline hook * fix exists check * avoid extra lines by wrapping arrow body in {} * one block more * fix errant `name` variable Co-authored-by: spalger # Conflicts: # test/api_integration/apis/kql_telemetry/kql_telemetry.ts # test/api_integration/apis/saved_objects/bulk_create.ts # test/api_integration/apis/saved_objects/bulk_get.ts # test/api_integration/apis/saved_objects/bulk_update.ts # test/api_integration/apis/saved_objects/create.ts # test/api_integration/apis/saved_objects/delete.ts # test/api_integration/apis/saved_objects/export.ts # test/api_integration/apis/saved_objects/find.ts # test/api_integration/apis/saved_objects/get.ts # test/api_integration/apis/saved_objects/import.ts # test/api_integration/apis/saved_objects/resolve.ts # test/api_integration/apis/saved_objects/resolve_import_errors.ts # test/api_integration/apis/saved_objects/update.ts # test/api_integration/apis/saved_objects_management/find.ts # test/api_integration/apis/saved_objects_management/get.ts # test/api_integration/apis/saved_objects_management/relationships.ts # test/api_integration/apis/shorten/index.js # test/api_integration/apis/stats/stats.js # test/api_integration/apis/suggestions/suggestions.js # test/functional/apps/discover/_discover.ts --- packages/kbn-test/src/kbn_archiver_cli.ts | 23 ++-------------- .../kbn-test/src/kbn_client/kbn_client.ts | 4 +-- .../kbn_client/kbn_client_import_export.ts | 26 ++++++++++--------- .../services/kibana_server/kibana_server.ts | 1 - test/functional/apps/discover/_data_grid.ts | 2 +- .../apps/discover/_data_grid_context.ts | 2 +- .../apps/discover/_data_grid_doc_table.ts | 2 +- .../apps/discover/_data_grid_field_data.ts | 2 +- test/functional/apps/discover/_discover.ts | 2 +- .../apps/discover/_discover_fields_api.ts | 2 +- test/functional/apps/discover/_doc_table.ts | 2 +- test/functional/apps/discover/_field_data.ts | 2 +- .../discover/_field_data_with_fields_api.ts | 2 +- .../apps/discover/_filter_editor.ts | 2 +- test/functional/apps/discover/_inspector.ts | 2 +- .../functional/apps/discover/_large_string.ts | 4 ++- .../apps/discover/_saved_queries.ts | 2 +- .../functional/apps/discover/_shared_links.ts | 2 +- .../functional/apps/visualize/legacy/index.ts | 2 +- .../functional/page_objects/visualize_page.ts | 4 ++- .../apps/ccs/ccs_discover.js | 1 - 21 files changed, 38 insertions(+), 53 deletions(-) diff --git a/packages/kbn-test/src/kbn_archiver_cli.ts b/packages/kbn-test/src/kbn_archiver_cli.ts index 04581a83546686..6eb1d4bf68dadc 100644 --- a/packages/kbn-test/src/kbn_archiver_cli.ts +++ b/packages/kbn-test/src/kbn_archiver_cli.ts @@ -43,13 +43,11 @@ export function runKbnArchiverCli() { new RunWithCommands({ description: 'Import/export saved objects from archives, for testing', globalFlags: { - string: ['config', 'space', 'kibana-url', 'dir'], + string: ['config', 'space', 'kibana-url'], help: ` --space space id to operate on, defaults to the default space --config optional path to an FTR config file that will be parsed and used for defaults --kibana-url set the url that kibana can be reached at, uses the "servers.kibana" setting from --config by default - --dir directory that contains exports to be imported, or where exports will be saved, uses the "kbnArchiver.directory" - setting from --config by default `, }, async extendContext({ log, flags }) { @@ -79,23 +77,6 @@ export function runKbnArchiverCli() { ); } - let importExportDir; - if (flags.dir) { - if (typeof flags.dir !== 'string') { - throw createFlagError('expected --dir to be a string'); - } - - importExportDir = flags.dir; - } else if (config) { - importExportDir = config.get('kbnArchiver.directory'); - } - - if (!importExportDir) { - throw createFlagError( - '--config does not include a kbnArchiver.directory, specify it or include --dir flag' - ); - } - const space = flags.space; if (!(space === undefined || typeof space === 'string')) { throw createFlagError('--space must be a string'); @@ -106,7 +87,7 @@ export function runKbnArchiverCli() { kbnClient: new KbnClient({ log, url: kibanaUrl, - importExportDir, + importExportBaseDir: process.cwd(), }), }; }, diff --git a/packages/kbn-test/src/kbn_client/kbn_client.ts b/packages/kbn-test/src/kbn_client/kbn_client.ts index 3fa74412c1a8bf..a0f7ff3759d403 100644 --- a/packages/kbn-test/src/kbn_client/kbn_client.ts +++ b/packages/kbn-test/src/kbn_client/kbn_client.ts @@ -21,7 +21,7 @@ export interface KbnClientOptions { certificateAuthorities?: Buffer[]; log: ToolingLog; uiSettingDefaults?: UiSettingValues; - importExportDir?: string; + importExportBaseDir?: string; } export class KbnClient { @@ -64,7 +64,7 @@ export class KbnClient { this.log, this.requester, this.savedObjects, - options.importExportDir + options.importExportBaseDir ); } diff --git a/packages/kbn-test/src/kbn_client/kbn_client_import_export.ts b/packages/kbn-test/src/kbn_client/kbn_client_import_export.ts index fe67fbb70fa3ca..5fd30929fecf68 100644 --- a/packages/kbn-test/src/kbn_client/kbn_client_import_export.ts +++ b/packages/kbn-test/src/kbn_client/kbn_client_import_export.ts @@ -8,10 +8,11 @@ import { inspect } from 'util'; import Fs from 'fs/promises'; +import { existsSync } from 'fs'; import Path from 'path'; import FormData from 'form-data'; -import { ToolingLog, isAxiosResponseError, createFailError } from '@kbn/dev-utils'; +import { ToolingLog, isAxiosResponseError, createFailError, REPO_ROOT } from '@kbn/dev-utils'; import { KbnClientRequester, uriencode, ReqOptions } from './kbn_client_requester'; import { KbnClientSavedObjects } from './kbn_client_saved_objects'; @@ -39,7 +40,7 @@ export class KbnClientImportExport { public readonly log: ToolingLog, public readonly requester: KbnClientRequester, public readonly savedObjects: KbnClientSavedObjects, - public readonly dir?: string + public readonly baseDir: string = REPO_ROOT ) {} private resolvePath(path: string) { @@ -47,18 +48,19 @@ export class KbnClientImportExport { path = `${path}.json`; } - if (!this.dir && !Path.isAbsolute(path)) { + const absolutePath = Path.resolve(this.baseDir, path); + if (!existsSync(absolutePath)) { throw new Error( - 'unable to resolve relative path to import/export without a configured dir, either path absolute path or specify --dir' + `unable to resolve path [${path}] to import/export, resolved relative to [${this.baseDir}]` ); } - return this.dir ? Path.resolve(this.dir, path) : path; + return absolutePath; } - async load(name: string, options?: { space?: string }) { - const src = this.resolvePath(name); - this.log.debug('resolved import for', name, 'to', src); + async load(path: string, options?: { space?: string }) { + const src = this.resolvePath(path); + this.log.debug('resolved import for', path, 'to', src); const objects = await parseArchive(src); this.log.info('importing', objects.length, 'saved objects', { space: options?.space }); @@ -91,8 +93,8 @@ export class KbnClientImportExport { } } - async unload(name: string, options?: { space?: string }) { - const src = this.resolvePath(name); + async unload(path: string, options?: { space?: string }) { + const src = this.resolvePath(path); this.log.debug('unloading docs from archive at', src); const objects = await parseArchive(src); @@ -110,8 +112,8 @@ export class KbnClientImportExport { this.log.success(deleted, 'saved objects deleted'); } - async save(name: string, options: { types: string[]; space?: string }) { - const dest = this.resolvePath(name); + async save(path: string, options: { types: string[]; space?: string }) { + const dest = this.resolvePath(path); this.log.debug('saving export to', dest); const resp = await this.req(options.space, { diff --git a/test/common/services/kibana_server/kibana_server.ts b/test/common/services/kibana_server/kibana_server.ts index 63803bd511bd14..f20fa4cafa55e2 100644 --- a/test/common/services/kibana_server/kibana_server.ts +++ b/test/common/services/kibana_server/kibana_server.ts @@ -22,7 +22,6 @@ export function KibanaServerProvider({ getService }: FtrProviderContext): KbnCli url, certificateAuthorities: config.get('servers.kibana.certificateAuthorities'), uiSettingDefaults: defaults, - importExportDir: config.get('kbnArchiver.directory'), }); if (defaults) { diff --git a/test/functional/apps/discover/_data_grid.ts b/test/functional/apps/discover/_data_grid.ts index eaefb3ba38f050..efd97fce3f7f5d 100644 --- a/test/functional/apps/discover/_data_grid.ts +++ b/test/functional/apps/discover/_data_grid.ts @@ -24,7 +24,7 @@ export default function ({ before(async function () { await kibanaServer.savedObjects.clean({ types: ['search', 'index-pattern'] }); - await kibanaServer.importExport.load('discover'); + await kibanaServer.importExport.load('test/functional/fixtures/kbn_archiver/discover.json'); await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/logstash_functional'); await kibanaServer.uiSettings.replace(defaultSettings); await PageObjects.common.navigateToApp('discover'); diff --git a/test/functional/apps/discover/_data_grid_context.ts b/test/functional/apps/discover/_data_grid_context.ts index bfce69ca08d552..3d9e01e1dee19d 100644 --- a/test/functional/apps/discover/_data_grid_context.ts +++ b/test/functional/apps/discover/_data_grid_context.ts @@ -36,7 +36,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { describe('discover data grid context tests', () => { before(async () => { await kibanaServer.savedObjects.clean({ types: ['search', 'index-pattern'] }); - await kibanaServer.importExport.load('discover'); + await kibanaServer.importExport.load('test/functional/fixtures/kbn_archiver/discover.json'); await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/logstash_functional'); await PageObjects.timePicker.setDefaultAbsoluteRangeViaUiSettings(); await kibanaServer.uiSettings.update(defaultSettings); diff --git a/test/functional/apps/discover/_data_grid_doc_table.ts b/test/functional/apps/discover/_data_grid_doc_table.ts index b18bfae2068225..ec5a51c55265bf 100644 --- a/test/functional/apps/discover/_data_grid_doc_table.ts +++ b/test/functional/apps/discover/_data_grid_doc_table.ts @@ -28,7 +28,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { before(async function () { log.debug('load kibana index with default index pattern'); await kibanaServer.savedObjects.clean({ types: ['search', 'index-pattern'] }); - await kibanaServer.importExport.load('discover'); + await kibanaServer.importExport.load('test/functional/fixtures/kbn_archiver/discover.json'); await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/logstash_functional'); await kibanaServer.uiSettings.replace(defaultSettings); await PageObjects.timePicker.setDefaultAbsoluteRangeViaUiSettings(); diff --git a/test/functional/apps/discover/_data_grid_field_data.ts b/test/functional/apps/discover/_data_grid_field_data.ts index 8dad479d8a689d..a6033be89ebf71 100644 --- a/test/functional/apps/discover/_data_grid_field_data.ts +++ b/test/functional/apps/discover/_data_grid_field_data.ts @@ -27,7 +27,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { this.tags('includeFirefox'); before(async function () { await kibanaServer.savedObjects.clean({ types: ['search', 'index-pattern'] }); - await kibanaServer.importExport.load('discover'); + await kibanaServer.importExport.load('test/functional/fixtures/kbn_archiver/discover.json'); await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/logstash_functional'); await PageObjects.timePicker.setDefaultAbsoluteRangeViaUiSettings(); await kibanaServer.uiSettings.update(defaultSettings); diff --git a/test/functional/apps/discover/_discover.ts b/test/functional/apps/discover/_discover.ts index 199cdefa8841a2..53682cdb2bf305 100644 --- a/test/functional/apps/discover/_discover.ts +++ b/test/functional/apps/discover/_discover.ts @@ -30,7 +30,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { before(async function () { log.debug('load kibana index with default index pattern'); - await kibanaServer.importExport.load('discover'); + await kibanaServer.importExport.load('test/functional/fixtures/kbn_archiver/discover.json'); log.info( `\n### SAVED OBJECT TYPES IN index: [.kibana]: \n\t${await savedObjectInfo.types()}` ); diff --git a/test/functional/apps/discover/_discover_fields_api.ts b/test/functional/apps/discover/_discover_fields_api.ts index d3df79dc8658b6..614a0794ffb3b2 100644 --- a/test/functional/apps/discover/_discover_fields_api.ts +++ b/test/functional/apps/discover/_discover_fields_api.ts @@ -23,7 +23,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { before(async function () { log.debug('load kibana index with default index pattern'); await kibanaServer.savedObjects.clean({ types: ['search', 'index-pattern'] }); - await kibanaServer.importExport.load('discover'); + await kibanaServer.importExport.load('test/functional/fixtures/kbn_archiver/discover.json'); await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/logstash_functional'); await kibanaServer.uiSettings.replace(defaultSettings); log.debug('discover'); diff --git a/test/functional/apps/discover/_doc_table.ts b/test/functional/apps/discover/_doc_table.ts index a70290f304db87..27f6a3ad4f1fbf 100644 --- a/test/functional/apps/discover/_doc_table.ts +++ b/test/functional/apps/discover/_doc_table.ts @@ -30,7 +30,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { before(async function () { log.debug('load kibana index with default index pattern'); await kibanaServer.savedObjects.clean({ types: ['search', 'index-pattern'] }); - await kibanaServer.importExport.load('discover'); + await kibanaServer.importExport.load('test/functional/fixtures/kbn_archiver/discover.json'); // and load a set of makelogs data await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/logstash_functional'); diff --git a/test/functional/apps/discover/_field_data.ts b/test/functional/apps/discover/_field_data.ts index e8698b984f6cc0..338d17ba31ff49 100644 --- a/test/functional/apps/discover/_field_data.ts +++ b/test/functional/apps/discover/_field_data.ts @@ -24,7 +24,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { this.tags('includeFirefox'); before(async function () { await kibanaServer.savedObjects.clean({ types: ['search', 'index-pattern'] }); - await kibanaServer.importExport.load('discover'); + await kibanaServer.importExport.load('test/functional/fixtures/kbn_archiver/discover.json'); await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/logstash_functional'); await kibanaServer.uiSettings.replace({ defaultIndex: 'logstash-*', diff --git a/test/functional/apps/discover/_field_data_with_fields_api.ts b/test/functional/apps/discover/_field_data_with_fields_api.ts index 7de123bb44f2a6..110e255d18c756 100644 --- a/test/functional/apps/discover/_field_data_with_fields_api.ts +++ b/test/functional/apps/discover/_field_data_with_fields_api.ts @@ -24,7 +24,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { this.tags('includeFirefox'); before(async function () { await kibanaServer.savedObjects.clean({ types: ['search', 'index-pattern'] }); - await kibanaServer.importExport.load('discover'); + await kibanaServer.importExport.load('test/functional/fixtures/kbn_archiver/discover.json'); await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/logstash_functional'); await kibanaServer.uiSettings.replace({ defaultIndex: 'logstash-*', diff --git a/test/functional/apps/discover/_filter_editor.ts b/test/functional/apps/discover/_filter_editor.ts index 4e9839d4e799ae..8bcb4382bb3bf3 100644 --- a/test/functional/apps/discover/_filter_editor.ts +++ b/test/functional/apps/discover/_filter_editor.ts @@ -25,7 +25,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { before(async function () { log.debug('load kibana index with default index pattern'); await kibanaServer.savedObjects.clean({ types: ['search', 'index-pattern'] }); - await kibanaServer.importExport.load('discover'); + await kibanaServer.importExport.load('test/functional/fixtures/kbn_archiver/discover.json'); // and load a set of makelogs data await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/logstash_functional'); diff --git a/test/functional/apps/discover/_inspector.ts b/test/functional/apps/discover/_inspector.ts index 9d4fd93eb3a9f0..17f358ec748714 100644 --- a/test/functional/apps/discover/_inspector.ts +++ b/test/functional/apps/discover/_inspector.ts @@ -34,7 +34,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { before(async () => { await kibanaServer.savedObjects.clean({ types: ['search', 'index-pattern'] }); - await kibanaServer.importExport.load('discover'); + await kibanaServer.importExport.load('test/functional/fixtures/kbn_archiver/discover.json'); await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/logstash_functional'); // delete .kibana index and update configDoc await kibanaServer.uiSettings.replace({ diff --git a/test/functional/apps/discover/_large_string.ts b/test/functional/apps/discover/_large_string.ts index 065711358d50e9..04ab35f9b77577 100644 --- a/test/functional/apps/discover/_large_string.ts +++ b/test/functional/apps/discover/_large_string.ts @@ -23,7 +23,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { before(async function () { await security.testUser.setRoles(['kibana_admin', 'kibana_large_strings']); - await kibanaServer.importExport.load('testlargestring'); + await kibanaServer.importExport.load( + 'test/functional/fixtures/kbn_archiver/testlargestring.json' + ); await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/hamlet'); await kibanaServer.uiSettings.replace({ defaultIndex: 'testlargestring', diff --git a/test/functional/apps/discover/_saved_queries.ts b/test/functional/apps/discover/_saved_queries.ts index 7b862851644f3c..52ebdb8c008c33 100644 --- a/test/functional/apps/discover/_saved_queries.ts +++ b/test/functional/apps/discover/_saved_queries.ts @@ -30,7 +30,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { before(async function () { log.debug('load kibana index with default index pattern'); await kibanaServer.savedObjects.clean({ types: ['search', 'index-pattern'] }); - await kibanaServer.importExport.load('discover'); + await kibanaServer.importExport.load('test/functional/fixtures/kbn_archiver/discover.json'); // and load a set of makelogs data await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/logstash_functional'); diff --git a/test/functional/apps/discover/_shared_links.ts b/test/functional/apps/discover/_shared_links.ts index 151e8555eea77e..62364739db311b 100644 --- a/test/functional/apps/discover/_shared_links.ts +++ b/test/functional/apps/discover/_shared_links.ts @@ -38,7 +38,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { log.debug('load kibana index with default index pattern'); await kibanaServer.savedObjects.clean({ types: ['search', 'index-pattern'] }); - await kibanaServer.importExport.load('discover'); + await kibanaServer.importExport.load('test/functional/fixtures/kbn_archiver/discover.json'); await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/logstash_functional'); await kibanaServer.uiSettings.replace({ diff --git a/test/functional/apps/visualize/legacy/index.ts b/test/functional/apps/visualize/legacy/index.ts index 4e43ec90d16999..d474287860b25e 100644 --- a/test/functional/apps/visualize/legacy/index.ts +++ b/test/functional/apps/visualize/legacy/index.ts @@ -23,7 +23,7 @@ export default function ({ getPageObjects, getService, loadTestFile }: FtrProvid await browser.setWindowSize(1280, 800); await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/logstash_functional'); await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/long_window_logstash'); - await kibanaServer.importExport.load('visualize'); + await kibanaServer.importExport.load('test/functional/fixtures/kbn_archiver/visualize.json'); await kibanaServer.uiSettings.replace({ defaultIndex: 'logstash-*', [UI_SETTINGS.FORMAT_BYTES_DEFAULT_PATTERN]: '0,0.[000]b', diff --git a/test/functional/page_objects/visualize_page.ts b/test/functional/page_objects/visualize_page.ts index f77db553e015f9..a11a254509e7a8 100644 --- a/test/functional/page_objects/visualize_page.ts +++ b/test/functional/page_objects/visualize_page.ts @@ -49,7 +49,9 @@ export class VisualizePageObject extends FtrService { public async initTests(isNewLibrary = false) { await this.kibanaServer.savedObjects.clean({ types: ['visualization'] }); - await this.kibanaServer.importExport.load('visualize'); + await this.kibanaServer.importExport.load( + 'test/functional/fixtures/kbn_archiver/visualize.json' + ); await this.kibanaServer.uiSettings.replace({ defaultIndex: 'logstash-*', diff --git a/x-pack/test/stack_functional_integration/apps/ccs/ccs_discover.js b/x-pack/test/stack_functional_integration/apps/ccs/ccs_discover.js index 83a759abe337d7..00fce236b5d172 100644 --- a/x-pack/test/stack_functional_integration/apps/ccs/ccs_discover.js +++ b/x-pack/test/stack_functional_integration/apps/ccs/ccs_discover.js @@ -227,7 +227,6 @@ export default ({ getService, getPageObjects }) => { url: process.env.TEST_KIBANA_URLDATA, certificateAuthorities: config.get('servers.kibana.certificateAuthorities'), uiSettingDefaults: kibanaServer.uiSettings, - importExportDir: config.get('kbnArchiver.directory'), }); const esArchiver = new EsArchiver({