[Security Solution] Use docker for endpoint tests (#73092)

* Copying api integration tests into their own directory

* Removing api integration tests and using ingest docker image

* Fixing typo

* Fixing type errors and empty string and reenabling tests

* Rebuilding docs

* Renaming url override variable

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

docs/developer/architecture/code-exploration.asciidoc

@@ -524,9 +524,9 @@ WARNING: Missing README.
 See Configuring security in Kibana.
 
 
-- {kib-repo}blob/{branch}/x-pack/plugins/security_solution[securitySolution]
+- {kib-repo}blob/{branch}/x-pack/plugins/security_solution/README.md[securitySolution]
 
-WARNING: Missing README.
+Welcome to the Kibana Security Solution plugin! This README will go over getting started with development and testing.
 
 
 - {kib-repo}blob/{branch}/x-pack/plugins/snapshot_restore/README.md[snapshotRestore]

x-pack/plugins/security_solution/README.md

@@ -0,0 +1,130 @@
+# Security Solution
+
+Welcome to the Kibana Security Solution plugin! This README will go over getting started with development and testing.
+
+## Development
+
+## Tests
+
+The endpoint specific tests leverage the ingest manager to install the endpoint package. Before the api integration
+and functional tests are run the ingest manager is initialized. This initialization process includes reaching out to
+a package registry service to install the endpoint package. The endpoint tests support three different ways to run
+the tests given the constraint on an available package registry.
+
+1. Using Docker
+2. Running your own local package registry
+3. Using the default external package registry
+
+These scenarios will be outlined the sections below.
+
+### Endpoint API Integration Tests Location
+
+The endpoint api integration tests are located [here](../../test/security_solution_endpoint_api_int)
+
+### Endpoint Functional Tests Location
+
+The endpoint functional tests are located [here](../../test/security_solution_endpoint)
+
+### Using Docker
+
+To run the tests using the recommended docker image version you must have `docker` installed. The testing infrastructure
+will stand up a docker container using the image defined [here](../../test/ingest_manager_api_integration/config.ts#L15)
+
+Make sure you're in the Kibana root directory.
+
+#### Endpoint API Integration Tests
+
+In one terminal, run:
+
+```bash
+INGEST_MANAGEMENT_PACKAGE_REGISTRY_PORT=12345 yarn test:ftr:server --config x-pack/test/security_solution_endpoint_api_int/config.ts
+```
+
+In another terminal, run:
+
+```bash
+INGEST_MANAGEMENT_PACKAGE_REGISTRY_PORT=12345 yarn test:ftr:runner --config x-pack/test/security_solution_endpoint_api_int/config.ts
+```
+
+#### Endpoint Functional Tests
+
+In one terminal, run:
+
+```bash
+INGEST_MANAGEMENT_PACKAGE_REGISTRY_PORT=12345 yarn test:ftr:server --config x-pack/test/security_solution_endpoint/config.ts
+```
+
+In another terminal, run:
+
+```bash
+INGEST_MANAGEMENT_PACKAGE_REGISTRY_PORT=12345 yarn test:ftr:runner --config x-pack/test/security_solution_endpoint/config.ts
+```
+
+### Running your own package registry
+
+If you are doing endpoint package development it will be useful to run your own package registry to serve the latest package you're building.
+To do this use the following commands:
+
+Make sure you're in the Kibana root directory.
+
+#### Endpoint API Integration Tests
+
+In one terminal, run:
+
+```bash
+PACKAGE_REGISTRY_URL_OVERRIDE=<url to your package registry like http://localhost:8080> yarn test:ftr:server --config x-pack/test/security_solution_endpoint_api_int/config.ts
+```
+
+In another terminal, run:
+
+```bash
+PACKAGE_REGISTRY_URL_OVERRIDE=<url to your package registry like http://localhost:8080>  yarn test:ftr:runner --config x-pack/test/security_solution_endpoint_api_int/config.ts
+```
+
+#### Endpoint Functional Tests
+
+In one terminal, run:
+
+```bash
+PACKAGE_REGISTRY_URL_OVERRIDE=<url to your package registry like http://localhost:8080> yarn test:ftr:server --config x-pack/test/security_solution_endpoint/config.ts
+```
+
+In another terminal, run:
+
+```bash
+PACKAGE_REGISTRY_URL_OVERRIDE=<url to your package registry like http://localhost:8080>  yarn test:ftr:runner --config x-pack/test/security_solution_endpoint/config.ts
+```
+
+### Using the default public registry
+
+If you don't have docker installed and don't want to run your own registry, you can run the tests using the ingest manager's default public package registry. The actual package registry used is [here](../../plugins/ingest_manager/common/constants/epm.ts#L9)
+
+Make sure you're in the Kibana root directory.
+
+#### Endpoint API Integration Tests
+
+In one terminal, run:
+
+```bash
+yarn test:ftr:server --config x-pack/test/security_solution_endpoint_api_int/config.ts
+```
+
+In another terminal, run:
+
+```bash
+yarn test:ftr:runner --config x-pack/test/security_solution_endpoint_api_int/config.ts
+```
+
+#### Endpoint Functional Tests
+
+In one terminal, run:
+
+```bash
+yarn test:ftr:server --config x-pack/test/security_solution_endpoint/config.ts
+```
+
+In another terminal, run:
+
+```bash
+yarn test:ftr:runner --config x-pack/test/security_solution_endpoint/config.ts
+```

x-pack/scripts/functional_tests.js

@@ -53,6 +53,7 @@ const onlyNotInCoverageTests = [
   require.resolve('../test/licensing_plugin/config.legacy.ts'),
   require.resolve('../test/endpoint_api_integration_no_ingest/config.ts'),
   require.resolve('../test/reporting_api_integration/config.js'),
+  require.resolve('../test/security_solution_endpoint_api_int/config.ts'),
   require.resolve('../test/ingest_manager_api_integration/config.ts'),
 ];
 

x-pack/test/api_integration/apis/index.js

@@ -28,7 +28,6 @@ export default function ({ loadTestFile }) {
     loadTestFile(require.resolve('./lens'));
     loadTestFile(require.resolve('./ml'));
     loadTestFile(require.resolve('./transform'));
-    loadTestFile(require.resolve('./endpoint'));
     loadTestFile(require.resolve('./lists'));
     loadTestFile(require.resolve('./upgrade_assistant'));
   });

x-pack/test/api_integration/services/index.ts

@@ -27,7 +27,6 @@ import { InfraOpsSourceConfigurationProvider } from './infraops_source_configura
 import { InfraLogSourceConfigurationProvider } from './infra_log_source_configuration';
 import { MachineLearningProvider } from './ml';
 import { IngestManagerProvider } from '../../common/services/ingest_manager';
-import { ResolverGeneratorProvider } from './resolver';
 import { TransformProvider } from './transform';
 
 export const services = {
@@ -48,6 +47,5 @@ export const services = {
   usageAPI: UsageAPIProvider,
   ml: MachineLearningProvider,
   ingestManager: IngestManagerProvider,
-  resolverGenerator: ResolverGeneratorProvider,
   transform: TransformProvider,
 };

x-pack/test/ingest_manager_api_integration/config.ts

@@ -9,6 +9,11 @@ import path from 'path';
 import { FtrConfigProviderContext } from '@kbn/test/types/ftr';
 import { defineDockerServersConfig } from '@kbn/test';
 
+// Docker image to use for Ingest Manager API integration tests.
+// This hash comes from the commit hash here: https://github.com/elastic/package-storage/commit/48f3935a72b0c5aacc6fec8ef36d559b089a238b
+export const dockerImage =
+  'docker.elastic.co/package-registry/distribution:48f3935a72b0c5aacc6fec8ef36d559b089a238b';
+
 export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const xPackAPITestsConfig = await readConfigFile(require.resolve('../api_integration/config.ts'));
 
@@ -29,10 +34,6 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
     )}:/packages/test-packages`,
   ];
 
-  // Docker image to use for Ingest Manager API integration tests.
-  const dockerImage =
-    'docker.elastic.co/package-registry/distribution:184b85f19e8fd14363e36150173d338ff9659f01';
-
   return {
     testFiles: [require.resolve('./apis')],
     servers: xPackAPITestsConfig.get('servers'),

x-pack/test/security_solution_endpoint/apps/endpoint/endpoint_list.ts

@@ -6,7 +6,7 @@
 
 import expect from '@kbn/expect';
 import { FtrProviderContext } from '../../ftr_provider_context';
-import { deleteMetadataStream } from '../../../api_integration/apis/endpoint/data_stream_helper';
+import { deleteMetadataStream } from '../../../security_solution_endpoint_api_int/apis/data_stream_helper';
 
 export default ({ getPageObjects, getService }: FtrProviderContext) => {
   const pageObjects = getPageObjects(['common', 'endpoint', 'header', 'endpointPageUtils']);

x-pack/test/security_solution_endpoint/apps/endpoint/index.ts

@@ -3,12 +3,28 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
+import { DEFAULT_REGISTRY_URL } from '../../../../plugins/ingest_manager/common';
 import { FtrProviderContext } from '../../ftr_provider_context';
+import {
+  isRegistryEnabled,
+  getRegistryUrl,
+} from '../../../security_solution_endpoint_api_int/registry';
+
+export default function (providerContext: FtrProviderContext) {
+  const { loadTestFile, getService } = providerContext;
 
-export default function ({ loadTestFile, getService }: FtrProviderContext) {
   describe('endpoint', function () {
     this.tags('ciGroup7');
     const ingestManager = getService('ingestManager');
+    const log = getService('log');
+
+    if (!isRegistryEnabled()) {
+      log.warning('These tests are being run with an external package registry');
+    }
+
+    const registryUrl = getRegistryUrl() ?? DEFAULT_REGISTRY_URL;
+    log.info(`Package registry URL for tests: ${registryUrl}`);
+
     before(async () => {
       await ingestManager.setup();
     });

x-pack/test/security_solution_endpoint/config.ts

@@ -8,6 +8,10 @@ import { resolve } from 'path';
 import { FtrConfigProviderContext } from '@kbn/test/types/ftr';
 import { pageObjects } from './page_objects';
 import { services } from './services';
+import {
+  getRegistryUrlAsArray,
+  createEndpointDockerConfig,
+} from '../security_solution_endpoint_api_int/registry';
 
 export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const xpackFunctionalConfig = await readConfigFile(require.resolve('../functional/config.js'));
@@ -16,6 +20,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
     ...xpackFunctionalConfig.getAll(),
     pageObjects,
     testFiles: [resolve(__dirname, './apps/endpoint')],
+    dockerServers: createEndpointDockerConfig(),
     junit: {
       reportName: 'X-Pack Endpoint Functional Tests',
     },
@@ -31,6 +36,8 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
       serverArgs: [
         ...xpackFunctionalConfig.get('kbnTestServer.serverArgs'),
         '--xpack.ingestManager.enabled=true',
+        // if you return an empty string here the kibana server will not start properly but an empty array works
+        ...getRegistryUrlAsArray(),
       ],
     },
   };

x-pack/test/api_integration/apis/endpoint/artifacts/index.ts → x-pack/test/security_solution_endpoint_api_int/apis/artifacts/index.ts

@@ -8,11 +8,8 @@ import expect from '@kbn/expect';
 import { createHash } from 'crypto';
 import { inflateSync } from 'zlib';
 
-import { FtrProviderContext } from '../../../ftr_provider_context';
-import {
-  getSupertestWithoutAuth,
-  setupIngest,
-} from '../../../../ingest_manager_api_integration/apis/fleet/agents/services';
+import { FtrProviderContext } from '../../../api_integration/ftr_provider_context';
+import { getSupertestWithoutAuth } from '../../../ingest_manager_api_integration/apis/fleet/agents/services';
 
 export default function (providerContext: FtrProviderContext) {
   const { getService } = providerContext;
@@ -22,7 +19,6 @@ export default function (providerContext: FtrProviderContext) {
   let agentAccessAPIKey: string;
 
   describe('artifact download', () => {
-    setupIngest(providerContext);
     before(async () => {
       await esArchiver.load('endpoint/artifacts/api_feature', { useCreate: true });
 

x-pack/test/api_integration/apis/endpoint/data_stream_helper.ts → x-pack/test/security_solution_endpoint_api_int/apis/data_stream_helper.ts

@@ -10,7 +10,7 @@ import {
   eventsIndexPattern,
   alertsIndexPattern,
   policyIndexPattern,
-} from '../../../../plugins/security_solution/common/endpoint/constants';
+} from '../../../plugins/security_solution/common/endpoint/constants';
 
 export async function deleteDataStream(getService: (serviceName: 'es') => Client, index: string) {
   const client = getService('es');

x-pack/test/security_solution_endpoint_api_int/apis/fixtures/package_registry_config.yml

@@ -0,0 +1,2 @@
+package_paths:
+  - /packages/production

x-pack/test/security_solution_endpoint_api_int/apis/index.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import { FtrProviderContext } from '../ftr_provider_context';
+import { isRegistryEnabled, getRegistryUrl } from '../registry';
+import { DEFAULT_REGISTRY_URL } from '../../../plugins/ingest_manager/common';
+
+export default function endpointAPIIntegrationTests(providerContext: FtrProviderContext) {
+  const { loadTestFile, getService } = providerContext;
+
+  describe('Endpoint plugin', function () {
+    const ingestManager = getService('ingestManager');
+
+    this.tags('ciGroup7');
+    const log = getService('log');
+
+    if (!isRegistryEnabled()) {
+      log.warning('These tests are being run with an external package registry');
+    }
+
+    const registryUrl = getRegistryUrl() ?? DEFAULT_REGISTRY_URL;
+    log.info(`Package registry URL for tests: ${registryUrl}`);
+
+    before(async () => {
+      await ingestManager.setup();
+    });
+    loadTestFile(require.resolve('./resolver'));
+    loadTestFile(require.resolve('./metadata'));
+    loadTestFile(require.resolve('./policy'));
+    loadTestFile(require.resolve('./artifacts'));
+  });
+}

x-pack/test/api_integration/apis/endpoint/metadata.ts → x-pack/test/security_solution_endpoint_api_int/apis/metadata.ts

@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 import expect from '@kbn/expect/expect.js';
-import { FtrProviderContext } from '../../ftr_provider_context';
+import { FtrProviderContext } from '../ftr_provider_context';
 import { deleteMetadataStream } from './data_stream_helper';
 
 /**

x-pack/test/api_integration/apis/endpoint/policy.ts → x-pack/test/security_solution_endpoint_api_int/apis/policy.ts

@@ -5,7 +5,7 @@
  */
 
 import expect from '@kbn/expect/expect.js';
-import { FtrProviderContext } from '../../ftr_provider_context';
+import { FtrProviderContext } from '../ftr_provider_context';
 import { deletePolicyStream } from './data_stream_helper';
 
 export default function ({ getService }: FtrProviderContext) {