[Ingest Manager] Rename data sources to package configs (#70259)

* Rename `datasource` saved object to `package_config` (SO type `ingest-datasource` to `ingest-package-config`) and adjust mappings: - Remove unused `processors` field on input & stream levels - Remove unnecessary `enabled` field on package config & input levels - Rename `agent_stream` field to `compiled_stream` - Reorder other fields so that important fields are closer to top & similar fields are grouped together Also, - Remove 7.9.0 migrations as we are not supporting an upgrade path from experimental to beta release * Pluralize `ingest-package-configs` for consistency * Rename `Datasource`-related types to `PackageConfig`, update all references (does not include actual type definition changes yet) * Rename `Datasource` schemas and rest spec typings to `PackageConfig` (does not include actual schema changes yet) * Change `datasources` on agent config typings and schemas to `package_configs` and update all references * Add back `enabled` field on package config and input levels. They are needed for current & future UI features. Also: - Match types and schemas with saved object mappings (`agent_stream` to `compiled_stream`, removal of `processors`) - Set `namespace` to be a required property on agent config and package config types, add validation support for it on UI * Rename server-side datasource references in file names, variable names, and routes * Update spec file and schema file * Update doc wording * Rename all instances of datasource in file paths and variable names on client sides, and for Endpoint too * Minor copy adjustments, fix i18n check * Replace datasource references in tests and fixtures; remove unused `ingest/policies` es archiver data * Fix tests * Fix test field name * Fix test fixtures fields again * Fix i18n
elastic · Jul 1, 2020 · 4f7da59 · 4f7da59
1 parent 0f418bb
commit 4f7da59
Show file tree

Hide file tree

Showing 139 changed files with 1,990 additions and 3,573 deletions.
diff --git a/x-pack/plugins/ingest_manager/README.md b/x-pack/plugins/ingest_manager/README.md
@@ -3,7 +3,7 @@
 ## Plugin
 
 - The plugin is disabled by default. See the TypeScript type for the [the available plugin configuration options](https://github.com/elastic/kibana/blob/master/x-pack/plugins/ingest_manager/common/types/index.ts#L9-L27)
-- Setting `xpack.ingestManager.enabled=true` enables the plugin including the EPM and Fleet features. It also adds the `DATASOURCE_API_ROUTES` and `AGENT_CONFIG_API_ROUTES` values in [`common/constants/routes.ts`](./common/constants/routes.ts)
+- Setting `xpack.ingestManager.enabled=true` enables the plugin including the EPM and Fleet features. It also adds the `PACKAGE_CONFIG_API_ROUTES` and `AGENT_CONFIG_API_ROUTES` values in [`common/constants/routes.ts`](./common/constants/routes.ts)
 - Adding `--xpack.ingestManager.epm.enabled=false` will disable the EPM API & UI
 - Adding `--xpack.ingestManager.fleet.enabled=false` will disable the Fleet API & UI
   - [code for adding the routes](https://github.com/elastic/kibana/blob/1f27d349533b1c2865c10c45b2cf705d7416fb36/x-pack/plugins/ingest_manager/server/plugin.ts#L115-L133)

diff --git a/x-pack/plugins/ingest_manager/common/constants/agent_config.ts b/x-pack/plugins/ingest_manager/common/constants/agent_config.ts
@@ -12,7 +12,7 @@ export const DEFAULT_AGENT_CONFIG = {
   namespace: 'default',
   description: 'Default agent configuration created by Kibana',
   status: AgentConfigStatus.Active,
-  datasources: [],
+  package_configs: [],
   is_default: true,
   monitoring_enabled: ['logs', 'metrics'] as Array<'logs' | 'metrics'>,
 };

diff --git a/x-pack/plugins/ingest_manager/common/constants/index.ts b/x-pack/plugins/ingest_manager/common/constants/index.ts
@@ -8,7 +8,7 @@ export * from './routes';
 
 export * from './agent';
 export * from './agent_config';
-export * from './datasource';
+export * from './package_config';
 export * from './epm';
 export * from './output';
 export * from './enrollment_api_key';

diff --git a/...st_manager/common/constants/datasource.ts → ...anager/common/constants/package_config.ts b/...st_manager/common/constants/datasource.ts → ...anager/common/constants/package_config.ts
@@ -4,4 +4,4 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-export const DATASOURCE_SAVED_OBJECT_TYPE = 'ingest-datasources';
+export const PACKAGE_CONFIG_SAVED_OBJECT_TYPE = 'ingest-package-configs';
diff --git a/x-pack/plugins/ingest_manager/common/constants/routes.ts b/x-pack/plugins/ingest_manager/common/constants/routes.ts
@@ -7,7 +7,7 @@
 export const API_ROOT = `/api/ingest_manager`;
 export const EPM_API_ROOT = `${API_ROOT}/epm`;
 export const DATA_STREAM_API_ROOT = `${API_ROOT}/data_streams`;
-export const DATASOURCE_API_ROOT = `${API_ROOT}/datasources`;
+export const PACKAGE_CONFIG_API_ROOT = `${API_ROOT}/package_configs`;
 export const AGENT_CONFIG_API_ROOT = `${API_ROOT}/agent_configs`;
 export const FLEET_API_ROOT = `${API_ROOT}/fleet`;
 
@@ -29,13 +29,13 @@ export const DATA_STREAM_API_ROUTES = {
   LIST_PATTERN: `${DATA_STREAM_API_ROOT}`,
 };
 
-// Datasource API routes
-export const DATASOURCE_API_ROUTES = {
-  LIST_PATTERN: `${DATASOURCE_API_ROOT}`,
-  INFO_PATTERN: `${DATASOURCE_API_ROOT}/{datasourceId}`,
-  CREATE_PATTERN: `${DATASOURCE_API_ROOT}`,
-  UPDATE_PATTERN: `${DATASOURCE_API_ROOT}/{datasourceId}`,
-  DELETE_PATTERN: `${DATASOURCE_API_ROOT}/delete`,
+// Package config API routes
+export const PACKAGE_CONFIG_API_ROUTES = {
+  LIST_PATTERN: `${PACKAGE_CONFIG_API_ROOT}`,
+  INFO_PATTERN: `${PACKAGE_CONFIG_API_ROOT}/{packageConfigId}`,
+  CREATE_PATTERN: `${PACKAGE_CONFIG_API_ROOT}`,
+  UPDATE_PATTERN: `${PACKAGE_CONFIG_API_ROOT}/{packageConfigId}`,
+  DELETE_PATTERN: `${PACKAGE_CONFIG_API_ROOT}/delete`,
 };
 
 // Agent config API routes

diff --git a/x-pack/plugins/ingest_manager/common/openapi/spec_oas3.json b/x-pack/plugins/ingest_manager/common/openapi/spec_oas3.json
@@ -60,7 +60,7 @@
                           "namespace": "default",
                           "description": "Default agent configuration created by Kibana",
                           "status": "active",
-                          "datasources": ["8a5679b0-8fbf-11ea-b2ce-01c4a6127154"],
+                          "packageConfigs": ["8a5679b0-8fbf-11ea-b2ce-01c4a6127154"],
                           "is_default": true,
                           "monitoring_enabled": ["logs", "metrics"],
                           "revision": 2,
@@ -175,7 +175,7 @@
                         "namespace": "default",
                         "description": "Default agent configuration created by Kibana",
                         "status": "active",
-                        "datasources": [
+                        "packageConfigs": [
                           {
                             "id": "8a5679b0-8fbf-11ea-b2ce-01c4a6127154",
                             "name": "system-1",
@@ -750,7 +750,7 @@
                         "namespace": "UPDATED namespace",
                         "updated_on": "Fri Feb 28 2020 16:22:31 GMT-0500 (Eastern Standard Time)",
                         "updated_by": "elastic",
-                        "datasources": []
+                        "packageConfigs": []
                       },
                       "success": true
                     }
@@ -922,9 +922,9 @@
       },
       "parameters": []
     },
-    "/datasources": {
+    "/packageConfigs": {
       "get": {
-        "summary": "Datasources - List",
+        "summary": "PackageConfigs - List",
         "tags": [],
         "responses": {
           "200": {
@@ -937,7 +937,7 @@
                     "items": {
                       "type": "array",
                       "items": {
-                        "$ref": "#/components/schemas/Datasource"
+                        "$ref": "#/components/schemas/PackageConfig"
                       }
                     },
                     "total": {
@@ -1166,14 +1166,14 @@
             }
           }
         },
-        "operationId": "get-datasources",
+        "operationId": "get-packageConfigs",
         "security": [],
         "parameters": []
       },
       "parameters": [],
       "post": {
-        "summary": "Datasources - Create",
-        "operationId": "post-datasources",
+        "summary": "PackageConfigs - Create",
+        "operationId": "post-packageConfigs",
         "responses": {
           "200": {
             "description": "OK"
@@ -1183,7 +1183,7 @@
           "content": {
             "application/json": {
               "schema": {
-                "$ref": "#/components/schemas/NewDatasource"
+                "$ref": "#/components/schemas/NewPackageConfig"
               },
               "examples": {
                 "example-1": {
@@ -1237,9 +1237,9 @@
         ]
       }
     },
-    "/datasources/{datasourceId}": {
+    "/packageConfigs/{packageConfigId}": {
       "get": {
-        "summary": "Datasources - Info",
+        "summary": "PackageConfigs - Info",
         "tags": [],
         "responses": {
           "200": {
@@ -1250,7 +1250,7 @@
                   "type": "object",
                   "properties": {
                     "item": {
-                      "$ref": "#/components/schemas/Datasource"
+                      "$ref": "#/components/schemas/PackageConfig"
                     },
                     "success": {
                       "type": "boolean"
@@ -1262,21 +1262,21 @@
             }
           }
         },
-        "operationId": "get-datasources-datasourceId"
+        "operationId": "get-packageConfigs-packageConfigId"
       },
       "parameters": [
         {
           "schema": {
             "type": "string"
           },
-          "name": "datasourceId",
+          "name": "packageConfigId",
           "in": "path",
           "required": true
         }
       ],
       "put": {
-        "summary": "Datasources - Update",
-        "operationId": "put-datasources-datasourceId",
+        "summary": "PackageConfigs - Update",
+        "operationId": "put-packageConfigs-packageConfigId",
         "responses": {
           "200": {
             "description": "OK",
@@ -1286,7 +1286,7 @@
                   "type": "object",
                   "properties": {
                     "item": {
-                      "$ref": "#/components/schemas/Datasource"
+                      "$ref": "#/components/schemas/PackageConfig"
                     },
                     "sucess": {
                       "type": "boolean"
@@ -1724,9 +1724,7 @@
                         "license": "basic",
                         "description": "This is the Elastic Endpoint package.",
                         "type": "solution",
-                        "categories": [
-                          "security"
-                        ],
+                        "categories": ["security"],
                         "release": "beta",
                         "requirement": {
                           "kibana": {
@@ -1826,10 +1824,10 @@
                             "path": "telemetry"
                           }
                         ],
-                        "datasources": [
+                        "packageConfigs": [
                           {
                             "name": "endpoint",
-                            "title": "Endpoint data source",
+                            "title": "Endpoint package config",
                             "description": "Interact with the endpoint.",
                             "inputs": null,
                             "multiple": false
@@ -2121,7 +2119,7 @@
                           }
                         },
                         {
-                          "description": "The log package should be used to create data sources for all type of logs for which an package doesn't exist yet.\n",
+                          "description": "The log package should be used to create package configs for all type of logs for which an package doesn't exist yet.\n",
                           "download": "/epr/log/log-0.9.0.tar.gz",
                           "icons": [
                             {
@@ -2782,7 +2780,7 @@
                           },
                           "actions": [
                             {
-                              "data": "{\"config\":{\"id\":\"ae556400-5e39-11ea-8b49-f9747e466f7b\",\"outputs\":{\"default\":{\"type\":\"elasticsearch\",\"hosts\":[\"http://localhost:9200\"],\"api_key\":\"\",\"api_token\":\"6ckkp3ABz7e_XRqr3LM8:gQuDfUNSRgmY0iziYqP9Hw\"}},\"datasources\":[]}}",
+                              "data": "{\"config\":{\"id\":\"ae556400-5e39-11ea-8b49-f9747e466f7b\",\"outputs\":{\"default\":{\"type\":\"elasticsearch\",\"hosts\":[\"http://localhost:9200\"],\"api_key\":\"\",\"api_token\":\"6ckkp3ABz7e_XRqr3LM8:gQuDfUNSRgmY0iziYqP9Hw\"}},\"packageConfigs\":[]}}",
                               "created_at": "2020-03-04T20:02:56.149Z",
                               "id": "6a95c00a-d76d-4931-97c3-0bf935272d7d",
                               "type": "CONFIG_CHANGE"
@@ -2967,7 +2965,7 @@
                                   "api_key": "Z-XkgHIBvwtjzIKtSCTh:AejRqdKpQx6z-6dqSI1LHg"
                                 }
                               },
-                              "datasources": [
+                              "packageConfigs": [
                                 {
                                   "id": "33d6bd70-a5e0-11ea-a587-5f886c8a849f",
                                   "name": "system-1",
@@ -3690,7 +3688,7 @@
                 "type": "string",
                 "enum": ["active", "inactive"]
               },
-              "datasources": {
+              "packageConfigs": {
                 "oneOf": [
                   {
                     "items": {
@@ -3699,7 +3697,7 @@
                   },
                   {
                     "items": {
-                      "$ref": "#/components/schemas/Datasource"
+                      "$ref": "#/components/schemas/PackageConfig"
                     }
                   }
                 ],
@@ -3723,8 +3721,8 @@
           }
         ]
       },
-      "Datasource": {
-        "title": "Datasource",
+      "PackageConfig": {
+        "title": "PackageConfig",
         "allOf": [
           {
             "type": "object",
@@ -3743,7 +3741,7 @@
             "required": ["id", "revision"]
           },
           {
-            "$ref": "#/components/schemas/NewDatasource"
+            "$ref": "#/components/schemas/NewPackageConfig"
           }
         ],
         "x-examples": {
@@ -3765,8 +3763,8 @@
           }
         }
       },
-      "NewDatasource": {
-        "title": "NewDatasource",
+      "NewPackageConfig": {
+        "title": "NewPackageConfig",
         "type": "object",
         "x-examples": {
           "example-1": {

diff --git a/x-pack/plugins/ingest_manager/common/services/index.ts b/x-pack/plugins/ingest_manager/common/services/index.ts
@@ -6,8 +6,8 @@
 import * as AgentStatusKueryHelper from './agent_status';
 
 export * from './routes';
-export { packageToConfigDatasourceInputs, packageToConfigDatasource } from './package_to_config';
-export { storedDatasourcesToAgentInputs } from './datasources_to_agent_inputs';
+export { packageToPackageConfigInputs, packageToPackageConfig } from './package_to_config';
+export { storedPackageConfigsToAgentInputs } from './package_configs_to_agent_inputs';
 export { configToYaml } from './config_to_yaml';
 export { AgentStatusKueryHelper };
 export { decodeCloudId } from './decode_cloud_id';