[Ingest Manager] Support updated package output structure (#69864) (#…

…70263) * Update EPM package registry typings to reflect registry changes * Change `dataset.id` references to `dataset.name` * Fix RegistryStream * Fix packageToConfigDatasourceInputs service * Fix assignPackageStream service * Fix validateDatasource service * Fix configure data source components * Fix variable * Fix stream template installation * Add support for `input[].dataset.type` and change `stream.dataset` mapping to be object containing `name` with instead of just a string * Nest package information under `meta` in agent config yaml * Move `dataset.type` to stream level instead of input level * Make single call to fetch registry package information instead of doing it per stream * Fix type issues * Update endpoint test assertion Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com> Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
elastic · Jun 29, 2020 · 2e07eb4 · 2e07eb4
1 parent f0002f4
commit 2e07eb4
Show file tree

Hide file tree

Showing 27 changed files with 461 additions and 248 deletions.
diff --git a/x-pack/plugins/ingest_manager/common/services/config_to_yaml.ts b/x-pack/plugins/ingest_manager/common/services/config_to_yaml.ts
@@ -16,7 +16,7 @@ const CONFIG_KEYS_ORDER = [
   'inputs',
   'enabled',
   'use_output',
-  'package',
+  'meta',
   'input',
 ];
 

diff --git a/x-pack/plugins/ingest_manager/common/services/datasources_to_agent_inputs.test.ts b/x-pack/plugins/ingest_manager/common/services/datasources_to_agent_inputs.test.ts
@@ -39,7 +39,7 @@ describe('Ingest Manager - storedDatasourcesToAgentInputs', () => {
       {
         id: 'test-logs-foo',
         enabled: true,
-        dataset: 'foo',
+        dataset: { name: 'foo', type: 'logs' },
         vars: {
           fooVar: { value: 'foo-value' },
           fooVar2: { value: [1, 2] },
@@ -52,7 +52,7 @@ describe('Ingest Manager - storedDatasourcesToAgentInputs', () => {
       {
         id: 'test-logs-bar',
         enabled: true,
-        dataset: 'bar',
+        dataset: { name: 'bar', type: 'logs' },
         vars: {
           barVar: { value: 'bar-value' },
           barVar2: { value: [1, 2] },
@@ -101,23 +101,41 @@ describe('Ingest Manager - storedDatasourcesToAgentInputs', () => {
   });
 
   it('returns agent inputs', () => {
-    expect(storedDatasourcesToAgentInputs([{ ...mockDatasource, inputs: [mockInput] }])).toEqual([
+    expect(
+      storedDatasourcesToAgentInputs([
+        {
+          ...mockDatasource,
+          package: {
+            name: 'mock-package',
+            title: 'Mock package',
+            version: '0.0.0',
+          },
+          inputs: [mockInput],
+        },
+      ])
+    ).toEqual([
       {
         id: 'some-uuid',
         name: 'mock-datasource',
         type: 'test-logs',
         dataset: { namespace: 'default' },
         use_output: 'default',
+        meta: {
+          package: {
+            name: 'mock-package',
+            version: '0.0.0',
+          },
+        },
         streams: [
           {
             id: 'test-logs-foo',
-            dataset: { name: 'foo' },
+            dataset: { name: 'foo', type: 'logs' },
             fooKey: 'fooValue1',
             fooKey2: ['fooValue2'],
           },
           {
             id: 'test-logs-bar',
-            dataset: { name: 'bar' },
+            dataset: { name: 'bar', type: 'logs' },
           },
         ],
       },
@@ -147,7 +165,7 @@ describe('Ingest Manager - storedDatasourcesToAgentInputs', () => {
         streams: [
           {
             id: 'test-logs-foo',
-            dataset: { name: 'foo' },
+            dataset: { name: 'foo', type: 'logs' },
             fooKey: 'fooValue1',
             fooKey2: ['fooValue2'],
           },

diff --git a/x-pack/plugins/ingest_manager/common/services/datasources_to_agent_inputs.ts b/x-pack/plugins/ingest_manager/common/services/datasources_to_agent_inputs.ts
@@ -24,7 +24,9 @@ export const storedDatasourcesToAgentInputs = (
         id: datasource.id || datasource.name,
         name: datasource.name,
         type: input.type,
-        dataset: { namespace: datasource.namespace || 'default' },
+        dataset: {
+          namespace: datasource.namespace || 'default',
+        },
         use_output: DEFAULT_OUTPUT.name,
         ...Object.entries(input.config || {}).reduce((acc, [key, { value }]) => {
           acc[key] = value;
@@ -35,7 +37,7 @@ export const storedDatasourcesToAgentInputs = (
           .map((stream) => {
             const fullStream: FullAgentConfigInputStream = {
               id: stream.id,
-              dataset: { name: stream.dataset },
+              dataset: stream.dataset,
               ...stream.agent_stream,
               ...Object.entries(stream.config || {}).reduce((acc, [key, { value }]) => {
                 acc[key] = value;
@@ -50,9 +52,11 @@ export const storedDatasourcesToAgentInputs = (
       };
 
       if (datasource.package) {
-        fullInput.package = {
-          name: datasource.package.name,
-          version: datasource.package.version,
+        fullInput.meta = {
+          package: {
+            name: datasource.package.name,
+            version: datasource.package.version,
+          },
         };
       }