Add support for input[].dataset.type and change stream.dataset ma…

…pping to be object containing `name` with instead of just a string
elastic · Jun 24, 2020 · f34e642 · f34e642
1 parent fe5a6ae
commit f34e642
Show file tree

Hide file tree

Showing 15 changed files with 105 additions and 50 deletions.
diff --git a/x-pack/plugins/ingest_manager/common/services/datasources_to_agent_inputs.test.ts b/x-pack/plugins/ingest_manager/common/services/datasources_to_agent_inputs.test.ts
@@ -39,7 +39,7 @@ describe('Ingest Manager - storedDatasourcesToAgentInputs', () => {
       {
         id: 'test-logs-foo',
         enabled: true,
-        dataset: 'foo',
+        dataset: { name: 'foo' },
         vars: {
           fooVar: { value: 'foo-value' },
           fooVar2: { value: [1, 2] },
@@ -52,7 +52,7 @@ describe('Ingest Manager - storedDatasourcesToAgentInputs', () => {
       {
         id: 'test-logs-bar',
         enabled: true,
-        dataset: 'bar',
+        dataset: { name: 'bar' },
         vars: {
           barVar: { value: 'bar-value' },
           barVar2: { value: [1, 2] },

diff --git a/x-pack/plugins/ingest_manager/common/services/datasources_to_agent_inputs.ts b/x-pack/plugins/ingest_manager/common/services/datasources_to_agent_inputs.ts
@@ -24,7 +24,10 @@ export const storedDatasourcesToAgentInputs = (
         id: datasource.id || datasource.name,
         name: datasource.name,
         type: input.type,
-        dataset: { namespace: datasource.namespace || 'default' },
+        dataset: {
+          namespace: datasource.namespace || 'default',
+          ...(input.dataset?.type ? { type: input.dataset.type } : {}),
+        },
         use_output: DEFAULT_OUTPUT.name,
         ...Object.entries(input.config || {}).reduce((acc, [key, { value }]) => {
           acc[key] = value;
@@ -35,7 +38,7 @@ export const storedDatasourcesToAgentInputs = (
           .map((stream) => {
             const fullStream: FullAgentConfigInputStream = {
               id: stream.id,
-              dataset: { name: stream.dataset },
+              dataset: stream.dataset,
               ...stream.agent_stream,
               ...Object.entries(stream.config || {}).reduce((acc, [key, { value }]) => {
                 acc[key] = value;

diff --git a/x-pack/plugins/ingest_manager/common/services/package_to_config.test.ts b/x-pack/plugins/ingest_manager/common/services/package_to_config.test.ts
@@ -82,15 +82,17 @@ describe('Ingest Manager - packageToConfig', () => {
       ).toEqual([
         {
           type: 'foo',
+          dataset: { type: 'logs' },
           enabled: true,
-          streams: [{ id: 'foo-foo', enabled: true, dataset: 'foo' }],
+          streams: [{ id: 'foo-foo', enabled: true, dataset: { name: 'foo' } }],
         },
         {
           type: 'bar',
+          dataset: { type: 'logs' },
           enabled: true,
           streams: [
-            { id: 'bar-bar', enabled: true, dataset: 'bar' },
-            { id: 'bar-bar2', enabled: true, dataset: 'bar2' },
+            { id: 'bar-bar', enabled: true, dataset: { name: 'bar' } },
+            { id: 'bar-bar2', enabled: true, dataset: { name: 'bar2' } },
           ],
         },
       ]);
@@ -136,30 +138,32 @@ describe('Ingest Manager - packageToConfig', () => {
       ).toEqual([
         {
           type: 'foo',
+          dataset: { type: 'logs' },
           enabled: true,
           streams: [
             {
               id: 'foo-foo',
               enabled: true,
-              dataset: 'foo',
+              dataset: { name: 'foo' },
               vars: { 'var-name': { value: 'foo-var-value' } },
             },
           ],
         },
         {
           type: 'bar',
+          dataset: { type: 'logs' },
           enabled: true,
           streams: [
             {
               id: 'bar-bar',
               enabled: true,
-              dataset: 'bar',
+              dataset: { name: 'bar' },
               vars: { 'var-name': { type: 'text', value: 'bar-var-value' } },
             },
             {
               id: 'bar-bar2',
               enabled: true,
-              dataset: 'bar2',
+              dataset: { name: 'bar2' },
               vars: { 'var-name': { type: 'yaml', value: 'bar2-var-value' } },
             },
           ],
@@ -247,6 +251,7 @@ describe('Ingest Manager - packageToConfig', () => {
       ).toEqual([
         {
           type: 'foo',
+          dataset: { type: 'logs' },
           enabled: true,
           vars: {
             'foo-input-var-name': { value: 'foo-input-var-value' },
@@ -257,7 +262,7 @@ describe('Ingest Manager - packageToConfig', () => {
             {
               id: 'foo-foo',
               enabled: true,
-              dataset: 'foo',
+              dataset: { name: 'foo' },
               vars: {
                 'var-name': { value: 'foo-var-value' },
               },
@@ -266,6 +271,7 @@ describe('Ingest Manager - packageToConfig', () => {
         },
         {
           type: 'bar',
+          dataset: { type: 'logs' },
           enabled: true,
           vars: {
             'bar-input-var-name': { value: ['value1', 'value2'] },
@@ -275,15 +281,15 @@ describe('Ingest Manager - packageToConfig', () => {
             {
               id: 'bar-bar',
               enabled: true,
-              dataset: 'bar',
+              dataset: { name: 'bar' },
               vars: {
                 'var-name': { value: 'bar-var-value' },
               },
             },
             {
               id: 'bar-bar2',
               enabled: true,
-              dataset: 'bar2',
+              dataset: { name: 'bar2' },
               vars: {
                 'var-name': { value: 'bar2-var-value' },
               },
@@ -292,20 +298,21 @@ describe('Ingest Manager - packageToConfig', () => {
         },
         {
           type: 'with-disabled-streams',
+          dataset: { type: 'logs' },
           enabled: false,
           streams: [
             {
               id: 'with-disabled-streams-disabled',
               enabled: false,
-              dataset: 'disabled',
+              dataset: { name: 'disabled' },
               vars: {
                 'var-name': { value: [] },
               },
             },
             {
               id: 'with-disabled-streams-disabled2',
               enabled: false,
-              dataset: 'disabled2',
+              dataset: { name: 'disabled2' },
             },
           ],
         },

diff --git a/x-pack/plugins/ingest_manager/common/services/package_to_config.ts b/x-pack/plugins/ingest_manager/common/services/package_to_config.ts
@@ -3,6 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
+import { pluck, uniq } from 'lodash';
 import {
   PackageInfo,
   RegistryConfigTemplate,
@@ -16,18 +17,19 @@ import {
   NewDatasource,
 } from '../types';
 
-const findStreamsForInputType = (
+const getStreamsForInputType = (
   inputType: string,
   packageInfo: PackageInfo
-): Array<RegistryStream & { dataset: { name: string } }> => {
-  const streams: Array<RegistryStream & { dataset: { name: string } }> = [];
+): Array<RegistryStream & { dataset: { type: string; name: string } }> => {
+  const streams: Array<RegistryStream & { dataset: { type: string; name: string } }> = [];
 
   (packageInfo.datasets || []).forEach((dataset) => {
     (dataset.streams || []).forEach((stream) => {
       if (stream.input === inputType) {
         streams.push({
           ...stream,
           dataset: {
+            type: dataset.type,
             name: dataset.name,
           },
         });
@@ -38,6 +40,11 @@ const findStreamsForInputType = (
   return streams;
 };
 
+const getDatasetTypeForInputType = (inputType: string, packageInfo: PackageInfo): string => {
+  const datasetTypes = uniq(pluck(getStreamsForInputType(inputType, packageInfo), 'dataset.type'));
+  return datasetTypes[0];
+};
+
 /*
  * This service creates a datasource inputs definition from defaults provided in package info
  */
@@ -70,14 +77,14 @@ export const packageToConfigDatasourceInputs = (packageInfo: PackageInfo): Datas
       };
 
       // Map each package input stream into datasource input stream
-      const streams: DatasourceInputStream[] = findStreamsForInputType(
+      const streams: DatasourceInputStream[] = getStreamsForInputType(
         packageInput.type,
         packageInfo
       ).map((packageStream) => {
         const stream: DatasourceInputStream = {
           id: `${packageInput.type}-${packageStream.dataset.name}`,
           enabled: packageStream.enabled === false ? false : true,
-          dataset: packageStream.dataset.name,
+          dataset: { name: packageStream.dataset.name },
         };
         if (packageStream.vars && packageStream.vars.length) {
           stream.vars = packageStream.vars.reduce(varsReducer, {});
@@ -91,6 +98,12 @@ export const packageToConfigDatasourceInputs = (packageInfo: PackageInfo): Datas
         streams,
       };
 
+      const datasetType = getDatasetTypeForInputType(packageInput.type, packageInfo);
+
+      if (datasetType) {
+        input.dataset = { type: datasetType };
+      }
+
       if (packageInput.vars && packageInput.vars.length) {
         input.vars = packageInput.vars.reduce(varsReducer, {});
       }

diff --git a/x-pack/plugins/ingest_manager/common/types/models/datasource.ts b/x-pack/plugins/ingest_manager/common/types/models/datasource.ts
@@ -20,7 +20,9 @@ export type DatasourceConfigRecord = Record<string, DatasourceConfigRecordEntry>
 export interface NewDatasourceInputStream {
   id: string;
   enabled: boolean;
-  dataset: string;
+  dataset: {
+    name: string;
+  };
   processors?: string[];
   config?: DatasourceConfigRecord;
   vars?: DatasourceConfigRecord;
@@ -32,6 +34,9 @@ export interface DatasourceInputStream extends NewDatasourceInputStream {
 
 export interface NewDatasourceInput {
   type: string;
+  dataset?: {
+    type: string;
+  };
   enabled: boolean;
   processors?: string[];
   config?: DatasourceConfigRecord;

diff --git a/...anager/sections/agent_config/create_datasource_page/components/datasource_input_panel.tsx b/...anager/sections/agent_config/create_datasource_page/components/datasource_input_panel.tsx
@@ -177,7 +177,7 @@ export const DatasourceInputPanel: React.FunctionComponent<{
         <EuiFlexGroup direction="column">
           {packageInputStreams.map((packageInputStream) => {
             const datasourceInputStream = datasourceInput.streams.find(
-              (stream) => stream.dataset === packageInputStream.dataset.name
+              (stream) => stream.dataset.name === packageInputStream.dataset.name
             );
             return datasourceInputStream ? (
               <EuiFlexItem key={packageInputStream.dataset.name}>
@@ -186,7 +186,7 @@ export const DatasourceInputPanel: React.FunctionComponent<{
                   datasourceInputStream={datasourceInputStream}
                   updateDatasourceInputStream={(updatedStream: Partial<DatasourceInputStream>) => {
                     const indexOfUpdatedStream = datasourceInput.streams.findIndex(
-                      (stream) => stream.dataset === packageInputStream.dataset.name
+                      (stream) => stream.dataset.name === packageInputStream.dataset.name
                     );
                     const newStreams = [...datasourceInput.streams];
                     newStreams[indexOfUpdatedStream] = {

diff --git a/...manager/sections/agent_config/create_datasource_page/services/validate_datasource.test.ts b/...manager/sections/agent_config/create_datasource_page/services/validate_datasource.test.ts
@@ -158,7 +158,7 @@ describe('Ingest Manager - validateDatasource()', () => {
         streams: [
           {
             id: 'foo-foo',
-            dataset: 'foo',
+            dataset: { name: 'foo' },
             enabled: true,
             vars: { 'var-name': { value: 'test_yaml: value', type: 'yaml' } },
           },
@@ -174,13 +174,13 @@ describe('Ingest Manager - validateDatasource()', () => {
         streams: [
           {
             id: 'bar-bar',
-            dataset: 'bar',
+            dataset: { name: 'bar' },
             enabled: true,
             vars: { 'var-name': { value: 'test_yaml: value', type: 'yaml' } },
           },
           {
             id: 'bar-bar2',
-            dataset: 'bar2',
+            dataset: { name: 'bar2' },
             enabled: true,
             vars: { 'var-name': { value: undefined, type: 'text' } },
           },
@@ -197,13 +197,13 @@ describe('Ingest Manager - validateDatasource()', () => {
         streams: [
           {
             id: 'with-disabled-streams-disabled',
-            dataset: 'disabled',
+            dataset: { name: 'disabled' },
             enabled: false,
             vars: { 'var-name': { value: undefined, type: 'text' } },
           },
           {
             id: 'with-disabled-streams-disabled-without-vars',
-            dataset: 'disabled2',
+            dataset: { name: 'disabled2' },
             enabled: false,
           },
         ],
@@ -217,7 +217,7 @@ describe('Ingest Manager - validateDatasource()', () => {
         streams: [
           {
             id: 'with-no-stream-vars-bar',
-            dataset: 'bar',
+            dataset: { name: 'bar' },
             enabled: true,
           },
         ],
@@ -240,7 +240,7 @@ describe('Ingest Manager - validateDatasource()', () => {
         streams: [
           {
             id: 'foo-foo',
-            dataset: 'foo',
+            dataset: { name: 'foo' },
             enabled: true,
             vars: { 'var-name': { value: 'invalidyaml: test\n foo bar:', type: 'yaml' } },
           },
@@ -256,13 +256,13 @@ describe('Ingest Manager - validateDatasource()', () => {
         streams: [
           {
             id: 'bar-bar',
-            dataset: 'bar',
+            dataset: { name: 'bar' },
             enabled: true,
             vars: { 'var-name': { value: '    \n\n', type: 'yaml' } },
           },
           {
             id: 'bar-bar2',
-            dataset: 'bar2',
+            dataset: { name: 'bar2' },
             enabled: true,
             vars: { 'var-name': { value: undefined, type: 'text' } },
           },
@@ -279,7 +279,7 @@ describe('Ingest Manager - validateDatasource()', () => {
         streams: [
           {
             id: 'with-disabled-streams-disabled',
-            dataset: 'disabled',
+            dataset: { name: 'disabled' },
             enabled: false,
             vars: {
               'var-name': {
@@ -290,7 +290,7 @@ describe('Ingest Manager - validateDatasource()', () => {
           },
           {
             id: 'with-disabled-streams-disabled-without-vars',
-            dataset: 'disabled2',
+            dataset: { name: 'disabled2' },
             enabled: false,
           },
         ],
@@ -304,7 +304,7 @@ describe('Ingest Manager - validateDatasource()', () => {
         streams: [
           {
             id: 'with-no-stream-vars-bar',
-            dataset: 'bar',
+            dataset: { name: 'bar' },
             enabled: true,
           },
         ],

diff --git a/...gest_manager/sections/agent_config/create_datasource_page/services/validate_datasource.ts b/...gest_manager/sections/agent_config/create_datasource_page/services/validate_datasource.ts
@@ -124,7 +124,7 @@ export const validateDatasource = (
         if (stream.vars) {
           const streamVarsByName = (
             (
-              registryStreamsByDataset[stream.dataset].find(
+              registryStreamsByDataset[stream.dataset.name].find(
                 (registryStream) => registryStream.input === input.type
               ) || {}
             ).vars || []