From 5628ef3efc5f5f08dd0fdc97333fcde3912519b4 Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Mon, 29 Jun 2020 23:05:51 +0200 Subject: [PATCH] Update package output structure (#514) This change contains several changes to the output generated for the package API endpoint: * dataset.id is renamed to dataset.name * datasources is renamed to config_templates. Currently only in the output, packages can follow later. * config_templates (aka datasources) do not pregenerate the content anymore to simply config building in Kibana * template_content is removed Further change is done to use `Dataset(s)` instead of `DataSet(s)` in the code for consistency. --- testdata/generated/package.json | 50 ++--- .../package/datasources/1.0.0/index.json | 206 ++++++------------ .../package/default_pipeline/0.0.2/index.json | 50 ++--- .../ecs_style_dataset/0.0.1/index.json | 22 +- .../package/example/1.0.0/index.json | 50 ++--- .../package/multiple_false/0.0.1/index.json | 50 ++--- .../no_stream_configs/1.0.0/index.json | 4 +- .../package/reference/1.0.0/index.json | 63 +++--- .../package/yamlpipeline/1.0.0/index.json | 5 +- .../example/1.0.0/dataset/foo/manifest.yml | 2 +- util/dataset.go | 41 ++-- util/package.go | 32 +-- 12 files changed, 204 insertions(+), 371 deletions(-) diff --git a/testdata/generated/package.json b/testdata/generated/package.json index 7e1f28e6f..e037a0388 100644 --- a/testdata/generated/package.json +++ b/testdata/generated/package.json @@ -60,12 +60,25 @@ "/package/example/1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-plaintext.json", "/package/example/1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-tcp.json" ], + "config_templates": [ + { + "name": "logs", + "title": "Logs datasource", + "description": "Datasource for your log files.", + "inputs": [ + { + "type": "foo" + } + ], + "multiple": true + } + ], "datasets": [ { - "id": "bar", + "type": "logs", + "name": "bar.noo", "title": "Foo", "release": "experimental", - "type": "logs", "ingest_pipeline": "pipeline-entry", "streams": [ { @@ -80,6 +93,7 @@ "show_user": false } ], + "template_path": "stream.yml.hbs", "enabled": true } ], @@ -87,38 +101,6 @@ "path": "foo" } ], - "datasources": [ - { - "name": "logs", - "title": "Logs datasource", - "description": "Datasource for your log files.", - "inputs": [ - { - "type": "foo", - "streams": [ - { - "input": "foo", - "vars": [ - { - "name": "paths", - "type": "text", - "description": "Path to log files to be collected", - "multi": true, - "required": true, - "show_user": false - } - ], - "dataset": "bar", - "template_path": "stream.yml.hbs", - "template": "foo: bar\n", - "enabled": true - } - ] - } - ], - "multiple": true - } - ], "owner": { "github": "ruflin" } diff --git a/testdata/generated/package/datasources/1.0.0/index.json b/testdata/generated/package/datasources/1.0.0/index.json index 52e5b437b..a5388a797 100644 --- a/testdata/generated/package/datasources/1.0.0/index.json +++ b/testdata/generated/package/datasources/1.0.0/index.json @@ -36,12 +36,69 @@ "/package/datasources/1.0.0/dataset/examplelog2/agent/stream/stream.yml.hbs", "/package/datasources/1.0.0/dataset/examplemetric/agent/stream/stream.yml.hbs" ], + "config_templates": [ + { + "name": "nginx", + "title": "Datasource title", + "description": "Details about the data source.", + "inputs": [ + { + "type": "nginx/metrics", + "vars": [ + { + "name": "hosts", + "type": "text", + "description": "Nginx hosts", + "multi": true, + "required": true, + "show_user": false, + "default": [ + "http://127.0.0.1" + ] + }, + { + "name": "period", + "type": "duration", + "description": "Collection period. Valid values: 10s, 5m, 2h", + "multi": false, + "required": false, + "show_user": false, + "default": "10s" + }, + { + "name": "username", + "type": "text", + "multi": false, + "required": false, + "show_user": false + }, + { + "name": "password", + "type": "password", + "multi": false, + "required": false, + "show_user": false + } + ], + "description": "Collecting metrics for nginx." + }, + { + "type": "logs", + "description": "Collect nginx logs." + }, + { + "type": "syslog" + } + ], + "multiple": true + } + ], "datasets": [ { - "id": "datasources.examplelog1", + "type": "logs", + "name": "datasources.examplelog1", "title": "Example dataset with inputs", "release": "experimental", - "type": "logs", "streams": [ { "input": "logs", @@ -75,10 +132,10 @@ "path": "examplelog1" }, { - "id": "datasources.examplelog2", + "type": "logs", + "name": "datasources.examplelog2", "title": "Example dataset with inputs", "release": "experimental", - "type": "logs", "streams": [ { "input": "logs", @@ -95,6 +152,7 @@ ] } ], + "template_path": "stream.yml.hbs", "title": "Title of the stream", "description": "Description of the stream with more details.", "enabled": true @@ -104,10 +162,10 @@ "path": "examplelog2" }, { - "id": "datasources.examplemetric", + "type": "metrics", + "name": "datasources.examplemetric", "title": "Example dataset with inputs", "release": "experimental", - "type": "metrics", "streams": [ { "input": "nginx/metrics", @@ -130,141 +188,5 @@ "package": "datasources", "path": "examplemetric" } - ], - "datasources": [ - { - "name": "nginx", - "title": "Datasource title", - "description": "Details about the data source.", - "inputs": [ - { - "type": "nginx/metrics", - "vars": [ - { - "name": "hosts", - "type": "text", - "description": "Nginx hosts", - "multi": true, - "required": true, - "show_user": false, - "default": [ - "http://127.0.0.1" - ] - }, - { - "name": "period", - "type": "duration", - "description": "Collection period. Valid values: 10s, 5m, 2h", - "multi": false, - "required": false, - "show_user": false, - "default": "10s" - }, - { - "name": "username", - "type": "text", - "multi": false, - "required": false, - "show_user": false - }, - { - "name": "password", - "type": "password", - "multi": false, - "required": false, - "show_user": false - } - ], - "description": "Collecting metrics for nginx.", - "streams": [ - { - "input": "nginx/metrics", - "vars": [ - { - "name": "url", - "type": "text", - "description": "Paths to the nginx access log file.", - "multi": false, - "required": true, - "show_user": false, - "default": "localhost" - } - ], - "dataset": "datasources.examplemetric", - "template_path": "stream.yml.hbs", - "template": "metric: foo\n", - "title": "Title of the stream", - "description": "Not enabled data source.", - "enabled": false - } - ] - }, - { - "type": "logs", - "description": "Collect nginx logs.", - "streams": [ - { - "input": "logs", - "vars": [ - { - "name": "paths", - "type": "text", - "description": "Paths to the nginx error log file.", - "multi": true, - "required": true, - "show_user": false, - "default": [ - "/var/log/nginx/error.log*" - ] - } - ], - "dataset": "datasources.examplelog1", - "template_path": "logs.yml", - "template": "foo: bar\n", - "title": "Title of the stream", - "description": "Description of the stream with more details.", - "enabled": true - }, - { - "input": "logs", - "vars": [ - { - "name": "paths", - "type": "text", - "description": "Paths to the nginx access log file.", - "multi": true, - "required": true, - "show_user": false, - "default": [ - "/var/log/nginx/access.log*" - ] - } - ], - "dataset": "datasources.examplelog2", - "template_path": "stream.yml.hbs", - "template": "foo: bar\n", - "title": "Title of the stream", - "description": "Description of the stream with more details.", - "enabled": true - } - ] - }, - { - "type": "syslog", - "streams": [ - { - "input": "syslog", - "dataset": "datasources.examplelog1", - "template_path": "syslog.yml", - "template": "syslog: bar\n", - "title": "Title of the stream", - "description": "Description of the stream with more details.", - "enabled": true - } - ] - } - ], - "multiple": true - } ] } \ No newline at end of file diff --git a/testdata/generated/package/default_pipeline/0.0.2/index.json b/testdata/generated/package/default_pipeline/0.0.2/index.json index 178ea6345..401f77a22 100644 --- a/testdata/generated/package/default_pipeline/0.0.2/index.json +++ b/testdata/generated/package/default_pipeline/0.0.2/index.json @@ -31,12 +31,25 @@ "/package/default_pipeline/0.0.2/dataset/foo/agent/stream/stream.yml.hbs", "/package/default_pipeline/0.0.2/dataset/foo/elasticsearch/ingest-pipeline/default.json" ], + "config_templates": [ + { + "name": "logs", + "title": "Logs datasource", + "description": "Datasource for your log files.", + "inputs": [ + { + "type": "logs" + } + ], + "multiple": true + } + ], "datasets": [ { - "id": "default_pipeline.foo", + "type": "logs", + "name": "default_pipeline.foo", "title": "Foo", "release": "experimental", - "type": "logs", "ingest_pipeline": "default", "streams": [ { @@ -51,43 +64,12 @@ "show_user": false } ], + "template_path": "stream.yml.hbs", "enabled": true } ], "package": "default_pipeline", "path": "foo" } - ], - "datasources": [ - { - "name": "logs", - "title": "Logs datasource", - "description": "Datasource for your log files.", - "inputs": [ - { - "type": "logs", - "streams": [ - { - "input": "logs", - "vars": [ - { - "name": "paths", - "type": "text", - "description": "Path to log files to be collected", - "multi": true, - "required": true, - "show_user": false - } - ], - "dataset": "default_pipeline.foo", - "template_path": "stream.yml.hbs", - "template": "foo: bar\n", - "enabled": true - } - ] - } - ], - "multiple": true - } ] } \ No newline at end of file diff --git a/testdata/generated/package/ecs_style_dataset/0.0.1/index.json b/testdata/generated/package/ecs_style_dataset/0.0.1/index.json index 2ca6a20f3..8fe8cfde0 100644 --- a/testdata/generated/package/ecs_style_dataset/0.0.1/index.json +++ b/testdata/generated/package/ecs_style_dataset/0.0.1/index.json @@ -28,17 +28,7 @@ "/package/ecs_style_dataset/0.0.1/dataset/foo/manifest.yml", "/package/ecs_style_dataset/0.0.1/dataset/foo/fields/fields.yml" ], - "datasets": [ - { - "id": "ecs_style_dataset.foo", - "title": "Foo", - "release": "experimental", - "type": "logs", - "package": "ecs_style_dataset", - "path": "foo" - } - ], - "datasources": [ + "config_templates": [ { "name": "logs", "title": "Logs datasource", @@ -50,5 +40,15 @@ ], "multiple": true } + ], + "datasets": [ + { + "type": "logs", + "name": "ecs_style_dataset.foo", + "title": "Foo", + "release": "experimental", + "package": "ecs_style_dataset", + "path": "foo" + } ] } \ No newline at end of file diff --git a/testdata/generated/package/example/1.0.0/index.json b/testdata/generated/package/example/1.0.0/index.json index 7e1f28e6f..e037a0388 100644 --- a/testdata/generated/package/example/1.0.0/index.json +++ b/testdata/generated/package/example/1.0.0/index.json @@ -60,12 +60,25 @@ "/package/example/1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-plaintext.json", "/package/example/1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-tcp.json" ], + "config_templates": [ + { + "name": "logs", + "title": "Logs datasource", + "description": "Datasource for your log files.", + "inputs": [ + { + "type": "foo" + } + ], + "multiple": true + } + ], "datasets": [ { - "id": "bar", + "type": "logs", + "name": "bar.noo", "title": "Foo", "release": "experimental", - "type": "logs", "ingest_pipeline": "pipeline-entry", "streams": [ { @@ -80,6 +93,7 @@ "show_user": false } ], + "template_path": "stream.yml.hbs", "enabled": true } ], @@ -87,38 +101,6 @@ "path": "foo" } ], - "datasources": [ - { - "name": "logs", - "title": "Logs datasource", - "description": "Datasource for your log files.", - "inputs": [ - { - "type": "foo", - "streams": [ - { - "input": "foo", - "vars": [ - { - "name": "paths", - "type": "text", - "description": "Path to log files to be collected", - "multi": true, - "required": true, - "show_user": false - } - ], - "dataset": "bar", - "template_path": "stream.yml.hbs", - "template": "foo: bar\n", - "enabled": true - } - ] - } - ], - "multiple": true - } - ], "owner": { "github": "ruflin" } diff --git a/testdata/generated/package/multiple_false/0.0.1/index.json b/testdata/generated/package/multiple_false/0.0.1/index.json index 3abfbafc2..4f675e972 100644 --- a/testdata/generated/package/multiple_false/0.0.1/index.json +++ b/testdata/generated/package/multiple_false/0.0.1/index.json @@ -30,12 +30,25 @@ "/package/multiple_false/0.0.1/dataset/foo/agent/stream/stream.yml.hbs", "/package/multiple_false/0.0.1/dataset/foo/elasticsearch/ingest-pipeline/default.json" ], + "config_templates": [ + { + "name": "logs", + "title": "Logs datasource", + "description": "Datasource for your log files.", + "inputs": [ + { + "type": "logs" + } + ], + "multiple": false + } + ], "datasets": [ { - "id": "multiple_false.foo", + "type": "logs", + "name": "multiple_false.foo", "title": "Foo", "release": "experimental", - "type": "logs", "ingest_pipeline": "default", "streams": [ { @@ -50,43 +63,12 @@ "show_user": false } ], + "template_path": "stream.yml.hbs", "enabled": true } ], "package": "multiple_false", "path": "foo" } - ], - "datasources": [ - { - "name": "logs", - "title": "Logs datasource", - "description": "Datasource for your log files.", - "inputs": [ - { - "type": "logs", - "streams": [ - { - "input": "logs", - "vars": [ - { - "name": "paths", - "type": "text", - "description": "Path to log files to be collected", - "multi": true, - "required": true, - "show_user": false - } - ], - "dataset": "multiple_false.foo", - "template_path": "stream.yml.hbs", - "template": "foo: bar\n", - "enabled": true - } - ] - } - ], - "multiple": false - } ] } \ No newline at end of file diff --git a/testdata/generated/package/no_stream_configs/1.0.0/index.json b/testdata/generated/package/no_stream_configs/1.0.0/index.json index 846d7a428..1e31870d4 100644 --- a/testdata/generated/package/no_stream_configs/1.0.0/index.json +++ b/testdata/generated/package/no_stream_configs/1.0.0/index.json @@ -31,10 +31,10 @@ ], "datasets": [ { - "id": "no_stream_configs.log", + "type": "logs", + "name": "no_stream_configs.log", "title": "Log Yaml pipeline", "release": "experimental", - "type": "logs", "package": "no_stream_configs", "path": "log" } diff --git a/testdata/generated/package/reference/1.0.0/index.json b/testdata/generated/package/reference/1.0.0/index.json index d334aa275..32c12f7d0 100644 --- a/testdata/generated/package/reference/1.0.0/index.json +++ b/testdata/generated/package/reference/1.0.0/index.json @@ -40,37 +40,7 @@ "/package/reference/1.0.0/dataset/reference/manifest.yml", "/package/reference/1.0.0/dataset/reference/fields/base-fields.yml" ], - "datasets": [ - { - "id": "reference.reference", - "title": "Reference Logs Title", - "release": "beta", - "type": "logs", - "streams": [ - { - "input": "logs", - "vars": [ - { - "name": "paths", - "type": "text", - "title": "Example variable title", - "description": "Description around how a variable should be used, what values it can contain and it can even contain **Markdown** or links.\n", - "multi": true, - "required": true, - "show_user": false, - "default": "foo" - } - ], - "title": "Title reference stream", - "description": "Collecting the nginx access logs from file.", - "enabled": true - } - ], - "package": "reference", - "path": "reference" - } - ], - "datasources": [ + "config_templates": [ { "name": "nginx", "title": "Nginx logs and metrics.", @@ -99,6 +69,37 @@ "multiple": true } ], + "datasets": [ + { + "type": "logs", + "name": "reference.reference", + "title": "Reference Logs Title", + "release": "beta", + "streams": [ + { + "input": "logs", + "vars": [ + { + "name": "paths", + "type": "text", + "title": "Example variable title", + "description": "Description around how a variable should be used, what values it can contain and it can even contain **Markdown** or links.\n", + "multi": true, + "required": true, + "show_user": false, + "default": "foo" + } + ], + "template_path": "stream.yml.hbs", + "title": "Title reference stream", + "description": "Collecting the nginx access logs from file.", + "enabled": true + } + ], + "package": "reference", + "path": "reference" + } + ], "owner": { "github": "ruflin" } diff --git a/testdata/generated/package/yamlpipeline/1.0.0/index.json b/testdata/generated/package/yamlpipeline/1.0.0/index.json index bc5d6aa24..fda37ee15 100644 --- a/testdata/generated/package/yamlpipeline/1.0.0/index.json +++ b/testdata/generated/package/yamlpipeline/1.0.0/index.json @@ -34,10 +34,10 @@ ], "datasets": [ { - "id": "yamlpipeline.log", + "type": "logs", + "name": "yamlpipeline.log", "title": "Log Yaml pipeline", "release": "experimental", - "type": "logs", "ingest_pipeline": "pipeline-entry", "streams": [ { @@ -51,6 +51,7 @@ "show_user": false } ], + "template_path": "stream.yml.hbs", "title": "Yamlpipline example logs", "description": "Yamlpipeline example", "enabled": true diff --git a/testdata/package/example/1.0.0/dataset/foo/manifest.yml b/testdata/package/example/1.0.0/dataset/foo/manifest.yml index 0c6b02a09..89203c344 100644 --- a/testdata/package/example/1.0.0/dataset/foo/manifest.yml +++ b/testdata/package/example/1.0.0/dataset/foo/manifest.yml @@ -1,5 +1,5 @@ # This dataset has a different id then the path -id: bar +name: bar.noo title: Foo diff --git a/util/dataset.go b/util/dataset.go index 44fe749fd..2fe87f192 100644 --- a/util/dataset.go +++ b/util/dataset.go @@ -31,11 +31,13 @@ var validTypes = map[string]string{ "events": "Events", } -type DataSet struct { - ID string `config:"id" json:"id,omitempty" yaml:"id,omitempty"` +type Dataset struct { + // Name and type of the dataset. This is linked to dataset.name and dataset.type fields. + Type string `config:"type" json:"type" validate:"required"` + Name string `config:"name" json:"name,omitempty" yaml:"name,omitempty"` + Title string `config:"title" json:"title" validate:"required"` Release string `config:"release" json:"release"` - Type string `config:"type" json:"type" validate:"required"` IngestPipeline string `config:"ingest_pipeline,omitempty" config:"ingest_pipeline" json:"ingest_pipeline,omitempty" yaml:"ingest_pipeline,omitempty"` Streams []Stream `config:"streams" json:"streams,omitempty" yaml:"streams,omitempty" ` Package string `json:"package,omitempty" yaml:"package,omitempty"` @@ -60,11 +62,10 @@ type Stream struct { Vars []Variable `config:"vars" json:"vars,omitempty" yaml:"vars,omitempty"` Dataset string `config:"dataset" json:"dataset,omitempty" yaml:"dataset,omitempty"` // TODO: This might cause issues when consuming the json as the key contains . (had been an issue in the past if I remember correctly) - TemplatePath string `config:"template_path" json:"template_path,omitempty" yaml:"template_path,omitempty"` - TemplateContent string `json:"template,omitempty" yaml:"template,omitempty"` // This is always generated in the json output - Title string `config:"title" json:"title,omitempty" yaml:"title,omitempty"` - Description string `config:"description" json:"description,omitempty" yaml:"description,omitempty"` - Enabled *bool `config:"enabled" json:"enabled,omitempty" yaml:"enabled,omitempty"` + TemplatePath string `config:"template_path" json:"template_path,omitempty" yaml:"template_path,omitempty"` + Title string `config:"title" json:"title,omitempty" yaml:"title,omitempty"` + Description string `config:"description" json:"description,omitempty" yaml:"description,omitempty"` + Enabled *bool `config:"enabled" json:"enabled,omitempty" yaml:"enabled,omitempty"` } type Variable struct { @@ -83,7 +84,7 @@ type fieldEntry struct { aType string } -func NewDataset(basePath string, p *Package) (*DataSet, error) { +func NewDataset(basePath string, p *Package) (*Dataset, error) { // Check if manifest exists manifestPath := filepath.Join(basePath, "manifest.yml") _, err := os.Stat(manifestPath) @@ -97,7 +98,7 @@ func NewDataset(basePath string, p *Package) (*DataSet, error) { if err != nil { return nil, errors.Wrapf(err, "error creating new manifest config %s", manifestPath) } - var d = &DataSet{ + var d = &Dataset{ Package: p.Name, // This is the name of the directory of the dataset Path: datasetPath, @@ -111,8 +112,8 @@ func NewDataset(basePath string, p *Package) (*DataSet, error) { } // if id is not set, {package}.{datasetPath} is the default - if d.ID == "" { - d.ID = p.Name + "." + datasetPath + if d.Name == "" { + d.Name = p.Name + "." + datasetPath } if d.Release == "" { @@ -124,6 +125,10 @@ func NewDataset(basePath string, p *Package) (*DataSet, error) { for i, _ := range d.Streams { if d.Streams[i].Enabled == nil { d.Streams[i].Enabled = &trueValue + // TODO: validate that the template path actually exists + if d.Streams[i].TemplatePath == "" { + d.Streams[i].TemplatePath = "stream.yml.hbs" + } } } @@ -133,15 +138,15 @@ func NewDataset(basePath string, p *Package) (*DataSet, error) { return d, nil } -func (d *DataSet) Validate() error { +func (d *Dataset) Validate() error { pipelineDir := filepath.Join(d.BasePath, "elasticsearch", DirIngestPipeline) paths, err := filepath.Glob(filepath.Join(pipelineDir, "*")) if err != nil { return err } - if strings.Contains(d.ID, "-") { - return fmt.Errorf("dataset name is not allowed to contain `-`: %s", d.ID) + if strings.Contains(d.Name, "-") { + return fmt.Errorf("dataset name is not allowed to contain `-`: %s", d.Name) } if !d.validType() { @@ -159,7 +164,7 @@ func (d *DataSet) Validate() error { } if d.IngestPipeline == "" && len(paths) > 0 { - return fmt.Errorf("unused pipelines in the package (dataSetID: %s): %s", d.ID, strings.Join(paths, ",")) + return fmt.Errorf("unused pipelines in the package (dataSetID: %s): %s", d.Name, strings.Join(paths, ",")) } // In case an ingest pipeline is set, check if it is around @@ -204,7 +209,7 @@ func (d *DataSet) Validate() error { return nil } -func (d *DataSet) validType() bool { +func (d *Dataset) validType() bool { _, exists := validTypes[d.Type] return exists } @@ -234,7 +239,7 @@ func validateIngestPipelineFile(pipelinePath string) error { } // validateRequiredFields method loads fields from all files and checks if required fields are present. -func (d *DataSet) validateRequiredFields() error { +func (d *Dataset) validateRequiredFields() error { fieldsDirPath := filepath.Join(d.BasePath, "fields") // Collect fields from all files diff --git a/util/package.go b/util/package.go index c0de00f51..0cefc258a 100644 --- a/util/package.go +++ b/util/package.go @@ -6,7 +6,6 @@ package util import ( "fmt" - "io/ioutil" "os" "path" "path/filepath" @@ -64,8 +63,8 @@ type Package struct { Requirement Requirement `config:"requirement" json:"requirement"` Screenshots []Image `config:"screenshots,omitempty" json:"screenshots,omitempty" yaml:"screenshots,omitempty"` Assets []string `config:"assets,omitempty" json:"assets,omitempty" yaml:"assets,omitempty"` - DataSets []*DataSet `config:"datasets,omitempty" json:"datasets,omitempty" yaml:"datasets,omitempty"` - Datasources []Datasource `config:"datasources,omitempty" json:"datasources,omitempty" yaml:"datasources,omitempty"` + Datasources []Datasource `config:"datasources,omitempty" json:"config_templates,omitempty" yaml:"datasources,omitempty"` + Datasets []*Dataset `config:"datasets,omitempty" json:"datasets,omitempty" yaml:"datasets,omitempty"` Owner *Owner `config:"owner,omitempty" json:"owner,omitempty" yaml:"owner,omitempty"` // Local path to the package dir @@ -439,32 +438,9 @@ func (p *Package) LoadDataSets() error { return err } - // Iterate through all datasources and inputs to find the matching streams and add them to the output. - for dK, datasource := range p.Datasources { - for iK, _ := range datasource.Inputs { - for _, stream := range d.Streams { - if stream.Input == p.Datasources[dK].Inputs[iK].Type { - if stream.TemplatePath == "" { - stream.TemplatePath = "stream.yml.hbs" - } - stream.Dataset = d.ID - streamTemplate := filepath.Join(datasetBasePath, "agent", "stream", stream.TemplatePath) - - streamTemplateData, err := ioutil.ReadFile(streamTemplate) - if err != nil { - return err - } - - stream.TemplateContent = string(streamTemplateData) - - // Add template to stream - p.Datasources[dK].Inputs[iK].Streams = append(p.Datasources[dK].Inputs[iK].Streams, stream) - } - } - } - } + // TODO: Validate that each input specified in a stream also is defined in the package - p.DataSets = append(p.DataSets, d) + p.Datasets = append(p.Datasets, d) } return nil