diff --git a/docs/api/package.json b/docs/api/package.json index b2604993c..8973b7c5f 100644 --- a/docs/api/package.json +++ b/docs/api/package.json @@ -55,6 +55,19 @@ "/package/example/1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-plaintext.json", "/package/example/1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-tcp.json" ], + "config_templates": [ + { + "name": "logs", + "title": "Logs datasource", + "description": "Datasource for your log files.", + "inputs": [ + { + "type": "foo" + } + ], + "multiple": true + } + ], "datasets": [ { "type": "logs", @@ -83,19 +96,6 @@ "path": "foo" } ], - "config_templates": [ - { - "name": "logs", - "title": "Logs datasource", - "description": "Datasource for your log files.", - "inputs": [ - { - "type": "foo" - } - ], - "multiple": true - } - ], "owner": { "github": "ruflin" } diff --git a/docs/api/package/datasources/1.0.0/index.json b/docs/api/package/datasources/1.0.0/index.json index ea3f1ffda..8f52ecd35 100644 --- a/docs/api/package/datasources/1.0.0/index.json +++ b/docs/api/package/datasources/1.0.0/index.json @@ -31,12 +31,69 @@ "/package/datasources/1.0.0/dataset/examplelog2/agent/stream/stream.yml.hbs", "/package/datasources/1.0.0/dataset/examplemetric/agent/stream/stream.yml.hbs" ], + "config_templates": [ + { + "name": "nginx", + "title": "Datasource title", + "description": "Details about the data source.", + "inputs": [ + { + "type": "nginx/metrics", + "vars": [ + { + "name": "hosts", + "type": "text", + "description": "Nginx hosts", + "multi": true, + "required": true, + "show_user": false, + "default": [ + "http://127.0.0.1" + ] + }, + { + "name": "period", + "type": "duration", + "description": "Collection period. Valid values: 10s, 5m, 2h", + "multi": false, + "required": false, + "show_user": false, + "default": "10s" + }, + { + "name": "username", + "type": "text", + "multi": false, + "required": false, + "show_user": false + }, + { + "name": "password", + "type": "password", + "multi": false, + "required": false, + "show_user": false + } + ], + "description": "Collecting metrics for nginx." + }, + { + "type": "logs", + "description": "Collect nginx logs." + }, + { + "type": "syslog" + } + ], + "multiple": true + } + ], "datasets": [ { - "id": "datasources.examplelog1", + "type": "logs", + "name": "datasources.examplelog1", "title": "Example dataset with inputs", "release": "experimental", - "type": "logs", "streams": [ { "input": "logs", @@ -82,10 +139,10 @@ "path": "examplelog1" }, { - "id": "datasources.examplelog2", + "type": "logs", + "name": "datasources.examplelog2", "title": "Example dataset with inputs", "release": "experimental", - "type": "logs", "streams": [ { "input": "logs", @@ -114,6 +171,7 @@ } } ], + "template_path": "stream.yml.hbs", "title": "Title of the stream", "description": "Description of the stream with more details.", "enabled": true @@ -123,10 +181,10 @@ "path": "examplelog2" }, { - "id": "datasources.examplemetric", + "type": "metrics", + "name": "datasources.examplemetric", "title": "Example dataset with inputs", "release": "experimental", - "type": "metrics", "streams": [ { "input": "nginx/metrics", @@ -149,165 +207,5 @@ "package": "datasources", "path": "examplemetric" } - ], - "datasources": [ - { - "name": "nginx", - "title": "Datasource title", - "description": "Details about the data source.", - "inputs": [ - { - "type": "nginx/metrics", - "vars": [ - { - "name": "hosts", - "type": "text", - "description": "Nginx hosts", - "multi": true, - "required": true, - "show_user": false, - "default": [ - "http://127.0.0.1" - ] - }, - { - "name": "period", - "type": "duration", - "description": "Collection period. Valid values: 10s, 5m, 2h", - "multi": false, - "required": false, - "show_user": false, - "default": "10s" - }, - { - "name": "username", - "type": "text", - "multi": false, - "required": false, - "show_user": false - }, - { - "name": "password", - "type": "password", - "multi": false, - "required": false, - "show_user": false - } - ], - "description": "Collecting metrics for nginx.", - "streams": [ - { - "input": "nginx/metrics", - "vars": [ - { - "name": "url", - "type": "text", - "description": "Paths to the nginx access log file.", - "multi": false, - "required": true, - "show_user": false, - "default": "localhost" - } - ], - "dataset": "datasources.examplemetric", - "template_path": "stream.yml.hbs", - "template": "metric: foo\n", - "title": "Title of the stream", - "description": "Not enabled data source.", - "enabled": false - } - ] - }, - { - "type": "logs", - "description": "Collect nginx logs.", - "streams": [ - { - "input": "logs", - "vars": [ - { - "name": "paths", - "type": "text", - "description": "Paths to the nginx error log file.", - "multi": true, - "required": true, - "show_user": false, - "default": [ - "/var/log/nginx/error.log*" - ], - "os": { - "darwin": { - "default": [ - "/usr/local/var/log/nginx/error.log*" - ] - }, - "windows": { - "default": [ - "c:/programdata/nginx/logs/*error.log*" - ] - } - } - } - ], - "dataset": "datasources.examplelog1", - "template_path": "logs.yml", - "template": "foo: bar\n", - "title": "Title of the stream", - "description": "Description of the stream with more details.", - "enabled": true - }, - { - "input": "logs", - "vars": [ - { - "name": "paths", - "type": "text", - "description": "Paths to the nginx access log file.", - "multi": true, - "required": true, - "show_user": false, - "default": [ - "/var/log/nginx/access.log*" - ], - "os": { - "darwin": { - "default": [ - "/usr/local/var/log/nginx/access.log*" - ] - }, - "windows": { - "default": [ - "c:/programdata/nginx/logs/*access.log*" - ] - } - } - } - ], - "dataset": "datasources.examplelog2", - "template_path": "stream.yml.hbs", - "template": "foo: bar\n", - "title": "Title of the stream", - "description": "Description of the stream with more details.", - "enabled": true - } - ] - }, - { - "type": "syslog", - "streams": [ - { - "input": "syslog", - "dataset": "datasources.examplelog1", - "template_path": "syslog.yml", - "template": "syslog: bar\n", - "title": "Title of the stream", - "description": "Description of the stream with more details.", - "enabled": true - } - ] - } - ], - "multiple": true - } ] } \ No newline at end of file diff --git a/docs/api/package/default_pipeline/0.0.2/index.json b/docs/api/package/default_pipeline/0.0.2/index.json index b0a9297e5..5ae2498a5 100644 --- a/docs/api/package/default_pipeline/0.0.2/index.json +++ b/docs/api/package/default_pipeline/0.0.2/index.json @@ -25,12 +25,25 @@ "/package/default_pipeline/0.0.2/dataset/foo/agent/stream/stream.yml.hbs", "/package/default_pipeline/0.0.2/dataset/foo/elasticsearch/ingest-pipeline/default.json" ], + "config_templates": [ + { + "name": "logs", + "title": "Logs datasource", + "description": "Datasource for your log files.", + "inputs": [ + { + "type": "logs" + } + ], + "multiple": true + } + ], "datasets": [ { - "id": "default_pipeline.foo", + "type": "logs", + "name": "default_pipeline.foo", "title": "Foo", "release": "experimental", - "type": "logs", "ingest_pipeline": "default", "streams": [ { @@ -45,43 +58,12 @@ "show_user": false } ], + "template_path": "stream.yml.hbs", "enabled": true } ], "package": "default_pipeline", "path": "foo" } - ], - "datasources": [ - { - "name": "logs", - "title": "Logs datasource", - "description": "Datasource for your log files.", - "inputs": [ - { - "type": "logs", - "streams": [ - { - "input": "logs", - "vars": [ - { - "name": "paths", - "type": "text", - "description": "Path to log files to be collected", - "multi": true, - "required": true, - "show_user": false - } - ], - "dataset": "default_pipeline.foo", - "template_path": "stream.yml.hbs", - "template": "foo: bar\n", - "enabled": true - } - ] - } - ], - "multiple": true - } ] } \ No newline at end of file diff --git a/docs/api/package/ecs_style_dataset/0.0.1/index.json b/docs/api/package/ecs_style_dataset/0.0.1/index.json index 37953a3ea..24b805f60 100644 --- a/docs/api/package/ecs_style_dataset/0.0.1/index.json +++ b/docs/api/package/ecs_style_dataset/0.0.1/index.json @@ -23,17 +23,7 @@ "/package/ecs_style_dataset/0.0.1/dataset/foo/manifest.yml", "/package/ecs_style_dataset/0.0.1/dataset/foo/fields/fields.yml" ], - "datasets": [ - { - "id": "ecs_style_dataset.foo", - "title": "Foo", - "release": "experimental", - "type": "logs", - "package": "ecs_style_dataset", - "path": "foo" - } - ], - "datasources": [ + "config_templates": [ { "name": "logs", "title": "Logs datasource", @@ -45,5 +35,15 @@ ], "multiple": true } + ], + "datasets": [ + { + "type": "logs", + "name": "ecs_style_dataset.foo", + "title": "Foo", + "release": "experimental", + "package": "ecs_style_dataset", + "path": "foo" + } ] } \ No newline at end of file diff --git a/docs/api/package/example/1.0.0/index.json b/docs/api/package/example/1.0.0/index.json index 5c95bc6cc..8973b7c5f 100644 --- a/docs/api/package/example/1.0.0/index.json +++ b/docs/api/package/example/1.0.0/index.json @@ -55,12 +55,25 @@ "/package/example/1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-plaintext.json", "/package/example/1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-tcp.json" ], + "config_templates": [ + { + "name": "logs", + "title": "Logs datasource", + "description": "Datasource for your log files.", + "inputs": [ + { + "type": "foo" + } + ], + "multiple": true + } + ], "datasets": [ { - "id": "bar", + "type": "logs", + "name": "bar.noo", "title": "Foo", "release": "experimental", - "type": "logs", "ingest_pipeline": "pipeline-entry", "streams": [ { @@ -75,6 +88,7 @@ "show_user": false } ], + "template_path": "stream.yml.hbs", "enabled": true } ], @@ -82,38 +96,6 @@ "path": "foo" } ], - "datasources": [ - { - "name": "logs", - "title": "Logs datasource", - "description": "Datasource for your log files.", - "inputs": [ - { - "type": "foo", - "streams": [ - { - "input": "foo", - "vars": [ - { - "name": "paths", - "type": "text", - "description": "Path to log files to be collected", - "multi": true, - "required": true, - "show_user": false - } - ], - "dataset": "bar", - "template_path": "stream.yml.hbs", - "template": "foo: bar\n", - "enabled": true - } - ] - } - ], - "multiple": true - } - ], "owner": { "github": "ruflin" } diff --git a/docs/api/package/internal/1.2.0/index.json b/docs/api/package/internal/1.2.0/index.json index 07aecdab7..8626ba51e 100644 --- a/docs/api/package/internal/1.2.0/index.json +++ b/docs/api/package/internal/1.2.0/index.json @@ -6,7 +6,6 @@ "type": "integration", "download": "/epr/internal/internal-1.2.0.tar.gz", "path": "/package/internal/1.2.0", - "internal": true, "format_version": "1.0.0", "readme": "/package/internal/1.2.0/docs/README.md", "license": "basic", @@ -19,5 +18,6 @@ "assets": [ "/package/internal/1.2.0/manifest.yml", "/package/internal/1.2.0/docs/README.md" - ] + ], + "internal": true } \ No newline at end of file diff --git a/docs/api/package/multiple_false/0.0.1/index.json b/docs/api/package/multiple_false/0.0.1/index.json index f53905fc2..d9e757a59 100644 --- a/docs/api/package/multiple_false/0.0.1/index.json +++ b/docs/api/package/multiple_false/0.0.1/index.json @@ -25,12 +25,25 @@ "/package/multiple_false/0.0.1/dataset/foo/agent/stream/stream.yml.hbs", "/package/multiple_false/0.0.1/dataset/foo/elasticsearch/ingest-pipeline/default.json" ], + "config_templates": [ + { + "name": "logs", + "title": "Logs datasource", + "description": "Datasource for your log files.", + "inputs": [ + { + "type": "logs" + } + ], + "multiple": false + } + ], "datasets": [ { - "id": "multiple_false.foo", + "type": "logs", + "name": "multiple_false.foo", "title": "Foo", "release": "experimental", - "type": "logs", "ingest_pipeline": "default", "streams": [ { @@ -45,43 +58,12 @@ "show_user": false } ], + "template_path": "stream.yml.hbs", "enabled": true } ], "package": "multiple_false", "path": "foo" } - ], - "datasources": [ - { - "name": "logs", - "title": "Logs datasource", - "description": "Datasource for your log files.", - "inputs": [ - { - "type": "logs", - "streams": [ - { - "input": "logs", - "vars": [ - { - "name": "paths", - "type": "text", - "description": "Path to log files to be collected", - "multi": true, - "required": true, - "show_user": false - } - ], - "dataset": "multiple_false.foo", - "template_path": "stream.yml.hbs", - "template": "foo: bar\n", - "enabled": true - } - ] - } - ], - "multiple": false - } ] } \ No newline at end of file diff --git a/docs/api/package/no_stream_configs/1.0.0/index.json b/docs/api/package/no_stream_configs/1.0.0/index.json index 275b18555..eba1ddd01 100644 --- a/docs/api/package/no_stream_configs/1.0.0/index.json +++ b/docs/api/package/no_stream_configs/1.0.0/index.json @@ -26,10 +26,10 @@ ], "datasets": [ { - "id": "no_stream_configs.log", + "type": "logs", + "name": "no_stream_configs.log", "title": "Log Yaml pipeline", "release": "experimental", - "type": "logs", "package": "no_stream_configs", "path": "log" } diff --git a/util/package.go b/util/package.go index fdd9ebd0e..246ec14eb 100644 --- a/util/package.go +++ b/util/package.go @@ -39,11 +39,10 @@ type Package struct { Requirement Requirement `config:"requirement" json:"requirement"` Screenshots []Image `config:"screenshots,omitempty" json:"screenshots,omitempty" yaml:"screenshots,omitempty"` Assets []string `config:"assets,omitempty" json:"assets,omitempty" yaml:"assets,omitempty"` - DataSets []*Dataset `config:"datasets,omitempty" json:"datasets,omitempty" yaml:"datasets,omitempty"` - Datasources []Datasource `config:"datasources,omitempty" json:"datasources,omitempty" yaml:"datasources,omitempty"` + Datasources []Datasource `config:"datasources,omitempty" json:"config_templates,omitempty" yaml:"datasources,omitempty"` + Datasets []*Dataset `config:"datasets,omitempty" json:"datasets,omitempty" yaml:"datasets,omitempty"` Owner *Owner `config:"owner,omitempty" json:"owner,omitempty" yaml:"owner,omitempty"` Internal bool `config:"internal,omitempty" json:"internal,omitempty" yaml:"internal,omitempty"` - Datasets []*Dataset `config:"datasets,omitempty" json:"datasets,omitempty" yaml:"datasets,omitempty"` // Local path to the package dir BasePath string `json:"-" yaml:"-"`