Skip to content

Commit

Permalink
Update package output structure (#514)
Browse files Browse the repository at this point in the history 
This change contains several changes to the output generated for the package API endpoint:

* dataset.id is renamed to dataset.name
* datasources is renamed to config_templates. Currently only in the output, packages can follow later.
* config_templates (aka datasources) do not pregenerate the content anymore to simply config building in Kibana
* template_content is removed

Further change is done to use `Dataset(s)` instead of `DataSet(s)` in the code for consistency.
  • Loading branch information
@ruflin
ruflin authored Jun 29, 2020
1 parent 92064a8 commit 5628ef3
Show file tree
Hide file tree
Showing 12 changed files with 204 additions and 371 deletions.
50 changes: 16 additions & 34 deletions testdata/generated/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,12 +60,25 @@
"/package/example/1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-plaintext.json",
"/package/example/1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-tcp.json"
],
"config_templates": [
{
"name": "logs",
"title": "Logs datasource",
"description": "Datasource for your log files.",
"inputs": [
{
"type": "foo"
}
],
"multiple": true
}
],
"datasets": [
{
"id": "bar",
"type": "logs",
"name": "bar.noo",
"title": "Foo",
"release": "experimental",
"type": "logs",
"ingest_pipeline": "pipeline-entry",
"streams": [
{
Expand All @@ -80,45 +93,14 @@
"show_user": false
}
],
"template_path": "stream.yml.hbs",
"enabled": true
}
],
"package": "example",
"path": "foo"
}
],
"datasources": [
{
"name": "logs",
"title": "Logs datasource",
"description": "Datasource for your log files.",
"inputs": [
{
"type": "foo",
"streams": [
{
"input": "foo",
"vars": [
{
"name": "paths",
"type": "text",
"description": "Path to log files to be collected",
"multi": true,
"required": true,
"show_user": false
}
],
"dataset": "bar",
"template_path": "stream.yml.hbs",
"template": "foo: bar\n",
"enabled": true
}
]
}
],
"multiple": true
}
],
"owner": {
"github": "ruflin"
}
Expand Down
206 changes: 64 additions & 142 deletions testdata/generated/package/datasources/1.0.0/index.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,12 +36,69 @@
"/package/datasources/1.0.0/dataset/examplelog2/agent/stream/stream.yml.hbs",
"/package/datasources/1.0.0/dataset/examplemetric/agent/stream/stream.yml.hbs"
],
"config_templates": [
{
"name": "nginx",
"title": "Datasource title",
"description": "Details about the data source.",
"inputs": [
{
"type": "nginx/metrics",
"vars": [
{
"name": "hosts",
"type": "text",
"description": "Nginx hosts",
"multi": true,
"required": true,
"show_user": false,
"default": [
"http://127.0.0.1"
]
},
{
"name": "period",
"type": "duration",
"description": "Collection period. Valid values: 10s, 5m, 2h",
"multi": false,
"required": false,
"show_user": false,
"default": "10s"
},
{
"name": "username",
"type": "text",
"multi": false,
"required": false,
"show_user": false
},
{
"name": "password",
"type": "password",
"multi": false,
"required": false,
"show_user": false
}
],
"description": "Collecting metrics for nginx."
},
{
"type": "logs",
"description": "Collect nginx logs."
},
{
"type": "syslog"
}
],
"multiple": true
}
],
"datasets": [
{
"id": "datasources.examplelog1",
"type": "logs",
"name": "datasources.examplelog1",
"title": "Example dataset with inputs",
"release": "experimental",
"type": "logs",
"streams": [
{
"input": "logs",
Expand Down Expand Up @@ -75,10 +132,10 @@
"path": "examplelog1"
},
{
"id": "datasources.examplelog2",
"type": "logs",
"name": "datasources.examplelog2",
"title": "Example dataset with inputs",
"release": "experimental",
"type": "logs",
"streams": [
{
"input": "logs",
Expand All @@ -95,6 +152,7 @@
]
}
],
"template_path": "stream.yml.hbs",
"title": "Title of the stream",
"description": "Description of the stream with more details.",
"enabled": true
Expand All @@ -104,10 +162,10 @@
"path": "examplelog2"
},
{
"id": "datasources.examplemetric",
"type": "metrics",
"name": "datasources.examplemetric",
"title": "Example dataset with inputs",
"release": "experimental",
"type": "metrics",
"streams": [
{
"input": "nginx/metrics",
Expand All @@ -130,141 +188,5 @@
"package": "datasources",
"path": "examplemetric"
}
],
"datasources": [
{
"name": "nginx",
"title": "Datasource title",
"description": "Details about the data source.",
"inputs": [
{
"type": "nginx/metrics",
"vars": [
{
"name": "hosts",
"type": "text",
"description": "Nginx hosts",
"multi": true,
"required": true,
"show_user": false,
"default": [
"http://127.0.0.1"
]
},
{
"name": "period",
"type": "duration",
"description": "Collection period. Valid values: 10s, 5m, 2h",
"multi": false,
"required": false,
"show_user": false,
"default": "10s"
},
{
"name": "username",
"type": "text",
"multi": false,
"required": false,
"show_user": false
},
{
"name": "password",
"type": "password",
"multi": false,
"required": false,
"show_user": false
}
],
"description": "Collecting metrics for nginx.",
"streams": [
{
"input": "nginx/metrics",
"vars": [
{
"name": "url",
"type": "text",
"description": "Paths to the nginx access log file.",
"multi": false,
"required": true,
"show_user": false,
"default": "localhost"
}
],
"dataset": "datasources.examplemetric",
"template_path": "stream.yml.hbs",
"template": "metric: foo\n",
"title": "Title of the stream",
"description": "Not enabled data source.",
"enabled": false
}
]
},
{
"type": "logs",
"description": "Collect nginx logs.",
"streams": [
{
"input": "logs",
"vars": [
{
"name": "paths",
"type": "text",
"description": "Paths to the nginx error log file.",
"multi": true,
"required": true,
"show_user": false,
"default": [
"/var/log/nginx/error.log*"
]
}
],
"dataset": "datasources.examplelog1",
"template_path": "logs.yml",
"template": "foo: bar\n",
"title": "Title of the stream",
"description": "Description of the stream with more details.",
"enabled": true
},
{
"input": "logs",
"vars": [
{
"name": "paths",
"type": "text",
"description": "Paths to the nginx access log file.",
"multi": true,
"required": true,
"show_user": false,
"default": [
"/var/log/nginx/access.log*"
]
}
],
"dataset": "datasources.examplelog2",
"template_path": "stream.yml.hbs",
"template": "foo: bar\n",
"title": "Title of the stream",
"description": "Description of the stream with more details.",
"enabled": true
}
]
},
{
"type": "syslog",
"streams": [
{
"input": "syslog",
"dataset": "datasources.examplelog1",
"template_path": "syslog.yml",
"template": "syslog: bar\n",
"title": "Title of the stream",
"description": "Description of the stream with more details.",
"enabled": true
}
]
}
],
"multiple": true
}
]
}
50 changes: 16 additions & 34 deletions testdata/generated/package/default_pipeline/0.0.2/index.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,12 +31,25 @@
"/package/default_pipeline/0.0.2/dataset/foo/agent/stream/stream.yml.hbs",
"/package/default_pipeline/0.0.2/dataset/foo/elasticsearch/ingest-pipeline/default.json"
],
"config_templates": [
{
"name": "logs",
"title": "Logs datasource",
"description": "Datasource for your log files.",
"inputs": [
{
"type": "logs"
}
],
"multiple": true
}
],
"datasets": [
{
"id": "default_pipeline.foo",
"type": "logs",
"name": "default_pipeline.foo",
"title": "Foo",
"release": "experimental",
"type": "logs",
"ingest_pipeline": "default",
"streams": [
{
Expand All @@ -51,43 +64,12 @@
"show_user": false
}
],
"template_path": "stream.yml.hbs",
"enabled": true
}
],
"package": "default_pipeline",
"path": "foo"
}
],
"datasources": [
{
"name": "logs",
"title": "Logs datasource",
"description": "Datasource for your log files.",
"inputs": [
{
"type": "logs",
"streams": [
{
"input": "logs",
"vars": [
{
"name": "paths",
"type": "text",
"description": "Path to log files to be collected",
"multi": true,
"required": true,
"show_user": false
}
],
"dataset": "default_pipeline.foo",
"template_path": "stream.yml.hbs",
"template": "foo: bar\n",
"enabled": true
}
]
}
],
"multiple": true
}
]
}
Loading

0 comments on commit 5628ef3

Please sign in to comment.