diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 97606084d4c..25141a2e174 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -59,6 +59,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fail to start Filebat if none between `queue_url`, `bucket_arn` or `non_aws_bucket_name` is set for a configured aws-s3 input {issue}13911[13911] {pull}28666[28666] - All modules: Replace usages of deprecated ECS fields `process.ppid` and `log.original` with `process.parent.pid` and `event.original`. {pull}28620[28620] - Replace usages of `host.user.*` fields with `user.*` in `cisco`, `microsoft` and `oracle` modules. {pull}28620[28620] +- Remove `docker` input. Please use `filestream` input with `container` parser or `container` input. {pull}28817[28817] *Heartbeat* diff --git a/filebeat/docs/filebeat-options.asciidoc b/filebeat/docs/filebeat-options.asciidoc index ec8d8ef2faf..1c4e9d3dd44 100644 --- a/filebeat/docs/filebeat-options.asciidoc +++ b/filebeat/docs/filebeat-options.asciidoc @@ -67,7 +67,6 @@ You can configure {beatname_uc} to use the following inputs: * <<{beatname_lc}-input-azure-eventhub>> * <<{beatname_lc}-input-cloudfoundry>> * <<{beatname_lc}-input-container>> -* <<{beatname_lc}-input-docker>> * <<{beatname_lc}-input-filestream>> * <<{beatname_lc}-input-gcp-pubsub>> * <<{beatname_lc}-input-http_endpoint>> @@ -97,8 +96,6 @@ include::../../x-pack/filebeat/docs/inputs/input-cloudfoundry.asciidoc[] include::inputs/input-container.asciidoc[] -include::inputs/input-docker.asciidoc[] - include::inputs/input-filestream.asciidoc[] include::../../x-pack/filebeat/docs/inputs/input-gcp-pubsub.asciidoc[] diff --git a/filebeat/docs/inputs/input-docker.asciidoc b/filebeat/docs/inputs/input-docker.asciidoc deleted file mode 100644 index a1422e19fea..00000000000 --- a/filebeat/docs/inputs/input-docker.asciidoc +++ /dev/null @@ -1,108 +0,0 @@ -:type: docker - -[id="{beatname_lc}-input-{type}"] -=== Docker input - -++++ -Docker -++++ - -deprecated:[7.2.0, Use `container` input instead.] - -Use the `docker` input to read logs from Docker containers. - -This input searches for container logs under its path, and parse them into -common message lines, extracting timestamps too. Everything happens before line -filtering, multiline, and JSON decoding, so this input can be used in -combination with those settings. - -Example configuration: - -["source","yaml",subs="attributes"] ----- -{beatname_lc}.inputs: -- type: docker - containers.ids: <1> - - '8b6fe7dc9e067b58476dc57d6986dd96d7100430c5de3b109a99cd56ac655347' ----- - -<1> `containers.ids` is required. All other settings are optional. - -==== Configuration options - -The `docker` input supports the following configuration options plus the -<<{beatname_lc}-input-{type}-common-options>> described later. - -[[config-container-ids]] -===== `containers.ids` - -The list of Docker container IDs to read logs from. Specify -`containers.ids: '*'` to read from all containers. - -===== `containers.path` - -The base path where Docker logs are located. The default -is `/var/lib/docker/containers`. - -===== `containers.paths` - -The list of paths to read logs from. This can be used as an alternative to -`containers.ids` for other runtimes that use the same logging format as docker -but place their logs in different paths. For example when using CRI-O runtime in -Kubernetes the following configuration can be used: - -["source","yaml",subs="attributes"] ----- -{beatname_lc}.inputs: -- type: docker - containers.paths: - - /var/log/pods/${data.kubernetes.pod.uid}/${data.kubernetes.container.name}/*.log ----- - -When `containers.paths` is used, `containers.path` is ignored. - - -===== `containers.stream` - -Reads from the specified streams only: `all`, `stdout` or `stderr`. The default -is `all`. - -===== `combine_partial` - -Enable partial messages joining. Docker `json-file` driver splits log lines larger than 16k bytes, -end of line (`\n`) is present for common lines in the resulting file, while it's not the for the lines -that have been split. `combine_partial` joins them back together when enabled. It is enabled by default. - -===== `cri.parse_flags` - -Enable CRI flags parsing from the log file. CRI uses flags to signal a partial line, enabling this will -ensure partial lines are rejoined. It is disabled by default. - - -The following input configures {beatname_uc} to read the `stdout` stream from -all containers under the default Docker containers path: - -[source,yaml] ----- -- type: docker - combine_partial: true - containers: - path: "/var/lib/docker/containers" - stream: "stdout" - ids: - - "*" ----- - -===== `cri.force` - -Force CRI format parsing. This disables automatic format detection, use it when you know format is CRI -to gain some performance. This is false by default. - -include::../inputs/input-common-harvester-options.asciidoc[] - -include::../inputs/input-common-file-options.asciidoc[] - -[id="{beatname_lc}-input-{type}-common-options"] -include::../inputs/input-common-options.asciidoc[] - -:type!: \ No newline at end of file diff --git a/filebeat/include/list.go b/filebeat/include/list.go index 51b04d4f92c..30f56f9e724 100644 --- a/filebeat/include/list.go +++ b/filebeat/include/list.go @@ -22,7 +22,6 @@ package include import ( // Import packages that need to register themselves. _ "github.com/elastic/beats/v7/filebeat/input/container" - _ "github.com/elastic/beats/v7/filebeat/input/docker" _ "github.com/elastic/beats/v7/filebeat/input/log" _ "github.com/elastic/beats/v7/filebeat/input/mqtt" _ "github.com/elastic/beats/v7/filebeat/input/redis" diff --git a/filebeat/input/docker/config.go b/filebeat/input/docker/config.go deleted file mode 100644 index 8a79f57f68b..00000000000 --- a/filebeat/input/docker/config.go +++ /dev/null @@ -1,49 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package docker - -var defaultConfig = config{ - Partial: true, - Containers: containers{ - IDs: []string{}, - Path: "/var/lib/docker/containers", - Stream: "all", - }, -} - -type config struct { - // List of containers' log files to tail - Containers containers `config:"containers"` - - // Partial configures the input to join partial lines - Partial bool `config:"combine_partials"` - - // Enable CRI flags parsing (to be switched to default in 7.0) - CRIFlags bool `config:"cri.parse_flags"` - - // Fore CRI format (don't perform autodetection) - CRIForce bool `config:"cri.force"` -} - -type containers struct { - IDs []string `config:"ids"` - Path string `config:"path"` - - // Stream can be all, stdout or stderr - Stream string `config:"stream"` -} diff --git a/filebeat/input/docker/input.go b/filebeat/input/docker/input.go deleted file mode 100644 index 70df9ce2238..00000000000 --- a/filebeat/input/docker/input.go +++ /dev/null @@ -1,119 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package docker - -import ( - "fmt" - "path" - - "github.com/elastic/beats/v7/filebeat/channel" - "github.com/elastic/beats/v7/filebeat/input" - "github.com/elastic/beats/v7/filebeat/input/log" - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/common/cfgwarn" - "github.com/elastic/beats/v7/libbeat/logp" - - "github.com/pkg/errors" -) - -func init() { - err := input.Register("docker", NewInput) - if err != nil { - panic(err) - } -} - -// NewInput creates a new docker input -func NewInput( - cfg *common.Config, - outletFactory channel.Connector, - context input.Context, -) (input.Input, error) { - logger := logp.NewLogger("docker") - - cfgwarn.Deprecate("8.0.0", "'docker' input deprecated. Use 'container' input instead.") - - // Wrap log input with custom docker settings - config := defaultConfig - if err := cfg.Unpack(&config); err != nil { - return nil, errors.Wrap(err, "reading docker input config") - } - - // Docker input should make sure that no callers should ever pass empty strings as container IDs - // Hence we explicitly make sure that we catch such things and print stack traces in the event of - // an invocation so that it can be fixed. - var ids []string - for _, containerID := range config.Containers.IDs { - if containerID != "" { - ids = append(ids, containerID) - } else { - logger.Error("Docker container ID can't be empty for Docker input config") - logger.Debugw("Empty docker container ID was received", logp.Stack("stacktrace")) - } - } - - if len(ids) == 0 { - return nil, errors.New("Docker input requires at least one entry under 'containers.ids' or 'containers.paths'") - } - - for idx, containerID := range ids { - cfg.SetString("paths", idx, path.Join(config.Containers.Path, containerID, "*.log")) - } - - if err := checkStream(config.Containers.Stream); err != nil { - return nil, err - } - - if err := cfg.SetString("docker-json.stream", -1, config.Containers.Stream); err != nil { - return nil, errors.Wrap(err, "update input config") - } - - if err := cfg.SetBool("docker-json.partial", -1, config.Partial); err != nil { - return nil, errors.Wrap(err, "update input config") - } - - if err := cfg.SetBool("docker-json.cri_flags", -1, config.CRIFlags); err != nil { - return nil, errors.Wrap(err, "update input config") - } - - if config.CRIForce { - if err := cfg.SetString("docker-json.format", -1, "cri"); err != nil { - return nil, errors.Wrap(err, "update input config") - } - } - - // Add stream to meta to ensure different state per stream - if config.Containers.Stream != "all" { - if context.Meta == nil { - context.Meta = map[string]string{} - } - context.Meta["stream"] = config.Containers.Stream - } - - return log.NewInput(cfg, outletFactory, context) -} - -func checkStream(val string) error { - for _, s := range []string{"all", "stdout", "stderr"} { - if s == val { - return nil - } - } - - return fmt.Errorf("Invalid value for containers.stream: %s, supported values are: all, stdout, stderr", val) -} diff --git a/filebeat/input/docker/input_test.go b/filebeat/input/docker/input_test.go deleted file mode 100644 index 8459894aba7..00000000000 --- a/filebeat/input/docker/input_test.go +++ /dev/null @@ -1,35 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//go:build !integration -// +build !integration - -package docker - -import ( - "testing" - - "github.com/elastic/beats/v7/filebeat/input/inputtest" - "github.com/elastic/beats/v7/libbeat/common" -) - -func TestNewInputDone(t *testing.T) { - config := common.MapStr{ - "containers.ids": "fad130edd3d2", - } - inputtest.AssertNotStartedInputCanBeDone(t, NewInput, &config) -} diff --git a/filebeat/tests/system/test_registrar.py b/filebeat/tests/system/test_registrar.py index be78cd044af..02d07a32446 100644 --- a/filebeat/tests/system/test_registrar.py +++ b/filebeat/tests/system/test_registrar.py @@ -1346,20 +1346,14 @@ def test_registrar_meta(self): """ self.render_config_template( - type='docker', + type='container', input_raw=''' - containers: - path: {path} - stream: stdout - ids: - - container_id -- type: docker - containers: - path: {path} - stream: stderr - ids: - - container_id - '''.format(path=os.path.abspath(self.working_dir) + "/log/") + paths: {path} + stream: stdout +- type: container + paths: {path} + stream: stderr +'''.format(path=os.path.abspath(self.working_dir) + "/log/*/*.log") ) os.mkdir(self.working_dir + "/log/") os.mkdir(self.working_dir + "/log/container_id") diff --git a/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc b/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc index 1c3b23e9267..5a4a6dc8b3d 100644 --- a/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc @@ -114,7 +114,7 @@ it was uploaded. For example: `application/json`. ==== `encoding` The file encoding to use for reading data that contains international -characters. This only applies to non-JSON logs. See <<_encoding_5>>. +characters. This only applies to non-JSON logs. See <<_encoding_3>>. [float]