Unify default module configurations (elastic#6908)

Since we split module configurations into modules.d folder, we can keep configurations more verbose, so users have a quick reference when opening them for editing. This change unifies configuration files using the followin rules: * Write at least the following settings, in this order: `module`, `metricsets`, `period`, `hosts`. * Default metricsets list shown as it helps to know what's the list, or disable just one. Inline if there is only one metricset, one line per metricset if there is more than one. * Some modules have exceptions and include more definitions, or different defaults. * Comments are allowed (and encouraged), both to explain a setting and to suggest a full configuration. * `enabled` setting is not shown at all, modules are enabled by using `metricbeat modules enable <module name>`. Use comments to suggest settings that are disabled by default.
stevea78 · May 20, 2018 · ca731a4 · ca731a4
1 parent 4e09384
commit ca731a4
Show file tree

Hide file tree

Showing 72 changed files with 770 additions and 136 deletions.
diff --git a/metricbeat/Makefile b/metricbeat/Makefile
@@ -10,6 +10,7 @@ ES_BEATS?=..
 GOX_OS=netbsd linux windows
 GOX_FLAGS=-arch="amd64 386 arm ppc64 ppc64le"
 
+DOCS_BRANCH=$(shell grep doc-branch ../libbeat/docs/version.asciidoc | cut -c 14-)
 
 include ${ES_BEATS}/libbeat/scripts/Makefile
 
@@ -41,8 +42,8 @@ configs: python-env
 	@cat ${ES_BEATS}/metricbeat/_meta/setup.yml >> _meta/beat.yml
 	@cat ${ES_BEATS}/metricbeat/_meta/common.reference.yml > _meta/beat.reference.yml
 	@${PYTHON_ENV}/bin/python ${ES_BEATS}/script/config_collector.py --beat ${BEAT_NAME} --full $(PWD) >> _meta/beat.reference.yml
-	@rm -rf modules.d && mkdir -p modules.d
-	@for MODULE in `find module -maxdepth 1 -mindepth 1 -type d -exec basename {} \;`; do cp -a $(PWD)/module/$$MODULE/_meta/config.yml modules.d/$$MODULE.yml.disabled; done
+	@rm -rf modules.d
+	${PYTHON_ENV}/bin/python ${ES_BEATS}/metricbeat/scripts/modules_collector.py --docs_branch=$(DOCS_BRANCH)
 	@chmod go-w modules.d/*
 	@# Enable system by default:
 	@if [ -f modules.d/system.yml.disabled ]; then mv modules.d/system.yml.disabled modules.d/system.yml; fi

diff --git a/metricbeat/docs/modules/golang.asciidoc b/metricbeat/docs/modules/golang.asciidoc
@@ -20,7 +20,9 @@ in <<configuration-metricbeat>>. Here is an example configuration:
 ----
 metricbeat.modules:
 - module: golang
-  metricsets: ["expvar","heap"]
+  #metricsets:
+  #  - expvar
+  #  - heap
   period: 10s
   hosts: ["localhost:6060"]
   heap.path: "/debug/vars"

diff --git a/metricbeat/docs/modules/http.asciidoc b/metricbeat/docs/modules/http.asciidoc
@@ -26,7 +26,8 @@ in <<configuration-metricbeat>>. Here is an example configuration:
 ----
 metricbeat.modules:
 - module: http
-  metricsets: ["json"]
+  #metricsets:
+  #  - json
   period: 10s
   hosts: ["localhost:80"]
   namespace: "json_namespace"
@@ -39,15 +40,16 @@ metricbeat.modules:
   #dedot.enabled: false
 
 - module: http
-  metricsets: ["server"]
+  #metricsets:
+  #  - server
   host: "localhost"
   port: "8080"
   enabled: false
-#  paths:
-#    - path: "/foo"
-#      namespace: "foo"
-#      fields: # added to the the response in root. overwrites existing fields
-#        key: "value"
+  #paths:
+  #  - path: "/foo"
+  #    namespace: "foo"
+  #    fields: # added to the the response in root. overwrites existing fields
+  #      key: "value"
 ----
 
 This module supports TLS connection when using `ssl` config field, as described in <<configuration-ssl>>.

diff --git a/metricbeat/docs/modules/jolokia.asciidoc b/metricbeat/docs/modules/jolokia.asciidoc
@@ -19,28 +19,32 @@ in <<configuration-metricbeat>>. Here is an example configuration:
 ----
 metricbeat.modules:
 - module: jolokia
-  metricsets: ["jmx"]
+  #metricsets: ["jmx"]
   period: 10s
   hosts: ["localhost"]
   namespace: "metrics"
+  #path: "/jolokia/?ignoreErrors=true&canonicalNaming=false"
   jmx.mappings:
-    - mbean: 'java.lang:type=Runtime'
-      attributes:
-        - attr: Uptime
-          field: uptime
-    - mbean: 'java.lang:type=Memory'
-      attributes:
-        - attr: HeapMemoryUsage
-          field: memory.heap_usage
-        - attr: NonHeapMemoryUsage
-          field: memory.non_heap_usage
+    #- mbean: 'java.lang:type=Runtime'
+    #  attributes:
+    #    - attr: Uptime
+    #      field: uptime
+    #- mbean: 'java.lang:type=Memory'
+    #  attributes:
+    #    - attr: HeapMemoryUsage
+    #      field: memory.heap_usage
+    #    - attr: NonHeapMemoryUsage
+    #      field: memory.non_heap_usage
     # GC Metrics - this depends on what is available on your JVM
-    # - mbean: 'java.lang:type=GarbageCollector,name=ConcurrentMarkSweep'
-    #   attributes:
-    #     - attr: CollectionTime
-    #       field: gc.cms_collection_time
-    #     - attr: CollectionCount
-    #       field: gc.cms_collection_count
+    #- mbean: 'java.lang:type=GarbageCollector,name=ConcurrentMarkSweep'
+    #  attributes:
+    #    - attr: CollectionTime
+    #      field: gc.cms_collection_time
+    #    - attr: CollectionCount
+    #      field: gc.cms_collection_count
+
+  jmx.application:
+  jmx.instance:
 ----
 
 This module supports TLS connection when using `ssl` config field, as described in <<configuration-ssl>>.

diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml
@@ -212,7 +212,9 @@ metricbeat.modules:
 
 #------------------------------- Golang Module -------------------------------
 - module: golang
-  metricsets: ["expvar","heap"]
+  #metricsets:
+  #  - expvar
+  #  - heap
   period: 10s
   hosts: ["localhost:6060"]
   heap.path: "/debug/vars"
@@ -253,7 +255,8 @@ metricbeat.modules:
 
 #-------------------------------- HTTP Module --------------------------------
 - module: http
-  metricsets: ["json"]
+  #metricsets:
+  #  - json
   period: 10s
   hosts: ["localhost:80"]
   namespace: "json_namespace"
@@ -266,40 +269,45 @@ metricbeat.modules:
   #dedot.enabled: false
 
 - module: http
-  metricsets: ["server"]
+  #metricsets:
+  #  - server
   host: "localhost"
   port: "8080"
   enabled: false
-#  paths:
-#    - path: "/foo"
-#      namespace: "foo"
-#      fields: # added to the the response in root. overwrites existing fields
-#        key: "value"
+  #paths:
+  #  - path: "/foo"
+  #    namespace: "foo"
+  #    fields: # added to the the response in root. overwrites existing fields
+  #      key: "value"
 
 #------------------------------- Jolokia Module ------------------------------
 - module: jolokia
-  metricsets: ["jmx"]
+  #metricsets: ["jmx"]
   period: 10s
   hosts: ["localhost"]
   namespace: "metrics"
+  #path: "/jolokia/?ignoreErrors=true&canonicalNaming=false"
   jmx.mappings:
-    - mbean: 'java.lang:type=Runtime'
-      attributes:
-        - attr: Uptime
-          field: uptime
-    - mbean: 'java.lang:type=Memory'
-      attributes:
-        - attr: HeapMemoryUsage
-          field: memory.heap_usage
-        - attr: NonHeapMemoryUsage
-          field: memory.non_heap_usage
+    #- mbean: 'java.lang:type=Runtime'
+    #  attributes:
+    #    - attr: Uptime
+    #      field: uptime
+    #- mbean: 'java.lang:type=Memory'
+    #  attributes:
+    #    - attr: HeapMemoryUsage
+    #      field: memory.heap_usage
+    #    - attr: NonHeapMemoryUsage
+    #      field: memory.non_heap_usage
     # GC Metrics - this depends on what is available on your JVM
-    # - mbean: 'java.lang:type=GarbageCollector,name=ConcurrentMarkSweep'
-    #   attributes:
-    #     - attr: CollectionTime
-    #       field: gc.cms_collection_time
-    #     - attr: CollectionCount
-    #       field: gc.cms_collection_count
+    #- mbean: 'java.lang:type=GarbageCollector,name=ConcurrentMarkSweep'
+    #  attributes:
+    #    - attr: CollectionTime
+    #      field: gc.cms_collection_time
+    #    - attr: CollectionCount
+    #      field: gc.cms_collection_count
+
+  jmx.application:
+  jmx.instance:
 
 #-------------------------------- Kafka Module -------------------------------
 - module: kafka

diff --git a/metricbeat/module/aerospike/_meta/config.yml b/metricbeat/module/aerospike/_meta/config.yml
@@ -1,2 +1,5 @@
 - module: aerospike
+  #metricsets:
+  #  - namespace
+  period: 10s
   hosts: ["localhost:3000"]
diff --git a/metricbeat/module/apache/_meta/config.yml b/metricbeat/module/apache/_meta/config.yml
@@ -1,2 +1,5 @@
 - module: apache
+  #metricsets:
+  #  - status
+  period: 10s
   hosts: ["http://127.0.0.1"]
diff --git a/metricbeat/module/ceph/_meta/config.yml b/metricbeat/module/ceph/_meta/config.yml
@@ -1,7 +1,16 @@
 - module: ceph
-  metricsets: ["cluster_health", "cluster_status", "monitor_health"]
+  #metricsets:
+  #  - cluster_health
+  #  - cluster_status
+  #  - monitor_health
   period: 10s
+  hosts: ["localhost:5000"]
 
 - module: ceph
-  metricsets: ["cluster_disk", "osd_tree", "pool_disk"]
+  #metricsets:
+  #  - cluster_disk
+  #  - osd_tree
+  #  - osd_df
+  #  - pool_disk
   period: 1m
+  hosts: ["localhost:5000"]
diff --git a/metricbeat/module/couchbase/_meta/config.yml b/metricbeat/module/couchbase/_meta/config.yml
@@ -1,2 +1,7 @@
 - module: couchbase
+  #metricsets:
+  #  - bucket
+  #  - cluster
+  #  - node
+  period: 10s
   hosts: ["localhost:8091"]
diff --git a/metricbeat/module/docker/_meta/config.yml b/metricbeat/module/docker/_meta/config.yml
@@ -1,5 +1,20 @@
 - module: docker
+  #metricsets:
+  #  - container
+  #  - cpu
+  #  - diskio
+  #  - healthcheck
+  #  - info
+  #  - memory
+  #  - network
+  period: 10s
   hosts: ["unix:///var/run/docker.sock"]
 
   # Replace dots in labels with `_`. Set to false to keep dots
   labels.dedot: true
+
+  # To connect to Docker over TLS you must specify a client and CA certificate.
+  #ssl:
+    #certificate_authority: "/etc/pki/root/ca.pem"
+    #certificate:           "/etc/pki/client/cert.pem"
+    #key:                   "/etc/pki/client/cert.key"
diff --git a/metricbeat/module/dropwizard/_meta/config.yml b/metricbeat/module/dropwizard/_meta/config.yml
@@ -1,4 +1,7 @@
 - module: dropwizard
+  #metricsets:
+  #  - collector
+  period: 10s
   hosts: ["localhost:8080"]
   metrics_path: /metrics/metrics
   namespace: example
diff --git a/metricbeat/module/elasticsearch/_meta/config.yml b/metricbeat/module/elasticsearch/_meta/config.yml
@@ -1,2 +1,6 @@
 - module: elasticsearch
+  #metricsets:
+  #  - node
+  #  - node_stats
+  period: 10s
   hosts: ["localhost:9200"]
diff --git a/metricbeat/module/etcd/_meta/config.yml b/metricbeat/module/etcd/_meta/config.yml
@@ -1,2 +1,7 @@
 - module: etcd
+  #metricsets:
+  #  - leader
+  #  - self
+  #  - store
+  period: 10s
   hosts: ["localhost:2379"]
diff --git a/metricbeat/module/golang/_meta/config.yml b/metricbeat/module/golang/_meta/config.yml
@@ -1,5 +1,7 @@
 - module: golang
-  metricsets: ["expvar","heap"]
+  #metricsets:
+  #  - expvar
+  #  - heap
   period: 10s
   hosts: ["localhost:6060"]
   heap.path: "/debug/vars"

diff --git a/metricbeat/module/graphite/_meta/config.yml b/metricbeat/module/graphite/_meta/config.yml
@@ -1 +1,9 @@
 - module: graphite
+  #metricsets:
+  #  - server
+  #protocol: "udp"
+  #templates:
+  #  - filter: "test.*.bash.*" # This would match metrics like test.localhost.bash.stats
+  #    namespace: "test"
+  #    template: ".host.shell.metric*" # test.localhost.bash.stats would become metric=stats and tags host=localhost,shell=bash
+  #    delimiter: "_"
diff --git a/metricbeat/module/haproxy/_meta/config.yml b/metricbeat/module/haproxy/_meta/config.yml
@@ -1,2 +1,6 @@
 - module: haproxy
+  #metricsets:
+  #  - info
+  #  - stat
+  period: 10s
   hosts: ["tcp://127.0.0.1:14567"]
diff --git a/metricbeat/module/http/_meta/config.yml b/metricbeat/module/http/_meta/config.yml
@@ -1,5 +1,6 @@
 - module: http
-  metricsets: ["json"]
+  #metricsets:
+  #  - json
   period: 10s
   hosts: ["localhost:80"]
   namespace: "json_namespace"
@@ -12,12 +13,13 @@
   #dedot.enabled: false
 
 - module: http
-  metricsets: ["server"]
+  #metricsets:
+  #  - server
   host: "localhost"
   port: "8080"
   enabled: false
-#  paths:
-#    - path: "/foo"
-#      namespace: "foo"
-#      fields: # added to the the response in root. overwrites existing fields
-#        key: "value"
+  #paths:
+  #  - path: "/foo"
+  #    namespace: "foo"
+  #    fields: # added to the the response in root. overwrites existing fields
+  #      key: "value"
diff --git a/metricbeat/module/jolokia/_meta/config.yml b/metricbeat/module/jolokia/_meta/config.yml
@@ -1,23 +1,27 @@
 - module: jolokia
-  metricsets: ["jmx"]
+  #metricsets: ["jmx"]
   period: 10s
   hosts: ["localhost"]
   namespace: "metrics"
+  #path: "/jolokia/?ignoreErrors=true&canonicalNaming=false"
   jmx.mappings:
-    - mbean: 'java.lang:type=Runtime'
-      attributes:
-        - attr: Uptime
-          field: uptime
-    - mbean: 'java.lang:type=Memory'
-      attributes:
-        - attr: HeapMemoryUsage
-          field: memory.heap_usage
-        - attr: NonHeapMemoryUsage
-          field: memory.non_heap_usage
+    #- mbean: 'java.lang:type=Runtime'
+    #  attributes:
+    #    - attr: Uptime
+    #      field: uptime
+    #- mbean: 'java.lang:type=Memory'
+    #  attributes:
+    #    - attr: HeapMemoryUsage
+    #      field: memory.heap_usage
+    #    - attr: NonHeapMemoryUsage
+    #      field: memory.non_heap_usage
     # GC Metrics - this depends on what is available on your JVM
-    # - mbean: 'java.lang:type=GarbageCollector,name=ConcurrentMarkSweep'
-    #   attributes:
-    #     - attr: CollectionTime
-    #       field: gc.cms_collection_time
-    #     - attr: CollectionCount
-    #       field: gc.cms_collection_count
+    #- mbean: 'java.lang:type=GarbageCollector,name=ConcurrentMarkSweep'
+    #  attributes:
+    #    - attr: CollectionTime
+    #      field: gc.cms_collection_time
+    #    - attr: CollectionCount
+    #      field: gc.cms_collection_count
+
+  jmx.application:
+  jmx.instance: