Finish ECS migration for Packetbeat (#10193)

- Add aliases - Clean up config files - Remove unused fields from fields.common.yml - Remove time_zone from dashboards - Update navigation links on dashboards to include all dashboards - Add DNS / TLS to overview dashboard to highlight the capabilities - Update fields used in the documentation - Move RPC fields to NFS from Mongo (they were in the wrong package) - Add scale to responsetime - Consolidate changelog entries to point to issue Part of #7968
elastic · Jan 24, 2019 · a1d3a9b · a1d3a9b
1 parent ae6b463
commit a1d3a9b
Show file tree

Hide file tree

Showing 41 changed files with 1,843 additions and 1,317 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -75,21 +75,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Adjust Packetbeat `http` fields to ECS Beta 2 {pull}9645[9645]
   - `http.request.body` moves to `http.request.body.content`
   - `http.response.body` moves to `http.response.body.content`
-- Changed DNS protocol fields to align with ECS. {pull}9941[9941]
+- Changed Packetbeat fields to align with ECS. {issue}7968[7968]
 - Removed trailing dot from domain names reported by the DNS protocol. {pull}9941[9941]
-- Changed TLS protocol fields to align with ECS. {pull}9980[9980]
-- Changed ICMP protocol fields to align with ECS. {pull}10062[10062]
-- Changed DHCPv4 protocol fields to align with ECS. {pull}10089[10089]
-- Changed AMQP protocol fields to align with ECS. {pull}10090[10090]
-- Changed Redis protocol fields to align with ECS. {pull}10126[10126]
-- Changed pgsql protocol fields to align with ECS. {pull}10147[10147]
-- Changed HTTP protocol fields to align with ECS. {pull}9976[9976]
-- Changed MongoDB protocol fields to align with ECS. {pull}10158[10158]
-- Changed MySQL protocol fields to align with ECS. {pull}10155[10155]
-- Changed NFS protocol fields to align with ECS. {pull}10153[10153]
-- Changed Thrift protocol fields to align with ECS. {pull}10125[10125]
-- Changed Cassandra protocol fields to align with ECS. {pull}10093[10093]
-- Changed Memcache protocol fields to align with ECS. {pull}10189[10189]
 
 *Winlogbeat*
 

diff --git a/dev-tools/ecs-migration.yml b/dev-tools/ecs-migration.yml
@@ -1223,34 +1223,34 @@
 ## Shared
 - from: bytes_in
   to: source.bytes
-  alias: false
-  comment: Don't add an alias until all of Packetbeat stops using this field.
+  alias: true
   beat: packetbeat
 
 - from: bytes_out
   to: destination.bytes
-  alias: false
-  comment: Don't add an alias until all of Packetbeat stops using this field.
+  alias: true
   beat: packetbeat
 
 - from: notes
   to: error.message
-  alias: false
-  comment: Don't add an alias until all of Packetbeat stops using this field.
+  alias: true
   beat: packetbeat
 
 - from: responsetime
   to: event.duration
   alias: false
-  comment: >
-    Units changed from usec to nsec. Don't add an alias until all of Packetbeat
-    stops using this field.
+  scale: 1000000
+  comment: The units changed so no alias was added.
   beat: packetbeat
 
 - from: transport
   to: network.transport
-  alias: false
-  comment: Don't add an alias until all of Packetbeat stops using this field.
+  alias: true
+  beat: packetbeat
+
+- from: real_ip
+  to: network.forwarded_ip
+  alias: true
   beat: packetbeat
 
 ## Flows
@@ -1323,7 +1323,7 @@
 - from: method
   to: http.request.method
   alias: false
-  comment: method is used by other protocols.
+  comment: Field is used by serveral protocols.
   beat: packetbeat
 
 - from: path
@@ -1334,6 +1334,7 @@
 - from: real_ip
   to: network.forwarded_ip
   alias: false
+  comment: Field is used by serveral protocols.
   beat: packetbeat
 
 ## MySQL

diff --git a/packetbeat/_meta/beat.docker.yml b/packetbeat/_meta/beat.docker.yml
@@ -1,13 +1,14 @@
 packetbeat.interfaces.device: any
+packetbeat.interfaces.snaplen: 1514
+packetbeat.interfaces.type: af_packet
+packetbeat.interfaces.buffer_size_mb: 100
 
 packetbeat.flows:
   timeout: 30s
   period: 10s
 
 packetbeat.protocols.dns:
   ports: [53]
-  include_authorities: true
-  include_additionals: true
 
 packetbeat.protocols.http:
   ports: [80, 5601, 9200, 8080, 8081, 5000, 8002]

diff --git a/packetbeat/_meta/beat.reference.yml b/packetbeat/_meta/beat.reference.yml
@@ -467,30 +467,14 @@ packetbeat.protocols:
 
 #=========================== Monitored processes ==============================
 
-# Configure the processes to be monitored and how to find them. If a process is
-# monitored then Packetbeat attempts to use it's name to fill in the `proc` and
-# `client_proc` fields.
-# The processes can be found by searching their command line by a given string.
-#
-# Process matching is optional and can be enabled by uncommenting the following
-# lines.
-#
-#packetbeat.procs:
-#  enabled: false
-#  monitored:
-#    - process: mysqld
-#      cmdline_grep: mysqld
-#
-#    - process: pgsql
-#      cmdline_grep: postgres
-#
-#    - process: nginx
-#      cmdline_grep: nginx
-#
-#    - process: app
-#      cmdline_grep: gunicorn
-
-# Uncomment the following if you want to ignore transactions created
-# by the server on which the shipper is installed. This option is useful
-# to remove duplicates if shippers are installed on multiple servers.
-#packetbeat.ignore_outgoing: true
+# Packetbeat can enrich events with information about the process associated
+# the socket that sent or received the packet if Packetbeat is monitoring
+# traffic from the host machine. By default process enrichment is disabled.
+# This feature works on Linux and Windows.
+packetbeat.procs.enabled: false
+
+# If you want to ignore transactions created by the server on which the shipper
+# is installed you can enable this option. This option is useful to remove
+# duplicates if shippers are installed on multiple servers. Default value is
+# false.
+packetbeat.ignore_outgoing: false
diff --git a/packetbeat/_meta/beat.yml b/packetbeat/_meta/beat.yml
@@ -49,14 +49,6 @@ packetbeat.protocols:
   # the DNS protocol by commenting out the list of ports.
   ports: [53]
 
-  # include_authorities controls whether or not the dns.authorities field
-  # (authority resource records) is added to messages.
-  include_authorities: true
-
-  # include_additionals controls whether or not the dns.additionals field
-  # (additional resource records) is added to messages.
-  include_additionals: true
-
 - type: http
   # Configure the ports where to listen for HTTP traffic. You can disable
   # the HTTP protocol by commenting out the list of ports.

diff --git a/packetbeat/_meta/fields.common.yml b/packetbeat/_meta/fields.common.yml
@@ -4,27 +4,10 @@
       These fields contain data about the environment in which the
       transaction or flow was captured.
   fields:
-    - name: real_ip
-      type: ip
-      description: >
-        If the server initiating the transaction is a proxy, this field
-        contains the original client IP address.
-        For HTTP, for example, the IP address extracted from a configurable
-        HTTP header, by default `X-Forwarded-For`.
-
-        Unless this field is disabled, it always has a value, and it matches
-        the `client_ip` for non proxy clients.
-      format: Dotted notation.
-
-    - name: transport
-      description: >
-        The transport protocol used for the transaction. If not specified, then
-        tcp is assumed.
-      example: udp
-
     - name: type
       description: >
-        The type of the transaction (for example, HTTP, MySQL, Redis, or RUM) or "flow" in case of flows.
+        The type of the transaction (for example, HTTP, MySQL, Redis, or RUM) or
+        "flow" in case of flows.
       required: true
 
     - name: server.process.name
@@ -67,6 +50,28 @@
       description: >
         The time the client process started.
 
+    # Aliases
+    - name: real_ip
+      type: alias
+      path: network.forwarded_ip
+      migration: true
+      description: >
+        If the server initiating the transaction is a proxy, this field
+        contains the original client IP address.
+        For HTTP, for example, the IP address extracted from a configurable
+        HTTP header, by default `X-Forwarded-For`.
+
+        Unless this field is disabled, it always has a value, and it matches
+        the `client_ip` for non proxy clients.
+
+    - name: transport
+      type: alias
+      path: network.transport
+      migration: true
+      description: >
+        The transport protocol used for the transaction. If not specified, then
+        tcp is assumed.
+
 - key: flows_event
   title: "Flow Event"
   description: >
@@ -89,6 +94,7 @@
         this field will be an array with the outer tag's VLAN identifier listed
         first.
 
+    # Aliases
     - name: flow_id
       type: alias
       path: flow.id
@@ -129,7 +135,6 @@
   description: >
     These fields contain data about the transaction itself.
   fields:
-
     - name: status
       description: >
         The high level status of the transaction. The way to compute this
@@ -145,15 +150,16 @@
     - name: method
       description: >
         The command/verb/method of the transaction. For HTTP, this is the
-        method name (GET, POST, PUT, and so on), for SQL this is the verb (SELECT,
-        UPDATE, DELETE, and so on).
+        method name (GET, POST, PUT, and so on), for SQL this is the verb
+        (SELECT, UPDATE, DELETE, and so on).
 
     - name: resource
       description: >
         The logical resource that this transaction refers to. For HTTP, this is
-        the URL path up to the last slash (/). For example, if the URL is `/users/1`,
-        the resource is `/users`. For databases, the resource is typically the
-        table name. The field is not filled for all transaction types.
+        the URL path up to the last slash (/). For example, if the URL is
+        `/users/1`, the resource is `/users`. For databases, the resource is
+        typically the table name. The field is not filled for all transaction
+        types.
 
     - name: path
       required: true
@@ -176,9 +182,12 @@
         For Thrift-RPC, these are the parameters from the request.
 
     - name: notes
+      type: alias
+      path: error.message
       description: >
-        Messages from Packetbeat itself. This field usually contains error messages for
-        interpreting the raw data. This information can be helpful for troubleshooting.
+        Messages from Packetbeat itself. This field usually contains error
+        messages for interpreting the raw data. This information can be helpful
+        for troubleshooting.
 
 - key: raw
   title: Raw
@@ -203,61 +212,19 @@
   description: >
     These fields contain measurements related to the transaction.
   fields:
-    - name: responsetime
-      description: >
-        The wall clock time it took to complete the transaction.
-        The precision is in milliseconds.
-      type: long
-
-    - name: cpu_time
-      description: The CPU time it took to complete the transaction.
-      type: long
-
+    # Aliases
     - name: bytes_in
+      type: alias
+      path: source.bytes
       description: >
         The number of bytes of the request. Note that this size is
         the application layer message length, without the length of the IP or
         TCP headers.
-      type: long
-      format: bytes
 
     - name: bytes_out
+      type: alias
+      path: destination.bytes
       description: >
         The number of bytes of the response. Note that this size is
         the application layer message length, without the length of the IP or
         TCP headers.
-      type: long
-      format: bytes
-
-    - name: dnstime
-      type: long
-      description: >
-        The time it takes to query the name server for a given request.
-        This is typically used for RUM (real-user-monitoring) but can
-        also have values for server-to-server communication when DNS
-        is used for service discovery.
-        The precision is in microseconds.
-
-    - name: connecttime
-      type: long
-      description: >
-        The time it takes for the TCP connection to be established for
-        the given transaction.
-        The precision is in microseconds.
-
-    - name: loadtime
-      type: long
-      description: >
-        The time it takes for the content to be loaded. This is typically
-        used for RUM (real-user-monitoring) but it can make sense in other
-        cases as well.
-        The precision is in microseconds.
-
-    - name: domloadtime
-      type: long
-      description: >
-        In RUM (real-user-monitoring), the total time it takes for the
-        DOM to be loaded. In terms of the W3 Navigation Timing API, this is
-        the difference between `domContentLoadedEnd` and
-        `domContentLoadedStart`.
-