[filebeat] Fix ingest pipeline overwriting module field values

filebeat/module/kibana/log/config/log.yml

@@ -5,10 +5,15 @@ paths:
 {{ end }}
 exclude_files: [".gz$"]
 
-json.keys_under_root: false
-json.add_error_key: true
 processors:
-  - add_fields:
-      target: ''
-      fields:
-        ecs.version: 1.12.0
+  # non-ECS: same as json.keys_under_root: false, allows compatibility with non-ecs logs.
+- decode_json_fields:
+    fields: [message]
+    target: 'json'
+- add_fields:
+    target: ""
+    fields:
+      ecs.version: 1.12.0
+    when:
+      not:
+        has_fields: ['ecs.version']

filebeat/module/kibana/log/ingest/pipeline-7.yml

@@ -12,10 +12,12 @@ processors:
 - date:
     field: kibana.log.meta.@timestamp
     formats:
-    - ISO8601
+      - ISO8601
     target_field: '@timestamp'
 - remove:
     field: kibana.log.meta.@timestamp
+- remove:
+    field: message
 - rename:
     field: kibana.log.meta.message
     target_field: message
@@ -93,12 +95,11 @@ processors:
           ctx.event.type = "info";
         }
       }
-
 - set:
     field: event.outcome
     value: success
-    if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400"
+    if: 'ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400'
 - set:
     field: event.outcome
     value: failure
-    if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400"
+    if: 'ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400'

filebeat/module/kibana/log/ingest/pipeline-ecs.yml

@@ -1,16 +1,36 @@
-description: Pipeline for parsing Kibana ecs logs
+description: Pipeline for parsing Kibana ECS logs
 processors:
 - set:
     field: event.ingested
     value: '{{_ingest.timestamp}}'
 - set:
     copy_from: '@timestamp'
     field: event.created
-- script:
-    lang: painless
-    inline: 'ctx.json.keySet().each (key -> ctx[key] = ctx.json.get(key))'
-- remove:
-    field: json
+- rename:
+    field: message
+    target_field: _ecs_json_message
+    if: |-
+      def message = ctx.message;
+      return message != null
+          && message.startsWith('{')
+          && message.endsWith('}')
+          && message.contains('"@timestamp"')
+    ignore_missing: true
+- json:
+    field: _ecs_json_message
+    add_to_root: true
+    add_to_root_conflict_strategy: merge
+    allow_duplicate_keys: true
+    if: ctx.containsKey('_ecs_json_message')
+    on_failure:
+    - rename:
+        field: _ecs_json_message
+        target_field: message
+        ignore_missing: true
+    - set:
+        field: error.message
+        value: Error while parsing JSON
+        override: false
 - rename:
     field: http.request.headers
     target_field: kibana.log.meta.req.headers
@@ -27,3 +47,9 @@ processors:
     field: event.outcome
     value: failure
     if: 'ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400'
+- remove:
+    field: json
+    ignore_missing: true
+- remove:
+    field: _ecs_json_message
+    ignore_missing: true

filebeat/module/kibana/log/test/log.830.log-expected.json

@@ -21,6 +21,7 @@
         "kibana.log.meta.res.headers.x-opaque-id": "unknownId",
         "log.level": "DEBUG",
         "log.logger": "elasticsearch.query.data",
+        "log.offset": 0,
         "message": "200 - 201.0B\nPOST /.kibana_task_manager_8.3.0_001/_pit?keep_alive=10m",
         "process.pid": 78667,
         "service.type": "kibana",
@@ -37,6 +38,7 @@
         "input.type": "log",
         "log.level": "INFO",
         "log.logger": "savedobjects-service",
+        "log.offset": 935,
         "message": "[.kibana_task_manager] OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT -> OUTDATED_DOCUMENTS_SEARCH_READ. took: 5ms.",
         "process.pid": 78667,
         "service.type": "kibana",
@@ -67,6 +69,7 @@
         "kibana.log.meta.res.headers.x-opaque-id": "unknownId",
         "log.level": "DEBUG",
         "log.logger": "elasticsearch.query.data",
+        "log.offset": 1286,
         "message": "200 - 344.0B\nPOST /_search\n{\"sort\":{\"_shard_doc\":{\"order\":\"asc\"}},\"pit\":{\"id\":\"k4_qAwERLmtpYmFuYV84LjMuMF8wMDEWMFh6RkhHN2NUdDZ2cS16WjRsUUs1UQAWVjFzSkhLV21RNzJKY1NJYlRKQkh2QQAAAAAAAACGkhZNMWx0T1Nhd1M2MnNWbjJ3VTVYTDVRAAEWMFh6RkhHN2NUdDZ2cS16WjRsUUs1UQAA\",\"keep_alive\":\"10m\"},\"size\":1000,\"track_total_hits\":true,\"query\":{\"bool\":{\"should\":[{\"bool\":{\"must\":{\"term\":{\"type\":\"core-usage-stats\"}},\"must_not\":{\"term\":{\"migrationVersion.core-usage-stats\":\"7.14.1\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"legacy-url-alias\"}},\"must_not\":{\"term\":{\"migrationVersion.legacy-url-alias\":\"8.2.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"config\"}},\"must_not\":{\"term\":{\"migrationVersion.config\":\"8.1.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"task\"}},\"must_not\":{\"term\":{\"migrationVersion.task\":\"8.2.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"index-pattern\"}},\"must_not\":{\"term\":{\"migrationVersion.index-pattern\":\"8.0.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"space\"}},\"must_not\":{\"term\":{\"migrationVersion.space\":\"6.6.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"spaces-usage-stats\"}},\"must_not\":{\"term\":{\"migrationVersion.spaces-usage-stats\":\"7.14.1\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"exception-list-agnostic\"}},\"must_not\":{\"term\":{\"migrationVersion.exception-list-agnostic\":\"7.12.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"exception-list\"}},\"must_not\":{\"term\":{\"migrationVersion.exception-list\":\"8.0.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"action\"}},\"must_not\":{\"term\":{\"migrationVersion.action\":\"8.0.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"action_task_params\"}},\"must_not\":{\"term\":{\"migrationVersion.action_task_params\":\"8.0.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"query\"}},\"must_not\":{\"term\":{\"migrationVersion.query\":\"8.0.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"search-telemetry\"}},\"must_not\":{\"term\":{\"migrationVersion.search-telemetry\":\"7.12.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"search-session\"}},\"must_not\":{\"term\":{\"migrationVersion.search-session\":\"8.0.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"alert\"}},\"must_not\":{\"term\":{\"migrationVersion.alert\":\"8.2.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"ingest_manager_settings\"}},\"must_not\":{\"term\":{\"migrationVersion.ingest_manager_settings\":\"7.13.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"ingest-agent-policies\"}},\"must_not\":{\"term\":{\"migrationVersion.ingest-agent-policies\":\"7.12.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"ingest-outputs\"}},\"must_not\":{\"term\":{\"migrationVersion.ingest-outputs\":\"8.0.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"ingest-package-policies\"}},\"must_not\":{\"term\":{\"migrationVersion.ingest-package-policies\":\"8.2.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"epm-packages\"}},\"must_not\":{\"term\":{\"migrationVersion.epm-packages\":\"8.0.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"graph-workspace\"}},\"must_not\":{\"term\":{\"migrationVersion.graph-workspace\":\"8.0.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"tag\"}},\"must_not\":{\"term\":{\"migrationVersion.tag\":\"8.0.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"visualization\"}},\"must_not\":{\"term\":{\"migrationVersion.visualization\":\"8.1.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"canvas-element\"}},\"must_not\":{\"term\":{\"migrationVersion.canvas-element\":\"8.1.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"canvas-workpad\"}},\"must_not\":{\"term\":{\"migrationVersion.canvas-workpad\":\"8.1.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"canvas-workpad-template\"}},\"must_not\":{\"term\":{\"migrationVersion.canvas-workpad-template\":\"8.1.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"dashboard\"}},\"must_not\":{\"term\":{\"migrationVersion.dashboard\":\"8.3.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"search\"}},\"must_not\":{\"term\":{\"migrationVersion.search\":\"8.0.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"lens\"}},\"must_not\":{\"term\":{\"migrationVersion.lens\":\"8.3.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"map\"}},\"must_not\":{\"term\":{\"migrationVersion.map\":\"8.1.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"ml-job\"}},\"must_not\":{\"term\":{\"migrationVersion.ml-job\":\"7.10.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"ml-trained-model\"}},\"must_not\":{\"term\":{\"migrationVersion.ml-trained-model\":\"7.10.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"ml-module\"}},\"must_not\":{\"term\":{\"migrationVersion.ml-module\":\"7.10.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"cases-comments\"}},\"must_not\":{\"term\":{\"migrationVersion.cases-comments\":\"8.3.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"cases-configure\"}},\"must_not\":{\"term\":{\"migrationVersion.cases-configure\":\"8.0.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"cases-connector-mappings\"}},\"must_not\":{\"term\":{\"migrationVersion.cases-connector-mappings\":\"8.0.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"cases\"}},\"must_not\":{\"term\":{\"migrationVersion.cases\":\"8.1.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"cases-user-actions\"}},\"must_not\":{\"term\":{\"migrationVersion.cases-user-actions\":\"8.1.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"siem-ui-timeline-note\"}},\"must_not\":{\"term\":{\"migrationVersion.siem-ui-timeline-note\":\"8.0.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"siem-ui-timeline-pinned-event\"}},\"must_not\":{\"term\":{\"migrationVersion.siem-ui-timeline-pinned-event\":\"8.0.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"siem-detection-engine-rule-actions\"}},\"must_not\":{\"term\":{\"migrationVersion.siem-detection-engine-rule-actions\":\"8.0.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"siem-detection-engine-rule-execution-info\"}},\"must_not\":{\"term\":{\"migrationVersion.siem-detection-engine-rule-execution-info\":\"8.0.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"siem-ui-timeline\"}},\"must_not\":{\"term\":{\"migrationVersion.siem-ui-timeline\":\"8.0.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"endpoint:user-artifact-manifest\"}},\"must_not\":{\"term\":{\"migrationVersion.endpoint:user-artifact-manifest\":\"7.12.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"uptime-dynamic-settings\"}},\"must_not\":{\"term\":{\"migrationVersion.uptime-dynamic-settings\":\"8.2.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"infrastructure-ui-source\"}},\"must_not\":{\"term\":{\"migrationVersion.infrastructure-ui-source\":\"7.16.2\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"upgrade-assistant-telemetry\"}},\"must_not\":{\"term\":{\"migrationVersion.upgrade-assistant-telemetry\":\"7.16.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"apm-indices\"}},\"must_not\":{\"term\":{\"migrationVersion.apm-indices\":\"8.2.0\"}}}}]}}}",
         "process.pid": 78667,
         "service.type": "kibana",
@@ -83,6 +86,7 @@
         "input.type": "log",
         "log.level": "INFO",
         "log.logger": "savedobjects-service",
+        "log.offset": 9226,
         "message": "[.kibana_task_manager] UPDATE_TARGET_MAPPINGS -> UPDATE_TARGET_MAPPINGS_WAIT_FOR_TASK. took: 8ms.",
         "process.pid": 78667,
         "service.type": "kibana",