Zeek add support for 8601 timestamps

- Relates elastic/beats#25564
leehinman · Jun 10, 2021 · 5b8fdfd · 5b8fdfd
1 parent c335d60
commit 5b8fdfd
Show file tree

Hide file tree

Showing 76 changed files with 271 additions and 138 deletions.
diff --git a/packages/zeek/changelog.yml b/packages/zeek/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.7.3"
+  changes:
+    - description: adding support for ISO8601 timestamps
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/XXX
 - version: "0.7.2"
   changes:
     - description: adding back 0.7.0 changes

diff --git a/...ages/zeek/data_stream/capture_loss/_dev/test/pipeline/test-capture-loss.log-expected.json b/...ages/zeek/data_stream/capture_loss/_dev/test/pipeline/test-capture-loss.log-expected.json
@@ -15,7 +15,7 @@
                 }
             },
             "event": {
-                "ingested": "2021-04-23T19:56:22.333182200Z",
+                "ingested": "2021-06-10T17:49:23.711919200Z",
                 "type": "info",
                 "created": "2020-04-28T11:07:58.223Z",
                 "kind": "metric"
@@ -36,7 +36,7 @@
                 }
             },
             "event": {
-                "ingested": "2021-04-23T19:56:22.333229400Z",
+                "ingested": "2021-06-10T17:49:23.711939800Z",
                 "type": "info",
                 "created": "2020-04-28T11:07:58.223Z",
                 "kind": "metric"
@@ -57,7 +57,7 @@
                 }
             },
             "event": {
-                "ingested": "2021-04-23T19:56:22.333237400Z",
+                "ingested": "2021-06-10T17:49:23.711954300Z",
                 "type": "info",
                 "created": "2020-04-28T11:07:58.223Z",
                 "kind": "metric"
@@ -78,7 +78,7 @@
                 }
             },
             "event": {
-                "ingested": "2021-04-23T19:56:22.333243200Z",
+                "ingested": "2021-06-10T17:49:23.711965Z",
                 "type": "info",
                 "created": "2020-04-28T11:07:58.223Z",
                 "kind": "metric"
@@ -99,7 +99,7 @@
                 }
             },
             "event": {
-                "ingested": "2021-04-23T19:56:22.333248500Z",
+                "ingested": "2021-06-10T17:49:23.711975200Z",
                 "type": "info",
                 "created": "2020-04-28T11:07:58.223Z",
                 "kind": "metric"
@@ -128,7 +128,7 @@
                 }
             },
             "event": {
-                "ingested": "2021-04-23T19:56:22.333253200Z",
+                "ingested": "2021-06-10T17:49:23.711985400Z",
                 "original": "{\"ts\":1568132368.465338,\"ts_delta\":32.282249,\"peer\":\"bro\",\"gaps\":0,\"acks\":206,\"percent_lost\":0.0}",
                 "type": "info",
                 "created": "2020-04-28T11:07:58.223Z",

diff --git a/packages/zeek/data_stream/capture_loss/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/capture_loss/elasticsearch/ingest_pipeline/default.yml
@@ -53,6 +53,7 @@ processors:
       field: zeek.capture_loss.ts
       formats:
         - UNIX
+        - ISO8601
   - set:
       field: event.kind
       value: metric

diff --git a/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log b/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log
@@ -14,4 +14,5 @@
 {"ts":1617062400.703865,"uid":"C3pPjh1YRYcVDiZD3","id.orig_h":"10.156.0.2","id.orig_p":44944,"id.resp_h":"169.254.169.254","id.resp_p":80,"proto":"tcp","conn_state":"OTH","local_orig":true,"local_resp":false,"missed_bytes":0,"history":"C","orig_pkts":0,"orig_ip_bytes":0,"resp_pkts":0,"resp_ip_bytes":0}
 {"ts":1617062400.703851,"uid":"ChUxTmYLG37oO5qUb","id.orig_h":"10.156.0.2","id.orig_p":44942,"id.resp_h":"169.254.169.254","id.resp_p":80,"proto":"tcp","conn_state":"OTH","local_orig":true,"local_resp":false,"missed_bytes":0,"history":"C","orig_pkts":0,"orig_ip_bytes":0,"resp_pkts":0,"resp_ip_bytes":0}
 {"ts":1617062400.704467,"uid":"CpeAOT3B11CTXJgzw2","id.orig_h":"10.156.0.2","id.orig_p":44946,"id.resp_h":"169.254.169.254","id.resp_p":80,"proto":"tcp","conn_state":"OTH","local_orig":true,"local_resp":false,"missed_bytes":0,"history":"C","orig_pkts":0,"orig_ip_bytes":0,"resp_pkts":0,"resp_ip_bytes":0}
-{"preview":false,"offset":0,"result":{"_bkt":"main~0~0758E7C3-1D0C-4B2B-8CF0-682BFEA86CDC","_cd":"0:12","_indextime":"1608752616","_raw":"{\"ts\":1547188417.857497,\"uid\":\"CAcJw21BbVedgFnYH5\",\"id.orig_h\":\"4.4.2.2\",\"id.orig_p\":38334,\"id.resp_h\":\"8.8.8.8\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.076967,\"orig_bytes\":75,\"resp_bytes\":178,\"conn_state\":\"SF\",\"local_orig\":false,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Dd\",\"orig_pkts\":1,\"orig_ip_bytes\":103,\"resp_pkts\":1,\"resp_ip_bytes\":206,\"tunnel_parents\":[]}","_serial":"0","_si":["b590508aafed","main"],"_sourcetype":"access_log-too_small","_time":"2020-12-23 19:43:35.000 UTC","host":"Lees-MBP.localdomain","index":"main","linecount":"1","my_max":"1608759317","source":"/usr/local/var/log/httpd/access_log","sourcetype":"access_log-too_small","splunk_server":"b590508aafed"}}
+{"preview":false,"offset":0,"result":{"_bkt":"main~0~0758E7C3-1D0C-4B2B-8CF0-682BFEA86CDC","_cd":"0:12","_indextime":"1608752616","_raw":"{\"ts\":1547188417.857497,\"uid\":\"CAcJw21BbVedgFnYH5\",\"id.orig_h\":\"4.4.2.2\",\"id.orig_p\":38334,\"id.resp_h\":\"8.8.8.8\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.076967,\"orig_bytes\":75,\"resp_bytes\":178,\"conn_state\":\"SF\",\"local_orig\":false,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Dd\",\"orig_pkts\":1,\"orig_ip_bytes\":103,\"resp_pkts\":1,\"resp_ip_bytes\":206,\"tunnel_parents\":[]}","_serial":"0","_si":["b590508aafed","main"],"_sourcetype":"access_log-too_small","_time":"2020-12-23 19:43:35.000 UTC","host":"Lees-MBP.localdomain","index":"main","linecount":"1","my_max":"1608759317","source":"/usr/local/var/log/httpd/access_log","sourcetype":"access_log-too_small","splunk_server":"b590508aafed"}}
+{"ts":"2021-06-09T20:55:13.160328Z","uid":"C2KP1V3alRLoxl4JB9","id.orig_h":"10.0.2.15","id.orig_p":46408,"id.resp_h":"172.217.9.68","id.resp_p":80,"proto":"tcp","conn_state":"OTH","local_orig":true,"local_resp":false,"missed_bytes":0,"history":"C","orig_pkts":0,"orig_ip_bytes":0,"resp_pkts":0,"resp_ip_bytes":0}
diff --git a/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log-expected.json b/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log-expected.json
@@ -39,7 +39,7 @@
             },
             "event": {
                 "duration": 76967000,
-                "ingested": "2021-04-23T19:56:22.481489300Z",
+                "ingested": "2021-06-10T17:49:23.978658400Z",
                 "created": "2020-04-28T11:07:58.223Z",
                 "kind": "event",
                 "id": "CAcJw21BbVedgFnYH3",
@@ -117,7 +117,7 @@
             },
             "event": {
                 "duration": 76967000,
-                "ingested": "2021-04-23T19:56:22.481509800Z",
+                "ingested": "2021-06-10T17:49:23.978673500Z",
                 "created": "2020-04-28T11:07:58.223Z",
                 "kind": "event",
                 "id": "CAcJw21BbVedgFnYH4",
@@ -210,7 +210,7 @@
             },
             "event": {
                 "duration": 76967000,
-                "ingested": "2021-04-23T19:56:22.481515200Z",
+                "ingested": "2021-06-10T17:49:23.978681100Z",
                 "created": "2020-04-28T11:07:58.223Z",
                 "kind": "event",
                 "id": "CAcJw21BbVedgFnYH5",
@@ -273,7 +273,7 @@
                 "ip": "192.0.2.205"
             },
             "event": {
-                "ingested": "2021-04-23T19:56:22.481522400Z",
+                "ingested": "2021-06-10T17:49:23.978687600Z",
                 "id": "Cc6NJ3GRlfjE44I3h",
                 "category": "network",
                 "type": [
@@ -351,7 +351,7 @@
                 "ip": "10.156.0.2"
             },
             "event": {
-                "ingested": "2021-04-23T19:56:22.481526700Z",
+                "ingested": "2021-06-10T17:49:23.978693600Z",
                 "id": "CCicIg43lOtCQOxXnb",
                 "category": "network",
                 "type": [
@@ -430,7 +430,7 @@
             },
             "event": {
                 "duration": 103708982,
-                "ingested": "2021-04-23T19:56:22.481531Z",
+                "ingested": "2021-06-10T17:49:23.978698100Z",
                 "created": "2020-04-28T11:07:58.223Z",
                 "kind": "event",
                 "id": "C52mXBCPJ4pPGkhr1",
@@ -509,7 +509,7 @@
             },
             "event": {
                 "duration": 104128838,
-                "ingested": "2021-04-23T19:56:22.481535Z",
+                "ingested": "2021-06-10T17:49:23.978702Z",
                 "created": "2020-04-28T11:07:58.223Z",
                 "kind": "event",
                 "id": "CTzCky2CyLT5JJvHck",
@@ -588,7 +588,7 @@
             },
             "event": {
                 "duration": 104333878,
-                "ingested": "2021-04-23T19:56:22.481550900Z",
+                "ingested": "2021-06-10T17:49:23.978707500Z",
                 "created": "2020-04-28T11:07:58.223Z",
                 "kind": "event",
                 "id": "CIkS28PDxqQnN49m2",
@@ -649,7 +649,7 @@
             },
             "event": {
                 "duration": 26802063,
-                "ingested": "2021-04-23T19:56:22.481559200Z",
+                "ingested": "2021-06-10T17:49:23.978720800Z",
                 "created": "2020-04-28T11:07:58.223Z",
                 "kind": "event",
                 "id": "CezEGe4jeLNkayV976",
@@ -711,7 +711,7 @@
             },
             "event": {
                 "duration": 25056124,
-                "ingested": "2021-04-23T19:56:22.481564200Z",
+                "ingested": "2021-06-10T17:49:23.978748500Z",
                 "created": "2020-04-28T11:07:58.223Z",
                 "kind": "event",
                 "id": "CKSr3w18mmW6t7bXC4",
@@ -773,7 +773,7 @@
             },
             "event": {
                 "duration": 3319979,
-                "ingested": "2021-04-23T19:56:22.481568400Z",
+                "ingested": "2021-06-10T17:49:23.978755500Z",
                 "created": "2020-04-28T11:07:58.223Z",
                 "kind": "event",
                 "id": "CGUiHy4kLIF2ml95eg",
@@ -835,7 +835,7 @@
             },
             "event": {
                 "duration": 1111984,
-                "ingested": "2021-04-23T19:56:22.481572700Z",
+                "ingested": "2021-06-10T17:49:23.978779300Z",
                 "created": "2020-04-28T11:07:58.223Z",
                 "kind": "event",
                 "id": "CAOZZi4Qrio7gUVgVc",
@@ -897,7 +897,7 @@
             },
             "event": {
                 "duration": 908852,
-                "ingested": "2021-04-23T19:56:22.481576800Z",
+                "ingested": "2021-06-10T17:49:23.978786500Z",
                 "created": "2020-04-28T11:07:58.223Z",
                 "kind": "event",
                 "id": "Chx5fs3xQ5ALB72i4e",
@@ -958,7 +958,7 @@
                 "ip": "10.156.0.2"
             },
             "event": {
-                "ingested": "2021-04-23T19:56:22.481581200Z",
+                "ingested": "2021-06-10T17:49:23.978795400Z",
                 "id": "C3pPjh1YRYcVDiZD3",
                 "category": "network",
                 "type": [
@@ -1018,7 +1018,7 @@
                 "ip": "10.156.0.2"
             },
             "event": {
-                "ingested": "2021-04-23T19:56:22.481584800Z",
+                "ingested": "2021-06-10T17:49:23.978803800Z",
                 "id": "ChUxTmYLG37oO5qUb",
                 "category": "network",
                 "type": [
@@ -1078,7 +1078,7 @@
                 "ip": "10.156.0.2"
             },
             "event": {
-                "ingested": "2021-04-23T19:56:22.481588500Z",
+                "ingested": "2021-06-10T17:49:23.978809700Z",
                 "id": "CpeAOT3B11CTXJgzw2",
                 "category": "network",
                 "type": [
@@ -1189,7 +1189,7 @@
             },
             "event": {
                 "duration": 76967000,
-                "ingested": "2021-04-23T19:56:22.481592400Z",
+                "ingested": "2021-06-10T17:49:23.978819200Z",
                 "original": "{\"ts\":1547188417.857497,\"uid\":\"CAcJw21BbVedgFnYH5\",\"id.orig_h\":\"4.4.2.2\",\"id.orig_p\":38334,\"id.resp_h\":\"8.8.8.8\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.076967,\"orig_bytes\":75,\"resp_bytes\":178,\"conn_state\":\"SF\",\"local_orig\":false,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Dd\",\"orig_pkts\":1,\"orig_ip_bytes\":103,\"resp_pkts\":1,\"resp_ip_bytes\":206,\"tunnel_parents\":[]}",
                 "created": "2020-04-28T11:07:58.223Z",
                 "kind": "event",
@@ -1201,6 +1201,81 @@
                     "end"
                 ]
             }
+        },
+        {
+            "@timestamp": "2021-06-09T20:55:13.160Z",
+            "ecs": {
+                "version": "1.9.0"
+            },
+            "related": {
+                "ip": [
+                    "10.0.2.15",
+                    "172.217.9.68"
+                ]
+            },
+            "destination": {
+                "geo": {
+                    "continent_name": "North America",
+                    "country_name": "United States",
+                    "location": {
+                        "lon": -97.822,
+                        "lat": 37.751
+                    },
+                    "country_iso_code": "US"
+                },
+                "as": {
+                    "number": 15169,
+                    "organization": {
+                        "name": "Google LLC"
+                    }
+                },
+                "address": "172.217.9.68",
+                "port": 80,
+                "bytes": 0,
+                "ip": "172.217.9.68",
+                "packets": 0
+            },
+            "zeek": {
+                "session_id": "C2KP1V3alRLoxl4JB9",
+                "connection": {
+                    "local_resp": false,
+                    "local_orig": true,
+                    "missed_bytes": 0,
+                    "history": "C",
+                    "id": {},
+                    "state": "OTH",
+                    "state_message": "No SYN seen, just midstream traffic (a 'partial connection' that was not later closed)."
+                }
+            },
+            "source": {
+                "address": "10.0.2.15",
+                "port": 46408,
+                "bytes": 0,
+                "packets": 0,
+                "ip": "10.0.2.15"
+            },
+            "event": {
+                "ingested": "2021-06-10T17:49:23.978828700Z",
+                "id": "C2KP1V3alRLoxl4JB9",
+                "category": "network",
+                "type": [
+                    "connection",
+                    "info"
+                ],
+                "created": "2020-04-28T11:07:58.223Z",
+                "kind": "event"
+            },
+            "network": {
+                "community_id": "1:DzqI9CYXjMSYV8VoSAHtMNfMIeU=",
+                "transport": "tcp",
+                "bytes": 0,
+                "packets": 0,
+                "direction": "outbound"
+            },
+            "tags": [
+                "local_orig",
+                "local_resp"
+            ]
         }
     ]
 }
diff --git a/packages/zeek/data_stream/connection/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/connection/elasticsearch/ingest_pipeline/default.yml
@@ -159,6 +159,7 @@ processors:
       field: zeek.connection.ts
       formats:
         - UNIX
+        - ISO8601
   - remove:
       field: zeek.connection.ts
   - set:

diff --git a/packages/zeek/data_stream/dce_rpc/_dev/test/pipeline/test-dce-rpc.log-expected.json b/packages/zeek/data_stream/dce_rpc/_dev/test/pipeline/test-dce-rpc.log-expected.json
@@ -31,7 +31,7 @@
                 "ip": "172.16.133.6"
             },
             "event": {
-                "ingested": "2021-04-23T19:56:23.054640400Z",
+                "ingested": "2021-06-10T17:49:25.085274900Z",
                 "created": "2020-04-28T11:07:58.223Z",
                 "kind": "event",
                 "action": "BrowserrQueryOtherDomains",
@@ -95,7 +95,7 @@
                 "name": "Lees-MBP.localdomain"
             },
             "event": {
-                "ingested": "2021-04-23T19:56:23.054658400Z",
+                "ingested": "2021-06-10T17:49:25.085291200Z",
                 "original": "{\"ts\":1361916332.298338,\"uid\":\"CsNHVHa1lzFtvJzT8\",\"id.orig_h\":\"172.16.133.6\",\"id.orig_p\":1728,\"id.resp_h\":\"172.16.128.202\",\"id.resp_p\":445,\"rtt\":0.09211,\"named_pipe\":\"\\u005cPIPE\\u005cbrowser\",\"endpoint\":\"browser\",\"operation\":\"BrowserrQueryOtherDomains\"}",
                 "created": "2020-04-28T11:07:58.223Z",
                 "kind": "event",

diff --git a/packages/zeek/data_stream/dce_rpc/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/dce_rpc/elasticsearch/ingest_pipeline/default.yml
@@ -122,6 +122,7 @@ processors:
       field: zeek.dce_rpc.ts
       formats:
         - UNIX
+        - ISO8601
   - remove:
       field: zeek.dce_rpc.ts
   - append:

diff --git a/packages/zeek/data_stream/dhcp/_dev/test/pipeline/test-dhcp.log-expected.json b/packages/zeek/data_stream/dhcp/_dev/test/pipeline/test-dhcp.log-expected.json
@@ -60,7 +60,7 @@
                 "address": "192.168.199.132"
             },
             "event": {
-                "ingested": "2021-04-23T19:56:23.150661700Z",
+                "ingested": "2021-06-10T17:49:25.273076500Z",
                 "id": [
                     "CmWOt6VWaNGqXYcH6",
                     "CLObLo4YHn0u23Tp8a"
@@ -132,7 +132,7 @@
                 "address": "10.156.0.2"
             },
             "event": {
-                "ingested": "2021-04-23T19:56:23.150679900Z",
+                "ingested": "2021-06-10T17:49:25.273091700Z",
                 "id": [
                     "Ck0tsG4wsJxI3lIEZ"
                 ],
@@ -216,7 +216,7 @@
                 "address": "192.168.199.132"
             },
             "event": {
-                "ingested": "2021-04-23T19:56:23.150685Z",
+                "ingested": "2021-06-10T17:49:25.273101100Z",
                 "original": "{\"ts\":1476605498.771847,\"uids\":[\"CmWOt6VWaNGqXYcH6\",\"CLObLo4YHn0u23Tp8a\"],\"client_addr\":\"192.168.199.132\",\"server_addr\":\"192.168.199.254\",\"mac\":\"00:0c:29:03:df:ad\",\"host_name\":\"DESKTOP-2AEFM7G\",\"client_fqdn\":\"DESKTOP-2AEFM7G\",\"domain\":\"localdomain\",\"requested_addr\":\"192.168.199.132\",\"assigned_addr\":\"192.168.199.132\",\"lease_time\":1800.0,\"msg_types\":[\"REQUEST\",\"ACK\"],\"duration\":0.000161}",
                 "created": "2020-04-28T11:07:58.223Z",
                 "kind": "event",

diff --git a/packages/zeek/data_stream/dhcp/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/dhcp/elasticsearch/ingest_pipeline/default.yml
@@ -182,6 +182,7 @@ processors:
       field: zeek.dhcp.ts
       formats:
         - UNIX
+        - ISO8601
   - remove:
       field: zeek.dhcp.ts
   - set: