zeek add support for iso8601 timestamps

- relates elastic/beats#25564
leehinman · Jun 10, 2021 · 27ffdc0 · 27ffdc0
1 parent c5984cb
commit 27ffdc0
Show file tree

Hide file tree

Showing 76 changed files with 273 additions and 138 deletions.
diff --git a/packages/zeek/changelog.yml b/packages/zeek/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.7.5"
+  changes:
+    - description: Add support for ISO8601 timestamps
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/XXXX
 - version: "0.7.4"
   changes:
     - description: Add system test for httpjson Splunk input.

diff --git a/...ages/zeek/data_stream/capture_loss/_dev/test/pipeline/test-capture-loss.log-expected.json b/...ages/zeek/data_stream/capture_loss/_dev/test/pipeline/test-capture-loss.log-expected.json
@@ -15,7 +15,7 @@
                 }
             },
             "event": {
-                "ingested": "2021-06-08T07:47:55.330460300Z",
+                "ingested": "2021-06-10T19:23:45.653000800Z",
                 "original": "{\"ts\":1568132368.465338,\"ts_delta\":32.282249,\"peer\":\"bro\",\"gaps\":0,\"acks\":206,\"percent_lost\":0.0}",
                 "type": "info",
                 "created": "2020-04-28T11:07:58.223Z",
@@ -40,7 +40,7 @@
                 }
             },
             "event": {
-                "ingested": "2021-06-08T07:47:55.330466Z",
+                "ingested": "2021-06-10T19:23:45.653013300Z",
                 "original": "{\"ts\":1617062640.941952,\"ts_delta\":900.0005369186401,\"peer\":\"zeek\",\"gaps\":58475,\"acks\":65665,\"percent_lost\":89.05048351481003}",
                 "type": "info",
                 "created": "2020-04-28T11:07:58.223Z",
@@ -65,7 +65,7 @@
                 }
             },
             "event": {
-                "ingested": "2021-06-08T07:47:55.330473300Z",
+                "ingested": "2021-06-10T19:23:45.653022600Z",
                 "original": "{\"ts\":1617063540.942231,\"ts_delta\":900.0002789497376,\"peer\":\"zeek\",\"gaps\":54754,\"acks\":61818,\"percent_lost\":88.5729075673752}",
                 "type": "info",
                 "created": "2020-04-28T11:07:58.223Z",
@@ -90,7 +90,7 @@
                 }
             },
             "event": {
-                "ingested": "2021-06-08T07:47:55.330482600Z",
+                "ingested": "2021-06-10T19:23:45.653031700Z",
                 "original": "{\"ts\":1617064440.942597,\"ts_delta\":900.0003659725189,\"peer\":\"zeek\",\"gaps\":51022,\"acks\":57974,\"percent_lost\":88.00841756649533}",
                 "type": "info",
                 "created": "2020-04-28T11:07:58.223Z",
@@ -115,7 +115,7 @@
                 }
             },
             "event": {
-                "ingested": "2021-06-08T07:47:55.330489900Z",
+                "ingested": "2021-06-10T19:23:45.653040700Z",
                 "original": "{\"ts\":1617065340.942651,\"ts_delta\":900.0000541210175,\"peer\":\"zeek\",\"gaps\":55105,\"acks\":62497,\"percent_lost\":88.17223226714883}",
                 "type": "info",
                 "created": "2020-04-28T11:07:58.223Z",
@@ -148,7 +148,7 @@
                 }
             },
             "event": {
-                "ingested": "2021-06-08T07:47:55.330494Z",
+                "ingested": "2021-06-10T19:23:45.653049700Z",
                 "original": "{\"ts\":1568132368.465338,\"ts_delta\":32.282249,\"peer\":\"bro\",\"gaps\":0,\"acks\":206,\"percent_lost\":0.0}",
                 "type": "info",
                 "created": "2020-04-28T11:07:58.223Z",

diff --git a/packages/zeek/data_stream/capture_loss/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/capture_loss/elasticsearch/ingest_pipeline/default.yml
@@ -62,6 +62,7 @@ processors:
       field: zeek.capture_loss.ts
       formats:
         - UNIX
+        - ISO8601
   - set:
       field: event.kind
       value: metric

diff --git a/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log b/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log
@@ -14,4 +14,5 @@
 {"ts":1617062400.703865,"uid":"C3pPjh1YRYcVDiZD3","id.orig_h":"10.156.0.2","id.orig_p":44944,"id.resp_h":"169.254.169.254","id.resp_p":80,"proto":"tcp","conn_state":"OTH","local_orig":true,"local_resp":false,"missed_bytes":0,"history":"C","orig_pkts":0,"orig_ip_bytes":0,"resp_pkts":0,"resp_ip_bytes":0}
 {"ts":1617062400.703851,"uid":"ChUxTmYLG37oO5qUb","id.orig_h":"10.156.0.2","id.orig_p":44942,"id.resp_h":"169.254.169.254","id.resp_p":80,"proto":"tcp","conn_state":"OTH","local_orig":true,"local_resp":false,"missed_bytes":0,"history":"C","orig_pkts":0,"orig_ip_bytes":0,"resp_pkts":0,"resp_ip_bytes":0}
 {"ts":1617062400.704467,"uid":"CpeAOT3B11CTXJgzw2","id.orig_h":"10.156.0.2","id.orig_p":44946,"id.resp_h":"169.254.169.254","id.resp_p":80,"proto":"tcp","conn_state":"OTH","local_orig":true,"local_resp":false,"missed_bytes":0,"history":"C","orig_pkts":0,"orig_ip_bytes":0,"resp_pkts":0,"resp_ip_bytes":0}
-{"preview":false,"offset":0,"result":{"_bkt":"main~0~0758E7C3-1D0C-4B2B-8CF0-682BFEA86CDC","_cd":"0:12","_indextime":"1608752616","_raw":"{\"ts\":1547188417.857497,\"uid\":\"CAcJw21BbVedgFnYH5\",\"id.orig_h\":\"4.4.2.2\",\"id.orig_p\":38334,\"id.resp_h\":\"8.8.8.8\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.076967,\"orig_bytes\":75,\"resp_bytes\":178,\"conn_state\":\"SF\",\"local_orig\":false,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Dd\",\"orig_pkts\":1,\"orig_ip_bytes\":103,\"resp_pkts\":1,\"resp_ip_bytes\":206,\"tunnel_parents\":[]}","_serial":"0","_si":["b590508aafed","main"],"_sourcetype":"access_log-too_small","_time":"2020-12-23 19:43:35.000 UTC","host":"Lees-MBP.localdomain","index":"main","linecount":"1","my_max":"1608759317","source":"/usr/local/var/log/httpd/access_log","sourcetype":"access_log-too_small","splunk_server":"b590508aafed"}}
+{"preview":false,"offset":0,"result":{"_bkt":"main~0~0758E7C3-1D0C-4B2B-8CF0-682BFEA86CDC","_cd":"0:12","_indextime":"1608752616","_raw":"{\"ts\":1547188417.857497,\"uid\":\"CAcJw21BbVedgFnYH5\",\"id.orig_h\":\"4.4.2.2\",\"id.orig_p\":38334,\"id.resp_h\":\"8.8.8.8\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.076967,\"orig_bytes\":75,\"resp_bytes\":178,\"conn_state\":\"SF\",\"local_orig\":false,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Dd\",\"orig_pkts\":1,\"orig_ip_bytes\":103,\"resp_pkts\":1,\"resp_ip_bytes\":206,\"tunnel_parents\":[]}","_serial":"0","_si":["b590508aafed","main"],"_sourcetype":"access_log-too_small","_time":"2020-12-23 19:43:35.000 UTC","host":"Lees-MBP.localdomain","index":"main","linecount":"1","my_max":"1608759317","source":"/usr/local/var/log/httpd/access_log","sourcetype":"access_log-too_small","splunk_server":"b590508aafed"}}
+{"ts":"2021-06-09T20:55:13.160328Z","uid":"C2KP1V3alRLoxl4JB9","id.orig_h":"10.0.2.15","id.orig_p":46408,"id.resp_h":"172.217.9.68","id.resp_p":80,"proto":"tcp","conn_state":"OTH","local_orig":true,"local_resp":false,"missed_bytes":0,"history":"C","orig_pkts":0,"orig_ip_bytes":0,"resp_pkts":0,"resp_ip_bytes":0}