From c62b2c4ddfb9630b2af13591ec1e46afe9d817e6 Mon Sep 17 00:00:00 2001 From: Lee Hinman <57081003+leehinman@users.noreply.github.com> Date: Thu, 17 Jun 2021 10:07:25 -0500 Subject: [PATCH] zeek add support for iso8601 timestamps (#1118) - relates elastic/beats#25564 --- packages/zeek/changelog.yml | 5 + .../test-capture-loss.log-expected.json | 12 +- .../elasticsearch/ingest_pipeline/default.yml | 1 + .../_dev/test/pipeline/test-conn.log | 3 +- .../test/pipeline/test-conn.log-expected.json | 111 +++++++++++++++--- .../elasticsearch/ingest_pipeline/default.yml | 1 + .../pipeline/test-dce-rpc.log-expected.json | 4 +- .../elasticsearch/ingest_pipeline/default.yml | 1 + .../test/pipeline/test-dhcp.log-expected.json | 6 +- .../elasticsearch/ingest_pipeline/default.yml | 1 + .../test/pipeline/test-dnp3.log-expected.json | 4 +- .../elasticsearch/ingest_pipeline/default.yml | 1 + .../test/pipeline/test-dns.log-expected.json | 16 +-- .../elasticsearch/ingest_pipeline/default.yml | 1 + .../test/pipeline/test-dpd.log-expected.json | 4 +- .../elasticsearch/ingest_pipeline/default.yml | 1 + .../pipeline/test-files.log-expected.json | 18 +-- .../elasticsearch/ingest_pipeline/default.yml | 1 + .../test/pipeline/test-ftp.log-expected.json | 8 +- .../elasticsearch/ingest_pipeline/default.yml | 1 + .../test/pipeline/test-http.log-expected.json | 16 +-- .../elasticsearch/ingest_pipeline/default.yml | 1 + .../pipeline/test-intel.log-expected.json | 4 +- .../elasticsearch/ingest_pipeline/default.yml | 1 + .../test/pipeline/test-irc.log-expected.json | 8 +- .../elasticsearch/ingest_pipeline/default.yml | 1 + .../pipeline/test-kerberos.log-expected.json | 4 +- .../elasticsearch/ingest_pipeline/default.yml | 3 + .../pipeline/test-modbus.log-expected.json | 4 +- .../elasticsearch/ingest_pipeline/default.yml | 1 + .../pipeline/test-mysql.log-expected.json | 4 +- .../elasticsearch/ingest_pipeline/default.yml | 1 + .../pipeline/test-notice.log-expected.json | 10 +- .../elasticsearch/ingest_pipeline/default.yml | 1 + .../test/pipeline/test-ntlm.log-expected.json | 4 +- .../elasticsearch/ingest_pipeline/default.yml | 1 + .../test/pipeline/test-ocsp.log-expected.json | 6 +- .../elasticsearch/ingest_pipeline/default.yml | 4 + .../test/pipeline/test-pe.log-expected.json | 4 +- .../elasticsearch/ingest_pipeline/default.yml | 2 + .../pipeline/test-radius.log-expected.json | 4 +- .../elasticsearch/ingest_pipeline/default.yml | 1 + .../test/pipeline/test-rdp.log-expected.json | 4 +- .../elasticsearch/ingest_pipeline/default.yml | 1 + .../test/pipeline/test-rfb.log-expected.json | 4 +- .../elasticsearch/ingest_pipeline/default.yml | 1 + .../test/pipeline/test-sip.log-expected.json | 12 +- .../elasticsearch/ingest_pipeline/default.yml | 1 + .../pipeline/test-smb-cmd.log-expected.json | 4 +- .../elasticsearch/ingest_pipeline/default.yml | 1 + .../pipeline/test-smb-files.log-expected.json | 4 +- .../elasticsearch/ingest_pipeline/default.yml | 5 + .../test-smb-mapping.log-expected.json | 4 +- .../elasticsearch/ingest_pipeline/default.yml | 1 + .../test/pipeline/test-smtp.log-expected.json | 4 +- .../elasticsearch/ingest_pipeline/default.yml | 1 + .../test/pipeline/test-snmp.log-expected.json | 6 +- .../elasticsearch/ingest_pipeline/default.yml | 2 + .../pipeline/test-socks.log-expected.json | 4 +- .../elasticsearch/ingest_pipeline/default.yml | 1 + .../test/pipeline/test-ssh.log-expected.json | 10 +- .../elasticsearch/ingest_pipeline/default.yml | 1 + .../test/pipeline/test-ssl.log-expected.json | 20 ++-- .../elasticsearch/ingest_pipeline/default.yml | 3 + .../pipeline/test-stats.log-expected.json | 4 +- .../elasticsearch/ingest_pipeline/default.yml | 1 + .../elasticsearch/ingest_pipeline/default.yml | 1 + .../test-traceroute.log-expected.json | 4 +- .../elasticsearch/ingest_pipeline/default.yml | 1 + .../pipeline/test-tunnel.log-expected.json | 4 +- .../elasticsearch/ingest_pipeline/default.yml | 1 + .../pipeline/test-weird.log-expected.json | 6 +- .../elasticsearch/ingest_pipeline/default.yml | 1 + .../test/pipeline/test-x509.log-expected.json | 4 +- .../elasticsearch/ingest_pipeline/default.yml | 3 + packages/zeek/manifest.yml | 2 +- 76 files changed, 273 insertions(+), 138 deletions(-) diff --git a/packages/zeek/changelog.yml b/packages/zeek/changelog.yml index fdcdff8ac8c0..f74823c5afad 100644 --- a/packages/zeek/changelog.yml +++ b/packages/zeek/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.8.1" + changes: + - description: Add support for ISO8601 timestamps + type: enhancement + link: https://github.com/elastic/integrations/pull/1118 - version: "0.8.0" changes: - description: Update to ECS 1.10.0, adding processor fields and replacing default tags from . to - between words. diff --git a/packages/zeek/data_stream/capture_loss/_dev/test/pipeline/test-capture-loss.log-expected.json b/packages/zeek/data_stream/capture_loss/_dev/test/pipeline/test-capture-loss.log-expected.json index e8354ad47f59..cc70edc9d3b8 100644 --- a/packages/zeek/data_stream/capture_loss/_dev/test/pipeline/test-capture-loss.log-expected.json +++ b/packages/zeek/data_stream/capture_loss/_dev/test/pipeline/test-capture-loss.log-expected.json @@ -15,7 +15,7 @@ } }, "event": { - "ingested": "2021-06-14T08:33:52.840653500Z", + "ingested": "2021-06-17T13:41:22.759516Z", "original": "{\"ts\":1568132368.465338,\"ts_delta\":32.282249,\"peer\":\"bro\",\"gaps\":0,\"acks\":206,\"percent_lost\":0.0}", "type": "info", "created": "2020-04-28T11:07:58.223Z", @@ -40,7 +40,7 @@ } }, "event": { - "ingested": "2021-06-14T08:33:52.840677100Z", + "ingested": "2021-06-17T13:41:22.759526400Z", "original": "{\"ts\":1617062640.941952,\"ts_delta\":900.0005369186401,\"peer\":\"zeek\",\"gaps\":58475,\"acks\":65665,\"percent_lost\":89.05048351481003}", "type": "info", "created": "2020-04-28T11:07:58.223Z", @@ -65,7 +65,7 @@ } }, "event": { - "ingested": "2021-06-14T08:33:52.840684700Z", + "ingested": "2021-06-17T13:41:22.759530800Z", "original": "{\"ts\":1617063540.942231,\"ts_delta\":900.0002789497376,\"peer\":\"zeek\",\"gaps\":54754,\"acks\":61818,\"percent_lost\":88.5729075673752}", "type": "info", "created": "2020-04-28T11:07:58.223Z", @@ -90,7 +90,7 @@ } }, "event": { - "ingested": "2021-06-14T08:33:52.840750500Z", + "ingested": "2021-06-17T13:41:22.759536200Z", "original": "{\"ts\":1617064440.942597,\"ts_delta\":900.0003659725189,\"peer\":\"zeek\",\"gaps\":51022,\"acks\":57974,\"percent_lost\":88.00841756649533}", "type": "info", "created": "2020-04-28T11:07:58.223Z", @@ -115,7 +115,7 @@ } }, "event": { - "ingested": "2021-06-14T08:33:52.840757600Z", + "ingested": "2021-06-17T13:41:22.759543500Z", "original": "{\"ts\":1617065340.942651,\"ts_delta\":900.0000541210175,\"peer\":\"zeek\",\"gaps\":55105,\"acks\":62497,\"percent_lost\":88.17223226714883}", "type": "info", "created": "2020-04-28T11:07:58.223Z", @@ -148,7 +148,7 @@ } }, "event": { - "ingested": "2021-06-14T08:33:52.840763500Z", + "ingested": "2021-06-17T13:41:22.759548900Z", "original": "{\"ts\":1568132368.465338,\"ts_delta\":32.282249,\"peer\":\"bro\",\"gaps\":0,\"acks\":206,\"percent_lost\":0.0}", "type": "info", "created": "2020-04-28T11:07:58.223Z", diff --git a/packages/zeek/data_stream/capture_loss/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/capture_loss/elasticsearch/ingest_pipeline/default.yml index eade6dfa1fa4..9f7f8c50db41 100644 --- a/packages/zeek/data_stream/capture_loss/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/capture_loss/elasticsearch/ingest_pipeline/default.yml @@ -62,6 +62,7 @@ processors: field: zeek.capture_loss.ts formats: - UNIX + - ISO8601 - set: field: event.kind value: metric diff --git a/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log b/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log index 4eaf9853b742..b652a4259543 100644 --- a/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log +++ b/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log @@ -14,4 +14,5 @@ {"ts":1617062400.703865,"uid":"C3pPjh1YRYcVDiZD3","id.orig_h":"10.156.0.2","id.orig_p":44944,"id.resp_h":"169.254.169.254","id.resp_p":80,"proto":"tcp","conn_state":"OTH","local_orig":true,"local_resp":false,"missed_bytes":0,"history":"C","orig_pkts":0,"orig_ip_bytes":0,"resp_pkts":0,"resp_ip_bytes":0} {"ts":1617062400.703851,"uid":"ChUxTmYLG37oO5qUb","id.orig_h":"10.156.0.2","id.orig_p":44942,"id.resp_h":"169.254.169.254","id.resp_p":80,"proto":"tcp","conn_state":"OTH","local_orig":true,"local_resp":false,"missed_bytes":0,"history":"C","orig_pkts":0,"orig_ip_bytes":0,"resp_pkts":0,"resp_ip_bytes":0} {"ts":1617062400.704467,"uid":"CpeAOT3B11CTXJgzw2","id.orig_h":"10.156.0.2","id.orig_p":44946,"id.resp_h":"169.254.169.254","id.resp_p":80,"proto":"tcp","conn_state":"OTH","local_orig":true,"local_resp":false,"missed_bytes":0,"history":"C","orig_pkts":0,"orig_ip_bytes":0,"resp_pkts":0,"resp_ip_bytes":0} -{"preview":false,"offset":0,"result":{"_bkt":"main~0~0758E7C3-1D0C-4B2B-8CF0-682BFEA86CDC","_cd":"0:12","_indextime":"1608752616","_raw":"{\"ts\":1547188417.857497,\"uid\":\"CAcJw21BbVedgFnYH5\",\"id.orig_h\":\"4.4.2.2\",\"id.orig_p\":38334,\"id.resp_h\":\"8.8.8.8\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.076967,\"orig_bytes\":75,\"resp_bytes\":178,\"conn_state\":\"SF\",\"local_orig\":false,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Dd\",\"orig_pkts\":1,\"orig_ip_bytes\":103,\"resp_pkts\":1,\"resp_ip_bytes\":206,\"tunnel_parents\":[]}","_serial":"0","_si":["b590508aafed","main"],"_sourcetype":"access_log-too_small","_time":"2020-12-23 19:43:35.000 UTC","host":"Lees-MBP.localdomain","index":"main","linecount":"1","my_max":"1608759317","source":"/usr/local/var/log/httpd/access_log","sourcetype":"access_log-too_small","splunk_server":"b590508aafed"}} \ No newline at end of file +{"preview":false,"offset":0,"result":{"_bkt":"main~0~0758E7C3-1D0C-4B2B-8CF0-682BFEA86CDC","_cd":"0:12","_indextime":"1608752616","_raw":"{\"ts\":1547188417.857497,\"uid\":\"CAcJw21BbVedgFnYH5\",\"id.orig_h\":\"4.4.2.2\",\"id.orig_p\":38334,\"id.resp_h\":\"8.8.8.8\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.076967,\"orig_bytes\":75,\"resp_bytes\":178,\"conn_state\":\"SF\",\"local_orig\":false,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Dd\",\"orig_pkts\":1,\"orig_ip_bytes\":103,\"resp_pkts\":1,\"resp_ip_bytes\":206,\"tunnel_parents\":[]}","_serial":"0","_si":["b590508aafed","main"],"_sourcetype":"access_log-too_small","_time":"2020-12-23 19:43:35.000 UTC","host":"Lees-MBP.localdomain","index":"main","linecount":"1","my_max":"1608759317","source":"/usr/local/var/log/httpd/access_log","sourcetype":"access_log-too_small","splunk_server":"b590508aafed"}} +{"ts":"2021-06-09T20:55:13.160328Z","uid":"C2KP1V3alRLoxl4JB9","id.orig_h":"10.0.2.15","id.orig_p":46408,"id.resp_h":"172.217.9.68","id.resp_p":80,"proto":"tcp","conn_state":"OTH","local_orig":true,"local_resp":false,"missed_bytes":0,"history":"C","orig_pkts":0,"orig_ip_bytes":0,"resp_pkts":0,"resp_ip_bytes":0} \ No newline at end of file diff --git a/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log-expected.json b/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log-expected.json index ccb8a7be0c07..49cecba5ccf7 100644 --- a/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log-expected.json +++ b/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log-expected.json @@ -39,7 +39,7 @@ }, "event": { "duration": 76967000, - "ingested": "2021-06-14T08:33:53.000399100Z", + "ingested": "2021-06-17T13:41:23.047071700Z", "original": "{\"ts\":1547188415.857497,\"uid\":\"CAcJw21BbVedgFnYH3\",\"id.orig_h\":\"192.168.86.167\",\"id.orig_p\":38339,\"id.resp_h\":\"192.168.86.1\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.076967,\"orig_bytes\":75,\"resp_bytes\":178,\"conn_state\":\"SF\",\"local_orig\":true,\"local_resp\":true,\"missed_bytes\":0,\"history\":\"Dd\",\"orig_pkts\":1,\"orig_ip_bytes\":103,\"resp_pkts\":1,\"resp_ip_bytes\":206,\"tunnel_parents\":[]}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -119,7 +119,7 @@ }, "event": { "duration": 76967000, - "ingested": "2021-06-14T08:33:53.000418600Z", + "ingested": "2021-06-17T13:41:23.047082500Z", "original": "{\"ts\":1547188416.857497,\"uid\":\"CAcJw21BbVedgFnYH4\",\"id.orig_h\":\"192.168.86.167\",\"id.orig_p\":38340,\"id.resp_h\":\"8.8.8.8\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.076967,\"orig_bytes\":75,\"resp_bytes\":178,\"conn_state\":\"SF\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Dd\",\"orig_pkts\":1,\"orig_ip_bytes\":103,\"resp_pkts\":1,\"resp_ip_bytes\":206,\"tunnel_parents\":[]}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -214,7 +214,7 @@ }, "event": { "duration": 76967000, - "ingested": "2021-06-14T08:33:53.000424500Z", + "ingested": "2021-06-17T13:41:23.047090200Z", "original": "{\"ts\":1547188417.857497,\"uid\":\"CAcJw21BbVedgFnYH5\",\"id.orig_h\":\"4.4.2.2\",\"id.orig_p\":38334,\"id.resp_h\":\"8.8.8.8\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.076967,\"orig_bytes\":75,\"resp_bytes\":178,\"conn_state\":\"SF\",\"local_orig\":false,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Dd\",\"orig_pkts\":1,\"orig_ip_bytes\":103,\"resp_pkts\":1,\"resp_ip_bytes\":206,\"tunnel_parents\":[]}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -279,7 +279,7 @@ "ip": "192.0.2.205" }, "event": { - "ingested": "2021-06-14T08:33:53.000432100Z", + "ingested": "2021-06-17T13:41:23.047097900Z", "original": "{\"ts\":1551399000.57855,\"uid\":\"Cc6NJ3GRlfjE44I3h\",\"id.orig_h\":\"192.0.2.205\",\"id.orig_p\":3,\"id.resp_h\":\"198.51.100.249\",\"id.resp_p\":3,\"proto\":\"icmp\",\"conn_state\":\"OTH\",\"local_orig\":false,\"local_resp\":false,\"missed_bytes\":0,\"orig_pkts\":1,\"orig_ip_bytes\":107,\"resp_pkts\":0,\"resp_ip_bytes\":0,\"tunnel_parents\":[]}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -359,7 +359,7 @@ "ip": "10.156.0.2" }, "event": { - "ingested": "2021-06-14T08:33:53.000436800Z", + "ingested": "2021-06-17T13:41:23.047105600Z", "original": "{\"ts\":1617062400.404645,\"uid\":\"CCicIg43lOtCQOxXnb\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":56190,\"id.resp_h\":\"46.101.87.151\",\"id.resp_p\":443,\"proto\":\"tcp\",\"conn_state\":\"OTH\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"C\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":0,\"resp_ip_bytes\":0}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -440,7 +440,7 @@ }, "event": { "duration": 103708982, - "ingested": "2021-06-14T08:33:53.000440900Z", + "ingested": "2021-06-17T13:41:23.047113200Z", "original": "{\"ts\":1617062100.419397,\"uid\":\"C52mXBCPJ4pPGkhr1\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":60810,\"id.resp_h\":\"20.190.160.73\",\"id.resp_p\":443,\"proto\":\"tcp\",\"duration\":0.10370898246765137,\"orig_bytes\":0,\"resp_bytes\":5854,\"conn_state\":\"SHR\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"^hCcdafA\",\"orig_pkts\":1,\"orig_ip_bytes\":52,\"resp_pkts\":4,\"resp_ip_bytes\":267}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -521,7 +521,7 @@ }, "event": { "duration": 104128838, - "ingested": "2021-06-14T08:33:53.000445100Z", + "ingested": "2021-06-17T13:41:23.047120800Z", "original": "{\"ts\":1617062100.419603,\"uid\":\"CTzCky2CyLT5JJvHck\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":60804,\"id.resp_h\":\"20.190.160.73\",\"id.resp_p\":443,\"proto\":\"tcp\",\"duration\":0.10412883758544922,\"orig_bytes\":0,\"resp_bytes\":5854,\"conn_state\":\"SHR\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"^hCcdafA\",\"orig_pkts\":1,\"orig_ip_bytes\":52,\"resp_pkts\":4,\"resp_ip_bytes\":267}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -602,7 +602,7 @@ }, "event": { "duration": 104333878, - "ingested": "2021-06-14T08:33:53.000449700Z", + "ingested": "2021-06-17T13:41:23.047132500Z", "original": "{\"ts\":1617062100.419826,\"uid\":\"CIkS28PDxqQnN49m2\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":60802,\"id.resp_h\":\"20.190.160.73\",\"id.resp_p\":443,\"proto\":\"tcp\",\"duration\":0.10433387756347656,\"orig_bytes\":0,\"resp_bytes\":5854,\"conn_state\":\"SHR\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"^hCcdafA\",\"orig_pkts\":1,\"orig_ip_bytes\":52,\"resp_pkts\":4,\"resp_ip_bytes\":267}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -665,7 +665,7 @@ }, "event": { "duration": 26802063, - "ingested": "2021-06-14T08:33:53.000453600Z", + "ingested": "2021-06-17T13:41:23.047137100Z", "original": "{\"ts\":1617062390.563187,\"uid\":\"CezEGe4jeLNkayV976\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":38948,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.02680206298828125,\"orig_bytes\":0,\"resp_bytes\":241,\"conn_state\":\"SHR\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Cd\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":1,\"resp_ip_bytes\":269}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -729,7 +729,7 @@ }, "event": { "duration": 25056124, - "ingested": "2021-06-14T08:33:53.000457600Z", + "ingested": "2021-06-17T13:41:23.047142500Z", "original": "{\"ts\":1617062390.563442,\"uid\":\"CKSr3w18mmW6t7bXC4\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":40080,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.025056123733520509,\"orig_bytes\":0,\"resp_bytes\":276,\"conn_state\":\"SHR\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Cd\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":1,\"resp_ip_bytes\":304}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -793,7 +793,7 @@ }, "event": { "duration": 3319979, - "ingested": "2021-06-14T08:33:53.000461500Z", + "ingested": "2021-06-17T13:41:23.047150400Z", "original": "{\"ts\":1617062390.667048,\"uid\":\"CGUiHy4kLIF2ml95eg\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":41407,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.003319978713989258,\"orig_bytes\":0,\"resp_bytes\":133,\"conn_state\":\"SHR\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Cd\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":1,\"resp_ip_bytes\":161}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -857,7 +857,7 @@ }, "event": { "duration": 1111984, - "ingested": "2021-06-14T08:33:53.000465600Z", + "ingested": "2021-06-17T13:41:23.047158300Z", "original": "{\"ts\":1617062390.698943,\"uid\":\"CAOZZi4Qrio7gUVgVc\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":50487,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.0011119842529296876,\"orig_bytes\":0,\"resp_bytes\":202,\"conn_state\":\"SHR\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Cd\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":1,\"resp_ip_bytes\":230}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -921,7 +921,7 @@ }, "event": { "duration": 908852, - "ingested": "2021-06-14T08:33:53.000469700Z", + "ingested": "2021-06-17T13:41:23.047166500Z", "original": "{\"ts\":1617062390.699227,\"uid\":\"Chx5fs3xQ5ALB72i4e\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":49647,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.0009088516235351563,\"orig_bytes\":0,\"resp_bytes\":145,\"conn_state\":\"SHR\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Cd\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":1,\"resp_ip_bytes\":173}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -984,7 +984,7 @@ "ip": "10.156.0.2" }, "event": { - "ingested": "2021-06-14T08:33:53.000473600Z", + "ingested": "2021-06-17T13:41:23.047174Z", "original": "{\"ts\":1617062400.703865,\"uid\":\"C3pPjh1YRYcVDiZD3\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":44944,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":80,\"proto\":\"tcp\",\"conn_state\":\"OTH\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"C\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":0,\"resp_ip_bytes\":0}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -1046,7 +1046,7 @@ "ip": "10.156.0.2" }, "event": { - "ingested": "2021-06-14T08:33:53.000477300Z", + "ingested": "2021-06-17T13:41:23.047181400Z", "original": "{\"ts\":1617062400.703851,\"uid\":\"ChUxTmYLG37oO5qUb\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":44942,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":80,\"proto\":\"tcp\",\"conn_state\":\"OTH\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"C\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":0,\"resp_ip_bytes\":0}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -1108,7 +1108,7 @@ "ip": "10.156.0.2" }, "event": { - "ingested": "2021-06-14T08:33:53.000481100Z", + "ingested": "2021-06-17T13:41:23.047205800Z", "original": "{\"ts\":1617062400.704467,\"uid\":\"CpeAOT3B11CTXJgzw2\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":44946,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":80,\"proto\":\"tcp\",\"conn_state\":\"OTH\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"C\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":0,\"resp_ip_bytes\":0}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -1222,7 +1222,7 @@ }, "event": { "duration": 76967000, - "ingested": "2021-06-14T08:33:53.000485100Z", + "ingested": "2021-06-17T13:41:23.047213100Z", "original": "{\"ts\":1547188417.857497,\"uid\":\"CAcJw21BbVedgFnYH5\",\"id.orig_h\":\"4.4.2.2\",\"id.orig_p\":38334,\"id.resp_h\":\"8.8.8.8\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.076967,\"orig_bytes\":75,\"resp_bytes\":178,\"conn_state\":\"SF\",\"local_orig\":false,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Dd\",\"orig_pkts\":1,\"orig_ip_bytes\":103,\"resp_pkts\":1,\"resp_ip_bytes\":206,\"tunnel_parents\":[]}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -1234,6 +1234,83 @@ "end" ] } + }, + { + "@timestamp": "2021-06-09T20:55:13.160Z", + "ecs": { + "version": "1.10.0" + }, + "related": { + "ip": [ + "10.0.2.15", + "172.217.9.68" + ] + }, + "destination": { + "geo": { + "continent_name": "North America", + "country_name": "United States", + "location": { + "lon": -97.822, + "lat": 37.751 + }, + "country_iso_code": "US" + }, + "as": { + "number": 15169, + "organization": { + "name": "Google LLC" + } + }, + "address": "172.217.9.68", + "port": 80, + "bytes": 0, + "ip": "172.217.9.68", + "packets": 0 + }, + "zeek": { + "session_id": "C2KP1V3alRLoxl4JB9", + "connection": { + "local_resp": false, + "local_orig": true, + "missed_bytes": 0, + "history": "C", + "id": {}, + "state": "OTH", + "state_message": "No SYN seen, just midstream traffic (a 'partial connection' that was not later closed)." + } + }, + "source": { + "address": "10.0.2.15", + "port": 46408, + "bytes": 0, + "packets": 0, + "ip": "10.0.2.15" + }, + "event": { + "ingested": "2021-06-17T13:41:23.047220300Z", + "original": "{\"ts\":\"2021-06-09T20:55:13.160328Z\",\"uid\":\"C2KP1V3alRLoxl4JB9\",\"id.orig_h\":\"10.0.2.15\",\"id.orig_p\":46408,\"id.resp_h\":\"172.217.9.68\",\"id.resp_p\":80,\"proto\":\"tcp\",\"conn_state\":\"OTH\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"C\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":0,\"resp_ip_bytes\":0}", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", + "id": "C2KP1V3alRLoxl4JB9", + "category": "network", + "type": [ + "connection", + "info" + ] + }, + "tags": [ + "preserve_original_event", + "local_orig", + "local_resp" + ], + "network": { + "community_id": "1:DzqI9CYXjMSYV8VoSAHtMNfMIeU=", + "transport": "tcp", + "bytes": 0, + "packets": 0, + "direction": "outbound" + } } ] } \ No newline at end of file diff --git a/packages/zeek/data_stream/connection/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/connection/elasticsearch/ingest_pipeline/default.yml index d12b7dbaae4f..606b42cb905f 100644 --- a/packages/zeek/data_stream/connection/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/connection/elasticsearch/ingest_pipeline/default.yml @@ -168,6 +168,7 @@ processors: field: zeek.connection.ts formats: - UNIX + - ISO8601 - remove: field: zeek.connection.ts - set: diff --git a/packages/zeek/data_stream/dce_rpc/_dev/test/pipeline/test-dce-rpc.log-expected.json b/packages/zeek/data_stream/dce_rpc/_dev/test/pipeline/test-dce-rpc.log-expected.json index 783c493e7b22..055353a850ac 100644 --- a/packages/zeek/data_stream/dce_rpc/_dev/test/pipeline/test-dce-rpc.log-expected.json +++ b/packages/zeek/data_stream/dce_rpc/_dev/test/pipeline/test-dce-rpc.log-expected.json @@ -31,7 +31,7 @@ "ip": "172.16.133.6" }, "event": { - "ingested": "2021-06-14T08:33:53.596451900Z", + "ingested": "2021-06-17T13:41:24.152427Z", "original": "{\"ts\":1361916332.298338,\"uid\":\"CsNHVHa1lzFtvJzT8\",\"id.orig_h\":\"172.16.133.6\",\"id.orig_p\":1728,\"id.resp_h\":\"172.16.128.202\",\"id.resp_p\":445,\"rtt\":0.09211,\"named_pipe\":\"\\u005cPIPE\\u005cbrowser\",\"endpoint\":\"browser\",\"operation\":\"BrowserrQueryOtherDomains\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -102,7 +102,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-06-14T08:33:53.596467300Z", + "ingested": "2021-06-17T13:41:24.152434700Z", "original": "{\"ts\":1361916332.298338,\"uid\":\"CsNHVHa1lzFtvJzT8\",\"id.orig_h\":\"172.16.133.6\",\"id.orig_p\":1728,\"id.resp_h\":\"172.16.128.202\",\"id.resp_p\":445,\"rtt\":0.09211,\"named_pipe\":\"\\u005cPIPE\\u005cbrowser\",\"endpoint\":\"browser\",\"operation\":\"BrowserrQueryOtherDomains\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/dce_rpc/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/dce_rpc/elasticsearch/ingest_pipeline/default.yml index b84a991bd605..607f76663405 100644 --- a/packages/zeek/data_stream/dce_rpc/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/dce_rpc/elasticsearch/ingest_pipeline/default.yml @@ -132,6 +132,7 @@ processors: field: zeek.dce_rpc.ts formats: - UNIX + - ISO8601 - remove: field: zeek.dce_rpc.ts - append: diff --git a/packages/zeek/data_stream/dhcp/_dev/test/pipeline/test-dhcp.log-expected.json b/packages/zeek/data_stream/dhcp/_dev/test/pipeline/test-dhcp.log-expected.json index 88a6d2577dbf..5d254ef9e38e 100644 --- a/packages/zeek/data_stream/dhcp/_dev/test/pipeline/test-dhcp.log-expected.json +++ b/packages/zeek/data_stream/dhcp/_dev/test/pipeline/test-dhcp.log-expected.json @@ -63,7 +63,7 @@ "address": "192.168.199.132" }, "event": { - "ingested": "2021-06-14T08:33:53.695063600Z", + "ingested": "2021-06-17T13:41:24.346602100Z", "original": "{\"ts\":1476605498.771847,\"uids\":[\"CmWOt6VWaNGqXYcH6\",\"CLObLo4YHn0u23Tp8a\"],\"client_addr\":\"192.168.199.132\",\"server_addr\":\"192.168.199.254\",\"mac\":\"00:0c:29:03:df:ad\",\"host_name\":\"DESKTOP-2AEFM7G\",\"client_fqdn\":\"DESKTOP-2AEFM7G\",\"domain\":\"localdomain\",\"requested_addr\":\"192.168.199.132\",\"assigned_addr\":\"192.168.199.132\",\"lease_time\":1800.0,\"msg_types\":[\"REQUEST\",\"ACK\"],\"duration\":0.000161}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -139,7 +139,7 @@ "address": "10.156.0.2" }, "event": { - "ingested": "2021-06-14T08:33:53.695078400Z", + "ingested": "2021-06-17T13:41:24.346612200Z", "original": "{\"ts\":1617088722.072416,\"uids\":[\"Ck0tsG4wsJxI3lIEZ\"],\"client_addr\":\"10.156.0.2\",\"server_addr\":\"169.254.169.254\",\"mac\":\"42:01:0a:9c:00:02\",\"domain\":\"c.elastic-sa.internal\",\"assigned_addr\":\"10.156.0.2\",\"lease_time\":86400.0,\"msg_types\":[\"ACK\"],\"duration\":0.0}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -227,7 +227,7 @@ "address": "192.168.199.132" }, "event": { - "ingested": "2021-06-14T08:33:53.695084Z", + "ingested": "2021-06-17T13:41:24.346619200Z", "original": "{\"ts\":1476605498.771847,\"uids\":[\"CmWOt6VWaNGqXYcH6\",\"CLObLo4YHn0u23Tp8a\"],\"client_addr\":\"192.168.199.132\",\"server_addr\":\"192.168.199.254\",\"mac\":\"00:0c:29:03:df:ad\",\"host_name\":\"DESKTOP-2AEFM7G\",\"client_fqdn\":\"DESKTOP-2AEFM7G\",\"domain\":\"localdomain\",\"requested_addr\":\"192.168.199.132\",\"assigned_addr\":\"192.168.199.132\",\"lease_time\":1800.0,\"msg_types\":[\"REQUEST\",\"ACK\"],\"duration\":0.000161}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/dhcp/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/dhcp/elasticsearch/ingest_pipeline/default.yml index a4b34ca14c1a..775ee9b9e5e4 100644 --- a/packages/zeek/data_stream/dhcp/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/dhcp/elasticsearch/ingest_pipeline/default.yml @@ -192,6 +192,7 @@ processors: field: zeek.dhcp.ts formats: - UNIX + - ISO8601 - remove: field: zeek.dhcp.ts - set: diff --git a/packages/zeek/data_stream/dnp3/_dev/test/pipeline/test-dnp3.log-expected.json b/packages/zeek/data_stream/dnp3/_dev/test/pipeline/test-dnp3.log-expected.json index 5ded35302bf9..82e531747097 100644 --- a/packages/zeek/data_stream/dnp3/_dev/test/pipeline/test-dnp3.log-expected.json +++ b/packages/zeek/data_stream/dnp3/_dev/test/pipeline/test-dnp3.log-expected.json @@ -29,7 +29,7 @@ "ip": "127.0.0.1" }, "event": { - "ingested": "2021-06-14T08:33:53.808599400Z", + "ingested": "2021-06-17T13:41:24.610732400Z", "original": "{\"ts\":1227729908.705944,\"uid\":\"CQV6tj1w1t4WzQpHoe\",\"id.orig_h\":\"127.0.0.1\",\"id.orig_p\":42942,\"id.resp_h\":\"127.0.0.1\",\"id.resp_p\":20000,\"fc_request\":\"READ\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -98,7 +98,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-06-14T08:33:53.808615900Z", + "ingested": "2021-06-17T13:41:24.610742300Z", "original": "{\"ts\":1227729908.705944,\"uid\":\"CQV6tj1w1t4WzQpHoe\",\"id.orig_h\":\"127.0.0.1\",\"id.orig_p\":42942,\"id.resp_h\":\"127.0.0.1\",\"id.resp_p\":20000,\"fc_request\":\"READ\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/dnp3/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/dnp3/elasticsearch/ingest_pipeline/default.yml index c60c3a5c371e..439fc3fb0b74 100644 --- a/packages/zeek/data_stream/dnp3/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/dnp3/elasticsearch/ingest_pipeline/default.yml @@ -144,6 +144,7 @@ processors: field: zeek.dnp3.ts formats: - UNIX + - ISO8601 - remove: field: zeek.dnp3.ts - set: diff --git a/packages/zeek/data_stream/dns/_dev/test/pipeline/test-dns.log-expected.json b/packages/zeek/data_stream/dns/_dev/test/pipeline/test-dns.log-expected.json index 64d5465663c3..05c405fadccf 100644 --- a/packages/zeek/data_stream/dns/_dev/test/pipeline/test-dns.log-expected.json +++ b/packages/zeek/data_stream/dns/_dev/test/pipeline/test-dns.log-expected.json @@ -94,7 +94,7 @@ }, "event": { "duration": 7.6967E7, - "ingested": "2021-06-14T08:33:53.996053500Z", + "ingested": "2021-06-17T13:41:24.872453600Z", "original": "{\"ts\":1547188415.857497,\"uid\":\"CAcJw21BbVedgFnYH3\",\"id.orig_h\":\"192.168.86.167\",\"id.orig_p\":38339,\"id.resp_h\":\"192.168.86.1\",\"id.resp_p\":53,\"proto\":\"udp\",\"trans_id\":15209,\"rtt\":0.076967,\"query\":\"dd625ffb4fc54735b281862aa1cd6cd4.us-west1.gcp.cloud.es.io\",\"qclass\":1,\"qclass_name\":\"C_INTERNET\",\"qtype\":1,\"qtype_name\":\"A\",\"rcode\":0,\"rcode_name\":\"NOERROR\",\"AA\":false,\"TC\":false,\"RD\":true,\"RA\":true,\"Z\":0,\"answers\":[\"proxy-production-us-west1.gcp.cloud.es.io\",\"proxy-production-us-west1-v1-009.gcp.cloud.es.io\",\"35.199.178.4\"],\"TTLs\":[119.0,119.0,59.0],\"rejected\":false}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -164,7 +164,7 @@ ] }, "event": { - "ingested": "2021-06-14T08:33:53.996072100Z", + "ingested": "2021-06-17T13:41:24.872468500Z", "original": "{\"ts\":1567095830.680046,\"uid\":\"C19a1k4lTv46YMbeOk\",\"id.orig_h\":\"fe80::4ef:15cf:769f:ff21\",\"id.orig_p\":5353,\"id.resp_h\":\"ff02::fb\",\"id.resp_p\":5353,\"proto\":\"udp\",\"trans_id\":0,\"query\":\"_googlecast._tcp.local\",\"qclass\":1,\"qclass_name\":\"C_INTERNET\",\"qtype\":12,\"qtype_name\":\"PTR\",\"AA\":false,\"TC\":false,\"RD\":false,\"RA\":false,\"Z\":0,\"rejected\":false}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -245,7 +245,7 @@ ] }, "event": { - "ingested": "2021-06-14T08:33:53.996078500Z", + "ingested": "2021-06-17T13:41:24.872477Z", "original": "{\"ts\":1567095830.734329,\"uid\":\"CdiVAw7jJw6gsX5H\",\"id.orig_h\":\"192.168.86.237\",\"id.orig_p\":5353,\"id.resp_h\":\"224.0.0.251\",\"id.resp_p\":5353,\"proto\":\"udp\",\"trans_id\":0,\"query\":\"_googlecast._tcp.local\",\"rcode\":0,\"rcode_name\":\"NOERROR\",\"AA\":true,\"TC\":false,\"RD\":false,\"RA\":false,\"Z\":0,\"answers\":[\"bravia-4k-gb-5c89f865c9d569ab338815b35e3acc56._googlecast._tcp.local\"],\"TTLs\":[120.0],\"rejected\":false}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -346,7 +346,7 @@ ] }, "event": { - "ingested": "2021-06-14T08:33:53.996082800Z", + "ingested": "2021-06-17T13:41:24.872484900Z", "original": "{\"ts\":1617105592.091052,\"uid\":\"CpwXdW4LQaJkaIgpk\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":33438,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":53,\"proto\":\"udp\",\"trans_id\":58036,\"query\":\"manage.office.com\",\"rcode\":0,\"rcode_name\":\"NOERROR\",\"AA\":false,\"TC\":false,\"RD\":false,\"RA\":true,\"Z\":0,\"answers\":[\"manage.office.com.trafficmanager.net\",\"o365adtapiproddeu001.cloudapp.net\",\"51.116.158.62\"],\"TTLs\":[13.0,18.0,8.0],\"rejected\":false}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -445,7 +445,7 @@ ] }, "event": { - "ingested": "2021-06-14T08:33:53.996086500Z", + "ingested": "2021-06-17T13:41:24.872493Z", "original": "{\"ts\":1617105592.973919,\"uid\":\"CO5TE748RoJEZuOThl\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":60444,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":53,\"proto\":\"udp\",\"trans_id\":35744,\"query\":\"login.microsoftonline.com\",\"rcode\":0,\"rcode_name\":\"NOERROR\",\"AA\":false,\"TC\":false,\"RD\":false,\"RA\":true,\"Z\":0,\"answers\":[\"a.privatelink.msidentity.com\",\"prda.aadg.msidentity.com\",\"www.tm.a.prd.aadg.akadns.net\"],\"TTLs\":[296.0,287.0,287.0],\"rejected\":false}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -601,7 +601,7 @@ ] }, "event": { - "ingested": "2021-06-14T08:33:53.996090200Z", + "ingested": "2021-06-17T13:41:24.872505800Z", "original": "{\"ts\":1617105592.9742,\"uid\":\"CG1jsmeHcBCGnWXmk\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":44310,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":53,\"proto\":\"udp\",\"trans_id\":58458,\"query\":\"login.microsoftonline.com\",\"rcode\":0,\"rcode_name\":\"NOERROR\",\"AA\":false,\"TC\":false,\"RD\":false,\"RA\":true,\"Z\":0,\"answers\":[\"a.privatelink.msidentity.com\",\"prda.aadg.msidentity.com\",\"www.tm.a.prd.aadg.trafficmanager.net\",\"20.190.159.132\",\"40.126.31.143\",\"20.190.159.134\",\"40.126.31.1\",\"20.190.159.136\",\"40.126.31.135\",\"40.126.31.6\",\"20.190.159.138\"],\"TTLs\":[299.0,214.0,243.0,243.0,243.0,243.0,243.0,243.0,243.0,243.0,243.0],\"rejected\":false}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -702,7 +702,7 @@ ] }, "event": { - "ingested": "2021-06-14T08:33:53.996094600Z", + "ingested": "2021-06-17T13:41:24.872514200Z", "original": "{\"ts\":1617105593.106256,\"uid\":\"ChP0cl4j5mbXSZ9TGf\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":36364,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":53,\"proto\":\"udp\",\"trans_id\":8791,\"query\":\"manage.office.com\",\"rcode\":0,\"rcode_name\":\"NOERROR\",\"AA\":false,\"TC\":false,\"RD\":false,\"RA\":true,\"Z\":0,\"answers\":[\"manage.office.com.trafficmanager.net\",\"o365adtapiproddeu001.cloudapp.net\",\"51.116.158.62\"],\"TTLs\":[12.0,17.0,7.0],\"rejected\":false}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -791,7 +791,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-06-14T08:33:53.996098200Z", + "ingested": "2021-06-17T13:41:24.872574700Z", "original": "{\"ts\":1567095830.734329,\"uid\":\"CdiVAw7jJw6gsX5H\",\"id.orig_h\":\"192.168.86.237\",\"id.orig_p\":5353,\"id.resp_h\":\"224.0.0.251\",\"id.resp_p\":5353,\"proto\":\"udp\",\"trans_id\":0,\"query\":\"_googlecast._tcp.local\",\"rcode\":0,\"rcode_name\":\"NOERROR\",\"AA\":true,\"TC\":false,\"RD\":false,\"RA\":false,\"Z\":0,\"answers\":[\"bravia-4k-gb-5c89f865c9d569ab338815b35e3acc56._googlecast._tcp.local\"],\"TTLs\":[120.0],\"rejected\":false}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/dns/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/dns/elasticsearch/ingest_pipeline/default.yml index f073c52641cc..21bae086b2c8 100644 --- a/packages/zeek/data_stream/dns/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/dns/elasticsearch/ingest_pipeline/default.yml @@ -253,6 +253,7 @@ processors: field: zeek.dns.ts formats: - UNIX + - ISO8601 - remove: field: zeek.dns.ts diff --git a/packages/zeek/data_stream/dpd/_dev/test/pipeline/test-dpd.log-expected.json b/packages/zeek/data_stream/dpd/_dev/test/pipeline/test-dpd.log-expected.json index 7f6ae8aa5dd3..70ce0a63e312 100644 --- a/packages/zeek/data_stream/dpd/_dev/test/pipeline/test-dpd.log-expected.json +++ b/packages/zeek/data_stream/dpd/_dev/test/pipeline/test-dpd.log-expected.json @@ -29,7 +29,7 @@ "ip": "192.168.10.31" }, "event": { - "ingested": "2021-06-14T08:33:54.266077900Z", + "ingested": "2021-06-17T13:41:25.355192700Z", "original": "{\"ts\":1507567500.423033,\"uid\":\"CRrT7S1ccw9H6hzCR\",\"id.orig_h\":\"192.168.10.31\",\"id.orig_p\":49285,\"id.resp_h\":\"192.168.10.10\",\"id.resp_p\":445,\"proto\":\"tcp\",\"analyzer\":\"DCE_RPC\",\"failure_reason\":\"Binpac exception: binpac exception: \\u0026enforce violation : DCE_RPC_Header:rpc_vers\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -94,7 +94,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-06-14T08:33:54.266091900Z", + "ingested": "2021-06-17T13:41:25.355204Z", "original": "{\"ts\":1507567500.423033,\"uid\":\"CRrT7S1ccw9H6hzCR\",\"id.orig_h\":\"192.168.10.31\",\"id.orig_p\":49285,\"id.resp_h\":\"192.168.10.10\",\"id.resp_p\":445,\"proto\":\"tcp\",\"analyzer\":\"DCE_RPC\",\"failure_reason\":\"Binpac exception: binpac exception: \\u0026enforce violation : DCE_RPC_Header:rpc_vers\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/dpd/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/dpd/elasticsearch/ingest_pipeline/default.yml index 1f4718a4fc49..529d36589018 100644 --- a/packages/zeek/data_stream/dpd/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/dpd/elasticsearch/ingest_pipeline/default.yml @@ -127,6 +127,7 @@ processors: field: zeek.dpd.ts formats: - UNIX + - ISO8601 - remove: field: zeek.dpd.ts - geoip: diff --git a/packages/zeek/data_stream/files/_dev/test/pipeline/test-files.log-expected.json b/packages/zeek/data_stream/files/_dev/test/pipeline/test-files.log-expected.json index f09840db7c94..ba6b580ff171 100644 --- a/packages/zeek/data_stream/files/_dev/test/pipeline/test-files.log-expected.json +++ b/packages/zeek/data_stream/files/_dev/test/pipeline/test-files.log-expected.json @@ -57,7 +57,7 @@ "ip": "10.178.98.102" }, "event": { - "ingested": "2021-06-14T08:33:54.368499800Z", + "ingested": "2021-06-17T13:41:25.552705200Z", "original": "{\"ts\":1547688796.636812,\"fuid\":\"FMkioa222mEuM2RuQ9\",\"tx_hosts\":[\"35.199.178.4\"],\"rx_hosts\":[\"10.178.98.102\"],\"conn_uids\":[\"C8I0zn3r9EPbfLgta6\"],\"source\":\"SSL\",\"depth\":0,\"analyzers\":[\"X509\",\"MD5\",\"SHA1\"],\"mime_type\":\"application/pkix-cert\",\"duration\":0.0,\"local_orig\":false,\"is_orig\":false,\"seen_bytes\":947,\"missing_bytes\":0,\"overflow_bytes\":0,\"timedout\":false,\"md5\":\"79e4a9840d7d3a96d7c04fe2434c892e\",\"sha1\":\"a8985d3a65e5e5c4b2d7d66d40c6dd2fb19c5436\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -130,7 +130,7 @@ "ip": "10.178.98.102" }, "event": { - "ingested": "2021-06-14T08:33:54.368515700Z", + "ingested": "2021-06-17T13:41:25.552718200Z", "original": "{\"ts\":1547688801.566262,\"fuid\":\"FShtIS1gydeSFf8M63\",\"tx_hosts\":[\"17.134.127.250\"],\"rx_hosts\":[\"10.178.98.102\"],\"conn_uids\":[\"C6sjVo23iNApLnlAt6\"],\"source\":\"SSL\",\"depth\":0,\"analyzers\":[\"X509\",\"MD5\",\"SHA1\"],\"mime_type\":\"application/pkix-cert\",\"duration\":0.0,\"local_orig\":false,\"is_orig\":false,\"seen_bytes\":2089,\"missing_bytes\":0,\"overflow_bytes\":0,\"timedout\":false,\"md5\":\"b9742f12eb97eff531d94f7800c6706c\",\"sha1\":\"b88d13fe319d342e7a808ce3a0a1158111fc3c2a\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -203,7 +203,7 @@ "ip": "10.178.98.102" }, "event": { - "ingested": "2021-06-14T08:33:54.368520700Z", + "ingested": "2021-06-17T13:41:25.552726500Z", "original": "{\"ts\":1547688801.566262,\"fuid\":\"F9ip9a3MDAq3XLBOn2\",\"tx_hosts\":[\"17.134.127.250\"],\"rx_hosts\":[\"10.178.98.102\"],\"conn_uids\":[\"C6sjVo23iNApLnlAt6\"],\"source\":\"SSL\",\"depth\":0,\"analyzers\":[\"X509\",\"MD5\",\"SHA1\"],\"mime_type\":\"application/pkix-cert\",\"duration\":0.0,\"local_orig\":false,\"is_orig\":false,\"seen_bytes\":1092,\"missing_bytes\":0,\"overflow_bytes\":0,\"timedout\":false,\"md5\":\"48f0e38385112eeca5fc9ffd402eaecd\",\"sha1\":\"8e8321ca08b08e3726fe1d82996884eeb5f0d655\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -280,7 +280,7 @@ "ip": "10.156.0.2" }, "event": { - "ingested": "2021-06-14T08:33:54.368524100Z", + "ingested": "2021-06-17T13:41:25.552734300Z", "original": "{\"ts\":1617069763.671838,\"fuid\":\"Fe722V1qt2DSlqCiOa\",\"tx_hosts\":[\"104.154.89.105\"],\"rx_hosts\":[\"10.156.0.2\"],\"conn_uids\":[\"ClG5ErV7bkgKgOaV\"],\"source\":\"SSL\",\"depth\":0,\"analyzers\":[\"X509\",\"SHA256\",\"SHA1\",\"MD5\"],\"mime_type\":\"application/x-x509-user-cert\",\"duration\":0.0,\"local_orig\":false,\"is_orig\":false,\"seen_bytes\":893,\"missing_bytes\":0,\"overflow_bytes\":0,\"timedout\":false,\"md5\":\"5abbc8a9137f6924861596848b7d7794\",\"sha1\":\"99c1daf07c8d69a8a065492dcaae43c43ff13497\",\"sha256\":\"1de4074b4e38377f4367303f4a19c986a506180f22a6e53a68cc7679ea6d9c74\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -357,7 +357,7 @@ "ip": "10.156.0.2" }, "event": { - "ingested": "2021-06-14T08:33:54.368527400Z", + "ingested": "2021-06-17T13:41:25.552746800Z", "original": "{\"ts\":1617069773.678327,\"fuid\":\"FYszs61e8hIUWMWgL5\",\"tx_hosts\":[\"104.154.89.105\"],\"rx_hosts\":[\"10.156.0.2\"],\"conn_uids\":[\"CaB3fq3yLrKCbYLqr4\"],\"source\":\"SSL\",\"depth\":0,\"analyzers\":[\"X509\",\"SHA256\",\"SHA1\",\"MD5\"],\"mime_type\":\"application/x-x509-user-cert\",\"duration\":0.0,\"local_orig\":false,\"is_orig\":false,\"seen_bytes\":893,\"missing_bytes\":0,\"overflow_bytes\":0,\"timedout\":false,\"md5\":\"5abbc8a9137f6924861596848b7d7794\",\"sha1\":\"99c1daf07c8d69a8a065492dcaae43c43ff13497\",\"sha256\":\"1de4074b4e38377f4367303f4a19c986a506180f22a6e53a68cc7679ea6d9c74\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -434,7 +434,7 @@ "ip": "10.156.0.2" }, "event": { - "ingested": "2021-06-14T08:33:54.368530300Z", + "ingested": "2021-06-17T13:41:25.552793200Z", "original": "{\"ts\":1617069783.678588,\"fuid\":\"FdGWZq2wRIvCfjvdI5\",\"tx_hosts\":[\"104.154.89.105\"],\"rx_hosts\":[\"10.156.0.2\"],\"conn_uids\":[\"C0vhl91PPOI7LbrPZ8\"],\"source\":\"SSL\",\"depth\":0,\"analyzers\":[\"X509\",\"SHA256\",\"SHA1\",\"MD5\"],\"mime_type\":\"application/x-x509-user-cert\",\"duration\":0.0,\"local_orig\":false,\"is_orig\":false,\"seen_bytes\":893,\"missing_bytes\":0,\"overflow_bytes\":0,\"timedout\":false,\"md5\":\"5abbc8a9137f6924861596848b7d7794\",\"sha1\":\"99c1daf07c8d69a8a065492dcaae43c43ff13497\",\"sha256\":\"1de4074b4e38377f4367303f4a19c986a506180f22a6e53a68cc7679ea6d9c74\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -507,7 +507,7 @@ "ip": "10.156.0.2" }, "event": { - "ingested": "2021-06-14T08:33:54.368533300Z", + "ingested": "2021-06-17T13:41:25.552801900Z", "original": "{\"ts\":1617069792.519193,\"fuid\":\"FSMkdM3YUSoEVpLZN4\",\"tx_hosts\":[\"169.254.169.254\"],\"rx_hosts\":[\"10.156.0.2\"],\"conn_uids\":[\"CgbPEj2jf5Ca7Lw0x2\"],\"source\":\"HTTP\",\"depth\":0,\"analyzers\":[\"SHA1\",\"MD5\"],\"mime_type\":\"text/html\",\"duration\":0.00005316734313964844,\"local_orig\":false,\"is_orig\":false,\"seen_bytes\":1609,\"total_bytes\":1609,\"missing_bytes\":0,\"overflow_bytes\":0,\"timedout\":false,\"md5\":\"1ab1d3a926a99ccfc25acccc5b4289b4\",\"sha1\":\"1895628784b47ad8da112c699a1b21f5b49c2b80\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -584,7 +584,7 @@ "ip": "10.156.0.2" }, "event": { - "ingested": "2021-06-14T08:33:54.368536100Z", + "ingested": "2021-06-17T13:41:25.552810800Z", "original": "{\"ts\":1617069793.669729,\"fuid\":\"F1msmE2xRFsdvL2iI\",\"tx_hosts\":[\"104.154.89.105\"],\"rx_hosts\":[\"10.156.0.2\"],\"conn_uids\":[\"C0vua63rzjtLaiefyj\"],\"source\":\"SSL\",\"depth\":0,\"analyzers\":[\"X509\",\"SHA256\",\"SHA1\",\"MD5\"],\"mime_type\":\"application/x-x509-user-cert\",\"duration\":0.0,\"local_orig\":false,\"is_orig\":false,\"seen_bytes\":893,\"missing_bytes\":0,\"overflow_bytes\":0,\"timedout\":false,\"md5\":\"5abbc8a9137f6924861596848b7d7794\",\"sha1\":\"99c1daf07c8d69a8a065492dcaae43c43ff13497\",\"sha256\":\"1de4074b4e38377f4367303f4a19c986a506180f22a6e53a68cc7679ea6d9c74\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -668,7 +668,7 @@ "ip": "10.178.98.102" }, "event": { - "ingested": "2021-06-14T08:33:54.368538900Z", + "ingested": "2021-06-17T13:41:25.552819500Z", "original": "{\"ts\":1547688801.566262,\"fuid\":\"F9ip9a3MDAq3XLBOn2\",\"tx_hosts\":[\"17.134.127.250\"],\"rx_hosts\":[\"10.178.98.102\"],\"conn_uids\":[\"C6sjVo23iNApLnlAt6\"],\"source\":\"SSL\",\"depth\":0,\"analyzers\":[\"X509\",\"MD5\",\"SHA1\"],\"mime_type\":\"application/pkix-cert\",\"duration\":0.0,\"local_orig\":false,\"is_orig\":false,\"seen_bytes\":1092,\"missing_bytes\":0,\"overflow_bytes\":0,\"timedout\":false,\"md5\":\"48f0e38385112eeca5fc9ffd402eaecd\",\"sha1\":\"8e8321ca08b08e3726fe1d82996884eeb5f0d655\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/files/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/files/elasticsearch/ingest_pipeline/default.yml index 9a249c0275ad..380a7911874c 100644 --- a/packages/zeek/data_stream/files/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/files/elasticsearch/ingest_pipeline/default.yml @@ -100,6 +100,7 @@ processors: field: zeek.files.ts formats: - UNIX + - ISO8601 - remove: field: zeek.files.ts - script: diff --git a/packages/zeek/data_stream/ftp/_dev/test/pipeline/test-ftp.log-expected.json b/packages/zeek/data_stream/ftp/_dev/test/pipeline/test-ftp.log-expected.json index 4577a176fa4c..19c9c1afbe77 100644 --- a/packages/zeek/data_stream/ftp/_dev/test/pipeline/test-ftp.log-expected.json +++ b/packages/zeek/data_stream/ftp/_dev/test/pipeline/test-ftp.log-expected.json @@ -51,7 +51,7 @@ ] }, "event": { - "ingested": "2021-06-14T08:33:54.564436500Z", + "ingested": "2021-06-17T13:41:25.930768900Z", "original": "{\"ts\":1187379104.955342,\"uid\":\"CpQoCn3o28tke89zv9\",\"id.orig_h\":\"192.168.1.182\",\"id.orig_p\":62014,\"id.resp_h\":\"192.168.1.231\",\"id.resp_p\":21,\"user\":\"ftp\",\"password\":\"ftp\",\"command\":\"EPSV\",\"reply_code\":229,\"reply_msg\":\"Entering Extended Passive Mode (|||37100|)\",\"data_channel.passive\":true,\"data_channel.orig_h\":\"192.168.1.182\",\"data_channel.resp_h\":\"192.168.1.231\",\"data_channel.resp_p\":37100}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -119,7 +119,7 @@ ] }, "event": { - "ingested": "2021-06-14T08:33:54.564448500Z", + "ingested": "2021-06-17T13:41:25.930780100Z", "original": "{\"ts\":1187379105.01948,\"uid\":\"CpQoCn3o28tke89zv9\",\"id.orig_h\":\"192.168.1.182\",\"id.orig_p\":62014,\"id.resp_h\":\"192.168.1.231\",\"id.resp_p\":21,\"user\":\"ftp\",\"password\":\"ftp\",\"command\":\"RETR\",\"arg\":\"ftp://192.168.1.231/resume.doc\",\"file_size\":39424,\"reply_code\":226,\"reply_msg\":\"Transfer complete.\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -184,7 +184,7 @@ ] }, "event": { - "ingested": "2021-06-14T08:33:54.564452900Z", + "ingested": "2021-06-17T13:41:25.930788100Z", "original": "{\"ts\":1187379117.579203,\"uid\":\"CpQoCn3o28tke89zv9\",\"id.orig_h\":\"192.168.1.182\",\"id.orig_p\":62014,\"id.resp_h\":\"192.168.1.231\",\"id.resp_p\":21,\"user\":\"ftp\",\"password\":\"ftp\",\"command\":\"STOR\",\"arg\":\"ftp://192.168.1.231/uploads/README\",\"reply_code\":226,\"reply_msg\":\"Transfer complete.\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -257,7 +257,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-06-14T08:33:54.564456300Z", + "ingested": "2021-06-17T13:41:25.930795900Z", "original": "{\"ts\":1187379117.579203,\"uid\":\"CpQoCn3o28tke89zv9\",\"id.orig_h\":\"192.168.1.182\",\"id.orig_p\":62014,\"id.resp_h\":\"192.168.1.231\",\"id.resp_p\":21,\"user\":\"ftp\",\"password\":\"ftp\",\"command\":\"STOR\",\"arg\":\"ftp://192.168.1.231/uploads/README\",\"reply_code\":226,\"reply_msg\":\"Transfer complete.\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/ftp/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/ftp/elasticsearch/ingest_pipeline/default.yml index 9742671da41e..61f68668c307 100644 --- a/packages/zeek/data_stream/ftp/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/ftp/elasticsearch/ingest_pipeline/default.yml @@ -192,6 +192,7 @@ processors: field: zeek.ftp.ts formats: - UNIX + - ISO8601 - remove: field: zeek.ftp.ts - dot_expander: diff --git a/packages/zeek/data_stream/http/_dev/test/pipeline/test-http.log-expected.json b/packages/zeek/data_stream/http/_dev/test/pipeline/test-http.log-expected.json index 67304b5de587..b74bcb014f72 100644 --- a/packages/zeek/data_stream/http/_dev/test/pipeline/test-http.log-expected.json +++ b/packages/zeek/data_stream/http/_dev/test/pipeline/test-http.log-expected.json @@ -85,7 +85,7 @@ } }, "event": { - "ingested": "2021-06-14T08:33:54.712499Z", + "ingested": "2021-06-17T13:41:26.235215100Z", "original": "{\"ts\":1547687130.172944,\"uid\":\"CCNp8v1SNzY7v9d1Ih\",\"id.orig_h\":\"10.178.98.102\",\"id.orig_p\":62995,\"id.resp_h\":\"17.253.5.203\",\"username\":\"user\",\"id.resp_p\":80,\"trans_depth\":1,\"method\":\"GET\",\"host\":\"ocsp.apple.com\",\"uri\":\"/ocsp04-aaica02/ME4wTKADAgEAMEUwQzBBMAkGBSsOAwIaBQAEFNqvF+Za6oA4ceFRLsAWwEInjUhJBBQx6napI3Sl39T97qDBpp7GEQ4R7AIIUP1IOZZ86ns=\",\"version\":\"1.1\",\"user_agent\":\"com.apple.trustd/2.0\",\"request_body_len\":0,\"response_body_len\":3735,\"status_code\":200,\"status_msg\":\"OK\",\"tags\":[],\"resp_fuids\":[\"F5zuip1tSwASjNAHy7\"],\"resp_mime_types\":[\"application/ocsp-response\"]}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -194,7 +194,7 @@ } }, "event": { - "ingested": "2021-06-14T08:33:54.712511300Z", + "ingested": "2021-06-17T13:41:26.235224100Z", "original": "{\"ts\":1547707019.757479,\"uid\":\"CMnIaR2V8VXyu7EPs\",\"id.orig_h\":\"10.20.8.197\",\"id.orig_p\":35684,\"id.resp_h\":\"34.206.130.40\",\"id.resp_p\":80,\"trans_depth\":1,\"method\":\"GET\",\"host\":\"httpbin.org\",\"uri\":\"/ip\",\"version\":\"1.1\",\"user_agent\":\"curl/7.58.0\",\"request_body_len\":0,\"response_body_len\":32,\"status_code\":200,\"status_msg\":\"OK\",\"tags\":[],\"resp_fuids\":[\"FwGPlr1GcKUWWdkXoi\"],\"resp_mime_types\":[\"text/json\"]}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -292,7 +292,7 @@ } }, "event": { - "ingested": "2021-06-14T08:33:54.712516300Z", + "ingested": "2021-06-17T13:41:26.235233100Z", "original": "{\"ts\":1617081354.277591,\"uid\":\"CdqHhA1AsxBIjmVZ9\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":57896,\"id.resp_h\":\"23.55.163.58\",\"id.resp_p\":80,\"trans_depth\":1,\"version\":\"1.0\",\"request_body_len\":0,\"response_body_len\":503,\"status_code\":200,\"status_msg\":\"OK\",\"tags\":[],\"resp_fuids\":[\"FM01o54RU9pez8AJba\"],\"resp_mime_types\":[\"application/ocsp-response\"]}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -384,7 +384,7 @@ } }, "event": { - "ingested": "2021-06-14T08:33:54.712520100Z", + "ingested": "2021-06-17T13:41:26.235241400Z", "original": "{\"ts\":1617081355.599548,\"uid\":\"CxhRTwkHNRsHxBw34\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":55378,\"id.resp_h\":\"52.53.69.85\",\"id.resp_p\":80,\"trans_depth\":1,\"version\":\"1.1\",\"request_body_len\":0,\"response_body_len\":191,\"status_code\":301,\"status_msg\":\"Moved Permanently\",\"tags\":[],\"resp_fuids\":[\"FVGTq31RBgKGE02hx7\"],\"resp_mime_types\":[\"text/html\"]}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -473,7 +473,7 @@ } }, "event": { - "ingested": "2021-06-14T08:33:54.712523300Z", + "ingested": "2021-06-17T13:41:26.235247400Z", "original": "{\"ts\":1617081360.171904,\"uid\":\"CrI5Xg30caNXnNvEse\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":41960,\"id.resp_h\":\"23.55.163.48\",\"id.resp_p\":80,\"trans_depth\":1,\"version\":\"1.0\",\"request_body_len\":0,\"response_body_len\":503,\"status_code\":200,\"status_msg\":\"OK\",\"tags\":[],\"resp_fuids\":[\"F8vozz46VoxeAmqLv3\"],\"resp_mime_types\":[\"application/ocsp-response\"]}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -562,7 +562,7 @@ } }, "event": { - "ingested": "2021-06-14T08:33:54.712526400Z", + "ingested": "2021-06-17T13:41:26.235253Z", "original": "{\"ts\":1617081364.250251,\"uid\":\"C6oCGd24yB2dZ7y7z7\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":42164,\"id.resp_h\":\"23.55.163.48\",\"id.resp_p\":80,\"trans_depth\":1,\"version\":\"1.0\",\"request_body_len\":0,\"response_body_len\":503,\"status_code\":200,\"status_msg\":\"OK\",\"tags\":[],\"resp_fuids\":[\"F1imAq4yUjbwyK7NO2\"],\"resp_mime_types\":[\"application/ocsp-response\"]}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -651,7 +651,7 @@ } }, "event": { - "ingested": "2021-06-14T08:33:54.712529700Z", + "ingested": "2021-06-17T13:41:26.235261300Z", "original": "{\"ts\":1617081366.285075,\"uid\":\"C7DWRE1zsvxUK9RyW1\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":42292,\"id.resp_h\":\"23.55.163.48\",\"id.resp_p\":80,\"trans_depth\":1,\"version\":\"1.0\",\"request_body_len\":0,\"response_body_len\":503,\"status_code\":200,\"status_msg\":\"OK\",\"tags\":[],\"resp_fuids\":[\"FQhm6z1cISaOxMzzR6\"],\"resp_mime_types\":[\"application/ocsp-response\"]}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -757,7 +757,7 @@ } }, "event": { - "ingested": "2021-06-14T08:33:54.712533300Z", + "ingested": "2021-06-17T13:41:26.235269200Z", "original": "{\"ts\":1547707019.757479,\"uid\":\"CMnIaR2V8VXyu7EPs\",\"id.orig_h\":\"10.20.8.197\",\"id.orig_p\":35684,\"id.resp_h\":\"34.206.130.40\",\"id.resp_p\":80,\"trans_depth\":1,\"method\":\"GET\",\"host\":\"httpbin.org\",\"uri\":\"/ip\",\"version\":\"1.1\",\"user_agent\":\"curl/7.58.0\",\"request_body_len\":0,\"response_body_len\":32,\"status_code\":200,\"status_msg\":\"OK\",\"tags\":[],\"resp_fuids\":[\"FwGPlr1GcKUWWdkXoi\"],\"resp_mime_types\":[\"text/json\"]}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/http/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/http/elasticsearch/ingest_pipeline/default.yml index 523d54391841..3a24d854818e 100644 --- a/packages/zeek/data_stream/http/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/http/elasticsearch/ingest_pipeline/default.yml @@ -192,6 +192,7 @@ processors: field: zeek.http.ts formats: - UNIX + - ISO8601 - remove: field: zeek.http.ts - geoip: diff --git a/packages/zeek/data_stream/intel/_dev/test/pipeline/test-intel.log-expected.json b/packages/zeek/data_stream/intel/_dev/test/pipeline/test-intel.log-expected.json index e73863702004..9f436895af54 100644 --- a/packages/zeek/data_stream/intel/_dev/test/pipeline/test-intel.log-expected.json +++ b/packages/zeek/data_stream/intel/_dev/test/pipeline/test-intel.log-expected.json @@ -54,7 +54,7 @@ "ip": "192.168.1.1" }, "event": { - "ingested": "2021-06-14T08:33:54.987600100Z", + "ingested": "2021-06-17T13:41:26.699646500Z", "original": "{\"ts\":1573030980.989353,\"uid\":\"Ctefoj1tgOPt4D0EK2\",\"id.orig_h\":\"192.168.1.1\",\"id.orig_p\":37598,\"id.resp_h\":\"198.41.0.4\",\"id.resp_p\":53,\"seen.indicator\":\"198.41.0.4\",\"seen.indicator_type\":\"Intel::ADDR\",\"seen.where\":\"Conn::IN_RESP\",\"seen.node\":\"worker-1-2\",\"matched\":[\"Intel::ADDR\"],\"sources\":[\"ETPRO Rep: AbusedTLD Score: 127\"]}", "created": "2020-04-28T11:07:58.223Z", "kind": "enrichment", @@ -135,7 +135,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-06-14T08:33:54.987608900Z", + "ingested": "2021-06-17T13:41:26.699657200Z", "original": "{\"ts\":1573030980.989353,\"uid\":\"Ctefoj1tgOPt4D0EK2\",\"id.orig_h\":\"192.168.1.1\",\"id.orig_p\":37598,\"id.resp_h\":\"198.41.0.4\",\"id.resp_p\":53,\"seen.indicator\":\"198.41.0.4\",\"seen.indicator_type\":\"Intel::ADDR\",\"seen.where\":\"Conn::IN_RESP\",\"seen.node\":\"worker-1-2\",\"matched\":[\"Intel::ADDR\"],\"sources\":[\"ETPRO Rep: AbusedTLD Score: 127\"]}", "created": "2020-04-28T11:07:58.223Z", "kind": "enrichment", diff --git a/packages/zeek/data_stream/intel/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/intel/elasticsearch/ingest_pipeline/default.yml index bed977bf2420..e9d42c795e4f 100644 --- a/packages/zeek/data_stream/intel/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/intel/elasticsearch/ingest_pipeline/default.yml @@ -156,6 +156,7 @@ processors: field: zeek.intel.ts formats: - UNIX + - ISO8601 - remove: field: zeek.intel.ts # IP Geolocation Lookup diff --git a/packages/zeek/data_stream/irc/_dev/test/pipeline/test-irc.log-expected.json b/packages/zeek/data_stream/irc/_dev/test/pipeline/test-irc.log-expected.json index d429a0db16f0..cb4959fe9525 100644 --- a/packages/zeek/data_stream/irc/_dev/test/pipeline/test-irc.log-expected.json +++ b/packages/zeek/data_stream/irc/_dev/test/pipeline/test-irc.log-expected.json @@ -45,7 +45,7 @@ "ip": "10.180.156.249" }, "event": { - "ingested": "2021-06-14T08:33:55.077411700Z", + "ingested": "2021-06-17T13:41:26.894354200Z", "original": "{\"ts\":1387554250.647295,\"uid\":\"CNJBX5FQdL62VUUP1\",\"id.orig_h\":\"10.180.156.249\",\"id.orig_p\":45921,\"id.resp_h\":\"38.229.70.20\",\"id.resp_p\":8000,\"command\":\"USER\",\"value\":\"xxxxx\",\"addl\":\"+iw xxxxx XxxxxxXxxx \"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -125,7 +125,7 @@ ] }, "event": { - "ingested": "2021-06-14T08:33:55.077421100Z", + "ingested": "2021-06-17T13:41:26.894363Z", "original": "{\"ts\":1387554250.647295,\"uid\":\"CNJBX5FQdL62VUUP1\",\"id.orig_h\":\"10.180.156.249\",\"id.orig_p\":45921,\"id.resp_h\":\"38.229.70.20\",\"id.resp_p\":8000,\"user\":\"xxxxx\",\"command\":\"NICK\",\"value\":\"molochtest\",\"addl\":\"+iw xxxxx XxxxxxXxxx \"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -201,7 +201,7 @@ ] }, "event": { - "ingested": "2021-06-14T08:33:55.077424700Z", + "ingested": "2021-06-17T13:41:26.894373200Z", "original": "{\"ts\":1387554250.706387,\"uid\":\"CNJBX5FQdL62VUUP1\",\"id.orig_h\":\"10.180.156.249\",\"id.orig_p\":45921,\"id.resp_h\":\"38.229.70.20\",\"id.resp_p\":8000,\"nick\":\"molochtest\",\"user\":\"xxxxx\",\"command\":\"JOIN\",\"value\":\"#moloch-fpc\",\"addl\":\" with channel key: \\u0027-\\u0027\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -285,7 +285,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-06-14T08:33:55.077428Z", + "ingested": "2021-06-17T13:41:26.894424200Z", "original": "{\"ts\":1387554250.706387,\"uid\":\"CNJBX5FQdL62VUUP1\",\"id.orig_h\":\"10.180.156.249\",\"id.orig_p\":45921,\"id.resp_h\":\"38.229.70.20\",\"id.resp_p\":8000,\"nick\":\"molochtest\",\"user\":\"xxxxx\",\"command\":\"JOIN\",\"value\":\"#moloch-fpc\",\"addl\":\" with channel key: \\u0027-\\u0027\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/irc/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/irc/elasticsearch/ingest_pipeline/default.yml index d8d49dfe78e4..a6b2729471fa 100644 --- a/packages/zeek/data_stream/irc/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/irc/elasticsearch/ingest_pipeline/default.yml @@ -152,6 +152,7 @@ processors: field: zeek.irc.ts formats: - UNIX + - ISO8601 - remove: field: zeek.irc.ts - append: diff --git a/packages/zeek/data_stream/kerberos/_dev/test/pipeline/test-kerberos.log-expected.json b/packages/zeek/data_stream/kerberos/_dev/test/pipeline/test-kerberos.log-expected.json index 61e9e7c71a61..1f6953386d61 100644 --- a/packages/zeek/data_stream/kerberos/_dev/test/pipeline/test-kerberos.log-expected.json +++ b/packages/zeek/data_stream/kerberos/_dev/test/pipeline/test-kerberos.log-expected.json @@ -86,7 +86,7 @@ } }, "event": { - "ingested": "2021-06-14T08:33:55.216293800Z", + "ingested": "2021-06-17T13:41:27.217964800Z", "original": "{\"ts\":1507565599.590346,\"uid\":\"C56Flhb4WQBNkfMOl\",\"id.orig_h\":\"192.168.10.31\",\"id.orig_p\":49242,\"id.resp_h\":\"192.168.10.10\",\"id.resp_p\":88,\"request_type\":\"TGS\",\"client\":\"RonHD/CONTOSO.LOCAL\",\"service\":\"HOST/admin-pc\",\"success\":true,\"till\":2136422885.0,\"cipher\":\"aes256-cts-hmac-sha1-96\",\"forwardable\":true,\"renewable\":true,\"cert.client_subject\":\"CN=*.gcp.cloud.es.io,O=Elasticsearch\\u005c, Inc.,L=Mountain View,ST=California,C=US\",\"cert.server_subject\":\"CN=*.gcp.cloud.es.io,O=Elasticsearch\\u005c, Inc.,L=Mountain View,ST=California,C=US\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -201,7 +201,7 @@ } }, "event": { - "ingested": "2021-06-14T08:33:55.216304200Z", + "ingested": "2021-06-17T13:41:27.217975700Z", "original": "{\"ts\":1507565599.590346,\"uid\":\"C56Flhb4WQBNkfMOl\",\"id.orig_h\":\"192.168.10.31\",\"id.orig_p\":49242,\"id.resp_h\":\"192.168.10.10\",\"id.resp_p\":88,\"request_type\":\"TGS\",\"client\":\"RonHD/CONTOSO.LOCAL\",\"service\":\"HOST/admin-pc\",\"success\":true,\"till\":2136422885.0,\"cipher\":\"aes256-cts-hmac-sha1-96\",\"forwardable\":true,\"renewable\":true,\"cert.client_subject\":\"CN=*.gcp.cloud.es.io,O=Elasticsearch\\u005c, Inc.,L=Mountain View,ST=California,C=US\",\"cert.server_subject\":\"CN=*.gcp.cloud.es.io,O=Elasticsearch\\u005c, Inc.,L=Mountain View,ST=California,C=US\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/kerberos/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/kerberos/elasticsearch/ingest_pipeline/default.yml index fc1a00390601..4ebaffb5c4cb 100644 --- a/packages/zeek/data_stream/kerberos/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/kerberos/elasticsearch/ingest_pipeline/default.yml @@ -221,6 +221,7 @@ processors: field: zeek.kerberos.ts formats: - UNIX + - ISO8601 - remove: field: zeek.kerberos.ts - script: @@ -231,12 +232,14 @@ processors: target_field: zeek.kerberos.valid.until formats: - UNIX + - ISO8601 if: ctx.zeek.kerberos.valid?.until != null - date: field: zeek.kerberos.valid.from target_field: zeek.kerberos.valid.from formats: - UNIX + - ISO8601 if: ctx.zeek.kerberos.valid?.from != null - set: field: event.outcome diff --git a/packages/zeek/data_stream/modbus/_dev/test/pipeline/test-modbus.log-expected.json b/packages/zeek/data_stream/modbus/_dev/test/pipeline/test-modbus.log-expected.json index bece6f1a90f4..da68448373f3 100644 --- a/packages/zeek/data_stream/modbus/_dev/test/pipeline/test-modbus.log-expected.json +++ b/packages/zeek/data_stream/modbus/_dev/test/pipeline/test-modbus.log-expected.json @@ -28,7 +28,7 @@ "ip": "192.168.1.10" }, "event": { - "ingested": "2021-06-14T08:33:55.363513700Z", + "ingested": "2021-06-17T13:41:27.502246900Z", "original": "{\"ts\":1352718265.222457,\"uid\":\"CpIIXl4DFGswmjH2bl\",\"id.orig_h\":\"192.168.1.10\",\"id.orig_p\":64342,\"id.resp_h\":\"192.168.1.164\",\"id.resp_p\":502,\"func\":\"READ_COILS\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -96,7 +96,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-06-14T08:33:55.363525800Z", + "ingested": "2021-06-17T13:41:27.502255500Z", "original": "{\"ts\":1352718265.222457,\"uid\":\"CpIIXl4DFGswmjH2bl\",\"id.orig_h\":\"192.168.1.10\",\"id.orig_p\":64342,\"id.resp_h\":\"192.168.1.164\",\"id.resp_p\":502,\"func\":\"READ_COILS\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/modbus/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/modbus/elasticsearch/ingest_pipeline/default.yml index 14b7b72d4d1a..1c0021d69313 100644 --- a/packages/zeek/data_stream/modbus/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/modbus/elasticsearch/ingest_pipeline/default.yml @@ -145,6 +145,7 @@ processors: field: zeek.modbus.ts formats: - UNIX + - ISO8601 - remove: field: zeek.modbus.ts - append: diff --git a/packages/zeek/data_stream/mysql/_dev/test/pipeline/test-mysql.log-expected.json b/packages/zeek/data_stream/mysql/_dev/test/pipeline/test-mysql.log-expected.json index 1053500cfea9..1d96f40deb28 100644 --- a/packages/zeek/data_stream/mysql/_dev/test/pipeline/test-mysql.log-expected.json +++ b/packages/zeek/data_stream/mysql/_dev/test/pipeline/test-mysql.log-expected.json @@ -30,7 +30,7 @@ "ip": "192.168.0.254" }, "event": { - "ingested": "2021-06-14T08:33:55.465091Z", + "ingested": "2021-06-17T13:41:27.697262600Z", "original": "{\"ts\":1216281087.437392,\"uid\":\"C5Hol527kLMUw36hj3\",\"id.orig_h\":\"192.168.0.254\",\"id.orig_p\":56162,\"id.resp_h\":\"192.168.0.254\",\"id.resp_p\":3306,\"cmd\":\"query\",\"arg\":\"select count(*) from foo\",\"success\":true,\"rows\":1}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -102,7 +102,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-06-14T08:33:55.465117500Z", + "ingested": "2021-06-17T13:41:27.697272700Z", "original": "{\"ts\":1216281087.437392,\"uid\":\"C5Hol527kLMUw36hj3\",\"id.orig_h\":\"192.168.0.254\",\"id.orig_p\":56162,\"id.resp_h\":\"192.168.0.254\",\"id.resp_p\":3306,\"cmd\":\"query\",\"arg\":\"select count(*) from foo\",\"success\":true,\"rows\":1}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/mysql/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/mysql/elasticsearch/ingest_pipeline/default.yml index 038686a717f5..4c99817d851f 100644 --- a/packages/zeek/data_stream/mysql/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/mysql/elasticsearch/ingest_pipeline/default.yml @@ -144,6 +144,7 @@ processors: field: zeek.mysql.ts formats: - UNIX + - ISO8601 - remove: field: zeek.mysql.ts - append: diff --git a/packages/zeek/data_stream/notice/_dev/test/pipeline/test-notice.log-expected.json b/packages/zeek/data_stream/notice/_dev/test/pipeline/test-notice.log-expected.json index 1f4c7176551e..407aacbceb39 100644 --- a/packages/zeek/data_stream/notice/_dev/test/pipeline/test-notice.log-expected.json +++ b/packages/zeek/data_stream/notice/_dev/test/pipeline/test-notice.log-expected.json @@ -32,7 +32,7 @@ "ip": "172.16.238.1" }, "event": { - "ingested": "2021-06-14T08:33:55.565602900Z", + "ingested": "2021-06-17T13:41:27.914524200Z", "original": "{\"ts\":1320435875.879278,\"note\":\"SSH::Password_Guessing\",\"msg\":\"172.16.238.1 appears to be guessing SSH passwords (seen in 30 connections).\",\"sub\":\"Sampled servers: 172.16.238.136, 172.16.238.136, 172.16.238.136, 172.16.238.136, 172.16.238.136\",\"src\":\"172.16.238.1\",\"peer_descr\":\"bro\",\"actions\":[\"Notice::ACTION_LOG\"],\"suppress_for\":3600.0,\"dropped\":false}", "category": [ "intrusion_detection" @@ -121,7 +121,7 @@ "ip": "8.42.77.171" }, "event": { - "ingested": "2021-06-14T08:33:55.565623200Z", + "ingested": "2021-06-17T13:41:27.914532700Z", "original": "{\"ts\":1551393388.426472,\"note\":\"Scan::Port_Scan\",\"msg\":\"8.42.77.171 scanned at least 15 unique ports of host 207.154.238.205 in 0m0s\",\"sub\":\"remote\",\"src\":\"8.42.77.171\",\"dst\":\"207.154.238.205\",\"peer_descr\":\"bro\",\"actions\":[\"Notice::ACTION_LOG\"],\"suppress_for\":3600.0,\"dropped\":false}", "category": [ "intrusion_detection" @@ -157,7 +157,7 @@ "description": "The capture loss script detected an estimated loss rate above 88.306%" }, "event": { - "ingested": "2021-06-14T08:33:55.565638Z", + "ingested": "2021-06-17T13:41:27.914536200Z", "original": "{\"ts\":1617097740.958466,\"note\":\"CaptureLoss::Too_Much_Loss\",\"msg\":\"The capture loss script detected an estimated loss rate above 88.306%\",\"actions\":[\"Notice::ACTION_LOG\"],\"suppress_for\":3600.0}", "category": [ "intrusion_detection" @@ -236,7 +236,7 @@ ] }, "event": { - "ingested": "2021-06-14T08:33:55.565640200Z", + "ingested": "2021-06-17T13:41:27.914542100Z", "original": "{\"ts\":1617097929.601155,\"uid\":\"CmvrSS1wIiuOGYCbfi\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":48818,\"id.resp_h\":\"104.154.89.105\",\"id.resp_p\":443,\"fuid\":\"F39b0Bdfm3FW1rNS5\",\"proto\":\"tcp\",\"note\":\"SSL::Invalid_Server_Cert\",\"msg\":\"SSL certificate validation failed with (self signed certificate)\",\"sub\":\"CN=*.badssl.com,O=BadSSL,L=San Francisco,ST=California,C=US\",\"src\":\"10.156.0.2\",\"dst\":\"104.154.89.105\",\"p\":443,\"actions\":[\"Notice::ACTION_LOG\"],\"suppress_for\":3600.0}", "created": "2020-04-28T11:07:58.223Z", "kind": "alert", @@ -333,7 +333,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-06-14T08:33:55.565642300Z", + "ingested": "2021-06-17T13:41:27.914547Z", "original": "{\"ts\":1551393388.426472,\"note\":\"Scan::Port_Scan\",\"msg\":\"8.42.77.171 scanned at least 15 unique ports of host 207.154.238.205 in 0m0s\",\"sub\":\"remote\",\"src\":\"8.42.77.171\",\"dst\":\"207.154.238.205\",\"peer_descr\":\"bro\",\"actions\":[\"Notice::ACTION_LOG\"],\"suppress_for\":3600.0,\"dropped\":false}", "category": [ "intrusion_detection" diff --git a/packages/zeek/data_stream/notice/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/notice/elasticsearch/ingest_pipeline/default.yml index c136ae89b54e..bce86176c2f9 100644 --- a/packages/zeek/data_stream/notice/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/notice/elasticsearch/ingest_pipeline/default.yml @@ -227,6 +227,7 @@ processors: field: zeek.notice.ts formats: - UNIX + - ISO8601 - remove: field: zeek.notice.ts - geoip: diff --git a/packages/zeek/data_stream/ntlm/_dev/test/pipeline/test-ntlm.log-expected.json b/packages/zeek/data_stream/ntlm/_dev/test/pipeline/test-ntlm.log-expected.json index f6dbf6e5d89a..b07061a5c13a 100644 --- a/packages/zeek/data_stream/ntlm/_dev/test/pipeline/test-ntlm.log-expected.json +++ b/packages/zeek/data_stream/ntlm/_dev/test/pipeline/test-ntlm.log-expected.json @@ -48,7 +48,7 @@ ] }, "event": { - "ingested": "2021-06-14T08:33:55.719399500Z", + "ingested": "2021-06-17T13:41:28.203050500Z", "original": "{\"ts\":1508959117.814467,\"uid\":\"CHphiNUKDC20fsy09\",\"id.orig_h\":\"192.168.10.50\",\"id.orig_p\":46785,\"id.resp_h\":\"192.168.10.31\",\"id.resp_p\":445,\"username\":\"JeffV\",\"hostname\":\"ybaARon55QykXrgu\",\"domainname\":\"contoso.local\",\"server_nb_computer_name\":\"VICTIM-PC\",\"server_dns_computer_name\":\"Victim-PC.contoso.local\",\"server_tree_name\":\"contoso.local\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -123,7 +123,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-06-14T08:33:55.719407Z", + "ingested": "2021-06-17T13:41:28.203058Z", "original": "{\"ts\":1508959117.814467,\"uid\":\"CHphiNUKDC20fsy09\",\"id.orig_h\":\"192.168.10.50\",\"id.orig_p\":46785,\"id.resp_h\":\"192.168.10.31\",\"id.resp_p\":445,\"username\":\"JeffV\",\"hostname\":\"ybaARon55QykXrgu\",\"domainname\":\"contoso.local\",\"server_nb_computer_name\":\"VICTIM-PC\",\"server_dns_computer_name\":\"Victim-PC.contoso.local\",\"server_tree_name\":\"contoso.local\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/ntlm/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/ntlm/elasticsearch/ingest_pipeline/default.yml index 4a57c2769522..e0ec530cf80f 100644 --- a/packages/zeek/data_stream/ntlm/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/ntlm/elasticsearch/ingest_pipeline/default.yml @@ -164,6 +164,7 @@ processors: field: zeek.ntlm.ts formats: - UNIX + - ISO8601 - remove: field: zeek.ntlm.ts - append: diff --git a/packages/zeek/data_stream/ocsp/_dev/test/pipeline/test-ocsp.log-expected.json b/packages/zeek/data_stream/ocsp/_dev/test/pipeline/test-ocsp.log-expected.json index 94e91f58a63e..70e98ce2be8f 100644 --- a/packages/zeek/data_stream/ocsp/_dev/test/pipeline/test-ocsp.log-expected.json +++ b/packages/zeek/data_stream/ocsp/_dev/test/pipeline/test-ocsp.log-expected.json @@ -27,7 +27,7 @@ } }, "event": { - "ingested": "2021-06-14T08:33:55.811014800Z", + "ingested": "2021-06-17T13:41:28.417281800Z", "original": "{\"ts\":1307712421.847886,\"id\":\"FSEWoS3ff8FcTn3WLf\",\"hashAlgorithm\":\"sha1\",\"issuerNameHash\":\"14A7E219F46B93E141258F08BC85764671F136B0\",\"issuerKeyHash\":\"EEDD79C0D379B04D7E47BC70A6E7C62AAEBADEC9\",\"serialNumber\":\"9239D5348F40D1695A745470E1F23F43\",\"certStatus\":\"revoked\",\"revoketime\":1300220120.0,\"thisUpdate\":1307640343.0,\"nextUpdate\":1307985943.0}", "created": "2020-04-28T11:07:58.223Z", "kind": "event" @@ -63,7 +63,7 @@ } }, "event": { - "ingested": "2021-06-14T08:33:55.811020500Z", + "ingested": "2021-06-17T13:41:28.417291200Z", "original": "{\"ts\":1307562416.100084,\"id\":\"FdZBFMEYgAErVhoC8\",\"hashAlgorithm\":\"sha1\",\"issuerNameHash\":\"6C2BC55AAF8D96BF60ADF81D023F23B48A0059C2\",\"issuerKeyHash\":\"A5EF0B11CEC04103A34A659048B21CE0572D7D47\",\"serialNumber\":\"30119E6EF41BDBA3FEFE711DBE8F6191\",\"certStatus\":\"good\",\"thisUpdate\":1307549998.0,\"nextUpdate\":1308154798.0}", "created": "2020-04-28T11:07:58.223Z", "kind": "event" @@ -107,7 +107,7 @@ } }, "event": { - "ingested": "2021-06-14T08:33:55.811022800Z", + "ingested": "2021-06-17T13:41:28.417298200Z", "original": "{\"ts\":1307562416.100084,\"id\":\"FdZBFMEYgAErVhoC8\",\"hashAlgorithm\":\"sha1\",\"issuerNameHash\":\"6C2BC55AAF8D96BF60ADF81D023F23B48A0059C2\",\"issuerKeyHash\":\"A5EF0B11CEC04103A34A659048B21CE0572D7D47\",\"serialNumber\":\"30119E6EF41BDBA3FEFE711DBE8F6191\",\"certStatus\":\"good\",\"thisUpdate\":1307549998.0,\"nextUpdate\":1308154798.0}", "created": "2020-04-28T11:07:58.223Z", "kind": "event" diff --git a/packages/zeek/data_stream/ocsp/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/ocsp/elasticsearch/ingest_pipeline/default.yml index 98c809efd017..581b281cfb4c 100644 --- a/packages/zeek/data_stream/ocsp/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/ocsp/elasticsearch/ingest_pipeline/default.yml @@ -109,6 +109,7 @@ processors: field: zeek.ocsp.ts formats: - UNIX + - ISO8601 - remove: field: zeek.ocsp.ts - date: @@ -116,18 +117,21 @@ processors: target_field: zeek.ocsp.revoke.date formats: - UNIX + - ISO8601 if: ctx.zeek.ocsp.revoke?.date != null - date: field: zeek.ocsp.update.this target_field: zeek.ocsp.update.this formats: - UNIX + - ISO8601 if: ctx.zeek.ocsp.update?.this != null - date: field: zeek.ocsp.update.next target_field: zeek.ocsp.update.next formats: - UNIX + - ISO8601 if: ctx.zeek.ocsp.update?.next != null - append: field: related.hash diff --git a/packages/zeek/data_stream/pe/_dev/test/pipeline/test-pe.log-expected.json b/packages/zeek/data_stream/pe/_dev/test/pipeline/test-pe.log-expected.json index edb96018f99e..c0c08c2aae19 100644 --- a/packages/zeek/data_stream/pe/_dev/test/pipeline/test-pe.log-expected.json +++ b/packages/zeek/data_stream/pe/_dev/test/pipeline/test-pe.log-expected.json @@ -32,7 +32,7 @@ } }, "event": { - "ingested": "2021-06-14T08:33:55.887631800Z", + "ingested": "2021-06-17T13:41:28.617585900Z", "original": "{\"ts\":1507565599.578328,\"id\":\"FtIFnm3ZqI1s96P74l\",\"machine\":\"I386\",\"compile_ts\":1467139314.0,\"os\":\"Windows XP\",\"subsystem\":\"WINDOWS_CUI\",\"is_exe\":true,\"is_64bit\":false,\"uses_aslr\":true,\"uses_dep\":true,\"uses_code_integrity\":false,\"uses_seh\":true,\"has_import_table\":true,\"has_export_table\":false,\"has_cert_table\":true,\"has_debug_data\":false,\"section_names\":[\".text\",\".rdata\",\".data\",\".rsrc\",\".reloc\"]}", "category": [ "file" @@ -87,7 +87,7 @@ } }, "event": { - "ingested": "2021-06-14T08:33:55.887637500Z", + "ingested": "2021-06-17T13:41:28.618099300Z", "original": "{\"ts\":1507565599.578328,\"id\":\"FtIFnm3ZqI1s96P74l\",\"machine\":\"I386\",\"compile_ts\":1467139314.0,\"os\":\"Windows XP\",\"subsystem\":\"WINDOWS_CUI\",\"is_exe\":true,\"is_64bit\":false,\"uses_aslr\":true,\"uses_dep\":true,\"uses_code_integrity\":false,\"uses_seh\":true,\"has_import_table\":true,\"has_export_table\":false,\"has_cert_table\":true,\"has_debug_data\":false,\"section_names\":[\".text\",\".rdata\",\".data\",\".rsrc\",\".reloc\"]}", "category": [ "file" diff --git a/packages/zeek/data_stream/pe/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/pe/elasticsearch/ingest_pipeline/default.yml index c8cf24f22d94..b8a3eff22e24 100644 --- a/packages/zeek/data_stream/pe/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/pe/elasticsearch/ingest_pipeline/default.yml @@ -76,6 +76,7 @@ processors: field: zeek.pe.ts formats: - UNIX + - ISO8601 - remove: field: zeek.pe.ts - date: @@ -83,6 +84,7 @@ processors: target_field: zeek.pe.compile_time formats: - UNIX + - ISO8601 if: ctx.zeek.pe.compile_time != null - remove: field: diff --git a/packages/zeek/data_stream/radius/_dev/test/pipeline/test-radius.log-expected.json b/packages/zeek/data_stream/radius/_dev/test/pipeline/test-radius.log-expected.json index 9b79df85fc44..0114bbd376f6 100644 --- a/packages/zeek/data_stream/radius/_dev/test/pipeline/test-radius.log-expected.json +++ b/packages/zeek/data_stream/radius/_dev/test/pipeline/test-radius.log-expected.json @@ -41,7 +41,7 @@ ] }, "event": { - "ingested": "2021-06-14T08:33:55.962180900Z", + "ingested": "2021-06-17T13:41:28.802319600Z", "original": "{\"ts\":1217631137.916736,\"uid\":\"CRe9VD3flCDWbPmpIh\",\"id.orig_h\":\"10.0.0.1\",\"id.orig_p\":1645,\"id.resp_h\":\"10.0.0.100\",\"id.resp_p\":1812,\"username\":\"John.McGuirk\",\"mac\":\"00:14:22:e9:54:5e\",\"result\":\"success\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -109,7 +109,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-06-14T08:33:55.962189600Z", + "ingested": "2021-06-17T13:41:28.802330500Z", "original": "{\"ts\":1217631137.916736,\"uid\":\"CRe9VD3flCDWbPmpIh\",\"id.orig_h\":\"10.0.0.1\",\"id.orig_p\":1645,\"id.resp_h\":\"10.0.0.100\",\"id.resp_p\":1812,\"username\":\"John.McGuirk\",\"mac\":\"00:14:22:e9:54:5e\",\"result\":\"success\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/radius/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/radius/elasticsearch/ingest_pipeline/default.yml index 1a29092cd33a..1ed2f660445f 100644 --- a/packages/zeek/data_stream/radius/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/radius/elasticsearch/ingest_pipeline/default.yml @@ -140,6 +140,7 @@ processors: field: zeek.radius.ts formats: - UNIX + - ISO8601 - remove: field: zeek.radius.ts - append: diff --git a/packages/zeek/data_stream/rdp/_dev/test/pipeline/test-rdp.log-expected.json b/packages/zeek/data_stream/rdp/_dev/test/pipeline/test-rdp.log-expected.json index d4207624d25c..00461a8a4176 100644 --- a/packages/zeek/data_stream/rdp/_dev/test/pipeline/test-rdp.log-expected.json +++ b/packages/zeek/data_stream/rdp/_dev/test/pipeline/test-rdp.log-expected.json @@ -44,7 +44,7 @@ "established": true }, "event": { - "ingested": "2021-06-14T08:33:56.056252700Z", + "ingested": "2021-06-17T13:41:28.965248400Z", "original": "{\"ts\":1568132339.668952,\"uid\":\"C2PcYV7D3ntaHm056\",\"id.orig_h\":\"192.168.131.1\",\"id.orig_p\":33872,\"id.resp_h\":\"192.168.131.131\",\"id.resp_p\":3389,\"result\":\"encrypted\",\"security_protocol\":\"HYBRID\",\"cert_count\":0,\"ssl\":true}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -110,7 +110,7 @@ "established": true }, "event": { - "ingested": "2021-06-14T08:33:56.056258300Z", + "ingested": "2021-06-17T13:41:28.965256400Z", "original": "{\"ts\":1568132339.668952,\"uid\":\"C2PcYV7D3ntaHm056\",\"id.orig_h\":\"192.168.131.1\",\"id.orig_p\":33872,\"id.resp_h\":\"192.168.131.131\",\"id.resp_p\":3389,\"result\":\"encrypted\",\"security_protocol\":\"HYBRID\",\"cert_count\":0,\"ssl\":true}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/rdp/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/rdp/elasticsearch/ingest_pipeline/default.yml index b108ab51263c..dc3a00a3b50f 100644 --- a/packages/zeek/data_stream/rdp/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/rdp/elasticsearch/ingest_pipeline/default.yml @@ -173,6 +173,7 @@ processors: field: zeek.rdp.ts formats: - UNIX + - ISO8601 - remove: field: zeek.rdp.ts - convert: diff --git a/packages/zeek/data_stream/rfb/_dev/test/pipeline/test-rfb.log-expected.json b/packages/zeek/data_stream/rfb/_dev/test/pipeline/test-rfb.log-expected.json index c6e374db8f27..892bc5d57588 100644 --- a/packages/zeek/data_stream/rfb/_dev/test/pipeline/test-rfb.log-expected.json +++ b/packages/zeek/data_stream/rfb/_dev/test/pipeline/test-rfb.log-expected.json @@ -45,7 +45,7 @@ "ip": "192.168.1.123" }, "event": { - "ingested": "2021-06-14T08:33:56.146451600Z", + "ingested": "2021-06-17T13:41:29.132640500Z", "original": "{\"ts\":1328632534.517208,\"uid\":\"CXoIzM3wH3fUwXtKN1\",\"id.orig_h\":\"192.168.1.123\",\"id.orig_p\":58102,\"id.resp_h\":\"192.168.1.10\",\"id.resp_p\":5900,\"client_major_version\":\"003\",\"client_minor_version\":\"008\",\"server_major_version\":\"003\",\"server_minor_version\":\"008\",\"authentication_method\":\"VNC\",\"auth\":true,\"share_flag\":false,\"desktop_name\":\"\\u00a0\",\"width\":800,\"height\":600}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -128,7 +128,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-06-14T08:33:56.146457600Z", + "ingested": "2021-06-17T13:41:29.132648Z", "original": "{\"ts\":1328632534.517208,\"uid\":\"CXoIzM3wH3fUwXtKN1\",\"id.orig_h\":\"192.168.1.123\",\"id.orig_p\":58102,\"id.resp_h\":\"192.168.1.10\",\"id.resp_p\":5900,\"client_major_version\":\"003\",\"client_minor_version\":\"008\",\"server_major_version\":\"003\",\"server_minor_version\":\"008\",\"authentication_method\":\"VNC\",\"auth\":true,\"share_flag\":false,\"desktop_name\":\"\\u00a0\",\"width\":800,\"height\":600}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/rfb/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/rfb/elasticsearch/ingest_pipeline/default.yml index fcd8660b5826..e4d97c9ee79c 100644 --- a/packages/zeek/data_stream/rfb/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/rfb/elasticsearch/ingest_pipeline/default.yml @@ -153,6 +153,7 @@ processors: field: zeek.rfb.ts formats: - UNIX + - ISO8601 - remove: field: zeek.rfb.ts - append: diff --git a/packages/zeek/data_stream/sip/_dev/test/pipeline/test-sip.log-expected.json b/packages/zeek/data_stream/sip/_dev/test/pipeline/test-sip.log-expected.json index cf16bef01613..b5b629ba3dd8 100644 --- a/packages/zeek/data_stream/sip/_dev/test/pipeline/test-sip.log-expected.json +++ b/packages/zeek/data_stream/sip/_dev/test/pipeline/test-sip.log-expected.json @@ -81,7 +81,7 @@ ] }, "event": { - "ingested": "2021-06-14T08:33:56.254231500Z", + "ingested": "2021-06-17T13:41:29.354931500Z", "original": "{\"ts\":1361916159.055464,\"uid\":\"CPRLCB4eWHdjP852Bk\",\"id.orig_h\":\"172.16.133.19\",\"id.orig_p\":5060,\"id.resp_h\":\"74.63.41.218\",\"id.resp_p\":5060,\"trans_depth\":0,\"method\":\"REGISTER\",\"uri\":\"sip:newyork.voip.ms:5060\",\"request_from\":\"\\u0022AppNeta\\u0022 \u003csip:116954_Boston6@newyork.voip.ms\u003e\",\"request_to\":\"\u003csip:116954_Boston6@newyork.voip.ms\u003e\",\"response_from\":\"\\u0022AppNeta\\u0022 \u003csip:116954_Boston6@newyork.voip.ms\u003e\",\"response_to\":\"\u003csip:116954_Boston6@newyork.voip.ms\u003e;tag=as023f66a5\",\"call_id\":\"8694cd7e-976e4fc3-d76f6e38@172.16.133.19\",\"seq\":\"4127 REGISTER\",\"request_path\":[\"SIP/2.0/UDP 172.16.133.19:5060\"],\"response_path\":[\"SIP/2.0/UDP 172.16.133.19:5060\"],\"user_agent\":\"PolycomSoundStationIP-SSIP_5000-UA/3.2.4.0267\",\"status_code\":401,\"status_msg\":\"Unauthorized\",\"request_body_len\":0,\"response_body_len\":0}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -203,7 +203,7 @@ ] }, "event": { - "ingested": "2021-06-14T08:33:56.254237100Z", + "ingested": "2021-06-17T13:41:29.354938900Z", "original": "{\"ts\":1105725482.965944,\"uid\":\"ComJz236lSOcuOmix3\",\"id.orig_h\":\"200.57.7.204\",\"id.orig_p\":5061,\"id.resp_h\":\"200.57.7.195\",\"id.resp_p\":5060,\"trans_depth\":0,\"method\":\"INVITE\",\"uri\":\"sip:francisco@bestel.com:55060\",\"request_from\":\"\u003csip:200.57.7.195:55061;user=phone\u003e\",\"request_to\":\"\\u0022francisco@bestel.com\\u0022 \u003csip:francisco@bestel.com:55060\u003e\",\"response_from\":\"\u003csip:200.57.7.195:55061;user=phone\u003e\",\"response_to\":\"\\u0022francisco@bestel.com\\u0022 \u003csip:francisco@bestel.com:55060\u003e;tag=298852044\",\"call_id\":\"12013223@200.57.7.195\",\"seq\":\"1 INVITE\",\"request_path\":[\"SIP/2.0/UDP 200.57.7.195\",\"SIP/2.0/UDP 200.57.7.195:55061\"],\"response_path\":[\"SIP/2.0/UDP 200.57.7.195\",\"SIP/2.0/UDP 200.57.7.195:55061\",\"SIP/2.0/UDP 200.57.7.195\",\"SIP/2.0/UDP 200.57.7.195:55061\"],\"status_code\":180,\"status_msg\":\"Ringing\",\"request_body_len\":229,\"response_body_len\":0}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -321,7 +321,7 @@ ] }, "event": { - "ingested": "2021-06-14T08:33:56.254239200Z", + "ingested": "2021-06-17T13:41:29.354945200Z", "original": "{\"ts\":1105725487.022577,\"uid\":\"CJZDWgixtwqXctWEg\",\"id.orig_h\":\"200.57.7.205\",\"id.orig_p\":5061,\"id.resp_h\":\"200.57.7.195\",\"id.resp_p\":5060,\"trans_depth\":0,\"method\":\"REGISTER\",\"uri\":\"sip:Verso.com\",\"request_from\":\"Ivan \u003csip:Ivan@Verso.com\u003e\",\"request_to\":\"Ivan \u003csip:Ivan@Verso.com\u003e\",\"response_from\":\"\\u0022Ivan\\u0022 \u003csip:Ivan@Verso.com\u003e\",\"response_to\":\"\\u0022Ivan\\u0022 \u003csip:Ivan@Verso.com\u003e\",\"call_id\":\"46E1C3CB36304F84A020CF6DD3F96461@Verso.com\",\"seq\":\"37764 REGISTER\",\"request_path\":[\"SIP/2.0/UDP 200.57.7.205:5061;rport\"],\"response_path\":[\"SIP/2.0/UDP 200.57.7.205:5061;received=200.57.7.205;rport=5061\"],\"user_agent\":\"Verso Softphone release 1104w\",\"status_code\":200,\"status_msg\":\"OK\",\"request_body_len\":0,\"response_body_len\":0}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -409,7 +409,7 @@ ] }, "event": { - "ingested": "2021-06-14T08:33:56.254240900Z", + "ingested": "2021-06-17T13:41:29.354949700Z", "original": "{\"ts\":1617119416.928735,\"uid\":\"CR6XQH1Lf2mF9YG7H2\",\"id.orig_h\":\"193.107.216.13\",\"id.orig_p\":5083,\"id.resp_h\":\"10.156.0.2\",\"id.resp_p\":5060,\"trans_depth\":0,\"method\":\"OPTIONS\",\"uri\":\"sip:100@35.198.74.222\",\"request_from\":\"\\\"sipvicious\\\"\u003csip:90501@1.1.1.1\u003e\",\"request_to\":\"\\\"sipvicious\\\"\u003csip:90501@1.1.1.1\u003e\",\"call_id\":\"767538559354206383610151\",\"seq\":\"1 OPTIONS\",\"request_path\":[\"SIP/2.0/UDP 193.107.216.13:5083\"],\"response_path\":[],\"user_agent\":\"friendly-scanner\",\"request_body_len\":0}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -496,7 +496,7 @@ ] }, "event": { - "ingested": "2021-06-14T08:33:56.254242600Z", + "ingested": "2021-06-17T13:41:29.354955800Z", "original": "{\"ts\":1617119923.416653,\"uid\":\"Cf9QMt4ear7ZkX74ti\",\"id.orig_h\":\"45.134.144.100\",\"id.orig_p\":5170,\"id.resp_h\":\"10.156.0.2\",\"id.resp_p\":5060,\"trans_depth\":0,\"method\":\"OPTIONS\",\"uri\":\"sip:100@35.198.74.222\",\"request_from\":\"\\\"sipvicious\\\"\u003csip:100@1.1.1.1\u003e\",\"request_to\":\"\\\"sipvicious\\\"\u003csip:100@1.1.1.1\u003e\",\"call_id\":\"35848812076538877174452\",\"seq\":\"1 OPTIONS\",\"request_path\":[\"SIP/2.0/UDP 127.0.0.1:5170\"],\"response_path\":[],\"user_agent\":\"friendly-scanner\",\"request_body_len\":0}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -621,7 +621,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-06-14T08:33:56.254294Z", + "ingested": "2021-06-17T13:41:29.354963700Z", "original": "{\"ts\":1105725487.022577,\"uid\":\"CJZDWgixtwqXctWEg\",\"id.orig_h\":\"200.57.7.205\",\"id.orig_p\":5061,\"id.resp_h\":\"200.57.7.195\",\"id.resp_p\":5060,\"trans_depth\":0,\"method\":\"REGISTER\",\"uri\":\"sip:Verso.com\",\"request_from\":\"Ivan \u003csip:Ivan@Verso.com\u003e\",\"request_to\":\"Ivan \u003csip:Ivan@Verso.com\u003e\",\"response_from\":\"\\u0022Ivan\\u0022 \u003csip:Ivan@Verso.com\u003e\",\"response_to\":\"\\u0022Ivan\\u0022 \u003csip:Ivan@Verso.com\u003e\",\"call_id\":\"46E1C3CB36304F84A020CF6DD3F96461@Verso.com\",\"seq\":\"37764 REGISTER\",\"request_path\":[\"SIP/2.0/UDP 200.57.7.205:5061;rport\"],\"response_path\":[\"SIP/2.0/UDP 200.57.7.205:5061;received=200.57.7.205;rport=5061\"],\"user_agent\":\"Verso Softphone release 1104w\",\"status_code\":200,\"status_msg\":\"OK\",\"request_body_len\":0,\"response_body_len\":0}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/sip/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/sip/elasticsearch/ingest_pipeline/default.yml index 3bff74b9e99f..d72082b655aa 100644 --- a/packages/zeek/data_stream/sip/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/sip/elasticsearch/ingest_pipeline/default.yml @@ -185,6 +185,7 @@ processors: field: zeek.sip.ts formats: - UNIX + - ISO8601 - remove: field: zeek.sip.ts - grok: diff --git a/packages/zeek/data_stream/smb_cmd/_dev/test/pipeline/test-smb-cmd.log-expected.json b/packages/zeek/data_stream/smb_cmd/_dev/test/pipeline/test-smb-cmd.log-expected.json index a96d511eec93..98071e51df22 100644 --- a/packages/zeek/data_stream/smb_cmd/_dev/test/pipeline/test-smb-cmd.log-expected.json +++ b/packages/zeek/data_stream/smb_cmd/_dev/test/pipeline/test-smb-cmd.log-expected.json @@ -43,7 +43,7 @@ "ip": "172.16.133.6" }, "event": { - "ingested": "2021-06-14T08:33:56.479090400Z", + "ingested": "2021-06-17T13:41:29.789091900Z", "original": "{\"ts\":1361916332.020006,\"uid\":\"CbT8mpAXseu6Pt4R7\",\"id.orig_h\":\"172.16.133.6\",\"id.orig_p\":1728,\"id.resp_h\":\"172.16.128.202\",\"id.resp_p\":445,\"command\":\"NT_CREATE_ANDX\",\"argument\":\"\\u005cbrowser\",\"status\":\"SUCCESS\",\"rtt\":0.091141,\"version\":\"SMB1\",\"tree\":\"\\u005c\\u005cJSRVR20\\u005cIPC$\",\"tree_service\":\"IPC\",\"referenced_file.ts\":1361916332.020006,\"referenced_file.uid\":\"CbT8mpAXseu6Pt4R7\",\"referenced_file.id.orig_h\":\"172.16.133.6\",\"referenced_file.id.orig_p\":1728,\"referenced_file.id.resp_h\":\"172.16.128.202\",\"referenced_file.id.resp_p\":445,\"referenced_file.action\":\"SMB::FILE_OPEN\",\"referenced_file.name\":\"\\u005cbrowser\",\"referenced_file.size\":0}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -126,7 +126,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-06-14T08:33:56.479098800Z", + "ingested": "2021-06-17T13:41:29.789102600Z", "original": "{\"ts\":1361916332.020006,\"uid\":\"CbT8mpAXseu6Pt4R7\",\"id.orig_h\":\"172.16.133.6\",\"id.orig_p\":1728,\"id.resp_h\":\"172.16.128.202\",\"id.resp_p\":445,\"command\":\"NT_CREATE_ANDX\",\"argument\":\"\\u005cbrowser\",\"status\":\"SUCCESS\",\"rtt\":0.091141,\"version\":\"SMB1\",\"tree\":\"\\u005c\\u005cJSRVR20\\u005cIPC$\",\"tree_service\":\"IPC\",\"referenced_file.ts\":1361916332.020006,\"referenced_file.uid\":\"CbT8mpAXseu6Pt4R7\",\"referenced_file.id.orig_h\":\"172.16.133.6\",\"referenced_file.id.orig_p\":1728,\"referenced_file.id.resp_h\":\"172.16.128.202\",\"referenced_file.id.resp_p\":445,\"referenced_file.action\":\"SMB::FILE_OPEN\",\"referenced_file.name\":\"\\u005cbrowser\",\"referenced_file.size\":0}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/smb_cmd/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/smb_cmd/elasticsearch/ingest_pipeline/default.yml index a331b927a0d7..b1333f4d663d 100644 --- a/packages/zeek/data_stream/smb_cmd/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/smb_cmd/elasticsearch/ingest_pipeline/default.yml @@ -235,6 +235,7 @@ processors: field: zeek.smb_cmd.ts formats: - UNIX + - ISO8601 - remove: field: zeek.smb_cmd.ts - remove: diff --git a/packages/zeek/data_stream/smb_files/_dev/test/pipeline/test-smb-files.log-expected.json b/packages/zeek/data_stream/smb_files/_dev/test/pipeline/test-smb-files.log-expected.json index 954519b34953..f469ab3a93da 100644 --- a/packages/zeek/data_stream/smb_files/_dev/test/pipeline/test-smb-files.log-expected.json +++ b/packages/zeek/data_stream/smb_files/_dev/test/pipeline/test-smb-files.log-expected.json @@ -54,7 +54,7 @@ ] }, "event": { - "ingested": "2021-06-14T08:33:56.582650100Z", + "ingested": "2021-06-17T13:41:30.015268800Z", "original": "{\"ts\":1507565599.576942,\"uid\":\"C9YAaEzWLL62yWMn5\",\"id.orig_h\":\"192.168.10.31\",\"id.orig_p\":49239,\"id.resp_h\":\"192.168.10.30\",\"id.resp_p\":445,\"action\":\"SMB::FILE_OPEN\",\"path\":\"\\u005c\\u005cadmin-pc\\u005cADMIN$\",\"name\":\"PSEXESVC.exe\",\"size\":0,\"times.modified\":1507565599.607777,\"times.accessed\":1507565599.607777,\"times.created\":1507565599.607777,\"times.changed\":1507565599.607777}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -133,7 +133,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-06-14T08:33:56.582655700Z", + "ingested": "2021-06-17T13:41:30.015278500Z", "original": "{\"ts\":1507565599.576942,\"uid\":\"C9YAaEzWLL62yWMn5\",\"id.orig_h\":\"192.168.10.31\",\"id.orig_p\":49239,\"id.resp_h\":\"192.168.10.30\",\"id.resp_p\":445,\"action\":\"SMB::FILE_OPEN\",\"path\":\"\\u005c\\u005cadmin-pc\\u005cADMIN$\",\"name\":\"PSEXESVC.exe\",\"size\":0,\"times.modified\":1507565599.607777,\"times.accessed\":1507565599.607777,\"times.created\":1507565599.607777,\"times.changed\":1507565599.607777}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/smb_files/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/smb_files/elasticsearch/ingest_pipeline/default.yml index 571dfe7fd153..648edec38f12 100644 --- a/packages/zeek/data_stream/smb_files/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/smb_files/elasticsearch/ingest_pipeline/default.yml @@ -144,6 +144,7 @@ processors: field: zeek.smb_files.ts formats: - UNIX + - ISO8601 - remove: field: zeek.smb_files.ts - dot_expander: @@ -163,6 +164,7 @@ processors: target_field: zeek.smb_files.times.accessed formats: - UNIX + - ISO8601 if: ctx.zeek.smb_files.times?.accessed != null - set: field: file.accessed @@ -173,6 +175,7 @@ processors: target_field: zeek.smb_files.times.changed formats: - UNIX + - ISO8601 if: ctx.zeek.smb_files.times?.accessed != null - set: field: file.ctime @@ -183,6 +186,7 @@ processors: target_field: zeek.smb_files.times.created formats: - UNIX + - ISO8601 if: ctx.zeek.smb_files.times?.accessed != null - set: field: file.created @@ -193,6 +197,7 @@ processors: target_field: zeek.smb_files.times.modified formats: - UNIX + - ISO8601 if: ctx.zeek.smb_files.times?.accessed != null - set: field: file.mtime diff --git a/packages/zeek/data_stream/smb_mapping/_dev/test/pipeline/test-smb-mapping.log-expected.json b/packages/zeek/data_stream/smb_mapping/_dev/test/pipeline/test-smb-mapping.log-expected.json index 1b31f81d4ab6..a19f3776a9c7 100644 --- a/packages/zeek/data_stream/smb_mapping/_dev/test/pipeline/test-smb-mapping.log-expected.json +++ b/packages/zeek/data_stream/smb_mapping/_dev/test/pipeline/test-smb-mapping.log-expected.json @@ -29,7 +29,7 @@ "ip": "192.168.10.31" }, "event": { - "ingested": "2021-06-14T08:33:56.681444400Z", + "ingested": "2021-06-17T13:41:30.257634500Z", "original": "{\"ts\":1507565599.576613,\"uid\":\"C9YAaEzWLL62yWMn5\",\"id.orig_h\":\"192.168.10.31\",\"id.orig_p\":49239,\"id.resp_h\":\"192.168.10.30\",\"id.resp_p\":445,\"path\":\"\\u005c\\u005cadmin-pc\\u005cADMIN$\",\"share_type\":\"DISK\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -96,7 +96,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-06-14T08:33:56.681449100Z", + "ingested": "2021-06-17T13:41:30.257644Z", "original": "{\"ts\":1507565599.576613,\"uid\":\"C9YAaEzWLL62yWMn5\",\"id.orig_h\":\"192.168.10.31\",\"id.orig_p\":49239,\"id.resp_h\":\"192.168.10.30\",\"id.resp_p\":445,\"path\":\"\\u005c\\u005cadmin-pc\\u005cADMIN$\",\"share_type\":\"DISK\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/smb_mapping/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/smb_mapping/elasticsearch/ingest_pipeline/default.yml index d6b7dec4a5a3..6b87f9ca5bb0 100644 --- a/packages/zeek/data_stream/smb_mapping/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/smb_mapping/elasticsearch/ingest_pipeline/default.yml @@ -129,6 +129,7 @@ processors: field: zeek.smb_mapping.ts formats: - UNIX + - ISO8601 - remove: field: zeek.smb_mapping.ts - geoip: diff --git a/packages/zeek/data_stream/smtp/_dev/test/pipeline/test-smtp.log-expected.json b/packages/zeek/data_stream/smtp/_dev/test/pipeline/test-smtp.log-expected.json index f1834742be3b..405c37f4a97d 100644 --- a/packages/zeek/data_stream/smtp/_dev/test/pipeline/test-smtp.log-expected.json +++ b/packages/zeek/data_stream/smtp/_dev/test/pipeline/test-smtp.log-expected.json @@ -47,7 +47,7 @@ "established": true }, "event": { - "ingested": "2021-06-14T08:33:56.766552500Z", + "ingested": "2021-06-17T13:41:30.422971700Z", "original": "{\"ts\":1543877987.381899,\"uid\":\"CWWzPB3RjqhFf528c\",\"id.orig_h\":\"192.168.1.10\",\"id.orig_p\":33782,\"id.resp_h\":\"192.168.1.9\",\"id.resp_p\":25,\"trans_depth\":1,\"helo\":\"EXAMPLE.COM\",\"last_reply\":\"220 2.0.0 SMTP server ready\",\"path\":[\"192.168.1.9\"],\"tls\":true,\"fuids\":[],\"is_webmail\":false}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -116,7 +116,7 @@ "established": true }, "event": { - "ingested": "2021-06-14T08:33:56.766558600Z", + "ingested": "2021-06-17T13:41:30.422981500Z", "original": "{\"ts\":1543877987.381899,\"uid\":\"CWWzPB3RjqhFf528c\",\"id.orig_h\":\"192.168.1.10\",\"id.orig_p\":33782,\"id.resp_h\":\"192.168.1.9\",\"id.resp_p\":25,\"trans_depth\":1,\"helo\":\"EXAMPLE.COM\",\"last_reply\":\"220 2.0.0 SMTP server ready\",\"path\":[\"192.168.1.9\"],\"tls\":true,\"fuids\":[],\"is_webmail\":false}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/smtp/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/smtp/elasticsearch/ingest_pipeline/default.yml index 35b5c625a1f2..e4736d9762d1 100644 --- a/packages/zeek/data_stream/smtp/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/smtp/elasticsearch/ingest_pipeline/default.yml @@ -146,6 +146,7 @@ processors: field: zeek.smtp.ts formats: - UNIX + - ISO8601 - remove: field: zeek.smtp.ts - date: diff --git a/packages/zeek/data_stream/snmp/_dev/test/pipeline/test-snmp.log-expected.json b/packages/zeek/data_stream/snmp/_dev/test/pipeline/test-snmp.log-expected.json index 08a3f5f73ba9..3c9f6ff6b3e0 100644 --- a/packages/zeek/data_stream/snmp/_dev/test/pipeline/test-snmp.log-expected.json +++ b/packages/zeek/data_stream/snmp/_dev/test/pipeline/test-snmp.log-expected.json @@ -39,7 +39,7 @@ "ip": "192.168.1.2" }, "event": { - "ingested": "2021-06-14T08:33:56.855739900Z", + "ingested": "2021-06-17T13:41:30.671045500Z", "original": "{\"ts\":1543877948.916584,\"uid\":\"CnKW1B4w9fpRa6Nkf2\",\"id.orig_h\":\"192.168.1.2\",\"id.orig_p\":59696,\"id.resp_h\":\"192.168.1.1\",\"id.resp_p\":161,\"duration\":7.849924,\"version\":\"2c\",\"community\":\"public\",\"get_requests\":0,\"get_bulk_requests\":0,\"get_responses\":8,\"set_requests\":0,\"up_since\":1543631204.766508}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -114,7 +114,7 @@ "ip": "184.105.139.67" }, "event": { - "ingested": "2021-06-14T08:33:56.855745300Z", + "ingested": "2021-06-17T13:41:30.671055200Z", "original": "{\"ts\":1617080496.400704,\"uid\":\"CxtWIB4ECPW89F8mSi\",\"id.orig_h\":\"184.105.139.67\",\"id.orig_p\":37533,\"id.resp_h\":\"10.156.0.2\",\"id.resp_p\":161,\"duration\":0.0,\"version\":\"2c\",\"community\":\"public\",\"get_requests\":4,\"get_bulk_requests\":0,\"get_responses\":0,\"set_requests\":0}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -191,7 +191,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-06-14T08:33:56.855747200Z", + "ingested": "2021-06-17T13:41:30.671060900Z", "original": "{\"ts\":1543877948.916584,\"uid\":\"CnKW1B4w9fpRa6Nkf2\",\"id.orig_h\":\"192.168.1.2\",\"id.orig_p\":59696,\"id.resp_h\":\"192.168.1.1\",\"id.resp_p\":161,\"duration\":7.849924,\"version\":\"2c\",\"community\":\"public\",\"get_requests\":0,\"get_bulk_requests\":0,\"get_responses\":8,\"set_requests\":0,\"up_since\":1543631204.766508}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/snmp/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/snmp/elasticsearch/ingest_pipeline/default.yml index 3c9a5bf65f1c..7aaf24a5f16d 100644 --- a/packages/zeek/data_stream/snmp/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/snmp/elasticsearch/ingest_pipeline/default.yml @@ -145,6 +145,7 @@ processors: field: zeek.snmp.ts formats: - UNIX + - ISO8601 - remove: field: zeek.snmp.ts - date: @@ -152,6 +153,7 @@ processors: target_field: zeek.snmp.up_since formats: - UNIX + - ISO8601 if: ctx.zeek.snmp.up_since != null - geoip: field: destination.ip diff --git a/packages/zeek/data_stream/socks/_dev/test/pipeline/test-socks.log-expected.json b/packages/zeek/data_stream/socks/_dev/test/pipeline/test-socks.log-expected.json index 909a6aed28cf..041fea9a2c4c 100644 --- a/packages/zeek/data_stream/socks/_dev/test/pipeline/test-socks.log-expected.json +++ b/packages/zeek/data_stream/socks/_dev/test/pipeline/test-socks.log-expected.json @@ -36,7 +36,7 @@ "ip": "127.0.0.1" }, "event": { - "ingested": "2021-06-14T08:33:56.967302900Z", + "ingested": "2021-06-17T13:41:30.901747200Z", "original": "{\"ts\":1566508093.09494,\"uid\":\"Cmz4Cb4qCw1hGqYw1c\",\"id.orig_h\":\"127.0.0.1\",\"id.orig_p\":35368,\"id.resp_h\":\"127.0.0.1\",\"id.resp_p\":8080,\"version\":5,\"status\":\"succeeded\",\"request.name\":\"www.google.com\",\"request_p\":443,\"bound.host\":\"0.0.0.0\",\"bound_p\":0}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -111,7 +111,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-06-14T08:33:56.967311900Z", + "ingested": "2021-06-17T13:41:30.901758400Z", "original": "{\"ts\":1566508093.09494,\"uid\":\"Cmz4Cb4qCw1hGqYw1c\",\"id.orig_h\":\"127.0.0.1\",\"id.orig_p\":35368,\"id.resp_h\":\"127.0.0.1\",\"id.resp_p\":8080,\"version\":5,\"status\":\"succeeded\",\"request.name\":\"www.google.com\",\"request_p\":443,\"bound.host\":\"0.0.0.0\",\"bound_p\":0}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/socks/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/socks/elasticsearch/ingest_pipeline/default.yml index 66666f81978d..364f0d303553 100644 --- a/packages/zeek/data_stream/socks/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/socks/elasticsearch/ingest_pipeline/default.yml @@ -149,6 +149,7 @@ processors: field: zeek.socks.ts formats: - UNIX + - ISO8601 - remove: field: zeek.socks.ts - dot_expander: diff --git a/packages/zeek/data_stream/ssh/_dev/test/pipeline/test-ssh.log-expected.json b/packages/zeek/data_stream/ssh/_dev/test/pipeline/test-ssh.log-expected.json index 259939e1b53b..5f6c203bfccd 100644 --- a/packages/zeek/data_stream/ssh/_dev/test/pipeline/test-ssh.log-expected.json +++ b/packages/zeek/data_stream/ssh/_dev/test/pipeline/test-ssh.log-expected.json @@ -42,7 +42,7 @@ "ip": "192.168.1.2" }, "event": { - "ingested": "2021-06-14T08:33:57.059504700Z", + "ingested": "2021-06-17T13:41:31.082995600Z", "original": "{\"ts\":1562527532.904291,\"uid\":\"CajWfz1b3qnnWT0BU9\",\"id.orig_h\":\"192.168.1.2\",\"id.orig_p\":48380,\"id.resp_h\":\"192.168.1.1\",\"id.resp_p\":22,\"version\":2,\"auth_success\":false,\"auth_attempts\":2,\"client\":\"SSH-2.0-OpenSSH_7.9p1 Ubuntu-10\",\"server\":\"SSH-2.0-OpenSSH_6.6.1p1 Debian-4~bpo70+1\",\"cipher_alg\":\"chacha20-poly1305@openssh.com\",\"mac_alg\":\"umac-64-etm@openssh.com\",\"compression_alg\":\"none\",\"kex_alg\":\"curve25519-sha256@libssh.org\",\"host_key_alg\":\"ecdsa-sha2-nistp256\",\"host_key\":\"86:71:ac:9c:35:1c:28:29:05:81:48:ec:66:67:de:bd\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -112,7 +112,7 @@ "ip": "51.161.10.160" }, "event": { - "ingested": "2021-06-14T08:33:57.059510400Z", + "ingested": "2021-06-17T13:41:31.083007100Z", "original": "{\"ts\":1617123417.413634,\"uid\":\"COXxsJ3dlSh6ECRYQj\",\"id.orig_h\":\"51.161.10.160\",\"id.orig_p\":38204,\"id.resp_h\":\"10.156.0.2\",\"id.resp_p\":22,\"auth_attempts\":0,\"direction\":\"INBOUND\",\"client\":\"SSH-2.0-libssh-0.6.3\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -181,7 +181,7 @@ "ip": "113.53.238.195" }, "event": { - "ingested": "2021-06-14T08:33:57.059512500Z", + "ingested": "2021-06-17T13:41:31.083015Z", "original": "{\"ts\":1617123445.61524,\"uid\":\"CZPdXz1jfKSWzIDAeb\",\"id.orig_h\":\"113.53.238.195\",\"id.orig_p\":44164,\"id.resp_h\":\"10.156.0.2\",\"id.resp_p\":22,\"auth_attempts\":0,\"direction\":\"INBOUND\",\"client\":\"SSH-2.0-libssh-0.6.3\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -250,7 +250,7 @@ "ip": "34.86.35.26" }, "event": { - "ingested": "2021-06-14T08:33:57.059514200Z", + "ingested": "2021-06-17T13:41:31.083024Z", "original": "{\"ts\":1617123450.957272,\"uid\":\"Cha1rs3OamonAZ4Nz6\",\"id.orig_h\":\"34.86.35.26\",\"id.orig_p\":33953,\"id.resp_h\":\"10.156.0.2\",\"id.resp_p\":22,\"auth_attempts\":0,\"direction\":\"INBOUND\",\"client\":\"SSH-2.0-ZGrab ZGrab SSH Survey\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -330,7 +330,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-06-14T08:33:57.059515900Z", + "ingested": "2021-06-17T13:41:31.083084800Z", "original": "{\"ts\":1562527532.904291,\"uid\":\"CajWfz1b3qnnWT0BU9\",\"id.orig_h\":\"192.168.1.2\",\"id.orig_p\":48380,\"id.resp_h\":\"192.168.1.1\",\"id.resp_p\":22,\"version\":2,\"auth_success\":false,\"auth_attempts\":2,\"client\":\"SSH-2.0-OpenSSH_7.9p1 Ubuntu-10\",\"server\":\"SSH-2.0-OpenSSH_6.6.1p1 Debian-4~bpo70+1\",\"cipher_alg\":\"chacha20-poly1305@openssh.com\",\"mac_alg\":\"umac-64-etm@openssh.com\",\"compression_alg\":\"none\",\"kex_alg\":\"curve25519-sha256@libssh.org\",\"host_key_alg\":\"ecdsa-sha2-nistp256\",\"host_key\":\"86:71:ac:9c:35:1c:28:29:05:81:48:ec:66:67:de:bd\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/ssh/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/ssh/elasticsearch/ingest_pipeline/default.yml index 389da5395f5b..0c1db6c51e54 100644 --- a/packages/zeek/data_stream/ssh/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/ssh/elasticsearch/ingest_pipeline/default.yml @@ -157,6 +157,7 @@ processors: field: zeek.ssh.ts formats: - UNIX + - ISO8601 - remove: field: zeek.ssh.ts - geoip: diff --git a/packages/zeek/data_stream/ssl/_dev/test/pipeline/test-ssl.log-expected.json b/packages/zeek/data_stream/ssl/_dev/test/pipeline/test-ssl.log-expected.json index f9226c2a6c13..9ea19056036b 100644 --- a/packages/zeek/data_stream/ssl/_dev/test/pipeline/test-ssl.log-expected.json +++ b/packages/zeek/data_stream/ssl/_dev/test/pipeline/test-ssl.log-expected.json @@ -112,7 +112,7 @@ "version_protocol": "tls" }, "event": { - "ingested": "2021-06-14T08:33:57.209961700Z", + "ingested": "2021-06-17T13:41:31.352461300Z", "original": "{\"ts\":1547688736.805088,\"uid\":\"CAOvs1BMFCX2Eh0Y3\",\"id.orig_h\":\"10.178.98.102\",\"id.orig_p\":63199,\"id.resp_h\":\"35.199.178.4\",\"id.resp_p\":9243,\"version\":\"TLSv12\",\"cipher\":\"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\",\"curve\":\"secp256r1\",\"server_name\":\"dd625ffb4fc54735b281862aa1cd6cd4.us-west1.gcp.cloud.es.io\",\"resumed\":false,\"established\":true,\"cert_chain_fuids\":[\"FebkbHWVCV8rEEEne\",\"F4BDY41MGUBT6URZMd\",\"FWlfEfiHVkv8evDL3\"],\"client_cert_chain_fuids\":[],\"subject\":\"CN=*.gcp.cloud.es.io,O=Elasticsearch\\u005c, Inc.,L=Mountain View,ST=California,C=US\",\"issuer\":\"CN=DigiCert SHA2 Secure Server CA,O=DigiCert Inc,C=US\",\"validation_status\":\"ok\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -238,7 +238,7 @@ "version_protocol": "tls" }, "event": { - "ingested": "2021-06-14T08:33:57.209967300Z", + "ingested": "2021-06-17T13:41:31.352477300Z", "original": "{\"ts\":1547688736.80509,\"uid\":\"C3mki91FnnNtm0u1ok\",\"id.orig_h\":\"10.178.98.102\",\"id.orig_p\":63198,\"id.resp_h\":\"35.199.178.4\",\"id.resp_p\":9243,\"version\":\"TLSv12\",\"cipher\":\"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\",\"curve\":\"secp256r1\",\"server_name\":\"dd625ffb4fc54735b281862aa1cd6cd4.us-west1.gcp.cloud.es.io\",\"resumed\":false,\"established\":true,\"cert_chain_fuids\":[\"Fue9H32OmuitQk2zR\",\"FpbiBP215tk2xftxM6\",\"FEdROj1vUzTGw3BIUa\"],\"client_cert_chain_fuids\":[],\"subject\":\"CN=*.gcp.cloud.es.io,O=Elasticsearch\\u005c, Inc.,L=Mountain View,ST=California,C=US\",\"issuer\":\"CN=DigiCert SHA2 Secure Server CA,O=DigiCert Inc,C=US\",\"validation_status\":\"ok\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -364,7 +364,7 @@ "version_protocol": "tls" }, "event": { - "ingested": "2021-06-14T08:33:57.209969300Z", + "ingested": "2021-06-17T13:41:31.352483600Z", "original": "{\"ts\":1547688736.805527,\"uid\":\"CfGBt82PzCXzHa0iek\",\"id.orig_h\":\"10.178.98.102\",\"id.orig_p\":63197,\"id.resp_h\":\"35.199.178.4\",\"id.resp_p\":9243,\"version\":\"TLSv12\",\"cipher\":\"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\",\"curve\":\"secp256r1\",\"server_name\":\"dd625ffb4fc54735b281862aa1cd6cd4.us-west1.gcp.cloud.es.io\",\"resumed\":false,\"established\":true,\"cert_chain_fuids\":[\"FiFLYv3UjeWyv2gcW\",\"FvSsiB1Xi816EMagI9\",\"FWpPS4mjGaAhTRXLf\"],\"client_cert_chain_fuids\":[],\"subject\":\"CN=*.gcp.cloud.es.io,O=Elasticsearch\\u005c, Inc.,L=Mountain View,ST=California,C=US\",\"issuer\":\"CN=DigiCert SHA2 Secure Server CA,O=DigiCert Inc,C=US\",\"validation_status\":\"ok\"}{\"ts\":1602179457.352156,\"uid\":\"CK17Dl2SB8bZOVonSl\",\"id.orig_h\":\"10.0.0.1\",\"id.orig_p\":49228,\"id.resp_h\":\"192.168.50.1\",\"id.resp_p\":443,\"version\":\"TLSv12\",\"cipher\":\"TLS_RSA_WITH_AES_128_CBC_SHA256\",\"resumed\":false,\"established\":true,\"cert_chain_fuids\":[\"FOLwYQ6rs70bIMSf9\"],\"client_cert_chain_fuids\":[],\"subject\":\"CN=foo,OU=foo@bar,O=org,L=locality,C=LO\",\"issuer\":\"CN=CA,OU=CA@example.com,O=Example Corp,L=foo,C=HI\",\"validation_status\":\"self signed certificate\",\"ja3\":\"74927e242d6c3febf8cb9cab10a7f889\",\"ja3s\":\"80b3a14bccc8598a1f3bbe83e71f735f\",\"resp_certificate_sha1\":\"5dad8b55621b6b9c30679d9d61248dd132a83c94\",\"not_valid_before\":1562022421,\"not_valid_after\":1577748224}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -447,7 +447,7 @@ "version_protocol": "tls" }, "event": { - "ingested": "2021-06-14T08:33:57.209971Z", + "ingested": "2021-06-17T13:41:31.352486500Z", "original": "{\"ts\":1617091251.151303,\"uid\":\"CLQiVH1VcpvT3ruEak\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":52730,\"id.resp_h\":\"46.101.87.151\",\"id.resp_p\":443,\"version\":\"TLSv12\",\"cipher\":\"TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256\",\"resumed\":false,\"established\":false}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -523,7 +523,7 @@ "resumed": false }, "event": { - "ingested": "2021-06-14T08:33:57.209972700Z", + "ingested": "2021-06-17T13:41:31.352513700Z", "original": "{\"ts\":1617090955.826099,\"uid\":\"CBiXOC4IqYxMv1xzf9\",\"id.orig_h\":\"35.195.125.46\",\"id.orig_p\":52678,\"id.resp_h\":\"10.156.0.2\",\"id.resp_p\":443,\"server_name\":\"splunk-api.swiftcrypto.com\",\"resumed\":false,\"established\":false}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -604,7 +604,7 @@ "resumed": false }, "event": { - "ingested": "2021-06-14T08:33:57.209974300Z", + "ingested": "2021-06-17T13:41:31.352520Z", "original": "{\"ts\":1617091253.726384,\"uid\":\"C4jH9IqWGZwc1PPUh\",\"id.orig_h\":\"35.198.74.222\",\"id.orig_p\":53368,\"id.resp_h\":\"10.156.0.2\",\"id.resp_p\":443,\"server_name\":\"tickets.swiftcrypto.com\",\"resumed\":false,\"established\":false}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -685,7 +685,7 @@ "resumed": false }, "event": { - "ingested": "2021-06-14T08:33:57.209976600Z", + "ingested": "2021-06-17T13:41:31.352541300Z", "original": "{\"ts\":1617091253.91861,\"uid\":\"CXVMSq6Dainy4WFN9\",\"id.orig_h\":\"35.198.74.222\",\"id.orig_p\":53382,\"id.resp_h\":\"10.156.0.2\",\"id.resp_p\":443,\"server_name\":\"rundeck.swiftcrypto.com\",\"resumed\":false,\"established\":false}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -777,7 +777,7 @@ "version_protocol": "tls" }, "event": { - "ingested": "2021-06-14T08:33:57.209978400Z", + "ingested": "2021-06-17T13:41:31.352548400Z", "original": "{\"ts\":1617091254.325291,\"uid\":\"CsgtQe4AikDZBsIM6k\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":55120,\"id.resp_h\":\"104.154.89.105\",\"id.resp_p\":443,\"version\":\"TLSv12\",\"cipher\":\"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256\",\"curve\":\"secp256r1\",\"resumed\":false,\"established\":false,\"cert_chain_fuids\":[\"FeyRIk4nUtwwcUcnRf\"],\"client_cert_chain_fuids\":[],\"validation_status\":\"self signed certificate\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -853,7 +853,7 @@ "resumed": false }, "event": { - "ingested": "2021-06-14T08:33:57.209980Z", + "ingested": "2021-06-17T13:41:31.352555200Z", "original": "{\"ts\":1617091255.065602,\"uid\":\"CPGhJS3UPpcnR96NQc\",\"id.orig_h\":\"35.195.125.46\",\"id.orig_p\":53095,\"id.resp_h\":\"10.156.0.2\",\"id.resp_p\":443,\"server_name\":\"splunk-api.swiftcrypto.com\",\"resumed\":false,\"established\":false}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -987,7 +987,7 @@ "version_protocol": "tls" }, "event": { - "ingested": "2021-06-14T08:33:57.209981600Z", + "ingested": "2021-06-17T13:41:31.352559300Z", "original": "{\"ts\":1547688736.805527,\"uid\":\"CfGBt82PzCXzHa0iek\",\"id.orig_h\":\"10.178.98.102\",\"id.orig_p\":63197,\"id.resp_h\":\"35.199.178.4\",\"id.resp_p\":9243,\"version\":\"TLSv12\",\"cipher\":\"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\",\"curve\":\"secp256r1\",\"server_name\":\"dd625ffb4fc54735b281862aa1cd6cd4.us-west1.gcp.cloud.es.io\",\"resumed\":false,\"established\":true,\"cert_chain_fuids\":[\"FiFLYv3UjeWyv2gcW\",\"FvSsiB1Xi816EMagI9\",\"FWpPS4mjGaAhTRXLf\"],\"client_cert_chain_fuids\":[],\"subject\":\"CN=*.gcp.cloud.es.io,O=Elasticsearch\\u005c, Inc.,L=Mountain View,ST=California,C=US\",\"issuer\":\"CN=DigiCert SHA2 Secure Server CA,O=DigiCert Inc,C=US\",\"validation_status\":\"ok\"}{\"ts\":1602179457.352156,\"uid\":\"CK17Dl2SB8bZOVonSl\",\"id.orig_h\":\"10.0.0.1\",\"id.orig_p\":49228,\"id.resp_h\":\"192.168.50.1\",\"id.resp_p\":443,\"version\":\"TLSv12\",\"cipher\":\"TLS_RSA_WITH_AES_128_CBC_SHA256\",\"resumed\":false,\"established\":true,\"cert_chain_fuids\":[\"FOLwYQ6rs70bIMSf9\"],\"client_cert_chain_fuids\":[],\"subject\":\"CN=foo,OU=foo@bar,O=org,L=locality,C=LO\",\"issuer\":\"CN=CA,OU=CA@example.com,O=Example Corp,L=foo,C=HI\",\"validation_status\":\"self signed certificate\",\"ja3\":\"74927e242d6c3febf8cb9cab10a7f889\",\"ja3s\":\"80b3a14bccc8598a1f3bbe83e71f735f\",\"resp_certificate_sha1\":\"5dad8b55621b6b9c30679d9d61248dd132a83c94\",\"not_valid_before\":1562022421,\"not_valid_after\":1577748224}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/ssl/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/ssl/elasticsearch/ingest_pipeline/default.yml index 9534a134abe4..5c6ed03ccb1b 100644 --- a/packages/zeek/data_stream/ssl/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/ssl/elasticsearch/ingest_pipeline/default.yml @@ -162,6 +162,7 @@ processors: field: zeek.ssl.ts formats: - UNIX + - ISO8601 - remove: field: zeek.ssl.ts - date: @@ -170,12 +171,14 @@ processors: target_field: tls.server.not_before formats: - UNIX + - ISO8601 - date: if: ctx.tls?.server?.not_after != null field: tls.server.not_after target_field: tls.server.not_after formats: - UNIX + - ISO8601 - geoip: field: destination.ip target_field: destination.geo diff --git a/packages/zeek/data_stream/stats/_dev/test/pipeline/test-stats.log-expected.json b/packages/zeek/data_stream/stats/_dev/test/pipeline/test-stats.log-expected.json index 97e2b73f8ecc..8f80e93cadbd 100644 --- a/packages/zeek/data_stream/stats/_dev/test/pipeline/test-stats.log-expected.json +++ b/packages/zeek/data_stream/stats/_dev/test/pipeline/test-stats.log-expected.json @@ -54,7 +54,7 @@ } }, "event": { - "ingested": "2021-06-14T08:33:57.622527600Z", + "ingested": "2021-06-17T13:41:32.110766500Z", "original": "{\"ts\":1476605878.714844,\"peer\":\"bro\",\"mem\":94,\"pkts_proc\":296,\"bytes_recv\":39674,\"events_proc\":723,\"events_queued\":728,\"active_tcp_conns\":1,\"active_udp_conns\":3,\"active_icmp_conns\":0,\"tcp_conns\":6,\"udp_conns\":36,\"icmp_conns\":2,\"timers\":797,\"active_timers\":38,\"files\":0,\"active_files\":0,\"dns_requests\":0,\"active_dns_requests\":0,\"reassem_tcp_size\":0,\"reassem_file_size\":0,\"reassem_frag_size\":0,\"reassem_unknown_size\":0}", "created": "2020-04-28T11:07:58.223Z", "kind": "metric" @@ -125,7 +125,7 @@ } }, "event": { - "ingested": "2021-06-14T08:33:57.622533500Z", + "ingested": "2021-06-17T13:41:32.110778700Z", "original": "{\"ts\":1476605878.714844,\"peer\":\"bro\",\"mem\":94,\"pkts_proc\":296,\"bytes_recv\":39674,\"events_proc\":723,\"events_queued\":728,\"active_tcp_conns\":1,\"active_udp_conns\":3,\"active_icmp_conns\":0,\"tcp_conns\":6,\"udp_conns\":36,\"icmp_conns\":2,\"timers\":797,\"active_timers\":38,\"files\":0,\"active_files\":0,\"dns_requests\":0,\"active_dns_requests\":0,\"reassem_tcp_size\":0,\"reassem_file_size\":0,\"reassem_frag_size\":0,\"reassem_unknown_size\":0}", "created": "2020-04-28T11:07:58.223Z", "kind": "metric" diff --git a/packages/zeek/data_stream/stats/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/stats/elasticsearch/ingest_pipeline/default.yml index e0101e9eb452..0b63b69cc13d 100644 --- a/packages/zeek/data_stream/stats/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/stats/elasticsearch/ingest_pipeline/default.yml @@ -162,6 +162,7 @@ processors: field: zeek.stats.ts formats: - UNIX + - ISO8601 - remove: field: zeek.stats.ts - set: diff --git a/packages/zeek/data_stream/syslog/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/syslog/elasticsearch/ingest_pipeline/default.yml index d6c79cc93c29..5130fbebf38b 100644 --- a/packages/zeek/data_stream/syslog/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/syslog/elasticsearch/ingest_pipeline/default.yml @@ -133,6 +133,7 @@ processors: field: zeek.syslog.ts formats: - UNIX + - ISO8601 - remove: field: zeek.syslog.ts - geoip: diff --git a/packages/zeek/data_stream/traceroute/_dev/test/pipeline/test-traceroute.log-expected.json b/packages/zeek/data_stream/traceroute/_dev/test/pipeline/test-traceroute.log-expected.json index 58416cf901e0..bae35db20469 100644 --- a/packages/zeek/data_stream/traceroute/_dev/test/pipeline/test-traceroute.log-expected.json +++ b/packages/zeek/data_stream/traceroute/_dev/test/pipeline/test-traceroute.log-expected.json @@ -36,7 +36,7 @@ "ip": "192.168.1.1" }, "event": { - "ingested": "2021-06-14T08:33:57.709335600Z", + "ingested": "2021-06-17T13:41:32.292479300Z", "original": "{\"ts\":1361916158.650605,\"src\":\"192.168.1.1\",\"dst\":\"8.8.8.8\",\"proto\":\"udp\"}", "category": [ "network" @@ -104,7 +104,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-06-14T08:33:57.709340Z", + "ingested": "2021-06-17T13:41:32.292489Z", "original": "{\"ts\":1361916158.650605,\"src\":\"192.168.1.1\",\"dst\":\"8.8.8.8\",\"proto\":\"udp\"}", "category": [ "network" diff --git a/packages/zeek/data_stream/traceroute/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/traceroute/elasticsearch/ingest_pipeline/default.yml index 52170969edfe..69fbdaf2f750 100644 --- a/packages/zeek/data_stream/traceroute/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/traceroute/elasticsearch/ingest_pipeline/default.yml @@ -92,6 +92,7 @@ processors: field: zeek.traceroute.ts formats: - UNIX + - ISO8601 - remove: field: zeek.traceroute.ts - geoip: diff --git a/packages/zeek/data_stream/tunnel/_dev/test/pipeline/test-tunnel.log-expected.json b/packages/zeek/data_stream/tunnel/_dev/test/pipeline/test-tunnel.log-expected.json index 4ba134417825..681db32b062d 100644 --- a/packages/zeek/data_stream/tunnel/_dev/test/pipeline/test-tunnel.log-expected.json +++ b/packages/zeek/data_stream/tunnel/_dev/test/pipeline/test-tunnel.log-expected.json @@ -58,7 +58,7 @@ "ip": "132.16.146.79" }, "event": { - "ingested": "2021-06-14T08:33:57.788151200Z", + "ingested": "2021-06-17T13:41:32.449562700Z", "original": "{\"ts\":1544405666.743509,\"id.orig_h\":\"132.16.146.79\",\"id.orig_p\":0,\"id.resp_h\":\"132.16.110.133\",\"id.resp_p\":8080,\"tunnel_type\":\"Tunnel::HTTP\",\"action\":\"Tunnel::DISCOVER\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -143,7 +143,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-06-14T08:33:57.788156700Z", + "ingested": "2021-06-17T13:41:32.449571800Z", "original": "{\"ts\":1544405666.743509,\"id.orig_h\":\"132.16.146.79\",\"id.orig_p\":0,\"id.resp_h\":\"132.16.110.133\",\"id.resp_p\":8080,\"tunnel_type\":\"Tunnel::HTTP\",\"action\":\"Tunnel::DISCOVER\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/tunnel/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/tunnel/elasticsearch/ingest_pipeline/default.yml index d0fd459a5a8f..31dbd151b0c8 100644 --- a/packages/zeek/data_stream/tunnel/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/tunnel/elasticsearch/ingest_pipeline/default.yml @@ -128,6 +128,7 @@ processors: field: zeek.tunnel.ts formats: - UNIX + - ISO8601 - remove: field: zeek.tunnel.ts - geoip: diff --git a/packages/zeek/data_stream/weird/_dev/test/pipeline/test-weird.log-expected.json b/packages/zeek/data_stream/weird/_dev/test/pipeline/test-weird.log-expected.json index 21d729522c9a..c1e4b22191a8 100644 --- a/packages/zeek/data_stream/weird/_dev/test/pipeline/test-weird.log-expected.json +++ b/packages/zeek/data_stream/weird/_dev/test/pipeline/test-weird.log-expected.json @@ -30,7 +30,7 @@ "ip": "192.168.1.1" }, "event": { - "ingested": "2021-06-14T08:33:57.875668400Z", + "ingested": "2021-06-17T13:41:32.637383700Z", "original": "{\"ts\":1543877999.99354,\"uid\":\"C1ralPp062bkwWt4e\",\"id.orig_h\":\"192.168.1.1\",\"id.orig_p\":64521,\"id.resp_h\":\"192.168.1.2\",\"id.resp_p\":53,\"name\":\"dns_unmatched_reply\",\"notice\":false,\"peer\":\"worker-6\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -59,7 +59,7 @@ } }, "event": { - "ingested": "2021-06-14T08:33:57.875673200Z", + "ingested": "2021-06-17T13:41:32.637391Z", "original": "{\"ts\":1580227259.342809,\"name\":\"non_ip_packet_in_ethernet\",\"notice\":false,\"peer\":\"ens3f1-4\"}", "category": [ "network" @@ -115,7 +115,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-06-14T08:33:57.875675Z", + "ingested": "2021-06-17T13:41:32.637396200Z", "original": "{\"ts\":1543877999.99354,\"uid\":\"C1ralPp062bkwWt4e\",\"id.orig_h\":\"192.168.1.1\",\"id.orig_p\":64521,\"id.resp_h\":\"192.168.1.2\",\"id.resp_p\":53,\"name\":\"dns_unmatched_reply\",\"notice\":false,\"peer\":\"worker-6\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/weird/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/weird/elasticsearch/ingest_pipeline/default.yml index 5b3ec02f977b..5529031bcd5f 100644 --- a/packages/zeek/data_stream/weird/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/weird/elasticsearch/ingest_pipeline/default.yml @@ -128,6 +128,7 @@ processors: field: zeek.weird.ts formats: - UNIX + - ISO8601 - remove: field: zeek.weird.ts - geoip: diff --git a/packages/zeek/data_stream/x509/_dev/test/pipeline/test-x509.log-expected.json b/packages/zeek/data_stream/x509/_dev/test/pipeline/test-x509.log-expected.json index c2ce798f737b..045b3913fb65 100644 --- a/packages/zeek/data_stream/x509/_dev/test/pipeline/test-x509.log-expected.json +++ b/packages/zeek/data_stream/x509/_dev/test/pipeline/test-x509.log-expected.json @@ -210,7 +210,7 @@ "session_id": "FxZ6gZ3YR6vFlIocq3" }, "event": { - "ingested": "2021-06-14T08:33:57.958981500Z", + "ingested": "2021-06-17T13:41:32.842999Z", "original": "{\"ts\":1543867200.143484,\"id\":\"FxZ6gZ3YR6vFlIocq3\",\"certificate.version\":3,\"certificate.serial\":\"2D00003299D7071DB7D1708A42000000003299\",\"certificate.subject\":\"CN=www.bing.com\",\"certificate.issuer\":\"CN=Microsoft IT TLS CA 5,OU=Microsoft IT,O=Microsoft Corporation,L=Redmond,ST=Washington,C=US\",\"certificate.not_valid_before\":1500572828.0,\"certificate.not_valid_after\":1562780828.0,\"certificate.key_alg\":\"rsaEncryption\",\"certificate.sig_alg\":\"sha256WithRSAEncryption\",\"certificate.key_type\":\"rsa\",\"certificate.key_length\":2048,\"certificate.exponent\":\"65537\",\"san.dns\":[\"www.bing.com\",\"dict.bing.com.cn\",\"*.platform.bing.com\",\"*.bing.com\",\"bing.com\",\"ieonline.microsoft.com\",\"*.windowssearch.com\",\"cn.ieonline.microsoft.com\",\"*.origin.bing.com\",\"*.mm.bing.net\",\"*.api.bing.com\",\"ecn.dev.virtualearth.net\",\"*.cn.bing.net\",\"*.cn.bing.com\",\"ssl-api.bing.com\",\"ssl-api.bing.net\",\"*.api.bing.net\",\"*.bingapis.com\",\"bingsandbox.com\",\"feedback.microsoft.com\",\"insertmedia.bing.office.net\",\"r.bat.bing.com\",\"*.r.bat.bing.com\",\"*.dict.bing.com.cn\",\"*.dict.bing.com\",\"*.ssl.bing.com\",\"*.appex.bing.com\",\"*.platform.cn.bing.com\",\"wp.m.bing.com\",\"*.m.bing.com\",\"global.bing.com\",\"windowssearch.com\",\"search.msn.com\",\"*.bingsandbox.com\",\"*.api.tiles.ditu.live.com\",\"*.ditu.live.com\",\"*.t0.tiles.ditu.live.com\",\"*.t1.tiles.ditu.live.com\",\"*.t2.tiles.ditu.live.com\",\"*.t3.tiles.ditu.live.com\",\"*.tiles.ditu.live.com\",\"3d.live.com\",\"api.search.live.com\",\"beta.search.live.com\",\"cnweb.search.live.com\",\"dev.live.com\",\"ditu.live.com\",\"farecast.live.com\",\"image.live.com\",\"images.live.com\",\"local.live.com.au\",\"localsearch.live.com\",\"ls4d.search.live.com\",\"mail.live.com\",\"mapindia.live.com\",\"local.live.com\",\"maps.live.com\",\"maps.live.com.au\",\"mindia.live.com\",\"news.live.com\",\"origin.cnweb.search.live.com\",\"preview.local.live.com\",\"search.live.com\",\"test.maps.live.com\",\"video.live.com\",\"videos.live.com\",\"virtualearth.live.com\",\"wap.live.com\",\"webmaster.live.com\",\"webmasters.live.com\",\"www.local.live.com.au\",\"www.maps.live.com.au\"]}", "id": "FxZ6gZ3YR6vFlIocq3", "type": [ @@ -441,7 +441,7 @@ "session_id": "FxZ6gZ3YR6vFlIocq3" }, "event": { - "ingested": "2021-06-14T08:33:57.958987400Z", + "ingested": "2021-06-17T13:41:32.843006900Z", "original": "{\"ts\":1543867200.143484,\"id\":\"FxZ6gZ3YR6vFlIocq3\",\"certificate.version\":3,\"certificate.serial\":\"2D00003299D7071DB7D1708A42000000003299\",\"certificate.subject\":\"CN=www.bing.com\",\"certificate.issuer\":\"CN=Microsoft IT TLS CA 5,OU=Microsoft IT,O=Microsoft Corporation,L=Redmond,ST=Washington,C=US\",\"certificate.not_valid_before\":1500572828.0,\"certificate.not_valid_after\":1562780828.0,\"certificate.key_alg\":\"rsaEncryption\",\"certificate.sig_alg\":\"sha256WithRSAEncryption\",\"certificate.key_type\":\"rsa\",\"certificate.key_length\":2048,\"certificate.exponent\":\"65537\",\"san.dns\":[\"www.bing.com\",\"dict.bing.com.cn\",\"*.platform.bing.com\",\"*.bing.com\",\"bing.com\",\"ieonline.microsoft.com\",\"*.windowssearch.com\",\"cn.ieonline.microsoft.com\",\"*.origin.bing.com\",\"*.mm.bing.net\",\"*.api.bing.com\",\"ecn.dev.virtualearth.net\",\"*.cn.bing.net\",\"*.cn.bing.com\",\"ssl-api.bing.com\",\"ssl-api.bing.net\",\"*.api.bing.net\",\"*.bingapis.com\",\"bingsandbox.com\",\"feedback.microsoft.com\",\"insertmedia.bing.office.net\",\"r.bat.bing.com\",\"*.r.bat.bing.com\",\"*.dict.bing.com.cn\",\"*.dict.bing.com\",\"*.ssl.bing.com\",\"*.appex.bing.com\",\"*.platform.cn.bing.com\",\"wp.m.bing.com\",\"*.m.bing.com\",\"global.bing.com\",\"windowssearch.com\",\"search.msn.com\",\"*.bingsandbox.com\",\"*.api.tiles.ditu.live.com\",\"*.ditu.live.com\",\"*.t0.tiles.ditu.live.com\",\"*.t1.tiles.ditu.live.com\",\"*.t2.tiles.ditu.live.com\",\"*.t3.tiles.ditu.live.com\",\"*.tiles.ditu.live.com\",\"3d.live.com\",\"api.search.live.com\",\"beta.search.live.com\",\"cnweb.search.live.com\",\"dev.live.com\",\"ditu.live.com\",\"farecast.live.com\",\"image.live.com\",\"images.live.com\",\"local.live.com.au\",\"localsearch.live.com\",\"ls4d.search.live.com\",\"mail.live.com\",\"mapindia.live.com\",\"local.live.com\",\"maps.live.com\",\"maps.live.com.au\",\"mindia.live.com\",\"news.live.com\",\"origin.cnweb.search.live.com\",\"preview.local.live.com\",\"search.live.com\",\"test.maps.live.com\",\"video.live.com\",\"videos.live.com\",\"virtualearth.live.com\",\"wap.live.com\",\"webmaster.live.com\",\"webmasters.live.com\",\"www.local.live.com.au\",\"www.maps.live.com.au\"]}", "id": "FxZ6gZ3YR6vFlIocq3", "type": [ diff --git a/packages/zeek/data_stream/x509/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/x509/elasticsearch/ingest_pipeline/default.yml index 69d0ca04e3f9..44508b120953 100644 --- a/packages/zeek/data_stream/x509/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/x509/elasticsearch/ingest_pipeline/default.yml @@ -185,6 +185,7 @@ processors: field: zeek.x509.ts formats: - UNIX + - ISO8601 - remove: field: zeek.x509.ts - set: @@ -303,6 +304,7 @@ processors: target_field: zeek.x509.certificate.valid.from formats: - UNIX + - ISO8601 if: ctx.zeek.x509.certificate?.valid?.from != null - set: field: file.x509.not_before @@ -313,6 +315,7 @@ processors: target_field: zeek.x509.certificate.valid.until formats: - UNIX + - ISO8601 if: ctx.zeek.x509.certificate?.valid?.until != null - set: field: file.x509.not_after diff --git a/packages/zeek/manifest.yml b/packages/zeek/manifest.yml index f0bc2d2908ec..3c37fa8b1949 100644 --- a/packages/zeek/manifest.yml +++ b/packages/zeek/manifest.yml @@ -1,6 +1,6 @@ name: zeek title: Zeek -version: 0.8.0 +version: 0.8.1 release: beta description: Zeek Integration type: integration