diff --git a/packages/zeek/changelog.yml b/packages/zeek/changelog.yml index 19ecfc8d9ea6..34ef81d0ce59 100644 --- a/packages/zeek/changelog.yml +++ b/packages/zeek/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.7.3" + changes: + - description: adding support for ISO8601 timestamps + type: enhancement + link: https://github.com/elastic/integrations/pull/XXX - version: "0.7.2" changes: - description: adding back 0.7.0 changes diff --git a/packages/zeek/data_stream/capture_loss/_dev/test/pipeline/test-capture-loss.log-expected.json b/packages/zeek/data_stream/capture_loss/_dev/test/pipeline/test-capture-loss.log-expected.json index 8cb72b48bc7a..554fded4a42b 100644 --- a/packages/zeek/data_stream/capture_loss/_dev/test/pipeline/test-capture-loss.log-expected.json +++ b/packages/zeek/data_stream/capture_loss/_dev/test/pipeline/test-capture-loss.log-expected.json @@ -15,7 +15,7 @@ } }, "event": { - "ingested": "2021-04-23T19:56:22.333182200Z", + "ingested": "2021-06-10T17:49:23.711919200Z", "type": "info", "created": "2020-04-28T11:07:58.223Z", "kind": "metric" @@ -36,7 +36,7 @@ } }, "event": { - "ingested": "2021-04-23T19:56:22.333229400Z", + "ingested": "2021-06-10T17:49:23.711939800Z", "type": "info", "created": "2020-04-28T11:07:58.223Z", "kind": "metric" @@ -57,7 +57,7 @@ } }, "event": { - "ingested": "2021-04-23T19:56:22.333237400Z", + "ingested": "2021-06-10T17:49:23.711954300Z", "type": "info", "created": "2020-04-28T11:07:58.223Z", "kind": "metric" @@ -78,7 +78,7 @@ } }, "event": { - "ingested": "2021-04-23T19:56:22.333243200Z", + "ingested": "2021-06-10T17:49:23.711965Z", "type": "info", "created": "2020-04-28T11:07:58.223Z", "kind": "metric" @@ -99,7 +99,7 @@ } }, "event": { - "ingested": "2021-04-23T19:56:22.333248500Z", + "ingested": "2021-06-10T17:49:23.711975200Z", "type": "info", "created": "2020-04-28T11:07:58.223Z", "kind": "metric" @@ -128,7 +128,7 @@ } }, "event": { - "ingested": "2021-04-23T19:56:22.333253200Z", + "ingested": "2021-06-10T17:49:23.711985400Z", "original": "{\"ts\":1568132368.465338,\"ts_delta\":32.282249,\"peer\":\"bro\",\"gaps\":0,\"acks\":206,\"percent_lost\":0.0}", "type": "info", "created": "2020-04-28T11:07:58.223Z", diff --git a/packages/zeek/data_stream/capture_loss/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/capture_loss/elasticsearch/ingest_pipeline/default.yml index b09928978e00..850af642851c 100644 --- a/packages/zeek/data_stream/capture_loss/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/capture_loss/elasticsearch/ingest_pipeline/default.yml @@ -53,6 +53,7 @@ processors: field: zeek.capture_loss.ts formats: - UNIX + - ISO8601 - set: field: event.kind value: metric diff --git a/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log b/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log index 4eaf9853b742..b652a4259543 100644 --- a/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log +++ b/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log @@ -14,4 +14,5 @@ {"ts":1617062400.703865,"uid":"C3pPjh1YRYcVDiZD3","id.orig_h":"10.156.0.2","id.orig_p":44944,"id.resp_h":"169.254.169.254","id.resp_p":80,"proto":"tcp","conn_state":"OTH","local_orig":true,"local_resp":false,"missed_bytes":0,"history":"C","orig_pkts":0,"orig_ip_bytes":0,"resp_pkts":0,"resp_ip_bytes":0} {"ts":1617062400.703851,"uid":"ChUxTmYLG37oO5qUb","id.orig_h":"10.156.0.2","id.orig_p":44942,"id.resp_h":"169.254.169.254","id.resp_p":80,"proto":"tcp","conn_state":"OTH","local_orig":true,"local_resp":false,"missed_bytes":0,"history":"C","orig_pkts":0,"orig_ip_bytes":0,"resp_pkts":0,"resp_ip_bytes":0} {"ts":1617062400.704467,"uid":"CpeAOT3B11CTXJgzw2","id.orig_h":"10.156.0.2","id.orig_p":44946,"id.resp_h":"169.254.169.254","id.resp_p":80,"proto":"tcp","conn_state":"OTH","local_orig":true,"local_resp":false,"missed_bytes":0,"history":"C","orig_pkts":0,"orig_ip_bytes":0,"resp_pkts":0,"resp_ip_bytes":0} -{"preview":false,"offset":0,"result":{"_bkt":"main~0~0758E7C3-1D0C-4B2B-8CF0-682BFEA86CDC","_cd":"0:12","_indextime":"1608752616","_raw":"{\"ts\":1547188417.857497,\"uid\":\"CAcJw21BbVedgFnYH5\",\"id.orig_h\":\"4.4.2.2\",\"id.orig_p\":38334,\"id.resp_h\":\"8.8.8.8\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.076967,\"orig_bytes\":75,\"resp_bytes\":178,\"conn_state\":\"SF\",\"local_orig\":false,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Dd\",\"orig_pkts\":1,\"orig_ip_bytes\":103,\"resp_pkts\":1,\"resp_ip_bytes\":206,\"tunnel_parents\":[]}","_serial":"0","_si":["b590508aafed","main"],"_sourcetype":"access_log-too_small","_time":"2020-12-23 19:43:35.000 UTC","host":"Lees-MBP.localdomain","index":"main","linecount":"1","my_max":"1608759317","source":"/usr/local/var/log/httpd/access_log","sourcetype":"access_log-too_small","splunk_server":"b590508aafed"}} \ No newline at end of file +{"preview":false,"offset":0,"result":{"_bkt":"main~0~0758E7C3-1D0C-4B2B-8CF0-682BFEA86CDC","_cd":"0:12","_indextime":"1608752616","_raw":"{\"ts\":1547188417.857497,\"uid\":\"CAcJw21BbVedgFnYH5\",\"id.orig_h\":\"4.4.2.2\",\"id.orig_p\":38334,\"id.resp_h\":\"8.8.8.8\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.076967,\"orig_bytes\":75,\"resp_bytes\":178,\"conn_state\":\"SF\",\"local_orig\":false,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Dd\",\"orig_pkts\":1,\"orig_ip_bytes\":103,\"resp_pkts\":1,\"resp_ip_bytes\":206,\"tunnel_parents\":[]}","_serial":"0","_si":["b590508aafed","main"],"_sourcetype":"access_log-too_small","_time":"2020-12-23 19:43:35.000 UTC","host":"Lees-MBP.localdomain","index":"main","linecount":"1","my_max":"1608759317","source":"/usr/local/var/log/httpd/access_log","sourcetype":"access_log-too_small","splunk_server":"b590508aafed"}} +{"ts":"2021-06-09T20:55:13.160328Z","uid":"C2KP1V3alRLoxl4JB9","id.orig_h":"10.0.2.15","id.orig_p":46408,"id.resp_h":"172.217.9.68","id.resp_p":80,"proto":"tcp","conn_state":"OTH","local_orig":true,"local_resp":false,"missed_bytes":0,"history":"C","orig_pkts":0,"orig_ip_bytes":0,"resp_pkts":0,"resp_ip_bytes":0} \ No newline at end of file diff --git a/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log-expected.json b/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log-expected.json index d7e342d0c762..63b9ffc2d2f7 100644 --- a/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log-expected.json +++ b/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log-expected.json @@ -39,7 +39,7 @@ }, "event": { "duration": 76967000, - "ingested": "2021-04-23T19:56:22.481489300Z", + "ingested": "2021-06-10T17:49:23.978658400Z", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "id": "CAcJw21BbVedgFnYH3", @@ -117,7 +117,7 @@ }, "event": { "duration": 76967000, - "ingested": "2021-04-23T19:56:22.481509800Z", + "ingested": "2021-06-10T17:49:23.978673500Z", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "id": "CAcJw21BbVedgFnYH4", @@ -210,7 +210,7 @@ }, "event": { "duration": 76967000, - "ingested": "2021-04-23T19:56:22.481515200Z", + "ingested": "2021-06-10T17:49:23.978681100Z", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "id": "CAcJw21BbVedgFnYH5", @@ -273,7 +273,7 @@ "ip": "192.0.2.205" }, "event": { - "ingested": "2021-04-23T19:56:22.481522400Z", + "ingested": "2021-06-10T17:49:23.978687600Z", "id": "Cc6NJ3GRlfjE44I3h", "category": "network", "type": [ @@ -351,7 +351,7 @@ "ip": "10.156.0.2" }, "event": { - "ingested": "2021-04-23T19:56:22.481526700Z", + "ingested": "2021-06-10T17:49:23.978693600Z", "id": "CCicIg43lOtCQOxXnb", "category": "network", "type": [ @@ -430,7 +430,7 @@ }, "event": { "duration": 103708982, - "ingested": "2021-04-23T19:56:22.481531Z", + "ingested": "2021-06-10T17:49:23.978698100Z", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "id": "C52mXBCPJ4pPGkhr1", @@ -509,7 +509,7 @@ }, "event": { "duration": 104128838, - "ingested": "2021-04-23T19:56:22.481535Z", + "ingested": "2021-06-10T17:49:23.978702Z", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "id": "CTzCky2CyLT5JJvHck", @@ -588,7 +588,7 @@ }, "event": { "duration": 104333878, - "ingested": "2021-04-23T19:56:22.481550900Z", + "ingested": "2021-06-10T17:49:23.978707500Z", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "id": "CIkS28PDxqQnN49m2", @@ -649,7 +649,7 @@ }, "event": { "duration": 26802063, - "ingested": "2021-04-23T19:56:22.481559200Z", + "ingested": "2021-06-10T17:49:23.978720800Z", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "id": "CezEGe4jeLNkayV976", @@ -711,7 +711,7 @@ }, "event": { "duration": 25056124, - "ingested": "2021-04-23T19:56:22.481564200Z", + "ingested": "2021-06-10T17:49:23.978748500Z", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "id": "CKSr3w18mmW6t7bXC4", @@ -773,7 +773,7 @@ }, "event": { "duration": 3319979, - "ingested": "2021-04-23T19:56:22.481568400Z", + "ingested": "2021-06-10T17:49:23.978755500Z", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "id": "CGUiHy4kLIF2ml95eg", @@ -835,7 +835,7 @@ }, "event": { "duration": 1111984, - "ingested": "2021-04-23T19:56:22.481572700Z", + "ingested": "2021-06-10T17:49:23.978779300Z", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "id": "CAOZZi4Qrio7gUVgVc", @@ -897,7 +897,7 @@ }, "event": { "duration": 908852, - "ingested": "2021-04-23T19:56:22.481576800Z", + "ingested": "2021-06-10T17:49:23.978786500Z", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "id": "Chx5fs3xQ5ALB72i4e", @@ -958,7 +958,7 @@ "ip": "10.156.0.2" }, "event": { - "ingested": "2021-04-23T19:56:22.481581200Z", + "ingested": "2021-06-10T17:49:23.978795400Z", "id": "C3pPjh1YRYcVDiZD3", "category": "network", "type": [ @@ -1018,7 +1018,7 @@ "ip": "10.156.0.2" }, "event": { - "ingested": "2021-04-23T19:56:22.481584800Z", + "ingested": "2021-06-10T17:49:23.978803800Z", "id": "ChUxTmYLG37oO5qUb", "category": "network", "type": [ @@ -1078,7 +1078,7 @@ "ip": "10.156.0.2" }, "event": { - "ingested": "2021-04-23T19:56:22.481588500Z", + "ingested": "2021-06-10T17:49:23.978809700Z", "id": "CpeAOT3B11CTXJgzw2", "category": "network", "type": [ @@ -1189,7 +1189,7 @@ }, "event": { "duration": 76967000, - "ingested": "2021-04-23T19:56:22.481592400Z", + "ingested": "2021-06-10T17:49:23.978819200Z", "original": "{\"ts\":1547188417.857497,\"uid\":\"CAcJw21BbVedgFnYH5\",\"id.orig_h\":\"4.4.2.2\",\"id.orig_p\":38334,\"id.resp_h\":\"8.8.8.8\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.076967,\"orig_bytes\":75,\"resp_bytes\":178,\"conn_state\":\"SF\",\"local_orig\":false,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Dd\",\"orig_pkts\":1,\"orig_ip_bytes\":103,\"resp_pkts\":1,\"resp_ip_bytes\":206,\"tunnel_parents\":[]}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -1201,6 +1201,81 @@ "end" ] } + }, + { + "@timestamp": "2021-06-09T20:55:13.160Z", + "ecs": { + "version": "1.9.0" + }, + "related": { + "ip": [ + "10.0.2.15", + "172.217.9.68" + ] + }, + "destination": { + "geo": { + "continent_name": "North America", + "country_name": "United States", + "location": { + "lon": -97.822, + "lat": 37.751 + }, + "country_iso_code": "US" + }, + "as": { + "number": 15169, + "organization": { + "name": "Google LLC" + } + }, + "address": "172.217.9.68", + "port": 80, + "bytes": 0, + "ip": "172.217.9.68", + "packets": 0 + }, + "zeek": { + "session_id": "C2KP1V3alRLoxl4JB9", + "connection": { + "local_resp": false, + "local_orig": true, + "missed_bytes": 0, + "history": "C", + "id": {}, + "state": "OTH", + "state_message": "No SYN seen, just midstream traffic (a 'partial connection' that was not later closed)." + } + }, + "source": { + "address": "10.0.2.15", + "port": 46408, + "bytes": 0, + "packets": 0, + "ip": "10.0.2.15" + }, + "event": { + "ingested": "2021-06-10T17:49:23.978828700Z", + "id": "C2KP1V3alRLoxl4JB9", + "category": "network", + "type": [ + "connection", + "info" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event" + }, + "network": { + "community_id": "1:DzqI9CYXjMSYV8VoSAHtMNfMIeU=", + "transport": "tcp", + "bytes": 0, + "packets": 0, + "direction": "outbound" + }, + "tags": [ + "local_orig", + "local_resp" + ] } ] } \ No newline at end of file diff --git a/packages/zeek/data_stream/connection/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/connection/elasticsearch/ingest_pipeline/default.yml index abeac41bf3ca..0df7fd917f3f 100644 --- a/packages/zeek/data_stream/connection/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/connection/elasticsearch/ingest_pipeline/default.yml @@ -159,6 +159,7 @@ processors: field: zeek.connection.ts formats: - UNIX + - ISO8601 - remove: field: zeek.connection.ts - set: diff --git a/packages/zeek/data_stream/dce_rpc/_dev/test/pipeline/test-dce-rpc.log-expected.json b/packages/zeek/data_stream/dce_rpc/_dev/test/pipeline/test-dce-rpc.log-expected.json index 7f811dd1f82f..b90b9eff7cef 100644 --- a/packages/zeek/data_stream/dce_rpc/_dev/test/pipeline/test-dce-rpc.log-expected.json +++ b/packages/zeek/data_stream/dce_rpc/_dev/test/pipeline/test-dce-rpc.log-expected.json @@ -31,7 +31,7 @@ "ip": "172.16.133.6" }, "event": { - "ingested": "2021-04-23T19:56:23.054640400Z", + "ingested": "2021-06-10T17:49:25.085274900Z", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "action": "BrowserrQueryOtherDomains", @@ -95,7 +95,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-04-23T19:56:23.054658400Z", + "ingested": "2021-06-10T17:49:25.085291200Z", "original": "{\"ts\":1361916332.298338,\"uid\":\"CsNHVHa1lzFtvJzT8\",\"id.orig_h\":\"172.16.133.6\",\"id.orig_p\":1728,\"id.resp_h\":\"172.16.128.202\",\"id.resp_p\":445,\"rtt\":0.09211,\"named_pipe\":\"\\u005cPIPE\\u005cbrowser\",\"endpoint\":\"browser\",\"operation\":\"BrowserrQueryOtherDomains\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/dce_rpc/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/dce_rpc/elasticsearch/ingest_pipeline/default.yml index 44572c149c70..f4d561ab74fb 100644 --- a/packages/zeek/data_stream/dce_rpc/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/dce_rpc/elasticsearch/ingest_pipeline/default.yml @@ -122,6 +122,7 @@ processors: field: zeek.dce_rpc.ts formats: - UNIX + - ISO8601 - remove: field: zeek.dce_rpc.ts - append: diff --git a/packages/zeek/data_stream/dhcp/_dev/test/pipeline/test-dhcp.log-expected.json b/packages/zeek/data_stream/dhcp/_dev/test/pipeline/test-dhcp.log-expected.json index 202f7e118e18..687059388792 100644 --- a/packages/zeek/data_stream/dhcp/_dev/test/pipeline/test-dhcp.log-expected.json +++ b/packages/zeek/data_stream/dhcp/_dev/test/pipeline/test-dhcp.log-expected.json @@ -60,7 +60,7 @@ "address": "192.168.199.132" }, "event": { - "ingested": "2021-04-23T19:56:23.150661700Z", + "ingested": "2021-06-10T17:49:25.273076500Z", "id": [ "CmWOt6VWaNGqXYcH6", "CLObLo4YHn0u23Tp8a" @@ -132,7 +132,7 @@ "address": "10.156.0.2" }, "event": { - "ingested": "2021-04-23T19:56:23.150679900Z", + "ingested": "2021-06-10T17:49:25.273091700Z", "id": [ "Ck0tsG4wsJxI3lIEZ" ], @@ -216,7 +216,7 @@ "address": "192.168.199.132" }, "event": { - "ingested": "2021-04-23T19:56:23.150685Z", + "ingested": "2021-06-10T17:49:25.273101100Z", "original": "{\"ts\":1476605498.771847,\"uids\":[\"CmWOt6VWaNGqXYcH6\",\"CLObLo4YHn0u23Tp8a\"],\"client_addr\":\"192.168.199.132\",\"server_addr\":\"192.168.199.254\",\"mac\":\"00:0c:29:03:df:ad\",\"host_name\":\"DESKTOP-2AEFM7G\",\"client_fqdn\":\"DESKTOP-2AEFM7G\",\"domain\":\"localdomain\",\"requested_addr\":\"192.168.199.132\",\"assigned_addr\":\"192.168.199.132\",\"lease_time\":1800.0,\"msg_types\":[\"REQUEST\",\"ACK\"],\"duration\":0.000161}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/dhcp/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/dhcp/elasticsearch/ingest_pipeline/default.yml index 2040ba988228..21fd785ca212 100644 --- a/packages/zeek/data_stream/dhcp/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/dhcp/elasticsearch/ingest_pipeline/default.yml @@ -182,6 +182,7 @@ processors: field: zeek.dhcp.ts formats: - UNIX + - ISO8601 - remove: field: zeek.dhcp.ts - set: diff --git a/packages/zeek/data_stream/dnp3/_dev/test/pipeline/test-dnp3.log-expected.json b/packages/zeek/data_stream/dnp3/_dev/test/pipeline/test-dnp3.log-expected.json index 8a0d8a3d6ba3..12485cf8cf8c 100644 --- a/packages/zeek/data_stream/dnp3/_dev/test/pipeline/test-dnp3.log-expected.json +++ b/packages/zeek/data_stream/dnp3/_dev/test/pipeline/test-dnp3.log-expected.json @@ -29,7 +29,7 @@ "ip": "127.0.0.1" }, "event": { - "ingested": "2021-04-23T19:56:23.292654100Z", + "ingested": "2021-06-10T17:49:25.488211500Z", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "action": "read", @@ -91,7 +91,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-04-23T19:56:23.292667900Z", + "ingested": "2021-06-10T17:49:25.488226200Z", "original": "{\"ts\":1227729908.705944,\"uid\":\"CQV6tj1w1t4WzQpHoe\",\"id.orig_h\":\"127.0.0.1\",\"id.orig_p\":42942,\"id.resp_h\":\"127.0.0.1\",\"id.resp_p\":20000,\"fc_request\":\"READ\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/dnp3/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/dnp3/elasticsearch/ingest_pipeline/default.yml index fee837ebb9db..b54db973c129 100644 --- a/packages/zeek/data_stream/dnp3/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/dnp3/elasticsearch/ingest_pipeline/default.yml @@ -134,6 +134,7 @@ processors: field: zeek.dnp3.ts formats: - UNIX + - ISO8601 - remove: field: zeek.dnp3.ts - set: diff --git a/packages/zeek/data_stream/dns/_dev/test/pipeline/test-dns.log-expected.json b/packages/zeek/data_stream/dns/_dev/test/pipeline/test-dns.log-expected.json index eef13847171e..e7df8266c08b 100644 --- a/packages/zeek/data_stream/dns/_dev/test/pipeline/test-dns.log-expected.json +++ b/packages/zeek/data_stream/dns/_dev/test/pipeline/test-dns.log-expected.json @@ -86,7 +86,7 @@ }, "event": { "duration": 7.6967E7, - "ingested": "2021-04-23T19:56:23.442358600Z", + "ingested": "2021-06-10T17:49:25.835320400Z", "original": "{\"ts\":1547188415.857497,\"uid\":\"CAcJw21BbVedgFnYH3\",\"id.orig_h\":\"192.168.86.167\",\"id.orig_p\":38339,\"id.resp_h\":\"192.168.86.1\",\"id.resp_p\":53,\"proto\":\"udp\",\"trans_id\":15209,\"rtt\":0.076967,\"query\":\"dd625ffb4fc54735b281862aa1cd6cd4.us-west1.gcp.cloud.es.io\",\"qclass\":1,\"qclass_name\":\"C_INTERNET\",\"qtype\":1,\"qtype_name\":\"A\",\"rcode\":0,\"rcode_name\":\"NOERROR\",\"AA\":false,\"TC\":false,\"RD\":true,\"RA\":true,\"Z\":0,\"answers\":[\"proxy-production-us-west1.gcp.cloud.es.io\",\"proxy-production-us-west1-v1-009.gcp.cloud.es.io\",\"35.199.178.4\"],\"TTLs\":[119.0,119.0,59.0],\"rejected\":false}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -153,7 +153,7 @@ "ip": "fe80::4ef:15cf:769f:ff21" }, "event": { - "ingested": "2021-04-23T19:56:23.442375900Z", + "ingested": "2021-06-10T17:49:25.835331900Z", "original": "{\"ts\":1567095830.680046,\"uid\":\"C19a1k4lTv46YMbeOk\",\"id.orig_h\":\"fe80::4ef:15cf:769f:ff21\",\"id.orig_p\":5353,\"id.resp_h\":\"ff02::fb\",\"id.resp_p\":5353,\"proto\":\"udp\",\"trans_id\":0,\"query\":\"_googlecast._tcp.local\",\"qclass\":1,\"qclass_name\":\"C_INTERNET\",\"qtype\":12,\"qtype_name\":\"PTR\",\"AA\":false,\"TC\":false,\"RD\":false,\"RA\":false,\"Z\":0,\"rejected\":false}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -231,7 +231,7 @@ "ip": "192.168.86.237" }, "event": { - "ingested": "2021-04-23T19:56:23.442399400Z", + "ingested": "2021-06-10T17:49:25.835338900Z", "original": "{\"ts\":1567095830.734329,\"uid\":\"CdiVAw7jJw6gsX5H\",\"id.orig_h\":\"192.168.86.237\",\"id.orig_p\":5353,\"id.resp_h\":\"224.0.0.251\",\"id.resp_p\":5353,\"proto\":\"udp\",\"trans_id\":0,\"query\":\"_googlecast._tcp.local\",\"rcode\":0,\"rcode_name\":\"NOERROR\",\"AA\":true,\"TC\":false,\"RD\":false,\"RA\":false,\"Z\":0,\"answers\":[\"bravia-4k-gb-5c89f865c9d569ab338815b35e3acc56._googlecast._tcp.local\"],\"TTLs\":[120.0],\"rejected\":false}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -329,7 +329,7 @@ "ip": "10.156.0.2" }, "event": { - "ingested": "2021-04-23T19:56:23.442407200Z", + "ingested": "2021-06-10T17:49:25.835348700Z", "original": "{\"ts\":1617105592.091052,\"uid\":\"CpwXdW4LQaJkaIgpk\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":33438,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":53,\"proto\":\"udp\",\"trans_id\":58036,\"query\":\"manage.office.com\",\"rcode\":0,\"rcode_name\":\"NOERROR\",\"AA\":false,\"TC\":false,\"RD\":false,\"RA\":true,\"Z\":0,\"answers\":[\"manage.office.com.trafficmanager.net\",\"o365adtapiproddeu001.cloudapp.net\",\"51.116.158.62\"],\"TTLs\":[13.0,18.0,8.0],\"rejected\":false}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -425,7 +425,7 @@ "ip": "10.156.0.2" }, "event": { - "ingested": "2021-04-23T19:56:23.442412Z", + "ingested": "2021-06-10T17:49:25.835405800Z", "original": "{\"ts\":1617105592.973919,\"uid\":\"CO5TE748RoJEZuOThl\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":60444,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":53,\"proto\":\"udp\",\"trans_id\":35744,\"query\":\"login.microsoftonline.com\",\"rcode\":0,\"rcode_name\":\"NOERROR\",\"AA\":false,\"TC\":false,\"RD\":false,\"RA\":true,\"Z\":0,\"answers\":[\"a.privatelink.msidentity.com\",\"prda.aadg.msidentity.com\",\"www.tm.a.prd.aadg.akadns.net\"],\"TTLs\":[296.0,287.0,287.0],\"rejected\":false}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -578,7 +578,7 @@ "ip": "10.156.0.2" }, "event": { - "ingested": "2021-04-23T19:56:23.442415600Z", + "ingested": "2021-06-10T17:49:25.835416900Z", "original": "{\"ts\":1617105592.9742,\"uid\":\"CG1jsmeHcBCGnWXmk\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":44310,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":53,\"proto\":\"udp\",\"trans_id\":58458,\"query\":\"login.microsoftonline.com\",\"rcode\":0,\"rcode_name\":\"NOERROR\",\"AA\":false,\"TC\":false,\"RD\":false,\"RA\":true,\"Z\":0,\"answers\":[\"a.privatelink.msidentity.com\",\"prda.aadg.msidentity.com\",\"www.tm.a.prd.aadg.trafficmanager.net\",\"20.190.159.132\",\"40.126.31.143\",\"20.190.159.134\",\"40.126.31.1\",\"20.190.159.136\",\"40.126.31.135\",\"40.126.31.6\",\"20.190.159.138\"],\"TTLs\":[299.0,214.0,243.0,243.0,243.0,243.0,243.0,243.0,243.0,243.0,243.0],\"rejected\":false}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -676,7 +676,7 @@ "ip": "10.156.0.2" }, "event": { - "ingested": "2021-04-23T19:56:23.442421100Z", + "ingested": "2021-06-10T17:49:25.835426900Z", "original": "{\"ts\":1617105593.106256,\"uid\":\"ChP0cl4j5mbXSZ9TGf\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":36364,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":53,\"proto\":\"udp\",\"trans_id\":8791,\"query\":\"manage.office.com\",\"rcode\":0,\"rcode_name\":\"NOERROR\",\"AA\":false,\"TC\":false,\"RD\":false,\"RA\":true,\"Z\":0,\"answers\":[\"manage.office.com.trafficmanager.net\",\"o365adtapiproddeu001.cloudapp.net\",\"51.116.158.62\"],\"TTLs\":[12.0,17.0,7.0],\"rejected\":false}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -767,7 +767,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-04-23T19:56:23.442424900Z", + "ingested": "2021-06-10T17:49:25.835436300Z", "original": "{\"ts\":1567095830.734329,\"uid\":\"CdiVAw7jJw6gsX5H\",\"id.orig_h\":\"192.168.86.237\",\"id.orig_p\":5353,\"id.resp_h\":\"224.0.0.251\",\"id.resp_p\":5353,\"proto\":\"udp\",\"trans_id\":0,\"query\":\"_googlecast._tcp.local\",\"rcode\":0,\"rcode_name\":\"NOERROR\",\"AA\":true,\"TC\":false,\"RD\":false,\"RA\":false,\"Z\":0,\"answers\":[\"bravia-4k-gb-5c89f865c9d569ab338815b35e3acc56._googlecast._tcp.local\"],\"TTLs\":[120.0],\"rejected\":false}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/dns/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/dns/elasticsearch/ingest_pipeline/default.yml index b08ade877eae..05da856c3648 100644 --- a/packages/zeek/data_stream/dns/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/dns/elasticsearch/ingest_pipeline/default.yml @@ -243,6 +243,7 @@ processors: field: zeek.dns.ts formats: - UNIX + - ISO8601 - remove: field: zeek.dns.ts diff --git a/packages/zeek/data_stream/dpd/_dev/test/pipeline/test-dpd.log-expected.json b/packages/zeek/data_stream/dpd/_dev/test/pipeline/test-dpd.log-expected.json index 6a64634b8087..f0e12e3f3559 100644 --- a/packages/zeek/data_stream/dpd/_dev/test/pipeline/test-dpd.log-expected.json +++ b/packages/zeek/data_stream/dpd/_dev/test/pipeline/test-dpd.log-expected.json @@ -29,7 +29,7 @@ "ip": "192.168.10.31" }, "event": { - "ingested": "2021-04-23T19:56:23.750033600Z", + "ingested": "2021-06-10T17:49:26.333084300Z", "id": "CRrT7S1ccw9H6hzCR", "category": [ "network" @@ -87,7 +87,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-04-23T19:56:23.750047800Z", + "ingested": "2021-06-10T17:49:26.333099800Z", "original": "{\"ts\":1507567500.423033,\"uid\":\"CRrT7S1ccw9H6hzCR\",\"id.orig_h\":\"192.168.10.31\",\"id.orig_p\":49285,\"id.resp_h\":\"192.168.10.10\",\"id.resp_p\":445,\"proto\":\"tcp\",\"analyzer\":\"DCE_RPC\",\"failure_reason\":\"Binpac exception: binpac exception: \\u0026enforce violation : DCE_RPC_Header:rpc_vers\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/dpd/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/dpd/elasticsearch/ingest_pipeline/default.yml index aafe4356c67e..8bd93b4d947d 100644 --- a/packages/zeek/data_stream/dpd/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/dpd/elasticsearch/ingest_pipeline/default.yml @@ -117,6 +117,7 @@ processors: field: zeek.dpd.ts formats: - UNIX + - ISO8601 - remove: field: zeek.dpd.ts - geoip: diff --git a/packages/zeek/data_stream/files/_dev/test/pipeline/test-files.log-expected.json b/packages/zeek/data_stream/files/_dev/test/pipeline/test-files.log-expected.json index 53538a9879c6..df8194931ed3 100644 --- a/packages/zeek/data_stream/files/_dev/test/pipeline/test-files.log-expected.json +++ b/packages/zeek/data_stream/files/_dev/test/pipeline/test-files.log-expected.json @@ -57,7 +57,7 @@ "ip": "10.178.98.102" }, "event": { - "ingested": "2021-04-23T19:56:23.863999500Z", + "ingested": "2021-06-10T17:49:26.521194800Z", "id": "C8I0zn3r9EPbfLgta6", "category": [ "file" @@ -126,7 +126,7 @@ "ip": "10.178.98.102" }, "event": { - "ingested": "2021-04-23T19:56:23.864014900Z", + "ingested": "2021-06-10T17:49:26.521212500Z", "id": "C6sjVo23iNApLnlAt6", "category": [ "file" @@ -195,7 +195,7 @@ "ip": "10.178.98.102" }, "event": { - "ingested": "2021-04-23T19:56:23.864019700Z", + "ingested": "2021-06-10T17:49:26.521222Z", "id": "C6sjVo23iNApLnlAt6", "category": [ "file" @@ -268,7 +268,7 @@ "ip": "10.156.0.2" }, "event": { - "ingested": "2021-04-23T19:56:23.864034100Z", + "ingested": "2021-06-10T17:49:26.521230700Z", "id": "ClG5ErV7bkgKgOaV", "category": [ "file" @@ -341,7 +341,7 @@ "ip": "10.156.0.2" }, "event": { - "ingested": "2021-04-23T19:56:23.864044100Z", + "ingested": "2021-06-10T17:49:26.521239300Z", "id": "CaB3fq3yLrKCbYLqr4", "category": [ "file" @@ -414,7 +414,7 @@ "ip": "10.156.0.2" }, "event": { - "ingested": "2021-04-23T19:56:23.864048800Z", + "ingested": "2021-06-10T17:49:26.521248Z", "id": "C0vhl91PPOI7LbrPZ8", "category": [ "file" @@ -483,7 +483,7 @@ "ip": "10.156.0.2" }, "event": { - "ingested": "2021-04-23T19:56:23.864052400Z", + "ingested": "2021-06-10T17:49:26.521256700Z", "id": "CgbPEj2jf5Ca7Lw0x2", "category": [ "file" @@ -556,7 +556,7 @@ "ip": "10.156.0.2" }, "event": { - "ingested": "2021-04-23T19:56:23.864055400Z", + "ingested": "2021-06-10T17:49:26.521264400Z", "id": "C0vua63rzjtLaiefyj", "category": [ "file" @@ -633,7 +633,7 @@ "ip": "10.178.98.102" }, "event": { - "ingested": "2021-04-23T19:56:23.864058100Z", + "ingested": "2021-06-10T17:49:26.521270400Z", "original": "{\"ts\":1547688801.566262,\"fuid\":\"F9ip9a3MDAq3XLBOn2\",\"tx_hosts\":[\"17.134.127.250\"],\"rx_hosts\":[\"10.178.98.102\"],\"conn_uids\":[\"C6sjVo23iNApLnlAt6\"],\"source\":\"SSL\",\"depth\":0,\"analyzers\":[\"X509\",\"MD5\",\"SHA1\"],\"mime_type\":\"application/pkix-cert\",\"duration\":0.0,\"local_orig\":false,\"is_orig\":false,\"seen_bytes\":1092,\"missing_bytes\":0,\"overflow_bytes\":0,\"timedout\":false,\"md5\":\"48f0e38385112eeca5fc9ffd402eaecd\",\"sha1\":\"8e8321ca08b08e3726fe1d82996884eeb5f0d655\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/files/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/files/elasticsearch/ingest_pipeline/default.yml index 8ba137682e50..473477cab243 100644 --- a/packages/zeek/data_stream/files/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/files/elasticsearch/ingest_pipeline/default.yml @@ -90,6 +90,7 @@ processors: field: zeek.files.ts formats: - UNIX + - ISO8601 - remove: field: zeek.files.ts - script: diff --git a/packages/zeek/data_stream/ftp/_dev/test/pipeline/test-ftp.log-expected.json b/packages/zeek/data_stream/ftp/_dev/test/pipeline/test-ftp.log-expected.json index 269e03042bd5..5276d84a47eb 100644 --- a/packages/zeek/data_stream/ftp/_dev/test/pipeline/test-ftp.log-expected.json +++ b/packages/zeek/data_stream/ftp/_dev/test/pipeline/test-ftp.log-expected.json @@ -43,7 +43,7 @@ "ip": "192.168.1.182" }, "event": { - "ingested": "2021-04-23T19:56:24.111927700Z", + "ingested": "2021-06-10T17:49:26.909467600Z", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "action": "EPSV", @@ -112,7 +112,7 @@ ] }, "event": { - "ingested": "2021-04-23T19:56:24.111942800Z", + "ingested": "2021-06-10T17:49:26.909484400Z", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "action": "RETR", @@ -168,7 +168,7 @@ "ip": "192.168.1.182" }, "event": { - "ingested": "2021-04-23T19:56:24.111947300Z", + "ingested": "2021-06-10T17:49:26.909495Z", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "action": "STOR", @@ -242,7 +242,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-04-23T19:56:24.111950500Z", + "ingested": "2021-06-10T17:49:26.909509200Z", "original": "{\"ts\":1187379117.579203,\"uid\":\"CpQoCn3o28tke89zv9\",\"id.orig_h\":\"192.168.1.182\",\"id.orig_p\":62014,\"id.resp_h\":\"192.168.1.231\",\"id.resp_p\":21,\"user\":\"ftp\",\"password\":\"ftp\",\"command\":\"STOR\",\"arg\":\"ftp://192.168.1.231/uploads/README\",\"reply_code\":226,\"reply_msg\":\"Transfer complete.\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/ftp/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/ftp/elasticsearch/ingest_pipeline/default.yml index b9334969a5d0..ce4e9570437b 100644 --- a/packages/zeek/data_stream/ftp/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/ftp/elasticsearch/ingest_pipeline/default.yml @@ -182,6 +182,7 @@ processors: field: zeek.ftp.ts formats: - UNIX + - ISO8601 - remove: field: zeek.ftp.ts - dot_expander: diff --git a/packages/zeek/data_stream/http/_dev/test/pipeline/test-http.log-expected.json b/packages/zeek/data_stream/http/_dev/test/pipeline/test-http.log-expected.json index 16ca364fe765..ec82cb8de9f2 100644 --- a/packages/zeek/data_stream/http/_dev/test/pipeline/test-http.log-expected.json +++ b/packages/zeek/data_stream/http/_dev/test/pipeline/test-http.log-expected.json @@ -82,7 +82,7 @@ } }, "event": { - "ingested": "2021-04-23T19:56:24.247391800Z", + "ingested": "2021-06-10T17:49:27.174325300Z", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "action": "get", @@ -187,7 +187,7 @@ } }, "event": { - "ingested": "2021-04-23T19:56:24.247408200Z", + "ingested": "2021-06-10T17:49:27.174335800Z", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "action": "get", @@ -277,7 +277,7 @@ "ip": "10.156.0.2" }, "event": { - "ingested": "2021-04-23T19:56:24.247412500Z", + "ingested": "2021-06-10T17:49:27.174342700Z", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "id": "CdqHhA1AsxBIjmVZ9", @@ -365,7 +365,7 @@ "ip": "10.156.0.2" }, "event": { - "ingested": "2021-04-23T19:56:24.247415800Z", + "ingested": "2021-06-10T17:49:27.174349900Z", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "id": "CxhRTwkHNRsHxBw34", @@ -450,7 +450,7 @@ "ip": "10.156.0.2" }, "event": { - "ingested": "2021-04-23T19:56:24.247418600Z", + "ingested": "2021-06-10T17:49:27.174356700Z", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "id": "CrI5Xg30caNXnNvEse", @@ -535,7 +535,7 @@ "ip": "10.156.0.2" }, "event": { - "ingested": "2021-04-23T19:56:24.247422Z", + "ingested": "2021-06-10T17:49:27.174363800Z", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "id": "C6oCGd24yB2dZ7y7z7", @@ -620,7 +620,7 @@ "ip": "10.156.0.2" }, "event": { - "ingested": "2021-04-23T19:56:24.247424800Z", + "ingested": "2021-06-10T17:49:27.174462Z", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "id": "C7DWRE1zsvxUK9RyW1", @@ -726,7 +726,7 @@ } }, "event": { - "ingested": "2021-04-23T19:56:24.247427600Z", + "ingested": "2021-06-10T17:49:27.174486Z", "original": "{\"ts\":1547707019.757479,\"uid\":\"CMnIaR2V8VXyu7EPs\",\"id.orig_h\":\"10.20.8.197\",\"id.orig_p\":35684,\"id.resp_h\":\"34.206.130.40\",\"id.resp_p\":80,\"trans_depth\":1,\"method\":\"GET\",\"host\":\"httpbin.org\",\"uri\":\"/ip\",\"version\":\"1.1\",\"user_agent\":\"curl/7.58.0\",\"request_body_len\":0,\"response_body_len\":32,\"status_code\":200,\"status_msg\":\"OK\",\"tags\":[],\"resp_fuids\":[\"FwGPlr1GcKUWWdkXoi\"],\"resp_mime_types\":[\"text/json\"]}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/http/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/http/elasticsearch/ingest_pipeline/default.yml index 471e756bf69b..ea08d53a3aa3 100644 --- a/packages/zeek/data_stream/http/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/http/elasticsearch/ingest_pipeline/default.yml @@ -182,6 +182,7 @@ processors: field: zeek.http.ts formats: - UNIX + - ISO8601 - remove: field: zeek.http.ts - geoip: diff --git a/packages/zeek/data_stream/intel/_dev/test/pipeline/test-intel.log-expected.json b/packages/zeek/data_stream/intel/_dev/test/pipeline/test-intel.log-expected.json index 6e04d5df7d4e..34a0bf747dfb 100644 --- a/packages/zeek/data_stream/intel/_dev/test/pipeline/test-intel.log-expected.json +++ b/packages/zeek/data_stream/intel/_dev/test/pipeline/test-intel.log-expected.json @@ -54,7 +54,7 @@ "ip": "192.168.1.1" }, "event": { - "ingested": "2021-04-23T19:56:24.549488500Z", + "ingested": "2021-06-10T17:49:27.629199900Z", "original": "{\"ts\":1573030980.989353,\"uid\":\"Ctefoj1tgOPt4D0EK2\",\"id.orig_h\":\"192.168.1.1\",\"id.orig_p\":37598,\"id.resp_h\":\"198.41.0.4\",\"id.resp_p\":53,\"seen.indicator\":\"198.41.0.4\",\"seen.indicator_type\":\"Intel::ADDR\",\"seen.where\":\"Conn::IN_RESP\",\"seen.node\":\"worker-1-2\",\"matched\":[\"Intel::ADDR\"],\"sources\":[\"ETPRO Rep: AbusedTLD Score: 127\"]}", "created": "2020-04-28T11:07:58.223Z", "kind": "enrichment", @@ -129,7 +129,7 @@ "ip": "192.168.1.1" }, "event": { - "ingested": "2021-04-23T19:56:24.549496200Z", + "ingested": "2021-06-10T17:49:27.629215200Z", "original": "{\"ts\":1573030980.989353,\"uid\":\"Ctefoj1tgOPt4D0EK2\",\"id.orig_h\":\"192.168.1.1\",\"id.orig_p\":37598,\"id.resp_h\":\"198.41.0.4\",\"id.resp_p\":53,\"seen.indicator\":\"198.41.0.4\",\"seen.indicator_type\":\"Intel::ADDR\",\"seen.where\":\"Conn::IN_RESP\",\"seen.node\":\"worker-1-2\",\"matched\":[\"Intel::ADDR\"],\"sources\":[\"ETPRO Rep: AbusedTLD Score: 127\"]}", "created": "2020-04-28T11:07:58.223Z", "kind": "enrichment", diff --git a/packages/zeek/data_stream/intel/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/intel/elasticsearch/ingest_pipeline/default.yml index f7512534f3e6..4175d15fac73 100644 --- a/packages/zeek/data_stream/intel/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/intel/elasticsearch/ingest_pipeline/default.yml @@ -146,6 +146,7 @@ processors: field: zeek.intel.ts formats: - UNIX + - ISO8601 - remove: field: zeek.intel.ts # IP Geolocation Lookup diff --git a/packages/zeek/data_stream/irc/_dev/test/pipeline/test-irc.log-expected.json b/packages/zeek/data_stream/irc/_dev/test/pipeline/test-irc.log-expected.json index 6a8860f05740..741105e4fdfb 100644 --- a/packages/zeek/data_stream/irc/_dev/test/pipeline/test-irc.log-expected.json +++ b/packages/zeek/data_stream/irc/_dev/test/pipeline/test-irc.log-expected.json @@ -45,7 +45,7 @@ "ip": "10.180.156.249" }, "event": { - "ingested": "2021-04-23T19:56:24.648599600Z", + "ingested": "2021-06-10T17:49:27.803787900Z", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "action": "USER", @@ -113,7 +113,7 @@ "ip": "10.180.156.249" }, "event": { - "ingested": "2021-04-23T19:56:24.648608300Z", + "ingested": "2021-06-10T17:49:27.803798100Z", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "action": "NICK", @@ -185,7 +185,7 @@ "ip": "10.180.156.249" }, "event": { - "ingested": "2021-04-23T19:56:24.648611Z", + "ingested": "2021-06-10T17:49:27.803805Z", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "action": "JOIN", @@ -270,7 +270,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-04-23T19:56:24.648613100Z", + "ingested": "2021-06-10T17:49:27.803814700Z", "original": "{\"ts\":1387554250.706387,\"uid\":\"CNJBX5FQdL62VUUP1\",\"id.orig_h\":\"10.180.156.249\",\"id.orig_p\":45921,\"id.resp_h\":\"38.229.70.20\",\"id.resp_p\":8000,\"nick\":\"molochtest\",\"user\":\"xxxxx\",\"command\":\"JOIN\",\"value\":\"#moloch-fpc\",\"addl\":\" with channel key: \\u0027-\\u0027\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/irc/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/irc/elasticsearch/ingest_pipeline/default.yml index 2c54a34a138e..31a61f118684 100644 --- a/packages/zeek/data_stream/irc/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/irc/elasticsearch/ingest_pipeline/default.yml @@ -142,6 +142,7 @@ processors: field: zeek.irc.ts formats: - UNIX + - ISO8601 - remove: field: zeek.irc.ts - append: diff --git a/packages/zeek/data_stream/kerberos/_dev/test/pipeline/test-kerberos.log-expected.json b/packages/zeek/data_stream/kerberos/_dev/test/pipeline/test-kerberos.log-expected.json index 738f8949ae2c..92b717ee7da3 100644 --- a/packages/zeek/data_stream/kerberos/_dev/test/pipeline/test-kerberos.log-expected.json +++ b/packages/zeek/data_stream/kerberos/_dev/test/pipeline/test-kerberos.log-expected.json @@ -83,7 +83,7 @@ } }, "event": { - "ingested": "2021-04-23T19:56:24.803165600Z", + "ingested": "2021-06-10T17:49:28.128230900Z", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "action": "TGS", @@ -194,7 +194,7 @@ } }, "event": { - "ingested": "2021-04-23T19:56:24.803185700Z", + "ingested": "2021-06-10T17:49:28.128242300Z", "original": "{\"ts\":1507565599.590346,\"uid\":\"C56Flhb4WQBNkfMOl\",\"id.orig_h\":\"192.168.10.31\",\"id.orig_p\":49242,\"id.resp_h\":\"192.168.10.10\",\"id.resp_p\":88,\"request_type\":\"TGS\",\"client\":\"RonHD/CONTOSO.LOCAL\",\"service\":\"HOST/admin-pc\",\"success\":true,\"till\":2136422885.0,\"cipher\":\"aes256-cts-hmac-sha1-96\",\"forwardable\":true,\"renewable\":true,\"cert.client_subject\":\"CN=*.gcp.cloud.es.io,O=Elasticsearch\\u005c, Inc.,L=Mountain View,ST=California,C=US\",\"cert.server_subject\":\"CN=*.gcp.cloud.es.io,O=Elasticsearch\\u005c, Inc.,L=Mountain View,ST=California,C=US\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/kerberos/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/kerberos/elasticsearch/ingest_pipeline/default.yml index fcd8c533700b..725fdd9b9ea9 100644 --- a/packages/zeek/data_stream/kerberos/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/kerberos/elasticsearch/ingest_pipeline/default.yml @@ -211,6 +211,7 @@ processors: field: zeek.kerberos.ts formats: - UNIX + - ISO8601 - remove: field: zeek.kerberos.ts - script: @@ -221,12 +222,14 @@ processors: target_field: zeek.kerberos.valid.until formats: - UNIX + - ISO8601 if: ctx.zeek.kerberos.valid?.until != null - date: field: zeek.kerberos.valid.from target_field: zeek.kerberos.valid.from formats: - UNIX + - ISO8601 if: ctx.zeek.kerberos.valid?.from != null - set: field: event.outcome diff --git a/packages/zeek/data_stream/modbus/_dev/test/pipeline/test-modbus.log-expected.json b/packages/zeek/data_stream/modbus/_dev/test/pipeline/test-modbus.log-expected.json index 0053050d03ca..dc45ea0eec04 100644 --- a/packages/zeek/data_stream/modbus/_dev/test/pipeline/test-modbus.log-expected.json +++ b/packages/zeek/data_stream/modbus/_dev/test/pipeline/test-modbus.log-expected.json @@ -28,7 +28,7 @@ "ip": "192.168.1.10" }, "event": { - "ingested": "2021-04-23T19:56:24.947572900Z", + "ingested": "2021-06-10T17:49:28.368452Z", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "action": "READ_COILS", @@ -89,7 +89,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-04-23T19:56:24.947580700Z", + "ingested": "2021-06-10T17:49:28.368568100Z", "original": "{\"ts\":1352718265.222457,\"uid\":\"CpIIXl4DFGswmjH2bl\",\"id.orig_h\":\"192.168.1.10\",\"id.orig_p\":64342,\"id.resp_h\":\"192.168.1.164\",\"id.resp_p\":502,\"func\":\"READ_COILS\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/modbus/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/modbus/elasticsearch/ingest_pipeline/default.yml index 19e5cc7d0357..f71a2a2ad070 100644 --- a/packages/zeek/data_stream/modbus/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/modbus/elasticsearch/ingest_pipeline/default.yml @@ -135,6 +135,7 @@ processors: field: zeek.modbus.ts formats: - UNIX + - ISO8601 - remove: field: zeek.modbus.ts - append: diff --git a/packages/zeek/data_stream/mysql/_dev/test/pipeline/test-mysql.log-expected.json b/packages/zeek/data_stream/mysql/_dev/test/pipeline/test-mysql.log-expected.json index 583f1ad6530e..1059ec34e03d 100644 --- a/packages/zeek/data_stream/mysql/_dev/test/pipeline/test-mysql.log-expected.json +++ b/packages/zeek/data_stream/mysql/_dev/test/pipeline/test-mysql.log-expected.json @@ -30,7 +30,7 @@ "ip": "192.168.0.254" }, "event": { - "ingested": "2021-04-23T19:56:25.040662800Z", + "ingested": "2021-06-10T17:49:28.623407Z", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "action": "query", @@ -95,7 +95,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-04-23T19:56:25.040670800Z", + "ingested": "2021-06-10T17:49:28.623423700Z", "original": "{\"ts\":1216281087.437392,\"uid\":\"C5Hol527kLMUw36hj3\",\"id.orig_h\":\"192.168.0.254\",\"id.orig_p\":56162,\"id.resp_h\":\"192.168.0.254\",\"id.resp_p\":3306,\"cmd\":\"query\",\"arg\":\"select count(*) from foo\",\"success\":true,\"rows\":1}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/mysql/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/mysql/elasticsearch/ingest_pipeline/default.yml index 3c0ab2f0d45f..637386efcbb4 100644 --- a/packages/zeek/data_stream/mysql/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/mysql/elasticsearch/ingest_pipeline/default.yml @@ -134,6 +134,7 @@ processors: field: zeek.mysql.ts formats: - UNIX + - ISO8601 - remove: field: zeek.mysql.ts - append: diff --git a/packages/zeek/data_stream/notice/_dev/test/pipeline/test-notice.log-expected.json b/packages/zeek/data_stream/notice/_dev/test/pipeline/test-notice.log-expected.json index 679683764188..58a226c2b9ca 100644 --- a/packages/zeek/data_stream/notice/_dev/test/pipeline/test-notice.log-expected.json +++ b/packages/zeek/data_stream/notice/_dev/test/pipeline/test-notice.log-expected.json @@ -32,7 +32,7 @@ "ip": "172.16.238.1" }, "event": { - "ingested": "2021-04-23T19:56:25.173308800Z", + "ingested": "2021-06-10T17:49:28.822481300Z", "category": [ "intrusion_detection" ], @@ -117,7 +117,7 @@ "ip": "8.42.77.171" }, "event": { - "ingested": "2021-04-23T19:56:25.173315800Z", + "ingested": "2021-06-10T17:49:28.822498200Z", "category": [ "intrusion_detection" ], @@ -149,7 +149,7 @@ "description": "The capture loss script detected an estimated loss rate above 88.306%" }, "event": { - "ingested": "2021-04-23T19:56:25.173320100Z", + "ingested": "2021-06-10T17:49:28.822508600Z", "category": [ "intrusion_detection" ], @@ -217,7 +217,7 @@ "ip": "10.156.0.2" }, "event": { - "ingested": "2021-04-23T19:56:25.173322600Z", + "ingested": "2021-06-10T17:49:28.822518700Z", "id": "CmvrSS1wIiuOGYCbfi", "category": [ "intrusion_detection" @@ -314,7 +314,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-04-23T19:56:25.173325600Z", + "ingested": "2021-06-10T17:49:28.822528800Z", "original": "{\"ts\":1551393388.426472,\"note\":\"Scan::Port_Scan\",\"msg\":\"8.42.77.171 scanned at least 15 unique ports of host 207.154.238.205 in 0m0s\",\"sub\":\"remote\",\"src\":\"8.42.77.171\",\"dst\":\"207.154.238.205\",\"peer_descr\":\"bro\",\"actions\":[\"Notice::ACTION_LOG\"],\"suppress_for\":3600.0,\"dropped\":false}", "category": [ "intrusion_detection" diff --git a/packages/zeek/data_stream/notice/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/notice/elasticsearch/ingest_pipeline/default.yml index dd0d216d3f1b..e3b7fd53e0b0 100644 --- a/packages/zeek/data_stream/notice/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/notice/elasticsearch/ingest_pipeline/default.yml @@ -217,6 +217,7 @@ processors: field: zeek.notice.ts formats: - UNIX + - ISO8601 - remove: field: zeek.notice.ts - geoip: diff --git a/packages/zeek/data_stream/ntlm/_dev/test/pipeline/test-ntlm.log-expected.json b/packages/zeek/data_stream/ntlm/_dev/test/pipeline/test-ntlm.log-expected.json index 958ad85f93ce..66a4ab9fc331 100644 --- a/packages/zeek/data_stream/ntlm/_dev/test/pipeline/test-ntlm.log-expected.json +++ b/packages/zeek/data_stream/ntlm/_dev/test/pipeline/test-ntlm.log-expected.json @@ -40,7 +40,7 @@ "ip": "192.168.10.50" }, "event": { - "ingested": "2021-04-23T19:56:25.320525100Z", + "ingested": "2021-06-10T17:49:29.137475100Z", "id": "CHphiNUKDC20fsy09", "category": [ "network", @@ -116,7 +116,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-04-23T19:56:25.320530300Z", + "ingested": "2021-06-10T17:49:29.137491700Z", "original": "{\"ts\":1508959117.814467,\"uid\":\"CHphiNUKDC20fsy09\",\"id.orig_h\":\"192.168.10.50\",\"id.orig_p\":46785,\"id.resp_h\":\"192.168.10.31\",\"id.resp_p\":445,\"username\":\"JeffV\",\"hostname\":\"ybaARon55QykXrgu\",\"domainname\":\"contoso.local\",\"server_nb_computer_name\":\"VICTIM-PC\",\"server_dns_computer_name\":\"Victim-PC.contoso.local\",\"server_tree_name\":\"contoso.local\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/ntlm/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/ntlm/elasticsearch/ingest_pipeline/default.yml index 87b22e3f8ea4..5aed1fe762e1 100644 --- a/packages/zeek/data_stream/ntlm/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/ntlm/elasticsearch/ingest_pipeline/default.yml @@ -154,6 +154,7 @@ processors: field: zeek.ntlm.ts formats: - UNIX + - ISO8601 - remove: field: zeek.ntlm.ts - append: diff --git a/packages/zeek/data_stream/ocsp/_dev/test/pipeline/test-ocsp.log-expected.json b/packages/zeek/data_stream/ocsp/_dev/test/pipeline/test-ocsp.log-expected.json index c58bc24f95c8..22078d37c089 100644 --- a/packages/zeek/data_stream/ocsp/_dev/test/pipeline/test-ocsp.log-expected.json +++ b/packages/zeek/data_stream/ocsp/_dev/test/pipeline/test-ocsp.log-expected.json @@ -27,7 +27,7 @@ } }, "event": { - "ingested": "2021-04-23T19:56:25.410541800Z", + "ingested": "2021-06-10T17:49:29.356966900Z", "created": "2020-04-28T11:07:58.223Z", "kind": "event" }, @@ -59,7 +59,7 @@ } }, "event": { - "ingested": "2021-04-23T19:56:25.410547500Z", + "ingested": "2021-06-10T17:49:29.356995400Z", "created": "2020-04-28T11:07:58.223Z", "kind": "event" }, @@ -99,7 +99,7 @@ } }, "event": { - "ingested": "2021-04-23T19:56:25.410549800Z", + "ingested": "2021-06-10T17:49:29.356998700Z", "original": "{\"ts\":1307562416.100084,\"id\":\"FdZBFMEYgAErVhoC8\",\"hashAlgorithm\":\"sha1\",\"issuerNameHash\":\"6C2BC55AAF8D96BF60ADF81D023F23B48A0059C2\",\"issuerKeyHash\":\"A5EF0B11CEC04103A34A659048B21CE0572D7D47\",\"serialNumber\":\"30119E6EF41BDBA3FEFE711DBE8F6191\",\"certStatus\":\"good\",\"thisUpdate\":1307549998.0,\"nextUpdate\":1308154798.0}", "created": "2020-04-28T11:07:58.223Z", "kind": "event" diff --git a/packages/zeek/data_stream/ocsp/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/ocsp/elasticsearch/ingest_pipeline/default.yml index be7be34575cb..1ae98591210f 100644 --- a/packages/zeek/data_stream/ocsp/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/ocsp/elasticsearch/ingest_pipeline/default.yml @@ -99,6 +99,7 @@ processors: field: zeek.ocsp.ts formats: - UNIX + - ISO8601 - remove: field: zeek.ocsp.ts - date: @@ -106,18 +107,21 @@ processors: target_field: zeek.ocsp.revoke.date formats: - UNIX + - ISO8601 if: ctx.zeek.ocsp.revoke?.date != null - date: field: zeek.ocsp.update.this target_field: zeek.ocsp.update.this formats: - UNIX + - ISO8601 if: ctx.zeek.ocsp.update?.this != null - date: field: zeek.ocsp.update.next target_field: zeek.ocsp.update.next formats: - UNIX + - ISO8601 if: ctx.zeek.ocsp.update?.next != null - append: field: related.hash diff --git a/packages/zeek/data_stream/pe/_dev/test/pipeline/test-pe.log-expected.json b/packages/zeek/data_stream/pe/_dev/test/pipeline/test-pe.log-expected.json index b89ce628d27d..4897b1bc5b91 100644 --- a/packages/zeek/data_stream/pe/_dev/test/pipeline/test-pe.log-expected.json +++ b/packages/zeek/data_stream/pe/_dev/test/pipeline/test-pe.log-expected.json @@ -32,7 +32,7 @@ } }, "event": { - "ingested": "2021-04-23T19:56:25.485194600Z", + "ingested": "2021-06-10T17:49:29.548242600Z", "category": [ "file" ], @@ -83,7 +83,7 @@ } }, "event": { - "ingested": "2021-04-23T19:56:25.485200900Z", + "ingested": "2021-06-10T17:49:29.548271100Z", "original": "{\"ts\":1507565599.578328,\"id\":\"FtIFnm3ZqI1s96P74l\",\"machine\":\"I386\",\"compile_ts\":1467139314.0,\"os\":\"Windows XP\",\"subsystem\":\"WINDOWS_CUI\",\"is_exe\":true,\"is_64bit\":false,\"uses_aslr\":true,\"uses_dep\":true,\"uses_code_integrity\":false,\"uses_seh\":true,\"has_import_table\":true,\"has_export_table\":false,\"has_cert_table\":true,\"has_debug_data\":false,\"section_names\":[\".text\",\".rdata\",\".data\",\".rsrc\",\".reloc\"]}", "category": [ "file" diff --git a/packages/zeek/data_stream/pe/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/pe/elasticsearch/ingest_pipeline/default.yml index ce8d0119aa4d..3ff9e3010163 100644 --- a/packages/zeek/data_stream/pe/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/pe/elasticsearch/ingest_pipeline/default.yml @@ -66,6 +66,7 @@ processors: field: zeek.pe.ts formats: - UNIX + - ISO8601 - remove: field: zeek.pe.ts - date: @@ -73,6 +74,7 @@ processors: target_field: zeek.pe.compile_time formats: - UNIX + - ISO8601 if: ctx.zeek.pe.compile_time != null - remove: field: diff --git a/packages/zeek/data_stream/radius/_dev/test/pipeline/test-radius.log-expected.json b/packages/zeek/data_stream/radius/_dev/test/pipeline/test-radius.log-expected.json index c8fe6b9ff822..35333456b8d3 100644 --- a/packages/zeek/data_stream/radius/_dev/test/pipeline/test-radius.log-expected.json +++ b/packages/zeek/data_stream/radius/_dev/test/pipeline/test-radius.log-expected.json @@ -33,7 +33,7 @@ "ip": "10.0.0.1" }, "event": { - "ingested": "2021-04-23T19:56:25.555260Z", + "ingested": "2021-06-10T17:49:29.690733400Z", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "id": "CRe9VD3flCDWbPmpIh", @@ -102,7 +102,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-04-23T19:56:25.555267700Z", + "ingested": "2021-06-10T17:49:29.690752500Z", "original": "{\"ts\":1217631137.916736,\"uid\":\"CRe9VD3flCDWbPmpIh\",\"id.orig_h\":\"10.0.0.1\",\"id.orig_p\":1645,\"id.resp_h\":\"10.0.0.100\",\"id.resp_p\":1812,\"username\":\"John.McGuirk\",\"mac\":\"00:14:22:e9:54:5e\",\"result\":\"success\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/radius/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/radius/elasticsearch/ingest_pipeline/default.yml index f46ef06438f7..bc4415d301cc 100644 --- a/packages/zeek/data_stream/radius/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/radius/elasticsearch/ingest_pipeline/default.yml @@ -130,6 +130,7 @@ processors: field: zeek.radius.ts formats: - UNIX + - ISO8601 - remove: field: zeek.radius.ts - append: diff --git a/packages/zeek/data_stream/rdp/_dev/test/pipeline/test-rdp.log-expected.json b/packages/zeek/data_stream/rdp/_dev/test/pipeline/test-rdp.log-expected.json index 86ba69d1d916..2587d95a2aa6 100644 --- a/packages/zeek/data_stream/rdp/_dev/test/pipeline/test-rdp.log-expected.json +++ b/packages/zeek/data_stream/rdp/_dev/test/pipeline/test-rdp.log-expected.json @@ -36,7 +36,7 @@ "ip": "192.168.131.1" }, "event": { - "ingested": "2021-04-23T19:56:25.633749700Z", + "ingested": "2021-06-10T17:49:29.873141500Z", "id": "C2PcYV7D3ntaHm056", "category": [ "network" @@ -103,7 +103,7 @@ "established": true }, "event": { - "ingested": "2021-04-23T19:56:25.633754800Z", + "ingested": "2021-06-10T17:49:29.873156700Z", "original": "{\"ts\":1568132339.668952,\"uid\":\"C2PcYV7D3ntaHm056\",\"id.orig_h\":\"192.168.131.1\",\"id.orig_p\":33872,\"id.resp_h\":\"192.168.131.131\",\"id.resp_p\":3389,\"result\":\"encrypted\",\"security_protocol\":\"HYBRID\",\"cert_count\":0,\"ssl\":true}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/rdp/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/rdp/elasticsearch/ingest_pipeline/default.yml index fc9f9bd8a53e..3877343aa59c 100644 --- a/packages/zeek/data_stream/rdp/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/rdp/elasticsearch/ingest_pipeline/default.yml @@ -163,6 +163,7 @@ processors: field: zeek.rdp.ts formats: - UNIX + - ISO8601 - remove: field: zeek.rdp.ts - convert: diff --git a/packages/zeek/data_stream/rfb/_dev/test/pipeline/test-rfb.log-expected.json b/packages/zeek/data_stream/rfb/_dev/test/pipeline/test-rfb.log-expected.json index 1d341e31c2bf..c24b2066ec36 100644 --- a/packages/zeek/data_stream/rfb/_dev/test/pipeline/test-rfb.log-expected.json +++ b/packages/zeek/data_stream/rfb/_dev/test/pipeline/test-rfb.log-expected.json @@ -45,7 +45,7 @@ "ip": "192.168.1.123" }, "event": { - "ingested": "2021-04-23T19:56:25.723384600Z", + "ingested": "2021-06-10T17:49:30.093690400Z", "id": "CXoIzM3wH3fUwXtKN1", "category": [ "network" @@ -121,7 +121,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-04-23T19:56:25.723391300Z", + "ingested": "2021-06-10T17:49:30.093703700Z", "original": "{\"ts\":1328632534.517208,\"uid\":\"CXoIzM3wH3fUwXtKN1\",\"id.orig_h\":\"192.168.1.123\",\"id.orig_p\":58102,\"id.resp_h\":\"192.168.1.10\",\"id.resp_p\":5900,\"client_major_version\":\"003\",\"client_minor_version\":\"008\",\"server_major_version\":\"003\",\"server_minor_version\":\"008\",\"authentication_method\":\"VNC\",\"auth\":true,\"share_flag\":false,\"desktop_name\":\"\\u00a0\",\"width\":800,\"height\":600}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/rfb/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/rfb/elasticsearch/ingest_pipeline/default.yml index 554ccb775b0d..4db55970f79f 100644 --- a/packages/zeek/data_stream/rfb/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/rfb/elasticsearch/ingest_pipeline/default.yml @@ -143,6 +143,7 @@ processors: field: zeek.rfb.ts formats: - UNIX + - ISO8601 - remove: field: zeek.rfb.ts - append: diff --git a/packages/zeek/data_stream/sip/_dev/test/pipeline/test-sip.log-expected.json b/packages/zeek/data_stream/sip/_dev/test/pipeline/test-sip.log-expected.json index b4379b287201..f02a8129595d 100644 --- a/packages/zeek/data_stream/sip/_dev/test/pipeline/test-sip.log-expected.json +++ b/packages/zeek/data_stream/sip/_dev/test/pipeline/test-sip.log-expected.json @@ -70,7 +70,7 @@ "ip": "172.16.133.19" }, "event": { - "ingested": "2021-04-23T19:56:25.834066Z", + "ingested": "2021-06-10T17:49:30.314921200Z", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "action": "REGISTER", @@ -188,7 +188,7 @@ "ip": "200.57.7.204" }, "event": { - "ingested": "2021-04-23T19:56:25.834072100Z", + "ingested": "2021-06-10T17:49:30.314971300Z", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "action": "INVITE", @@ -302,7 +302,7 @@ "ip": "200.57.7.205" }, "event": { - "ingested": "2021-04-23T19:56:25.834128800Z", + "ingested": "2021-06-10T17:49:30.314983100Z", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "action": "REGISTER", @@ -386,7 +386,7 @@ "ip": "193.107.216.13" }, "event": { - "ingested": "2021-04-23T19:56:25.834134900Z", + "ingested": "2021-06-10T17:49:30.314993400Z", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "action": "OPTIONS", @@ -469,7 +469,7 @@ "ip": "45.134.144.100" }, "event": { - "ingested": "2021-04-23T19:56:25.834137600Z", + "ingested": "2021-06-10T17:49:30.315003200Z", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "action": "OPTIONS", @@ -598,7 +598,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-04-23T19:56:25.834139700Z", + "ingested": "2021-06-10T17:49:30.315013100Z", "original": "{\"ts\":1105725487.022577,\"uid\":\"CJZDWgixtwqXctWEg\",\"id.orig_h\":\"200.57.7.205\",\"id.orig_p\":5061,\"id.resp_h\":\"200.57.7.195\",\"id.resp_p\":5060,\"trans_depth\":0,\"method\":\"REGISTER\",\"uri\":\"sip:Verso.com\",\"request_from\":\"Ivan \u003csip:Ivan@Verso.com\u003e\",\"request_to\":\"Ivan \u003csip:Ivan@Verso.com\u003e\",\"response_from\":\"\\u0022Ivan\\u0022 \u003csip:Ivan@Verso.com\u003e\",\"response_to\":\"\\u0022Ivan\\u0022 \u003csip:Ivan@Verso.com\u003e\",\"call_id\":\"46E1C3CB36304F84A020CF6DD3F96461@Verso.com\",\"seq\":\"37764 REGISTER\",\"request_path\":[\"SIP/2.0/UDP 200.57.7.205:5061;rport\"],\"response_path\":[\"SIP/2.0/UDP 200.57.7.205:5061;received=200.57.7.205;rport=5061\"],\"user_agent\":\"Verso Softphone release 1104w\",\"status_code\":200,\"status_msg\":\"OK\",\"request_body_len\":0,\"response_body_len\":0}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/sip/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/sip/elasticsearch/ingest_pipeline/default.yml index 3d307c2b4f92..c91902c762a7 100644 --- a/packages/zeek/data_stream/sip/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/sip/elasticsearch/ingest_pipeline/default.yml @@ -175,6 +175,7 @@ processors: field: zeek.sip.ts formats: - UNIX + - ISO8601 - remove: field: zeek.sip.ts - grok: diff --git a/packages/zeek/data_stream/smb_cmd/_dev/test/pipeline/test-smb-cmd.log-expected.json b/packages/zeek/data_stream/smb_cmd/_dev/test/pipeline/test-smb-cmd.log-expected.json index fe6fbca61e50..35e2792c24af 100644 --- a/packages/zeek/data_stream/smb_cmd/_dev/test/pipeline/test-smb-cmd.log-expected.json +++ b/packages/zeek/data_stream/smb_cmd/_dev/test/pipeline/test-smb-cmd.log-expected.json @@ -43,7 +43,7 @@ "ip": "172.16.133.6" }, "event": { - "ingested": "2021-04-23T19:56:26.060942200Z", + "ingested": "2021-06-10T17:49:30.735302600Z", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "action": "NT_CREATE_ANDX", @@ -119,7 +119,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-04-23T19:56:26.060951300Z", + "ingested": "2021-06-10T17:49:30.735319300Z", "original": "{\"ts\":1361916332.020006,\"uid\":\"CbT8mpAXseu6Pt4R7\",\"id.orig_h\":\"172.16.133.6\",\"id.orig_p\":1728,\"id.resp_h\":\"172.16.128.202\",\"id.resp_p\":445,\"command\":\"NT_CREATE_ANDX\",\"argument\":\"\\u005cbrowser\",\"status\":\"SUCCESS\",\"rtt\":0.091141,\"version\":\"SMB1\",\"tree\":\"\\u005c\\u005cJSRVR20\\u005cIPC$\",\"tree_service\":\"IPC\",\"referenced_file.ts\":1361916332.020006,\"referenced_file.uid\":\"CbT8mpAXseu6Pt4R7\",\"referenced_file.id.orig_h\":\"172.16.133.6\",\"referenced_file.id.orig_p\":1728,\"referenced_file.id.resp_h\":\"172.16.128.202\",\"referenced_file.id.resp_p\":445,\"referenced_file.action\":\"SMB::FILE_OPEN\",\"referenced_file.name\":\"\\u005cbrowser\",\"referenced_file.size\":0}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/smb_cmd/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/smb_cmd/elasticsearch/ingest_pipeline/default.yml index 3c726db949aa..5689bc6164bc 100644 --- a/packages/zeek/data_stream/smb_cmd/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/smb_cmd/elasticsearch/ingest_pipeline/default.yml @@ -225,6 +225,7 @@ processors: field: zeek.smb_cmd.ts formats: - UNIX + - ISO8601 - remove: field: zeek.smb_cmd.ts - remove: diff --git a/packages/zeek/data_stream/smb_files/_dev/test/pipeline/test-smb-files.log-expected.json b/packages/zeek/data_stream/smb_files/_dev/test/pipeline/test-smb-files.log-expected.json index 69ee6c24bb08..1d058ba3fe68 100644 --- a/packages/zeek/data_stream/smb_files/_dev/test/pipeline/test-smb-files.log-expected.json +++ b/packages/zeek/data_stream/smb_files/_dev/test/pipeline/test-smb-files.log-expected.json @@ -46,7 +46,7 @@ "ip": "192.168.10.31" }, "event": { - "ingested": "2021-04-23T19:56:26.168856300Z", + "ingested": "2021-06-10T17:49:30.972528500Z", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "action": "SMB::FILE_OPEN", @@ -126,7 +126,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-04-23T19:56:26.168863Z", + "ingested": "2021-06-10T17:49:30.972546900Z", "original": "{\"ts\":1507565599.576942,\"uid\":\"C9YAaEzWLL62yWMn5\",\"id.orig_h\":\"192.168.10.31\",\"id.orig_p\":49239,\"id.resp_h\":\"192.168.10.30\",\"id.resp_p\":445,\"action\":\"SMB::FILE_OPEN\",\"path\":\"\\u005c\\u005cadmin-pc\\u005cADMIN$\",\"name\":\"PSEXESVC.exe\",\"size\":0,\"times.modified\":1507565599.607777,\"times.accessed\":1507565599.607777,\"times.created\":1507565599.607777,\"times.changed\":1507565599.607777}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/smb_files/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/smb_files/elasticsearch/ingest_pipeline/default.yml index f986f2cdea51..0aed512b6908 100644 --- a/packages/zeek/data_stream/smb_files/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/smb_files/elasticsearch/ingest_pipeline/default.yml @@ -134,6 +134,7 @@ processors: field: zeek.smb_files.ts formats: - UNIX + - ISO8601 - remove: field: zeek.smb_files.ts - dot_expander: @@ -153,6 +154,7 @@ processors: target_field: zeek.smb_files.times.accessed formats: - UNIX + - ISO8601 if: ctx.zeek.smb_files.times?.accessed != null - set: field: file.accessed @@ -163,6 +165,7 @@ processors: target_field: zeek.smb_files.times.changed formats: - UNIX + - ISO8601 if: ctx.zeek.smb_files.times?.accessed != null - set: field: file.ctime @@ -173,6 +176,7 @@ processors: target_field: zeek.smb_files.times.created formats: - UNIX + - ISO8601 if: ctx.zeek.smb_files.times?.accessed != null - set: field: file.created @@ -183,6 +187,7 @@ processors: target_field: zeek.smb_files.times.modified formats: - UNIX + - ISO8601 if: ctx.zeek.smb_files.times?.accessed != null - set: field: file.mtime diff --git a/packages/zeek/data_stream/smb_mapping/_dev/test/pipeline/test-smb-mapping.log-expected.json b/packages/zeek/data_stream/smb_mapping/_dev/test/pipeline/test-smb-mapping.log-expected.json index 9a899e272d47..5d9e2034b827 100644 --- a/packages/zeek/data_stream/smb_mapping/_dev/test/pipeline/test-smb-mapping.log-expected.json +++ b/packages/zeek/data_stream/smb_mapping/_dev/test/pipeline/test-smb-mapping.log-expected.json @@ -29,7 +29,7 @@ "ip": "192.168.10.31" }, "event": { - "ingested": "2021-04-23T19:56:26.260175700Z", + "ingested": "2021-06-10T17:49:31.178043700Z", "id": "C9YAaEzWLL62yWMn5", "category": [ "network" @@ -89,7 +89,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-04-23T19:56:26.260180800Z", + "ingested": "2021-06-10T17:49:31.178054200Z", "original": "{\"ts\":1507565599.576613,\"uid\":\"C9YAaEzWLL62yWMn5\",\"id.orig_h\":\"192.168.10.31\",\"id.orig_p\":49239,\"id.resp_h\":\"192.168.10.30\",\"id.resp_p\":445,\"path\":\"\\u005c\\u005cadmin-pc\\u005cADMIN$\",\"share_type\":\"DISK\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/smb_mapping/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/smb_mapping/elasticsearch/ingest_pipeline/default.yml index 43cbc0d729be..b09071c7774c 100644 --- a/packages/zeek/data_stream/smb_mapping/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/smb_mapping/elasticsearch/ingest_pipeline/default.yml @@ -119,6 +119,7 @@ processors: field: zeek.smb_mapping.ts formats: - UNIX + - ISO8601 - remove: field: zeek.smb_mapping.ts - geoip: diff --git a/packages/zeek/data_stream/smtp/_dev/test/pipeline/test-smtp.log-expected.json b/packages/zeek/data_stream/smtp/_dev/test/pipeline/test-smtp.log-expected.json index 2ea084127f06..c7bfd0135022 100644 --- a/packages/zeek/data_stream/smtp/_dev/test/pipeline/test-smtp.log-expected.json +++ b/packages/zeek/data_stream/smtp/_dev/test/pipeline/test-smtp.log-expected.json @@ -39,7 +39,7 @@ "ip": "192.168.1.10" }, "event": { - "ingested": "2021-04-23T19:56:26.337592400Z", + "ingested": "2021-06-10T17:49:31.349982Z", "id": "CWWzPB3RjqhFf528c", "category": [ "network" @@ -109,7 +109,7 @@ "established": true }, "event": { - "ingested": "2021-04-23T19:56:26.337598600Z", + "ingested": "2021-06-10T17:49:31.349998900Z", "original": "{\"ts\":1543877987.381899,\"uid\":\"CWWzPB3RjqhFf528c\",\"id.orig_h\":\"192.168.1.10\",\"id.orig_p\":33782,\"id.resp_h\":\"192.168.1.9\",\"id.resp_p\":25,\"trans_depth\":1,\"helo\":\"EXAMPLE.COM\",\"last_reply\":\"220 2.0.0 SMTP server ready\",\"path\":[\"192.168.1.9\"],\"tls\":true,\"fuids\":[],\"is_webmail\":false}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/smtp/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/smtp/elasticsearch/ingest_pipeline/default.yml index 1c5d19a70e81..ef925d8c8767 100644 --- a/packages/zeek/data_stream/smtp/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/smtp/elasticsearch/ingest_pipeline/default.yml @@ -136,6 +136,7 @@ processors: field: zeek.smtp.ts formats: - UNIX + - ISO8601 - remove: field: zeek.smtp.ts - date: diff --git a/packages/zeek/data_stream/snmp/_dev/test/pipeline/test-snmp.log-expected.json b/packages/zeek/data_stream/snmp/_dev/test/pipeline/test-snmp.log-expected.json index 9f16b0ecc894..168f714e5cc6 100644 --- a/packages/zeek/data_stream/snmp/_dev/test/pipeline/test-snmp.log-expected.json +++ b/packages/zeek/data_stream/snmp/_dev/test/pipeline/test-snmp.log-expected.json @@ -39,7 +39,7 @@ "ip": "192.168.1.2" }, "event": { - "ingested": "2021-04-23T19:56:26.428131500Z", + "ingested": "2021-06-10T17:49:31.570718300Z", "id": "CnKW1B4w9fpRa6Nkf2", "category": [ "network" @@ -110,7 +110,7 @@ "ip": "184.105.139.67" }, "event": { - "ingested": "2021-04-23T19:56:26.428137900Z", + "ingested": "2021-06-10T17:49:31.570733500Z", "id": "CxtWIB4ECPW89F8mSi", "category": [ "network" @@ -180,7 +180,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-04-23T19:56:26.428139800Z", + "ingested": "2021-06-10T17:49:31.570743Z", "original": "{\"ts\":1543877948.916584,\"uid\":\"CnKW1B4w9fpRa6Nkf2\",\"id.orig_h\":\"192.168.1.2\",\"id.orig_p\":59696,\"id.resp_h\":\"192.168.1.1\",\"id.resp_p\":161,\"duration\":7.849924,\"version\":\"2c\",\"community\":\"public\",\"get_requests\":0,\"get_bulk_requests\":0,\"get_responses\":8,\"set_requests\":0,\"up_since\":1543631204.766508}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/snmp/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/snmp/elasticsearch/ingest_pipeline/default.yml index 658a8afe7192..57123f9c7c5c 100644 --- a/packages/zeek/data_stream/snmp/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/snmp/elasticsearch/ingest_pipeline/default.yml @@ -135,6 +135,7 @@ processors: field: zeek.snmp.ts formats: - UNIX + - ISO8601 - remove: field: zeek.snmp.ts - date: @@ -142,6 +143,7 @@ processors: target_field: zeek.snmp.up_since formats: - UNIX + - ISO8601 if: ctx.zeek.snmp.up_since != null - geoip: field: destination.ip diff --git a/packages/zeek/data_stream/socks/_dev/test/pipeline/test-socks.log-expected.json b/packages/zeek/data_stream/socks/_dev/test/pipeline/test-socks.log-expected.json index a384957c2ad1..509911b37bd6 100644 --- a/packages/zeek/data_stream/socks/_dev/test/pipeline/test-socks.log-expected.json +++ b/packages/zeek/data_stream/socks/_dev/test/pipeline/test-socks.log-expected.json @@ -36,7 +36,7 @@ "ip": "127.0.0.1" }, "event": { - "ingested": "2021-04-23T19:56:26.527432600Z", + "ingested": "2021-06-10T17:49:31.766679400Z", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "id": "Cmz4Cb4qCw1hGqYw1c", @@ -104,7 +104,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-04-23T19:56:26.527438700Z", + "ingested": "2021-06-10T17:49:31.766695600Z", "original": "{\"ts\":1566508093.09494,\"uid\":\"Cmz4Cb4qCw1hGqYw1c\",\"id.orig_h\":\"127.0.0.1\",\"id.orig_p\":35368,\"id.resp_h\":\"127.0.0.1\",\"id.resp_p\":8080,\"version\":5,\"status\":\"succeeded\",\"request.name\":\"www.google.com\",\"request_p\":443,\"bound.host\":\"0.0.0.0\",\"bound_p\":0}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/socks/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/socks/elasticsearch/ingest_pipeline/default.yml index 28d93ec5f741..2ca7370402b8 100644 --- a/packages/zeek/data_stream/socks/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/socks/elasticsearch/ingest_pipeline/default.yml @@ -139,6 +139,7 @@ processors: field: zeek.socks.ts formats: - UNIX + - ISO8601 - remove: field: zeek.socks.ts - dot_expander: diff --git a/packages/zeek/data_stream/ssh/_dev/test/pipeline/test-ssh.log-expected.json b/packages/zeek/data_stream/ssh/_dev/test/pipeline/test-ssh.log-expected.json index e6b0d3b250b3..033a27481b4a 100644 --- a/packages/zeek/data_stream/ssh/_dev/test/pipeline/test-ssh.log-expected.json +++ b/packages/zeek/data_stream/ssh/_dev/test/pipeline/test-ssh.log-expected.json @@ -42,7 +42,7 @@ "ip": "192.168.1.2" }, "event": { - "ingested": "2021-04-23T19:56:26.636866800Z", + "ingested": "2021-06-10T17:49:31.966858800Z", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "id": "CajWfz1b3qnnWT0BU9", @@ -108,7 +108,7 @@ "ip": "51.161.10.160" }, "event": { - "ingested": "2021-04-23T19:56:26.636872200Z", + "ingested": "2021-06-10T17:49:31.966920200Z", "id": "COXxsJ3dlSh6ECRYQj", "category": [ "network" @@ -173,7 +173,7 @@ "ip": "113.53.238.195" }, "event": { - "ingested": "2021-04-23T19:56:26.636874100Z", + "ingested": "2021-06-10T17:49:31.966926900Z", "id": "CZPdXz1jfKSWzIDAeb", "category": [ "network" @@ -238,7 +238,7 @@ "ip": "34.86.35.26" }, "event": { - "ingested": "2021-04-23T19:56:26.636875700Z", + "ingested": "2021-06-10T17:49:31.966933700Z", "id": "Cha1rs3OamonAZ4Nz6", "category": [ "network" @@ -311,7 +311,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-04-23T19:56:26.636877300Z", + "ingested": "2021-06-10T17:49:31.966943500Z", "original": "{\"ts\":1562527532.904291,\"uid\":\"CajWfz1b3qnnWT0BU9\",\"id.orig_h\":\"192.168.1.2\",\"id.orig_p\":48380,\"id.resp_h\":\"192.168.1.1\",\"id.resp_p\":22,\"version\":2,\"auth_success\":false,\"auth_attempts\":2,\"client\":\"SSH-2.0-OpenSSH_7.9p1 Ubuntu-10\",\"server\":\"SSH-2.0-OpenSSH_6.6.1p1 Debian-4~bpo70+1\",\"cipher_alg\":\"chacha20-poly1305@openssh.com\",\"mac_alg\":\"umac-64-etm@openssh.com\",\"compression_alg\":\"none\",\"kex_alg\":\"curve25519-sha256@libssh.org\",\"host_key_alg\":\"ecdsa-sha2-nistp256\",\"host_key\":\"86:71:ac:9c:35:1c:28:29:05:81:48:ec:66:67:de:bd\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/ssh/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/ssh/elasticsearch/ingest_pipeline/default.yml index 5836500d09e3..b2a88d37f21b 100644 --- a/packages/zeek/data_stream/ssh/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/ssh/elasticsearch/ingest_pipeline/default.yml @@ -147,6 +147,7 @@ processors: field: zeek.ssh.ts formats: - UNIX + - ISO8601 - remove: field: zeek.ssh.ts - geoip: diff --git a/packages/zeek/data_stream/ssl/_dev/test/pipeline/test-ssl.log-expected.json b/packages/zeek/data_stream/ssl/_dev/test/pipeline/test-ssl.log-expected.json index fc33a82bbcc8..48ad820d7a26 100644 --- a/packages/zeek/data_stream/ssl/_dev/test/pipeline/test-ssl.log-expected.json +++ b/packages/zeek/data_stream/ssl/_dev/test/pipeline/test-ssl.log-expected.json @@ -109,7 +109,7 @@ "version_protocol": "tls" }, "event": { - "ingested": "2021-04-23T19:56:26.790039800Z", + "ingested": "2021-06-10T17:49:32.255615400Z", "id": "CAOvs1BMFCX2Eh0Y3", "category": [ "network" @@ -231,7 +231,7 @@ "version_protocol": "tls" }, "event": { - "ingested": "2021-04-23T19:56:26.790047400Z", + "ingested": "2021-06-10T17:49:32.255636500Z", "id": "C3mki91FnnNtm0u1ok", "category": [ "network" @@ -353,7 +353,7 @@ "version_protocol": "tls" }, "event": { - "ingested": "2021-04-23T19:56:26.790049400Z", + "ingested": "2021-06-10T17:49:32.255647500Z", "id": "CfGBt82PzCXzHa0iek", "category": [ "network" @@ -432,7 +432,7 @@ "version_protocol": "tls" }, "event": { - "ingested": "2021-04-23T19:56:26.790051300Z", + "ingested": "2021-06-10T17:49:32.255657700Z", "id": "CLQiVH1VcpvT3ruEak", "category": [ "network" @@ -504,7 +504,7 @@ "resumed": false }, "event": { - "ingested": "2021-04-23T19:56:26.790053600Z", + "ingested": "2021-06-10T17:49:32.255667900Z", "id": "CBiXOC4IqYxMv1xzf9", "category": [ "network" @@ -581,7 +581,7 @@ "resumed": false }, "event": { - "ingested": "2021-04-23T19:56:26.790055200Z", + "ingested": "2021-06-10T17:49:32.255678700Z", "id": "C4jH9IqWGZwc1PPUh", "category": [ "network" @@ -658,7 +658,7 @@ "resumed": false }, "event": { - "ingested": "2021-04-23T19:56:26.790058300Z", + "ingested": "2021-06-10T17:49:32.255685800Z", "id": "CXVMSq6Dainy4WFN9", "category": [ "network" @@ -746,7 +746,7 @@ "version_protocol": "tls" }, "event": { - "ingested": "2021-04-23T19:56:26.798594300Z", + "ingested": "2021-06-10T17:49:32.255696600Z", "id": "CsgtQe4AikDZBsIM6k", "category": [ "network" @@ -818,7 +818,7 @@ "resumed": false }, "event": { - "ingested": "2021-04-23T19:56:26.798609400Z", + "ingested": "2021-06-10T17:49:32.255705900Z", "id": "CPGhJS3UPpcnR96NQc", "category": [ "network" @@ -948,7 +948,7 @@ "version_protocol": "tls" }, "event": { - "ingested": "2021-04-23T19:56:26.798612200Z", + "ingested": "2021-06-10T17:49:32.255714100Z", "original": "{\"ts\":1547688736.805527,\"uid\":\"CfGBt82PzCXzHa0iek\",\"id.orig_h\":\"10.178.98.102\",\"id.orig_p\":63197,\"id.resp_h\":\"35.199.178.4\",\"id.resp_p\":9243,\"version\":\"TLSv12\",\"cipher\":\"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\",\"curve\":\"secp256r1\",\"server_name\":\"dd625ffb4fc54735b281862aa1cd6cd4.us-west1.gcp.cloud.es.io\",\"resumed\":false,\"established\":true,\"cert_chain_fuids\":[\"FiFLYv3UjeWyv2gcW\",\"FvSsiB1Xi816EMagI9\",\"FWpPS4mjGaAhTRXLf\"],\"client_cert_chain_fuids\":[],\"subject\":\"CN=*.gcp.cloud.es.io,O=Elasticsearch\\u005c, Inc.,L=Mountain View,ST=California,C=US\",\"issuer\":\"CN=DigiCert SHA2 Secure Server CA,O=DigiCert Inc,C=US\",\"validation_status\":\"ok\"}{\"ts\":1602179457.352156,\"uid\":\"CK17Dl2SB8bZOVonSl\",\"id.orig_h\":\"10.0.0.1\",\"id.orig_p\":49228,\"id.resp_h\":\"192.168.50.1\",\"id.resp_p\":443,\"version\":\"TLSv12\",\"cipher\":\"TLS_RSA_WITH_AES_128_CBC_SHA256\",\"resumed\":false,\"established\":true,\"cert_chain_fuids\":[\"FOLwYQ6rs70bIMSf9\"],\"client_cert_chain_fuids\":[],\"subject\":\"CN=foo,OU=foo@bar,O=org,L=locality,C=LO\",\"issuer\":\"CN=CA,OU=CA@example.com,O=Example Corp,L=foo,C=HI\",\"validation_status\":\"self signed certificate\",\"ja3\":\"74927e242d6c3febf8cb9cab10a7f889\",\"ja3s\":\"80b3a14bccc8598a1f3bbe83e71f735f\",\"resp_certificate_sha1\":\"5dad8b55621b6b9c30679d9d61248dd132a83c94\",\"not_valid_before\":1562022421,\"not_valid_after\":1577748224}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/ssl/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/ssl/elasticsearch/ingest_pipeline/default.yml index a9f771539d93..f8b4ab77516e 100644 --- a/packages/zeek/data_stream/ssl/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/ssl/elasticsearch/ingest_pipeline/default.yml @@ -152,6 +152,7 @@ processors: field: zeek.ssl.ts formats: - UNIX + - ISO8601 - remove: field: zeek.ssl.ts - date: @@ -160,12 +161,14 @@ processors: target_field: tls.server.not_before formats: - UNIX + - ISO8601 - date: if: ctx.tls?.server?.not_after != null field: tls.server.not_after target_field: tls.server.not_after formats: - UNIX + - ISO8601 - geoip: field: destination.ip target_field: destination.geo diff --git a/packages/zeek/data_stream/stats/_dev/test/pipeline/test-stats.log-expected.json b/packages/zeek/data_stream/stats/_dev/test/pipeline/test-stats.log-expected.json index a1511929ccf5..fdd12c5220bf 100644 --- a/packages/zeek/data_stream/stats/_dev/test/pipeline/test-stats.log-expected.json +++ b/packages/zeek/data_stream/stats/_dev/test/pipeline/test-stats.log-expected.json @@ -54,7 +54,7 @@ } }, "event": { - "ingested": "2021-04-23T19:56:27.258699Z", + "ingested": "2021-06-10T17:49:32.938070800Z", "created": "2020-04-28T11:07:58.223Z", "kind": "metric" } @@ -121,7 +121,7 @@ } }, "event": { - "ingested": "2021-04-23T19:56:27.258707200Z", + "ingested": "2021-06-10T17:49:32.938087800Z", "original": "{\"ts\":1476605878.714844,\"peer\":\"bro\",\"mem\":94,\"pkts_proc\":296,\"bytes_recv\":39674,\"events_proc\":723,\"events_queued\":728,\"active_tcp_conns\":1,\"active_udp_conns\":3,\"active_icmp_conns\":0,\"tcp_conns\":6,\"udp_conns\":36,\"icmp_conns\":2,\"timers\":797,\"active_timers\":38,\"files\":0,\"active_files\":0,\"dns_requests\":0,\"active_dns_requests\":0,\"reassem_tcp_size\":0,\"reassem_file_size\":0,\"reassem_frag_size\":0,\"reassem_unknown_size\":0}\r\n", "created": "2020-04-28T11:07:58.223Z", "kind": "metric" diff --git a/packages/zeek/data_stream/stats/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/stats/elasticsearch/ingest_pipeline/default.yml index 2384293986f7..45160e18bf60 100644 --- a/packages/zeek/data_stream/stats/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/stats/elasticsearch/ingest_pipeline/default.yml @@ -152,6 +152,7 @@ processors: field: zeek.stats.ts formats: - UNIX + - ISO8601 - remove: field: zeek.stats.ts - set: diff --git a/packages/zeek/data_stream/syslog/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/syslog/elasticsearch/ingest_pipeline/default.yml index c8bde430f545..af7617aaae79 100644 --- a/packages/zeek/data_stream/syslog/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/syslog/elasticsearch/ingest_pipeline/default.yml @@ -123,6 +123,7 @@ processors: field: zeek.syslog.ts formats: - UNIX + - ISO8601 - remove: field: zeek.syslog.ts - geoip: diff --git a/packages/zeek/data_stream/traceroute/_dev/test/pipeline/test-traceroute.log-expected.json b/packages/zeek/data_stream/traceroute/_dev/test/pipeline/test-traceroute.log-expected.json index e597e9fbd896..3320a29c7f68 100644 --- a/packages/zeek/data_stream/traceroute/_dev/test/pipeline/test-traceroute.log-expected.json +++ b/packages/zeek/data_stream/traceroute/_dev/test/pipeline/test-traceroute.log-expected.json @@ -36,7 +36,7 @@ "ip": "192.168.1.1" }, "event": { - "ingested": "2021-04-23T19:56:27.356117800Z", + "ingested": "2021-06-10T17:49:33.110110800Z", "category": [ "network" ], @@ -97,7 +97,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-04-23T19:56:27.356124800Z", + "ingested": "2021-06-10T17:49:33.110126700Z", "original": "{\"ts\":1361916158.650605,\"src\":\"192.168.1.1\",\"dst\":\"8.8.8.8\",\"proto\":\"udp\"}", "category": [ "network" diff --git a/packages/zeek/data_stream/traceroute/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/traceroute/elasticsearch/ingest_pipeline/default.yml index 8ac8f6691f9c..fdc15f2cbb16 100644 --- a/packages/zeek/data_stream/traceroute/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/traceroute/elasticsearch/ingest_pipeline/default.yml @@ -82,6 +82,7 @@ processors: field: zeek.traceroute.ts formats: - UNIX + - ISO8601 - remove: field: zeek.traceroute.ts - geoip: diff --git a/packages/zeek/data_stream/tunnel/_dev/test/pipeline/test-tunnel.log-expected.json b/packages/zeek/data_stream/tunnel/_dev/test/pipeline/test-tunnel.log-expected.json index 3ed54a8ec771..5db945cbb7a2 100644 --- a/packages/zeek/data_stream/tunnel/_dev/test/pipeline/test-tunnel.log-expected.json +++ b/packages/zeek/data_stream/tunnel/_dev/test/pipeline/test-tunnel.log-expected.json @@ -59,7 +59,7 @@ }, "event": { "action": "Tunnel::DISCOVER", - "ingested": "2021-04-23T19:56:27.433629700Z", + "ingested": "2021-06-10T17:49:33.297396500Z", "category": [ "network" ], @@ -136,7 +136,7 @@ "ip": "132.16.146.79" }, "event": { - "ingested": "2021-04-23T19:56:27.433636400Z", + "ingested": "2021-06-10T17:49:33.297412800Z", "original": "{\"ts\":1544405666.743509,\"id.orig_h\":\"132.16.146.79\",\"id.orig_p\":0,\"id.resp_h\":\"132.16.110.133\",\"id.resp_p\":8080,\"tunnel_type\":\"Tunnel::HTTP\",\"action\":\"Tunnel::DISCOVER\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/tunnel/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/tunnel/elasticsearch/ingest_pipeline/default.yml index 0011461ea48b..8927c0efef92 100644 --- a/packages/zeek/data_stream/tunnel/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/tunnel/elasticsearch/ingest_pipeline/default.yml @@ -118,6 +118,7 @@ processors: field: zeek.tunnel.ts formats: - UNIX + - ISO8601 - remove: field: zeek.tunnel.ts - geoip: diff --git a/packages/zeek/data_stream/weird/_dev/test/pipeline/test-weird.log-expected.json b/packages/zeek/data_stream/weird/_dev/test/pipeline/test-weird.log-expected.json index 8090a996de1f..7ec31ba44f66 100644 --- a/packages/zeek/data_stream/weird/_dev/test/pipeline/test-weird.log-expected.json +++ b/packages/zeek/data_stream/weird/_dev/test/pipeline/test-weird.log-expected.json @@ -30,7 +30,7 @@ "ip": "192.168.1.1" }, "event": { - "ingested": "2021-04-23T19:56:27.521977200Z", + "ingested": "2021-06-10T17:49:33.473546400Z", "id": "C1ralPp062bkwWt4e", "category": [ "network" @@ -55,7 +55,7 @@ } }, "event": { - "ingested": "2021-04-23T19:56:27.521983300Z", + "ingested": "2021-06-10T17:49:33.473559900Z", "category": [ "network" ], @@ -104,7 +104,7 @@ "ip": "192.168.1.1" }, "event": { - "ingested": "2021-04-23T19:56:27.521985200Z", + "ingested": "2021-06-10T17:49:33.473563100Z", "original": "{\"ts\":1543877999.99354,\"uid\":\"C1ralPp062bkwWt4e\",\"id.orig_h\":\"192.168.1.1\",\"id.orig_p\":64521,\"id.resp_h\":\"192.168.1.2\",\"id.resp_p\":53,\"name\":\"dns_unmatched_reply\",\"notice\":false,\"peer\":\"worker-6\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/weird/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/weird/elasticsearch/ingest_pipeline/default.yml index fd3a9dd66b79..5563e2b76961 100644 --- a/packages/zeek/data_stream/weird/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/weird/elasticsearch/ingest_pipeline/default.yml @@ -118,6 +118,7 @@ processors: field: zeek.weird.ts formats: - UNIX + - ISO8601 - remove: field: zeek.weird.ts - geoip: diff --git a/packages/zeek/data_stream/x509/_dev/test/pipeline/test-x509.log-expected.json b/packages/zeek/data_stream/x509/_dev/test/pipeline/test-x509.log-expected.json index f310ae9e87e5..11af09bf01cd 100644 --- a/packages/zeek/data_stream/x509/_dev/test/pipeline/test-x509.log-expected.json +++ b/packages/zeek/data_stream/x509/_dev/test/pipeline/test-x509.log-expected.json @@ -210,7 +210,7 @@ "session_id": "FxZ6gZ3YR6vFlIocq3" }, "event": { - "ingested": "2021-04-23T19:56:27.603067300Z", + "ingested": "2021-06-10T17:49:33.665774Z", "id": "FxZ6gZ3YR6vFlIocq3", "type": [ "info" @@ -437,7 +437,7 @@ "session_id": "FxZ6gZ3YR6vFlIocq3" }, "event": { - "ingested": "2021-04-23T19:56:27.603073Z", + "ingested": "2021-06-10T17:49:33.665789900Z", "original": "{\"ts\":1543867200.143484,\"id\":\"FxZ6gZ3YR6vFlIocq3\",\"certificate.version\":3,\"certificate.serial\":\"2D00003299D7071DB7D1708A42000000003299\",\"certificate.subject\":\"CN=www.bing.com\",\"certificate.issuer\":\"CN=Microsoft IT TLS CA 5,OU=Microsoft IT,O=Microsoft Corporation,L=Redmond,ST=Washington,C=US\",\"certificate.not_valid_before\":1500572828.0,\"certificate.not_valid_after\":1562780828.0,\"certificate.key_alg\":\"rsaEncryption\",\"certificate.sig_alg\":\"sha256WithRSAEncryption\",\"certificate.key_type\":\"rsa\",\"certificate.key_length\":2048,\"certificate.exponent\":\"65537\",\"san.dns\":[\"www.bing.com\",\"dict.bing.com.cn\",\"*.platform.bing.com\",\"*.bing.com\",\"bing.com\",\"ieonline.microsoft.com\",\"*.windowssearch.com\",\"cn.ieonline.microsoft.com\",\"*.origin.bing.com\",\"*.mm.bing.net\",\"*.api.bing.com\",\"ecn.dev.virtualearth.net\",\"*.cn.bing.net\",\"*.cn.bing.com\",\"ssl-api.bing.com\",\"ssl-api.bing.net\",\"*.api.bing.net\",\"*.bingapis.com\",\"bingsandbox.com\",\"feedback.microsoft.com\",\"insertmedia.bing.office.net\",\"r.bat.bing.com\",\"*.r.bat.bing.com\",\"*.dict.bing.com.cn\",\"*.dict.bing.com\",\"*.ssl.bing.com\",\"*.appex.bing.com\",\"*.platform.cn.bing.com\",\"wp.m.bing.com\",\"*.m.bing.com\",\"global.bing.com\",\"windowssearch.com\",\"search.msn.com\",\"*.bingsandbox.com\",\"*.api.tiles.ditu.live.com\",\"*.ditu.live.com\",\"*.t0.tiles.ditu.live.com\",\"*.t1.tiles.ditu.live.com\",\"*.t2.tiles.ditu.live.com\",\"*.t3.tiles.ditu.live.com\",\"*.tiles.ditu.live.com\",\"3d.live.com\",\"api.search.live.com\",\"beta.search.live.com\",\"cnweb.search.live.com\",\"dev.live.com\",\"ditu.live.com\",\"farecast.live.com\",\"image.live.com\",\"images.live.com\",\"local.live.com.au\",\"localsearch.live.com\",\"ls4d.search.live.com\",\"mail.live.com\",\"mapindia.live.com\",\"local.live.com\",\"maps.live.com\",\"maps.live.com.au\",\"mindia.live.com\",\"news.live.com\",\"origin.cnweb.search.live.com\",\"preview.local.live.com\",\"search.live.com\",\"test.maps.live.com\",\"video.live.com\",\"videos.live.com\",\"virtualearth.live.com\",\"wap.live.com\",\"webmaster.live.com\",\"webmasters.live.com\",\"www.local.live.com.au\",\"www.maps.live.com.au\"]}", "id": "FxZ6gZ3YR6vFlIocq3", "type": [ diff --git a/packages/zeek/data_stream/x509/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/x509/elasticsearch/ingest_pipeline/default.yml index 2d72e70ca6fc..18fdb753a31f 100644 --- a/packages/zeek/data_stream/x509/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/x509/elasticsearch/ingest_pipeline/default.yml @@ -175,6 +175,7 @@ processors: field: zeek.x509.ts formats: - UNIX + - ISO8601 - remove: field: zeek.x509.ts - set: @@ -293,6 +294,7 @@ processors: target_field: zeek.x509.certificate.valid.from formats: - UNIX + - ISO8601 if: ctx.zeek.x509.certificate?.valid?.from != null - set: field: file.x509.not_before @@ -303,6 +305,7 @@ processors: target_field: zeek.x509.certificate.valid.until formats: - UNIX + - ISO8601 if: ctx.zeek.x509.certificate?.valid?.until != null - set: field: file.x509.not_after diff --git a/packages/zeek/manifest.yml b/packages/zeek/manifest.yml index b3f5471d4d71..1ce20bd1d7bf 100644 --- a/packages/zeek/manifest.yml +++ b/packages/zeek/manifest.yml @@ -1,6 +1,6 @@ name: zeek title: Zeek -version: 0.7.2 +version: 0.7.3 release: beta description: Zeek Integration type: integration