diff --git a/packages/zeek/_dev/build/docs/README.md b/packages/zeek/_dev/build/docs/README.md index 149e2d2312d9..f9607ff0780c 100644 --- a/packages/zeek/_dev/build/docs/README.md +++ b/packages/zeek/_dev/build/docs/README.md @@ -124,6 +124,13 @@ Zeek notices. {{fields "notice"}} +### ntp + +The `ntp` dataset collects the Zeek ntp.log file, which contains +NTP data. + +{{fields "ntp"}} + ### ntlm The `ntlm` dataset collects the Zeek ntlm.log file, which contains NT @@ -166,6 +173,13 @@ Remote Framebuffer (RFB) data. {{fields "rfb"}} +### signature + +The `signature` dataset collects the Zeek signature.log file, which contains +Zeek signature matches. + +{{fields "signature"}} + ### sip The `sip` dataset collects the Zeek sip.log file, which contains SIP diff --git a/packages/zeek/_dev/deploy/docker/sample_logs/ntp.log b/packages/zeek/_dev/deploy/docker/sample_logs/ntp.log new file mode 100644 index 000000000000..9799c888dba7 --- /dev/null +++ b/packages/zeek/_dev/deploy/docker/sample_logs/ntp.log @@ -0,0 +1,2 @@ +{"ts":1602116947.977,"uid":"CqlPpF1AQVLMPgGiL5","id.orig_h":"130.118.205.62","id.orig_p":38461,"id.resp_h":"208.79.89.249","id.resp_p":123,"version":4,"mode":3,"stratum":0,"poll":1,"precision":1,"root_delay":0,"root_disp":0,"ref_id":"\\x00\\x00\\x00\\x00","ref_time":0,"org_time":0,"rec_time":0,"xmt_time":1602116947.215,"num_exts":0} +{"ts":1602116948.081,"uid":"CqlPpF1AQVLMPgGiL5","id.orig_h":"130.118.205.62","id.orig_p":38461,"id.resp_h":"208.79.89.249","id.resp_p":123,"version":4,"mode":4,"stratum":2,"poll":8,"precision":5.960464477539063e-8,"root_delay":0.00921630859375,"root_disp":0.0212249755859375,"ref_id":"127.67.113.92","ref_time":1602116655.942,"org_time":1602116947.215,"rec_time":1602116947.964,"xmt_time":1602116947.964,"num_exts":0} diff --git a/packages/zeek/_dev/deploy/docker/sample_logs/signature.log b/packages/zeek/_dev/deploy/docker/sample_logs/signature.log new file mode 100644 index 000000000000..4725117d90e6 --- /dev/null +++ b/packages/zeek/_dev/deploy/docker/sample_logs/signature.log @@ -0,0 +1 @@ +{"ts": 1611852809.869245,"uid": "CbjAXE4CBxJ8W7VoJg","src_addr": "124.51.137.154","src_port": 51617,"dst_addr": "160.218.27.63","dst_port": 445,"note": "Signatures::Sensitive_Signature","sig_id": "my-second-sig","event_msg": "124.51.137.154: TCP traffic","sub_msg": ""} diff --git a/packages/zeek/changelog.yml b/packages/zeek/changelog.yml index 5b91092089d2..0f3a64035a0c 100644 --- a/packages/zeek/changelog.yml +++ b/packages/zeek/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.3.0" + changes: + - description: Add Sigature and NTP data streams + type: enhancement + link: https://github.com/elastic/integrations/pull/1515 - version: '1.2.1' changes: - description: update to ECS 1.11.0 diff --git a/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log-expected.json b/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log-expected.json index acab4e77c63d..86da77298912 100644 --- a/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log-expected.json +++ b/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log-expected.json @@ -22,12 +22,11 @@ "session_id": "CAcJw21BbVedgFnYH3", "connection": { "local_resp": true, - "local_orig": true, "missed_bytes": 0, "history": "Dd", - "id": {}, "state": "SF", - "state_message": "Normal establishment and termination." + "state_message": "Normal establishment and termination.", + "local_orig": true } }, "source": { @@ -39,7 +38,7 @@ }, "event": { "duration": 76967000, - "ingested": "2021-06-17T13:41:23.047071700Z", + "ingested": "2021-08-10T12:16:43.225275533Z", "original": "{\"ts\":1547188415.857497,\"uid\":\"CAcJw21BbVedgFnYH3\",\"id.orig_h\":\"192.168.86.167\",\"id.orig_p\":38339,\"id.resp_h\":\"192.168.86.1\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.076967,\"orig_bytes\":75,\"resp_bytes\":178,\"conn_state\":\"SF\",\"local_orig\":true,\"local_resp\":true,\"missed_bytes\":0,\"history\":\"Dd\",\"orig_pkts\":1,\"orig_ip_bytes\":103,\"resp_pkts\":1,\"resp_ip_bytes\":206,\"tunnel_parents\":[]}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -102,12 +101,11 @@ "session_id": "CAcJw21BbVedgFnYH4", "connection": { "local_resp": false, - "local_orig": true, "missed_bytes": 0, "history": "Dd", - "id": {}, "state": "SF", - "state_message": "Normal establishment and termination." + "state_message": "Normal establishment and termination.", + "local_orig": true } }, "source": { @@ -119,7 +117,7 @@ }, "event": { "duration": 76967000, - "ingested": "2021-06-17T13:41:23.047082500Z", + "ingested": "2021-08-10T12:16:43.225299658Z", "original": "{\"ts\":1547188416.857497,\"uid\":\"CAcJw21BbVedgFnYH4\",\"id.orig_h\":\"192.168.86.167\",\"id.orig_p\":38340,\"id.resp_h\":\"8.8.8.8\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.076967,\"orig_bytes\":75,\"resp_bytes\":178,\"conn_state\":\"SF\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Dd\",\"orig_pkts\":1,\"orig_ip_bytes\":103,\"resp_pkts\":1,\"resp_ip_bytes\":206,\"tunnel_parents\":[]}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -182,12 +180,11 @@ "session_id": "CAcJw21BbVedgFnYH5", "connection": { "local_resp": false, - "local_orig": false, "missed_bytes": 0, "history": "Dd", - "id": {}, "state": "SF", - "state_message": "Normal establishment and termination." + "state_message": "Normal establishment and termination.", + "local_orig": false } }, "source": { @@ -214,7 +211,7 @@ }, "event": { "duration": 76967000, - "ingested": "2021-06-17T13:41:23.047090200Z", + "ingested": "2021-08-10T12:16:43.225307774Z", "original": "{\"ts\":1547188417.857497,\"uid\":\"CAcJw21BbVedgFnYH5\",\"id.orig_h\":\"4.4.2.2\",\"id.orig_p\":38334,\"id.resp_h\":\"8.8.8.8\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.076967,\"orig_bytes\":75,\"resp_bytes\":178,\"conn_state\":\"SF\",\"local_orig\":false,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Dd\",\"orig_pkts\":1,\"orig_ip_bytes\":103,\"resp_pkts\":1,\"resp_ip_bytes\":206,\"tunnel_parents\":[]}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -261,15 +258,14 @@ "session_id": "Cc6NJ3GRlfjE44I3h", "connection": { "local_resp": false, - "local_orig": false, "missed_bytes": 0, - "id": {}, "state": "OTH", "icmp": { "type": 3, "code": 3 }, - "state_message": "No SYN seen, just midstream traffic (a 'partial connection' that was not later closed)." + "state_message": "No SYN seen, just midstream traffic (a 'partial connection' that was not later closed).", + "local_orig": false } }, "source": { @@ -279,7 +275,7 @@ "ip": "192.0.2.205" }, "event": { - "ingested": "2021-06-17T13:41:23.047097900Z", + "ingested": "2021-08-10T12:16:43.225310341Z", "original": "{\"ts\":1551399000.57855,\"uid\":\"Cc6NJ3GRlfjE44I3h\",\"id.orig_h\":\"192.0.2.205\",\"id.orig_p\":3,\"id.resp_h\":\"198.51.100.249\",\"id.resp_p\":3,\"proto\":\"icmp\",\"conn_state\":\"OTH\",\"local_orig\":false,\"local_resp\":false,\"missed_bytes\":0,\"orig_pkts\":1,\"orig_ip_bytes\":107,\"resp_pkts\":0,\"resp_ip_bytes\":0,\"tunnel_parents\":[]}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -343,12 +339,11 @@ "session_id": "CCicIg43lOtCQOxXnb", "connection": { "local_resp": false, - "local_orig": true, "missed_bytes": 0, "history": "C", - "id": {}, "state": "OTH", - "state_message": "No SYN seen, just midstream traffic (a 'partial connection' that was not later closed)." + "state_message": "No SYN seen, just midstream traffic (a 'partial connection' that was not later closed).", + "local_orig": true } }, "source": { @@ -359,7 +354,7 @@ "ip": "10.156.0.2" }, "event": { - "ingested": "2021-06-17T13:41:23.047105600Z", + "ingested": "2021-08-10T12:16:43.225312808Z", "original": "{\"ts\":1617062400.404645,\"uid\":\"CCicIg43lOtCQOxXnb\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":56190,\"id.resp_h\":\"46.101.87.151\",\"id.resp_p\":443,\"proto\":\"tcp\",\"conn_state\":\"OTH\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"C\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":0,\"resp_ip_bytes\":0}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -423,12 +418,11 @@ "session_id": "C52mXBCPJ4pPGkhr1", "connection": { "local_resp": false, - "local_orig": true, "missed_bytes": 0, "history": "^hCcdafA", - "id": {}, "state": "SHR", - "state_message": "Responder sent a SYN ACK followed by a FIN, we never saw a SYN from the originator." + "state_message": "Responder sent a SYN ACK followed by a FIN, we never saw a SYN from the originator.", + "local_orig": true } }, "source": { @@ -440,7 +434,7 @@ }, "event": { "duration": 103708982, - "ingested": "2021-06-17T13:41:23.047113200Z", + "ingested": "2021-08-10T12:16:43.225315271Z", "original": "{\"ts\":1617062100.419397,\"uid\":\"C52mXBCPJ4pPGkhr1\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":60810,\"id.resp_h\":\"20.190.160.73\",\"id.resp_p\":443,\"proto\":\"tcp\",\"duration\":0.10370898246765137,\"orig_bytes\":0,\"resp_bytes\":5854,\"conn_state\":\"SHR\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"^hCcdafA\",\"orig_pkts\":1,\"orig_ip_bytes\":52,\"resp_pkts\":4,\"resp_ip_bytes\":267}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -504,12 +498,11 @@ "session_id": "CTzCky2CyLT5JJvHck", "connection": { "local_resp": false, - "local_orig": true, "missed_bytes": 0, "history": "^hCcdafA", - "id": {}, "state": "SHR", - "state_message": "Responder sent a SYN ACK followed by a FIN, we never saw a SYN from the originator." + "state_message": "Responder sent a SYN ACK followed by a FIN, we never saw a SYN from the originator.", + "local_orig": true } }, "source": { @@ -521,7 +514,7 @@ }, "event": { "duration": 104128838, - "ingested": "2021-06-17T13:41:23.047120800Z", + "ingested": "2021-08-10T12:16:43.225317743Z", "original": "{\"ts\":1617062100.419603,\"uid\":\"CTzCky2CyLT5JJvHck\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":60804,\"id.resp_h\":\"20.190.160.73\",\"id.resp_p\":443,\"proto\":\"tcp\",\"duration\":0.10412883758544922,\"orig_bytes\":0,\"resp_bytes\":5854,\"conn_state\":\"SHR\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"^hCcdafA\",\"orig_pkts\":1,\"orig_ip_bytes\":52,\"resp_pkts\":4,\"resp_ip_bytes\":267}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -585,12 +578,11 @@ "session_id": "CIkS28PDxqQnN49m2", "connection": { "local_resp": false, - "local_orig": true, "missed_bytes": 0, "history": "^hCcdafA", - "id": {}, "state": "SHR", - "state_message": "Responder sent a SYN ACK followed by a FIN, we never saw a SYN from the originator." + "state_message": "Responder sent a SYN ACK followed by a FIN, we never saw a SYN from the originator.", + "local_orig": true } }, "source": { @@ -602,7 +594,7 @@ }, "event": { "duration": 104333878, - "ingested": "2021-06-17T13:41:23.047132500Z", + "ingested": "2021-08-10T12:16:43.225320249Z", "original": "{\"ts\":1617062100.419826,\"uid\":\"CIkS28PDxqQnN49m2\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":60802,\"id.resp_h\":\"20.190.160.73\",\"id.resp_p\":443,\"proto\":\"tcp\",\"duration\":0.10433387756347656,\"orig_bytes\":0,\"resp_bytes\":5854,\"conn_state\":\"SHR\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"^hCcdafA\",\"orig_pkts\":1,\"orig_ip_bytes\":52,\"resp_pkts\":4,\"resp_ip_bytes\":267}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -648,12 +640,11 @@ "session_id": "CezEGe4jeLNkayV976", "connection": { "local_resp": false, - "local_orig": true, "missed_bytes": 0, "history": "Cd", - "id": {}, "state": "SHR", - "state_message": "Responder sent a SYN ACK followed by a FIN, we never saw a SYN from the originator." + "state_message": "Responder sent a SYN ACK followed by a FIN, we never saw a SYN from the originator.", + "local_orig": true } }, "source": { @@ -665,7 +656,7 @@ }, "event": { "duration": 26802063, - "ingested": "2021-06-17T13:41:23.047137100Z", + "ingested": "2021-08-10T12:16:43.225322698Z", "original": "{\"ts\":1617062390.563187,\"uid\":\"CezEGe4jeLNkayV976\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":38948,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.02680206298828125,\"orig_bytes\":0,\"resp_bytes\":241,\"conn_state\":\"SHR\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Cd\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":1,\"resp_ip_bytes\":269}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -712,12 +703,11 @@ "session_id": "CKSr3w18mmW6t7bXC4", "connection": { "local_resp": false, - "local_orig": true, "missed_bytes": 0, "history": "Cd", - "id": {}, "state": "SHR", - "state_message": "Responder sent a SYN ACK followed by a FIN, we never saw a SYN from the originator." + "state_message": "Responder sent a SYN ACK followed by a FIN, we never saw a SYN from the originator.", + "local_orig": true } }, "source": { @@ -729,7 +719,7 @@ }, "event": { "duration": 25056124, - "ingested": "2021-06-17T13:41:23.047142500Z", + "ingested": "2021-08-10T12:16:43.225325161Z", "original": "{\"ts\":1617062390.563442,\"uid\":\"CKSr3w18mmW6t7bXC4\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":40080,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.025056123733520509,\"orig_bytes\":0,\"resp_bytes\":276,\"conn_state\":\"SHR\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Cd\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":1,\"resp_ip_bytes\":304}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -776,12 +766,11 @@ "session_id": "CGUiHy4kLIF2ml95eg", "connection": { "local_resp": false, - "local_orig": true, "missed_bytes": 0, "history": "Cd", - "id": {}, "state": "SHR", - "state_message": "Responder sent a SYN ACK followed by a FIN, we never saw a SYN from the originator." + "state_message": "Responder sent a SYN ACK followed by a FIN, we never saw a SYN from the originator.", + "local_orig": true } }, "source": { @@ -793,7 +782,7 @@ }, "event": { "duration": 3319979, - "ingested": "2021-06-17T13:41:23.047150400Z", + "ingested": "2021-08-10T12:16:43.225327579Z", "original": "{\"ts\":1617062390.667048,\"uid\":\"CGUiHy4kLIF2ml95eg\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":41407,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.003319978713989258,\"orig_bytes\":0,\"resp_bytes\":133,\"conn_state\":\"SHR\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Cd\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":1,\"resp_ip_bytes\":161}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -840,12 +829,11 @@ "session_id": "CAOZZi4Qrio7gUVgVc", "connection": { "local_resp": false, - "local_orig": true, "missed_bytes": 0, "history": "Cd", - "id": {}, "state": "SHR", - "state_message": "Responder sent a SYN ACK followed by a FIN, we never saw a SYN from the originator." + "state_message": "Responder sent a SYN ACK followed by a FIN, we never saw a SYN from the originator.", + "local_orig": true } }, "source": { @@ -857,7 +845,7 @@ }, "event": { "duration": 1111984, - "ingested": "2021-06-17T13:41:23.047158300Z", + "ingested": "2021-08-10T12:16:43.225330390Z", "original": "{\"ts\":1617062390.698943,\"uid\":\"CAOZZi4Qrio7gUVgVc\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":50487,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.0011119842529296876,\"orig_bytes\":0,\"resp_bytes\":202,\"conn_state\":\"SHR\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Cd\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":1,\"resp_ip_bytes\":230}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -904,12 +892,11 @@ "session_id": "Chx5fs3xQ5ALB72i4e", "connection": { "local_resp": false, - "local_orig": true, "missed_bytes": 0, "history": "Cd", - "id": {}, "state": "SHR", - "state_message": "Responder sent a SYN ACK followed by a FIN, we never saw a SYN from the originator." + "state_message": "Responder sent a SYN ACK followed by a FIN, we never saw a SYN from the originator.", + "local_orig": true } }, "source": { @@ -921,7 +908,7 @@ }, "event": { "duration": 908852, - "ingested": "2021-06-17T13:41:23.047166500Z", + "ingested": "2021-08-10T12:16:43.225332830Z", "original": "{\"ts\":1617062390.699227,\"uid\":\"Chx5fs3xQ5ALB72i4e\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":49647,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.0009088516235351563,\"orig_bytes\":0,\"resp_bytes\":145,\"conn_state\":\"SHR\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Cd\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":1,\"resp_ip_bytes\":173}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -968,12 +955,11 @@ "session_id": "C3pPjh1YRYcVDiZD3", "connection": { "local_resp": false, - "local_orig": true, "missed_bytes": 0, "history": "C", - "id": {}, "state": "OTH", - "state_message": "No SYN seen, just midstream traffic (a 'partial connection' that was not later closed)." + "state_message": "No SYN seen, just midstream traffic (a 'partial connection' that was not later closed).", + "local_orig": true } }, "source": { @@ -984,7 +970,7 @@ "ip": "10.156.0.2" }, "event": { - "ingested": "2021-06-17T13:41:23.047174Z", + "ingested": "2021-08-10T12:16:43.225335244Z", "original": "{\"ts\":1617062400.703865,\"uid\":\"C3pPjh1YRYcVDiZD3\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":44944,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":80,\"proto\":\"tcp\",\"conn_state\":\"OTH\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"C\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":0,\"resp_ip_bytes\":0}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -1030,12 +1016,11 @@ "session_id": "ChUxTmYLG37oO5qUb", "connection": { "local_resp": false, - "local_orig": true, "missed_bytes": 0, "history": "C", - "id": {}, "state": "OTH", - "state_message": "No SYN seen, just midstream traffic (a 'partial connection' that was not later closed)." + "state_message": "No SYN seen, just midstream traffic (a 'partial connection' that was not later closed).", + "local_orig": true } }, "source": { @@ -1046,7 +1031,7 @@ "ip": "10.156.0.2" }, "event": { - "ingested": "2021-06-17T13:41:23.047181400Z", + "ingested": "2021-08-10T12:16:43.225337683Z", "original": "{\"ts\":1617062400.703851,\"uid\":\"ChUxTmYLG37oO5qUb\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":44942,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":80,\"proto\":\"tcp\",\"conn_state\":\"OTH\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"C\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":0,\"resp_ip_bytes\":0}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -1092,12 +1077,11 @@ "session_id": "CpeAOT3B11CTXJgzw2", "connection": { "local_resp": false, - "local_orig": true, "missed_bytes": 0, "history": "C", - "id": {}, "state": "OTH", - "state_message": "No SYN seen, just midstream traffic (a 'partial connection' that was not later closed)." + "state_message": "No SYN seen, just midstream traffic (a 'partial connection' that was not later closed).", + "local_orig": true } }, "source": { @@ -1108,7 +1092,7 @@ "ip": "10.156.0.2" }, "event": { - "ingested": "2021-06-17T13:41:23.047205800Z", + "ingested": "2021-08-10T12:16:43.225340124Z", "original": "{\"ts\":1617062400.704467,\"uid\":\"CpeAOT3B11CTXJgzw2\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":44946,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":80,\"proto\":\"tcp\",\"conn_state\":\"OTH\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"C\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":0,\"resp_ip_bytes\":0}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -1164,12 +1148,11 @@ "session_id": "CAcJw21BbVedgFnYH5", "connection": { "local_resp": false, - "local_orig": false, "missed_bytes": 0, "history": "Dd", - "id": {}, "state": "SF", - "state_message": "Normal establishment and termination." + "state_message": "Normal establishment and termination.", + "local_orig": false } }, "source": { @@ -1222,7 +1205,7 @@ }, "event": { "duration": 76967000, - "ingested": "2021-06-17T13:41:23.047213100Z", + "ingested": "2021-08-10T12:16:43.225342661Z", "original": "{\"ts\":1547188417.857497,\"uid\":\"CAcJw21BbVedgFnYH5\",\"id.orig_h\":\"4.4.2.2\",\"id.orig_p\":38334,\"id.resp_h\":\"8.8.8.8\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.076967,\"orig_bytes\":75,\"resp_bytes\":178,\"conn_state\":\"SF\",\"local_orig\":false,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Dd\",\"orig_pkts\":1,\"orig_ip_bytes\":103,\"resp_pkts\":1,\"resp_ip_bytes\":206,\"tunnel_parents\":[]}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -1272,12 +1255,11 @@ "session_id": "C2KP1V3alRLoxl4JB9", "connection": { "local_resp": false, - "local_orig": true, "missed_bytes": 0, "history": "C", - "id": {}, "state": "OTH", - "state_message": "No SYN seen, just midstream traffic (a 'partial connection' that was not later closed)." + "state_message": "No SYN seen, just midstream traffic (a 'partial connection' that was not later closed).", + "local_orig": true } }, "source": { @@ -1288,7 +1270,7 @@ "ip": "10.0.2.15" }, "event": { - "ingested": "2021-06-17T13:41:23.047220300Z", + "ingested": "2021-08-10T12:16:43.225345083Z", "original": "{\"ts\":\"2021-06-09T20:55:13.160328Z\",\"uid\":\"C2KP1V3alRLoxl4JB9\",\"id.orig_h\":\"10.0.2.15\",\"id.orig_p\":46408,\"id.resp_h\":\"172.217.9.68\",\"id.resp_p\":80,\"proto\":\"tcp\",\"conn_state\":\"OTH\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"C\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":0,\"resp_ip_bytes\":0}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/connection/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/connection/elasticsearch/ingest_pipeline/default.yml index 673d220e6a41..336d669e6f5d 100644 --- a/packages/zeek/data_stream/connection/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/connection/elasticsearch/ingest_pipeline/default.yml @@ -322,6 +322,7 @@ processors: } - remove: field: + - zeek.connection.id - zeek.connection.orig_bytes - zeek.connection.resp_bytes - zeek.connection.tunnel_parents diff --git a/packages/zeek/data_stream/ntp/_dev/test/pipeline/test-ntp.log b/packages/zeek/data_stream/ntp/_dev/test/pipeline/test-ntp.log new file mode 100644 index 000000000000..9799c888dba7 --- /dev/null +++ b/packages/zeek/data_stream/ntp/_dev/test/pipeline/test-ntp.log @@ -0,0 +1,2 @@ +{"ts":1602116947.977,"uid":"CqlPpF1AQVLMPgGiL5","id.orig_h":"130.118.205.62","id.orig_p":38461,"id.resp_h":"208.79.89.249","id.resp_p":123,"version":4,"mode":3,"stratum":0,"poll":1,"precision":1,"root_delay":0,"root_disp":0,"ref_id":"\\x00\\x00\\x00\\x00","ref_time":0,"org_time":0,"rec_time":0,"xmt_time":1602116947.215,"num_exts":0} +{"ts":1602116948.081,"uid":"CqlPpF1AQVLMPgGiL5","id.orig_h":"130.118.205.62","id.orig_p":38461,"id.resp_h":"208.79.89.249","id.resp_p":123,"version":4,"mode":4,"stratum":2,"poll":8,"precision":5.960464477539063e-8,"root_delay":0.00921630859375,"root_disp":0.0212249755859375,"ref_id":"127.67.113.92","ref_time":1602116655.942,"org_time":1602116947.215,"rec_time":1602116947.964,"xmt_time":1602116947.964,"num_exts":0} diff --git a/packages/zeek/data_stream/ntp/_dev/test/pipeline/test-ntp.log-config.yml b/packages/zeek/data_stream/ntp/_dev/test/pipeline/test-ntp.log-config.yml new file mode 100644 index 000000000000..3cabcf9fb82a --- /dev/null +++ b/packages/zeek/data_stream/ntp/_dev/test/pipeline/test-ntp.log-config.yml @@ -0,0 +1,6 @@ +dynamic_fields: + event.ingested: ".*" +fields: + "@timestamp": "2020-04-28T11:07:58.223Z" + tags: + - preserve_original_event diff --git a/packages/zeek/data_stream/ntp/_dev/test/pipeline/test-ntp.log-expected.json b/packages/zeek/data_stream/ntp/_dev/test/pipeline/test-ntp.log-expected.json new file mode 100644 index 000000000000..cb59e72ff23a --- /dev/null +++ b/packages/zeek/data_stream/ntp/_dev/test/pipeline/test-ntp.log-expected.json @@ -0,0 +1,176 @@ +{ + "expected": [ + { + "@timestamp": "2020-10-08T00:29:07.977Z", + "ecs": { + "version": "1.10.0" + }, + "related": { + "ip": [ + "130.118.205.62", + "208.79.89.249" + ] + }, + "destination": { + "geo": { + "continent_name": "North America", + "country_name": "United States", + "location": { + "lon": -97.822, + "lat": 37.751 + }, + "country_iso_code": "US" + }, + "as": { + "number": 25795, + "organization": { + "name": "ARP NETWORKS, INC." + } + }, + "address": "208.79.89.249", + "port": 123, + "ip": "208.79.89.249" + }, + "zeek": { + "session_id": "CqlPpF1AQVLMPgGiL5", + "ntp": { + "ref_id": "\\x00\\x00\\x00\\x00", + "rec_time": "1970-01-01T00:00:00.000Z", + "ref_time": "1970-01-01T00:00:00.000Z", + "root_delay": 0, + "precision": 1, + "poll": 1, + "version": 4, + "num_exts": 0, + "stratum": 0, + "mode": 3, + "root_disp": 0, + "org_time": "1970-01-01T00:00:00.000Z", + "xmt_time": "2020-10-08T00:29:07.215Z" + } + }, + "source": { + "geo": { + "continent_name": "North America", + "country_name": "United States", + "location": { + "lon": -97.822, + "lat": 37.751 + }, + "country_iso_code": "US" + }, + "address": "130.118.205.62", + "port": 38461, + "ip": "130.118.205.62" + }, + "event": { + "ingested": "2021-08-10T12:16:45.963639174Z", + "original": "{\"ts\":1602116947.977,\"uid\":\"CqlPpF1AQVLMPgGiL5\",\"id.orig_h\":\"130.118.205.62\",\"id.orig_p\":38461,\"id.resp_h\":\"208.79.89.249\",\"id.resp_p\":123,\"version\":4,\"mode\":3,\"stratum\":0,\"poll\":1,\"precision\":1,\"root_delay\":0,\"root_disp\":0,\"ref_id\":\"\\\\x00\\\\x00\\\\x00\\\\x00\",\"ref_time\":0,\"org_time\":0,\"rec_time\":0,\"xmt_time\":1602116947.215,\"num_exts\":0}", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", + "id": "CqlPpF1AQVLMPgGiL5", + "category": "network", + "type": [ + "connection", + "protocol", + "info" + ] + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "ntp", + "community_id": "1:IDiKR+C1G8mk7LQhFpp+4p1tHrk=", + "transport": "udp", + "type": "ipv4" + } + }, + { + "@timestamp": "2020-10-08T00:29:08.081Z", + "ecs": { + "version": "1.10.0" + }, + "related": { + "ip": [ + "130.118.205.62", + "208.79.89.249" + ] + }, + "destination": { + "geo": { + "continent_name": "North America", + "country_name": "United States", + "location": { + "lon": -97.822, + "lat": 37.751 + }, + "country_iso_code": "US" + }, + "as": { + "number": 25795, + "organization": { + "name": "ARP NETWORKS, INC." + } + }, + "address": "208.79.89.249", + "port": 123, + "ip": "208.79.89.249" + }, + "zeek": { + "session_id": "CqlPpF1AQVLMPgGiL5", + "ntp": { + "ref_id": "127.67.113.92", + "rec_time": "2020-10-08T00:29:07.964Z", + "ref_time": "2020-10-08T00:24:15.942Z", + "root_delay": 0.00921630859375, + "precision": 5.9604644775390625E-8, + "poll": 8, + "version": 4, + "num_exts": 0, + "stratum": 2, + "mode": 4, + "root_disp": 0.0212249755859375, + "org_time": "2020-10-08T00:29:07.215Z", + "xmt_time": "2020-10-08T00:29:07.964Z" + } + }, + "source": { + "geo": { + "continent_name": "North America", + "country_name": "United States", + "location": { + "lon": -97.822, + "lat": 37.751 + }, + "country_iso_code": "US" + }, + "address": "130.118.205.62", + "port": 38461, + "ip": "130.118.205.62" + }, + "event": { + "ingested": "2021-08-10T12:16:45.963645447Z", + "original": "{\"ts\":1602116948.081,\"uid\":\"CqlPpF1AQVLMPgGiL5\",\"id.orig_h\":\"130.118.205.62\",\"id.orig_p\":38461,\"id.resp_h\":\"208.79.89.249\",\"id.resp_p\":123,\"version\":4,\"mode\":4,\"stratum\":2,\"poll\":8,\"precision\":5.960464477539063e-8,\"root_delay\":0.00921630859375,\"root_disp\":0.0212249755859375,\"ref_id\":\"127.67.113.92\",\"ref_time\":1602116655.942,\"org_time\":1602116947.215,\"rec_time\":1602116947.964,\"xmt_time\":1602116947.964,\"num_exts\":0}", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", + "id": "CqlPpF1AQVLMPgGiL5", + "category": "network", + "type": [ + "connection", + "protocol", + "info" + ] + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "ntp", + "community_id": "1:IDiKR+C1G8mk7LQhFpp+4p1tHrk=", + "transport": "udp", + "type": "ipv4" + } + } + ] +} \ No newline at end of file diff --git a/packages/zeek/data_stream/ntp/_dev/test/system/test-logs-config.yml b/packages/zeek/data_stream/ntp/_dev/test/system/test-logs-config.yml new file mode 100644 index 000000000000..5cfff5002996 --- /dev/null +++ b/packages/zeek/data_stream/ntp/_dev/test/system/test-logs-config.yml @@ -0,0 +1,6 @@ +vars: + base_paths: + - "{{SERVICE_LOGS_DIR}}" +input: logfile +data_stream: + vars: ~ diff --git a/packages/zeek/data_stream/ntp/_dev/test/system/test-splunk-config.yml b/packages/zeek/data_stream/ntp/_dev/test/system/test-splunk-config.yml new file mode 100644 index 000000000000..dfa8f5c9201f --- /dev/null +++ b/packages/zeek/data_stream/ntp/_dev/test/system/test-splunk-config.yml @@ -0,0 +1,9 @@ +input: httpjson +service: splunk-mock +vars: + url: http://{{Hostname}}:{{Port}} + username: test + password: test +data_stream: + vars: + preserve_original_event: true diff --git a/packages/zeek/data_stream/ntp/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/ntp/agent/stream/httpjson.yml.hbs new file mode 100644 index 000000000000..a2f2528bfb26 --- /dev/null +++ b/packages/zeek/data_stream/ntp/agent/stream/httpjson.yml.hbs @@ -0,0 +1,63 @@ +config_version: 2 +interval: {{interval}} +{{#unless token}} +{{#if username}} +{{#if password}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +{{/if}} +{{/if}} +{{/unless}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +{{#unless username}} +{{#unless password}} +{{#if token}} + - set: + target: header.Authorization + value: {{token}} +{{/if}} +{{/unless}} +{{/unless}} +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/packages/zeek/data_stream/ntp/agent/stream/log.yml.hbs b/packages/zeek/data_stream/ntp/agent/stream/log.yml.hbs new file mode 100644 index 000000000000..d5c9b7811b0c --- /dev/null +++ b/packages/zeek/data_stream/ntp/agent/stream/log.yml.hbs @@ -0,0 +1,21 @@ +paths: +{{#each base_paths}} + {{#each ../filenames}} + - {{../this}}/{{this}} + {{/each}} +{{/each}} +exclude_files: [".gz$"] +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/packages/zeek/data_stream/ntp/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/ntp/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 000000000000..d0752c4975b4 --- /dev/null +++ b/packages/zeek/data_stream/ntp/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,195 @@ +--- +description: Pipeline for normalizing Zeek conn.log +processors: + - rename: + field: message + target_field: event.original + - json: + field: event.original + target_field: _temp_ + - pipeline: + if: ctx?._temp_?.result != null + name: '{{ IngestPipeline "third-party" }}' + - drop: + description: Drop if no timestamp (invalid json) + if: 'ctx?._temp_?.ts == null' + - rename: + field: _temp_ + target_field: zeek.ntp + ignore_failure: true + - set: + field: event.ingested + value: "{{_ingest.timestamp}}" +# Sets event.created from the @timestamp field generated by filebeat before being overwritten further down + - set: + field: event.created + copy_from: "@timestamp" + - set: + field: ecs.version + value: '1.10.0' + - set: + field: event.kind + value: event + - set: + field: event.category + value: network + - append: + field: event.type + value: + - connection + - protocol + - info + allow_duplicates: false + - dot_expander: + path: zeek.ntp + field: id.orig_p + ignore_failure: true + - dot_expander: + path: zeek.ntp + field: id.orig_h + ignore_failure: true + - dot_expander: + path: zeek.ntp + field: id.resp_h + ignore_failure: true + - dot_expander: + path: zeek.ntp + field: id.resp_p + ignore_failure: true + - rename: + field: zeek.ntp.id.orig_h + target_field: source.address + ignore_missing: true + - rename: + field: zeek.ntp.id.orig_p + target_field: source.port + ignore_missing: true + - rename: + field: zeek.ntp.id.resp_h + target_field: destination.address + ignore_missing: true + - rename: + field: zeek.ntp.id.resp_p + target_field: destination.port + ignore_missing: true + - rename: + field: zeek.ntp.uid + target_field: zeek.session_id + ignore_missing: true + - set: + field: source.ip + copy_from: source.address + if: ctx?.source?.address != null + - set: + field: destination.ip + copy_from: destination.address + if: ctx?.destination?.address != null + - set: + field: network.transport + value: udp + - set: + field: network.protocol + value: ntp + - set: + field: network.type + value: ipv4 + if: ctx.source?.ip.contains('.') + - set: + field: network.type + value: ipv6 + if: ctx.source?.ip.contains(':') + - community_id: + ignore_missing: true + - date: + field: zeek.ntp.ts + formats: + - UNIX + - ISO8601 + - date: + field: zeek.ntp.ref_time + target_field: zeek.ntp.ref_time + formats: + - UNIX + - date: + field: zeek.ntp.org_time + target_field: zeek.ntp.org_time + formats: + - UNIX + - date: + field: zeek.ntp.rec_time + target_field: zeek.ntp.rec_time + formats: + - UNIX + - date: + field: zeek.ntp.xmt_time + target_field: zeek.ntp.xmt_time + formats: + - UNIX + - set: + field: event.id + copy_from: zeek.session_id + if: ctx.zeek.session_id != null + - append: + field: related.ip + value: "{{source.ip}}" + if: ctx?.source?.ip != null + allow_duplicates: false + - append: + field: related.ip + value: "{{destination.ip}}" + if: ctx?.destination?.ip != null + allow_duplicates: false + - geoip: + field: destination.ip + target_field: destination.geo + ignore_missing: true + - geoip: + field: source.ip + target_field: source.geo + ignore_missing: true + - geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true + - geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true + - rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true + - rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true + - rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true + - rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true + - remove: + field: + - zeek.ntp.id + - zeek.ntp.ts + ignore_missing: true + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true +on_failure: + - set: + field: error.message + value: "{{ _ingest.on_failure_message }}" diff --git a/packages/zeek/data_stream/ntp/elasticsearch/ingest_pipeline/third-party.yml b/packages/zeek/data_stream/ntp/elasticsearch/ingest_pipeline/third-party.yml new file mode 100644 index 000000000000..5bc2247db25a --- /dev/null +++ b/packages/zeek/data_stream/ntp/elasticsearch/ingest_pipeline/third-party.yml @@ -0,0 +1,39 @@ +--- +description: Pipeline for parsing Zeek logs from third party api +processors: + - fingerprint: + fields: + - _temp_.result._cd + - _temp_.result._indextime + - _temp_.result._raw + - _temp_.result._time + - _temp_.result.host + - _temp_.result.source + target_field: '_id' + ignore_missing: true + - set: + field: event.original + copy_from: _temp_.result._raw + ignore_empty_value: true + - set: + field: host.name + copy_from: _temp_.result.host + ignore_empty_value: true + - set: + copy_from: _temp_.result.source + field: log.file.path + ignore_empty_value: true + - remove: + field: _temp_ + ignore_missing: true + - json: + field: event.original + target_field: _temp_ +on_failure: + - append: + field: error.message + value: >- + error in third party api pipeline: + error in [{{_ingest.on_failure_processor_type}}] processor{{#_ingest.on_failure_processor_tag}} + with tag [{{_ingest.on_failure_processor_tag }}]{{/_ingest.on_failure_processor_tag}} + {{ _ingest.on_failure_message }} diff --git a/packages/zeek/data_stream/ntp/fields/agent.yml b/packages/zeek/data_stream/ntp/fields/agent.yml new file mode 100644 index 000000000000..79a7a39864bd --- /dev/null +++ b/packages/zeek/data_stream/ntp/fields/agent.yml @@ -0,0 +1,180 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: "Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on." + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: "The cloud account or organization id used to identify different entities in a multi-tenant environment.\nExamples: AWS account id, Google Cloud ORG Id, or other unique identifier." + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: "Container fields are used for meta information about the specific container that is the source of information.\nThese fields help correlate data based containers from any runtime." + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: "A host is defined as a general computing instance.\nECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes." + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: "Name of the domain of which the host is a member.\nFor example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider." + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: "Hostname of the host.\nIt normally contains what the `hostname` command returns on the host machine." + - name: id + level: core + type: keyword + ignore_above: 1024 + description: "Unique host id.\nAs hostname is not always unique, use values that are meaningful in your environment.\nExample: The current usage of `beat.name`." + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: "Name of the host.\nIt can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use." + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: "Type of host.\nFor Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment." + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + diff --git a/packages/zeek/data_stream/ntp/fields/base-fields.yml b/packages/zeek/data_stream/ntp/fields/base-fields.yml new file mode 100644 index 000000000000..9790a9113a25 --- /dev/null +++ b/packages/zeek/data_stream/ntp/fields/base-fields.yml @@ -0,0 +1,20 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: event.module + type: constant_keyword + description: Event module + value: zeek +- name: event.dataset + type: constant_keyword + description: Event dataset + value: zeek.connection +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/packages/zeek/data_stream/ntp/fields/beats.yml b/packages/zeek/data_stream/ntp/fields/beats.yml new file mode 100644 index 000000000000..470f5fae484f --- /dev/null +++ b/packages/zeek/data_stream/ntp/fields/beats.yml @@ -0,0 +1,23 @@ +- description: Unique container id. + ignore_above: 1024 + name: container.id + type: keyword +- description: Type of Filebeat input. + name: input.type + type: keyword +- description: Full path to the log file this event came from. + example: /var/log/fun-times.log + ignore_above: 1024 + name: log.file.path + type: keyword +- description: Flags for the log file. + name: log.flags + type: keyword +- description: Offset of the entry in the log file. + name: log.offset + type: long +- description: List of keywords used to tag each event. + example: '["production", "env2"]' + ignore_above: 1024 + name: tags + type: keyword diff --git a/packages/zeek/data_stream/ntp/fields/ecs.yml b/packages/zeek/data_stream/ntp/fields/ecs.yml new file mode 100644 index 000000000000..6bae2883f4f7 --- /dev/null +++ b/packages/zeek/data_stream/ntp/fields/ecs.yml @@ -0,0 +1,227 @@ +- description: Destination network address. + ignore_above: 1024 + name: destination.address + type: keyword +- description: Unique number allocated to the autonomous system. + example: 15169 + name: destination.as.number + type: long +- description: Organization name. + example: Google LLC + ignore_above: 1024 + multi_fields: + - flat_name: destination.as.organization.name.text + name: text + norms: false + type: text + name: destination.as.organization.name + type: keyword +- description: Bytes sent from the destination to the source. + example: 184 + name: destination.bytes + type: long +- description: City name. + example: Montreal + ignore_above: 1024 + name: destination.geo.city_name + type: keyword +- description: Name of the continent. + example: North America + ignore_above: 1024 + name: destination.geo.continent_name + type: keyword +- description: Country ISO code. + example: CA + ignore_above: 1024 + name: destination.geo.country_iso_code + type: keyword +- description: Country name. + example: Canada + ignore_above: 1024 + name: destination.geo.country_name + type: keyword +- description: Longitude and latitude. + example: '{ "lon": -73.614830, "lat": 45.505918 }' + name: destination.geo.location + type: geo_point +- description: User-defined description of a location. + example: boston-dc + ignore_above: 1024 + name: destination.geo.name + type: keyword +- description: Region ISO code. + example: CA-QC + ignore_above: 1024 + name: destination.geo.region_iso_code + type: keyword +- description: Region name. + example: Quebec + ignore_above: 1024 + name: destination.geo.region_name + type: keyword +- description: IP address of the destination. + name: destination.ip + type: ip +- description: MAC address of the destination. + ignore_above: 1024 + name: destination.mac + type: keyword +- description: Packets sent from the destination to the source. + example: 12 + name: destination.packets + type: long +- description: Port of the destination. + name: destination.port + type: long +- description: ECS version this event conforms to. + example: 1.0.0 + ignore_above: 1024 + name: ecs.version + type: keyword +- description: Error message. + name: error.message + type: text +- description: Event category. The second categorization field in the hierarchy. + example: authentication + ignore_above: 1024 + name: event.category + type: keyword +- description: Time when the event was first read by an agent or by your pipeline. + example: "2016-05-23T08:05:34.857Z" + name: event.created + type: date +- description: Duration of the event in nanoseconds. + name: event.duration + type: long +- description: Unique ID to describe the event. + example: 8a4f500d + ignore_above: 1024 + name: event.id + type: keyword +- description: Timestamp when an event arrived in the central data store. + example: "2016-05-23T08:05:35.101Z" + name: event.ingested + type: date +- description: The kind of the event. The highest categorization field in the hierarchy. + example: alert + ignore_above: 1024 + name: event.kind + type: keyword +- description: Event type. The third categorization field in the hierarchy. + ignore_above: 1024 + name: event.type + type: keyword +- description: Host ip addresses. + name: host.ip + type: ip +- description: Total bytes transferred in both directions. + example: 368 + name: network.bytes + type: long +- description: A hash of source and destination IPs and ports. + example: 1:hO+sN4H+MG5MY/8hIrXPqc4ZQz0= + ignore_above: 1024 + name: network.community_id + type: keyword +- description: Direction of the network traffic. + example: inbound + ignore_above: 1024 + name: network.direction + type: keyword +- description: Total packets transferred in both directions. + example: 24 + name: network.packets + type: long +- description: L7 Network protocol name. + example: http + ignore_above: 1024 + name: network.protocol + type: keyword +- description: Protocol Name corresponding to the field `iana_number`. + example: tcp + ignore_above: 1024 + name: network.transport + type: keyword +- description: In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc + example: ipv4 + ignore_above: 1024 + name: network.type + type: keyword +- description: All of the IPs seen on your event. + name: related.ip + type: ip +- description: Source network address. + ignore_above: 1024 + name: source.address + type: keyword +- description: Unique number allocated to the autonomous system. + example: 15169 + name: source.as.number + type: long +- description: Organization name. + example: Google LLC + ignore_above: 1024 + multi_fields: + - flat_name: source.as.organization.name.text + name: text + norms: false + type: text + name: source.as.organization.name + type: keyword +- description: Bytes sent from the source to the destination. + example: 184 + name: source.bytes + type: long +- description: City name. + example: Montreal + ignore_above: 1024 + name: source.geo.city_name + type: keyword +- description: Name of the continent. + example: North America + ignore_above: 1024 + name: source.geo.continent_name + type: keyword +- description: Country ISO code. + example: CA + ignore_above: 1024 + name: source.geo.country_iso_code + type: keyword +- description: Country name. + example: Canada + ignore_above: 1024 + name: source.geo.country_name + type: keyword +- description: Longitude and latitude. + example: '{ "lon": -73.614830, "lat": 45.505918 }' + name: source.geo.location + type: geo_point +- description: User-defined description of a location. + example: boston-dc + ignore_above: 1024 + name: source.geo.name + type: keyword +- description: Region ISO code. + example: CA-QC + ignore_above: 1024 + name: source.geo.region_iso_code + type: keyword +- description: Region name. + example: Quebec + ignore_above: 1024 + name: source.geo.region_name + type: keyword +- description: IP address of the source. + name: source.ip + type: ip +- description: MAC address of the source. + ignore_above: 1024 + name: source.mac + type: keyword +- description: Packets sent from the source to the destination. + example: 12 + name: source.packets + type: long +- description: Port of the source. + name: source.port + type: long diff --git a/packages/zeek/data_stream/ntp/fields/fields.yml b/packages/zeek/data_stream/ntp/fields/fields.yml new file mode 100644 index 000000000000..022ae5dc500c --- /dev/null +++ b/packages/zeek/data_stream/ntp/fields/fields.yml @@ -0,0 +1,71 @@ +- name: zeek.ntp + type: group + default_field: false + description: > + Fields exported by the Zeek NTP log. + + fields: + - name: version + type: integer + description: > + The NTP version number (1, 2, 3, 4). + + - name: mode + type: integer + description: > + The NTP mode being used. + + - name: stratum + type: integer + description: > + The stratum (primary server, secondary server, etc.). + + - name: poll + type: double + description: > + The maximum interval between successive messages in seconds. + + - name: precision + type: double + description: > + The precision of the system clock in seconds. + + - name: root_delay + type: double + description: > + Total round-trip delay to the reference clock in seconds. + + - name: root_disp + type: double + description: > + Total dispersion to the reference clock in seconds. + + - name: ref_id + type: keyword + description: > + For stratum 0, 4 character string used for debugging. For stratum 1, ID assigned to the reference clock by IANA. Above stratum 1, when using IPv4, the IP address of the reference clock. Note that the NTP protocol did not originally specify a large enough field to represent IPv6 addresses, so they use the first four bytes of the MD5 hash of the reference clock’s IPv6 address (i.e. an IPv4 address here is not necessarily IPv4). + + - name: ref_time + type: date + description: > + Time when the system clock was last set or correct. + + - name: org_time + type: date + description: > + Time at the client when the request departed for the NTP server. + + - name: rec_time + type: date + description: > + Time at the server when the request arrived from the NTP client. + + - name: xmt_time + type: date + description: > + Time at the server when the response departed for the NTP client. + + - name: num_exts + type: integer + description: >- + Number of extension fields (which are not currently parsed). diff --git a/packages/zeek/data_stream/ntp/fields/package-fields.yml b/packages/zeek/data_stream/ntp/fields/package-fields.yml new file mode 100644 index 000000000000..4d6d6ea170fa --- /dev/null +++ b/packages/zeek/data_stream/ntp/fields/package-fields.yml @@ -0,0 +1,7 @@ +- name: zeek + type: group + fields: + - name: session_id + type: keyword + description: | + A unique identifier of the session diff --git a/packages/zeek/data_stream/ntp/manifest.yml b/packages/zeek/data_stream/ntp/manifest.yml new file mode 100644 index 000000000000..d7e880d777da --- /dev/null +++ b/packages/zeek/data_stream/ntp/manifest.yml @@ -0,0 +1,84 @@ +type: logs +title: Zeek ntp logs +streams: + - input: logfile + template_path: log.yml.hbs + title: Zeek conn.log + description: Collect Zeek ntp logs + vars: + - name: filenames + type: text + title: Filename of ntp log + multi: true + required: true + show_user: true + default: + - ntp.log + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - zeek-ntp + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + + - input: httpjson + title: Zeek ntp logs via Splunk Enterprise REST API + description: Collect Zeek ntp logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: "search sourcetype=\"conn-*\"" + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded + - zeek-ntp + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: >- + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. diff --git a/packages/zeek/data_stream/signature/_dev/test/pipeline/test-signature.log b/packages/zeek/data_stream/signature/_dev/test/pipeline/test-signature.log new file mode 100644 index 000000000000..4725117d90e6 --- /dev/null +++ b/packages/zeek/data_stream/signature/_dev/test/pipeline/test-signature.log @@ -0,0 +1 @@ +{"ts": 1611852809.869245,"uid": "CbjAXE4CBxJ8W7VoJg","src_addr": "124.51.137.154","src_port": 51617,"dst_addr": "160.218.27.63","dst_port": 445,"note": "Signatures::Sensitive_Signature","sig_id": "my-second-sig","event_msg": "124.51.137.154: TCP traffic","sub_msg": ""} diff --git a/packages/zeek/data_stream/signature/_dev/test/pipeline/test-signature.log-config.yml b/packages/zeek/data_stream/signature/_dev/test/pipeline/test-signature.log-config.yml new file mode 100644 index 000000000000..3cabcf9fb82a --- /dev/null +++ b/packages/zeek/data_stream/signature/_dev/test/pipeline/test-signature.log-config.yml @@ -0,0 +1,6 @@ +dynamic_fields: + event.ingested: ".*" +fields: + "@timestamp": "2020-04-28T11:07:58.223Z" + tags: + - preserve_original_event diff --git a/packages/zeek/data_stream/signature/_dev/test/pipeline/test-signature.log-expected.json b/packages/zeek/data_stream/signature/_dev/test/pipeline/test-signature.log-expected.json new file mode 100644 index 000000000000..44dd953d78b3 --- /dev/null +++ b/packages/zeek/data_stream/signature/_dev/test/pipeline/test-signature.log-expected.json @@ -0,0 +1,84 @@ +{ + "expected": [ + { + "destination": { + "geo": { + "continent_name": "Europe", + "country_name": "Czechia", + "location": { + "lon": 14.4112, + "lat": 50.0848 + }, + "country_iso_code": "CZ" + }, + "as": { + "number": 5610, + "organization": { + "name": "O2 Czech Republic, a.s." + } + }, + "address": "160.218.27.63", + "port": 445, + "ip": "160.218.27.63" + }, + "zeek": { + "signature": { + "note": "Signatures::Sensitive_Signature", + "sub_msg": "" + }, + "session_id": "CbjAXE4CBxJ8W7VoJg" + }, + "rule": { + "description": "124.51.137.154: TCP traffic", + "id": "my-second-sig" + }, + "source": { + "geo": { + "continent_name": "Asia", + "region_iso_code": "KR-26", + "city_name": "Busan", + "country_iso_code": "KR", + "country_name": "South Korea", + "region_name": "Busan", + "location": { + "lon": 129.0442, + "lat": 35.1003 + } + }, + "as": { + "number": 17858, + "organization": { + "name": "LG POWERCOMM" + } + }, + "address": "124.51.137.154", + "port": 51617, + "ip": "124.51.137.154" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "type": "ipv4" + }, + "@timestamp": "2021-01-28T16:53:29.869Z", + "ecs": { + "version": "1.10.0" + }, + "related": { + "ip": [ + "124.51.137.154", + "160.218.27.63" + ] + }, + "event": { + "ingested": "2021-08-10T12:16:47.210647593Z", + "original": "{\"ts\": 1611852809.869245,\"uid\": \"CbjAXE4CBxJ8W7VoJg\",\"src_addr\": \"124.51.137.154\",\"src_port\": 51617,\"dst_addr\": \"160.218.27.63\",\"dst_port\": 445,\"note\": \"Signatures::Sensitive_Signature\",\"sig_id\": \"my-second-sig\",\"event_msg\": \"124.51.137.154: TCP traffic\",\"sub_msg\": \"\"}", + "id": "CbjAXE4CBxJ8W7VoJg", + "category": "network", + "created": "2020-04-28T11:07:58.223Z", + "kind": "alert" + } + } + ] +} \ No newline at end of file diff --git a/packages/zeek/data_stream/signature/_dev/test/system/test-logs-config.yml b/packages/zeek/data_stream/signature/_dev/test/system/test-logs-config.yml new file mode 100644 index 000000000000..5cfff5002996 --- /dev/null +++ b/packages/zeek/data_stream/signature/_dev/test/system/test-logs-config.yml @@ -0,0 +1,6 @@ +vars: + base_paths: + - "{{SERVICE_LOGS_DIR}}" +input: logfile +data_stream: + vars: ~ diff --git a/packages/zeek/data_stream/signature/_dev/test/system/test-splunk-config.yml b/packages/zeek/data_stream/signature/_dev/test/system/test-splunk-config.yml new file mode 100644 index 000000000000..dfa8f5c9201f --- /dev/null +++ b/packages/zeek/data_stream/signature/_dev/test/system/test-splunk-config.yml @@ -0,0 +1,9 @@ +input: httpjson +service: splunk-mock +vars: + url: http://{{Hostname}}:{{Port}} + username: test + password: test +data_stream: + vars: + preserve_original_event: true diff --git a/packages/zeek/data_stream/signature/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/signature/agent/stream/httpjson.yml.hbs new file mode 100644 index 000000000000..a2f2528bfb26 --- /dev/null +++ b/packages/zeek/data_stream/signature/agent/stream/httpjson.yml.hbs @@ -0,0 +1,63 @@ +config_version: 2 +interval: {{interval}} +{{#unless token}} +{{#if username}} +{{#if password}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +{{/if}} +{{/if}} +{{/unless}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +{{#unless username}} +{{#unless password}} +{{#if token}} + - set: + target: header.Authorization + value: {{token}} +{{/if}} +{{/unless}} +{{/unless}} +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/packages/zeek/data_stream/signature/agent/stream/log.yml.hbs b/packages/zeek/data_stream/signature/agent/stream/log.yml.hbs new file mode 100644 index 000000000000..d5c9b7811b0c --- /dev/null +++ b/packages/zeek/data_stream/signature/agent/stream/log.yml.hbs @@ -0,0 +1,21 @@ +paths: +{{#each base_paths}} + {{#each ../filenames}} + - {{../this}}/{{this}} + {{/each}} +{{/each}} +exclude_files: [".gz$"] +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/packages/zeek/data_stream/signature/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/signature/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 000000000000..93753718286a --- /dev/null +++ b/packages/zeek/data_stream/signature/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,151 @@ +--- +description: Pipeline for normalizing Zeek conn.log +processors: + - rename: + field: message + target_field: event.original + - json: + field: event.original + target_field: _temp_ + - pipeline: + if: ctx?._temp_?.result != null + name: '{{ IngestPipeline "third-party" }}' + - drop: + description: Drop if no timestamp (invalid json) + if: 'ctx?._temp_?.ts == null' + - rename: + field: _temp_ + target_field: zeek.signature + ignore_failure: true + - set: + field: event.ingested + value: "{{_ingest.timestamp}}" +# Sets event.created from the @timestamp field generated by filebeat before being overwritten further down + - set: + field: event.created + copy_from: "@timestamp" + - set: + field: ecs.version + value: '1.10.0' + - set: + field: event.kind + value: alert + - set: + field: event.category + value: network + - rename: + field: zeek.signature.src_addr + target_field: source.address + ignore_missing: true + - rename: + field: zeek.signature.src_port + target_field: source.port + ignore_missing: true + - rename: + field: zeek.signature.dst_addr + target_field: destination.address + ignore_missing: true + - rename: + field: zeek.signature.dst_port + target_field: destination.port + ignore_missing: true + - rename: + field: zeek.signature.uid + target_field: zeek.session_id + ignore_missing: true + - rename: + field: zeek.signature.sig_id + target_field: rule.id + ignore_missing: true + - rename: + field: zeek.signature.event_msg + target_field: rule.description + ignore_missing: true + - set: + field: source.ip + copy_from: source.address + if: ctx?.source?.address != null + - set: + field: destination.ip + copy_from: destination.address + if: ctx?.destination?.address != null + - date: + field: zeek.signature.ts + formats: + - UNIX + - ISO8601 + - set: + field: event.id + copy_from: zeek.session_id + if: ctx.zeek.session_id != null + - set: + field: network.type + value: ipv4 + if: ctx.source?.ip.contains('.') + - set: + field: network.type + value: ipv6 + if: ctx.source?.ip.contains(':') + - append: + field: related.ip + value: "{{source.ip}}" + if: ctx?.source?.ip != null + allow_duplicates: false + - append: + field: related.ip + value: "{{destination.ip}}" + if: ctx?.destination?.ip != null + allow_duplicates: false + - geoip: + field: destination.ip + target_field: destination.geo + ignore_missing: true + - geoip: + field: source.ip + target_field: source.geo + ignore_missing: true + - geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true + - geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true + - rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true + - rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true + - rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true + - rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true + - remove: + field: + - zeek.signature.ts + ignore_missing: true + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true +on_failure: + - set: + field: error.message + value: "{{ _ingest.on_failure_message }}" diff --git a/packages/zeek/data_stream/signature/elasticsearch/ingest_pipeline/third-party.yml b/packages/zeek/data_stream/signature/elasticsearch/ingest_pipeline/third-party.yml new file mode 100644 index 000000000000..5bc2247db25a --- /dev/null +++ b/packages/zeek/data_stream/signature/elasticsearch/ingest_pipeline/third-party.yml @@ -0,0 +1,39 @@ +--- +description: Pipeline for parsing Zeek logs from third party api +processors: + - fingerprint: + fields: + - _temp_.result._cd + - _temp_.result._indextime + - _temp_.result._raw + - _temp_.result._time + - _temp_.result.host + - _temp_.result.source + target_field: '_id' + ignore_missing: true + - set: + field: event.original + copy_from: _temp_.result._raw + ignore_empty_value: true + - set: + field: host.name + copy_from: _temp_.result.host + ignore_empty_value: true + - set: + copy_from: _temp_.result.source + field: log.file.path + ignore_empty_value: true + - remove: + field: _temp_ + ignore_missing: true + - json: + field: event.original + target_field: _temp_ +on_failure: + - append: + field: error.message + value: >- + error in third party api pipeline: + error in [{{_ingest.on_failure_processor_type}}] processor{{#_ingest.on_failure_processor_tag}} + with tag [{{_ingest.on_failure_processor_tag }}]{{/_ingest.on_failure_processor_tag}} + {{ _ingest.on_failure_message }} diff --git a/packages/zeek/data_stream/signature/fields/agent.yml b/packages/zeek/data_stream/signature/fields/agent.yml new file mode 100644 index 000000000000..79a7a39864bd --- /dev/null +++ b/packages/zeek/data_stream/signature/fields/agent.yml @@ -0,0 +1,180 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: "Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on." + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: "The cloud account or organization id used to identify different entities in a multi-tenant environment.\nExamples: AWS account id, Google Cloud ORG Id, or other unique identifier." + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: "Container fields are used for meta information about the specific container that is the source of information.\nThese fields help correlate data based containers from any runtime." + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: "A host is defined as a general computing instance.\nECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes." + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: "Name of the domain of which the host is a member.\nFor example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider." + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: "Hostname of the host.\nIt normally contains what the `hostname` command returns on the host machine." + - name: id + level: core + type: keyword + ignore_above: 1024 + description: "Unique host id.\nAs hostname is not always unique, use values that are meaningful in your environment.\nExample: The current usage of `beat.name`." + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: "Name of the host.\nIt can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use." + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: "Type of host.\nFor Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment." + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + diff --git a/packages/zeek/data_stream/signature/fields/base-fields.yml b/packages/zeek/data_stream/signature/fields/base-fields.yml new file mode 100644 index 000000000000..9790a9113a25 --- /dev/null +++ b/packages/zeek/data_stream/signature/fields/base-fields.yml @@ -0,0 +1,20 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: event.module + type: constant_keyword + description: Event module + value: zeek +- name: event.dataset + type: constant_keyword + description: Event dataset + value: zeek.connection +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/packages/zeek/data_stream/signature/fields/beats.yml b/packages/zeek/data_stream/signature/fields/beats.yml new file mode 100644 index 000000000000..470f5fae484f --- /dev/null +++ b/packages/zeek/data_stream/signature/fields/beats.yml @@ -0,0 +1,23 @@ +- description: Unique container id. + ignore_above: 1024 + name: container.id + type: keyword +- description: Type of Filebeat input. + name: input.type + type: keyword +- description: Full path to the log file this event came from. + example: /var/log/fun-times.log + ignore_above: 1024 + name: log.file.path + type: keyword +- description: Flags for the log file. + name: log.flags + type: keyword +- description: Offset of the entry in the log file. + name: log.offset + type: long +- description: List of keywords used to tag each event. + example: '["production", "env2"]' + ignore_above: 1024 + name: tags + type: keyword diff --git a/packages/zeek/data_stream/signature/fields/ecs.yml b/packages/zeek/data_stream/signature/fields/ecs.yml new file mode 100644 index 000000000000..8cf67091cf13 --- /dev/null +++ b/packages/zeek/data_stream/signature/fields/ecs.yml @@ -0,0 +1,235 @@ +- description: Destination network address. + ignore_above: 1024 + name: destination.address + type: keyword +- description: Unique number allocated to the autonomous system. + example: 15169 + name: destination.as.number + type: long +- description: Organization name. + example: Google LLC + ignore_above: 1024 + multi_fields: + - flat_name: destination.as.organization.name.text + name: text + norms: false + type: text + name: destination.as.organization.name + type: keyword +- description: Bytes sent from the destination to the source. + example: 184 + name: destination.bytes + type: long +- description: City name. + example: Montreal + ignore_above: 1024 + name: destination.geo.city_name + type: keyword +- description: Name of the continent. + example: North America + ignore_above: 1024 + name: destination.geo.continent_name + type: keyword +- description: Country ISO code. + example: CA + ignore_above: 1024 + name: destination.geo.country_iso_code + type: keyword +- description: Country name. + example: Canada + ignore_above: 1024 + name: destination.geo.country_name + type: keyword +- description: Longitude and latitude. + example: '{ "lon": -73.614830, "lat": 45.505918 }' + name: destination.geo.location + type: geo_point +- description: User-defined description of a location. + example: boston-dc + ignore_above: 1024 + name: destination.geo.name + type: keyword +- description: Region ISO code. + example: CA-QC + ignore_above: 1024 + name: destination.geo.region_iso_code + type: keyword +- description: Region name. + example: Quebec + ignore_above: 1024 + name: destination.geo.region_name + type: keyword +- description: IP address of the destination. + name: destination.ip + type: ip +- description: MAC address of the destination. + ignore_above: 1024 + name: destination.mac + type: keyword +- description: Packets sent from the destination to the source. + example: 12 + name: destination.packets + type: long +- description: Port of the destination. + name: destination.port + type: long +- description: ECS version this event conforms to. + example: 1.0.0 + ignore_above: 1024 + name: ecs.version + type: keyword +- description: Error message. + name: error.message + type: text +- description: Event category. The second categorization field in the hierarchy. + example: authentication + ignore_above: 1024 + name: event.category + type: keyword +- description: Time when the event was first read by an agent or by your pipeline. + example: "2016-05-23T08:05:34.857Z" + name: event.created + type: date +- description: Duration of the event in nanoseconds. + name: event.duration + type: long +- description: Unique ID to describe the event. + example: 8a4f500d + ignore_above: 1024 + name: event.id + type: keyword +- description: Timestamp when an event arrived in the central data store. + example: "2016-05-23T08:05:35.101Z" + name: event.ingested + type: date +- description: The kind of the event. The highest categorization field in the hierarchy. + example: alert + ignore_above: 1024 + name: event.kind + type: keyword +- description: Event type. The third categorization field in the hierarchy. + ignore_above: 1024 + name: event.type + type: keyword +- description: Host ip addresses. + name: host.ip + type: ip +- description: Total bytes transferred in both directions. + example: 368 + name: network.bytes + type: long +- description: A hash of source and destination IPs and ports. + example: 1:hO+sN4H+MG5MY/8hIrXPqc4ZQz0= + ignore_above: 1024 + name: network.community_id + type: keyword +- description: Direction of the network traffic. + example: inbound + ignore_above: 1024 + name: network.direction + type: keyword +- description: Total packets transferred in both directions. + example: 24 + name: network.packets + type: long +- description: L7 Network protocol name. + example: http + ignore_above: 1024 + name: network.protocol + type: keyword +- description: Protocol Name corresponding to the field `iana_number`. + example: tcp + ignore_above: 1024 + name: network.transport + type: keyword +- description: In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc + example: ipv4 + ignore_above: 1024 + name: network.type + type: keyword +- description: All of the IPs seen on your event. + name: related.ip + type: ip +- description: Source network address. + ignore_above: 1024 + name: source.address + type: keyword +- description: Unique number allocated to the autonomous system. + example: 15169 + name: source.as.number + type: long +- description: Organization name. + example: Google LLC + ignore_above: 1024 + multi_fields: + - flat_name: source.as.organization.name.text + name: text + norms: false + type: text + name: source.as.organization.name + type: keyword +- description: Bytes sent from the source to the destination. + example: 184 + name: source.bytes + type: long +- description: City name. + example: Montreal + ignore_above: 1024 + name: source.geo.city_name + type: keyword +- description: Name of the continent. + example: North America + ignore_above: 1024 + name: source.geo.continent_name + type: keyword +- description: Country ISO code. + example: CA + ignore_above: 1024 + name: source.geo.country_iso_code + type: keyword +- description: Country name. + example: Canada + ignore_above: 1024 + name: source.geo.country_name + type: keyword +- description: Longitude and latitude. + example: '{ "lon": -73.614830, "lat": 45.505918 }' + name: source.geo.location + type: geo_point +- description: User-defined description of a location. + example: boston-dc + ignore_above: 1024 + name: source.geo.name + type: keyword +- description: Region ISO code. + example: CA-QC + ignore_above: 1024 + name: source.geo.region_iso_code + type: keyword +- description: Region name. + example: Quebec + ignore_above: 1024 + name: source.geo.region_name + type: keyword +- description: IP address of the source. + name: source.ip + type: ip +- description: MAC address of the source. + ignore_above: 1024 + name: source.mac + type: keyword +- description: Packets sent from the source to the destination. + example: 12 + name: source.packets + type: long +- description: Port of the source. + name: source.port + type: long +- description: A rule ID that is unique within the scope of an agent, observer, or other entity using the rule for detection of this event. + ignore_above: 1024 + name: rule.id + type: keyword +- description: The description of the rule generating the event. + ignore_above: 1024 + name: rule.description + type: keyword diff --git a/packages/zeek/data_stream/signature/fields/fields.yml b/packages/zeek/data_stream/signature/fields/fields.yml new file mode 100644 index 000000000000..6b3043bf65e7 --- /dev/null +++ b/packages/zeek/data_stream/signature/fields/fields.yml @@ -0,0 +1,36 @@ +- name: zeek.signature + type: group + default_field: false + description: > + Fields exported by the Zeek Signature log. + + fields: + - name: note + type: keyword + description: > + Notice associated with signature event. + + - name: sig_id + type: keyword + description: > + The name of the signature that matched. + + - name: event_msg + type: keyword + description: > + A more descriptive message of the signature-matching event. + + - name: sub_msg + type: keyword + description: > + Extracted payload data or extra message. + + - name: sig_count + type: integer + description: > + Number of sigs, usually from summary count. + + - name: host_count + type: integer + description: >- + Number of hosts, from a summary count. diff --git a/packages/zeek/data_stream/signature/fields/package-fields.yml b/packages/zeek/data_stream/signature/fields/package-fields.yml new file mode 100644 index 000000000000..4d6d6ea170fa --- /dev/null +++ b/packages/zeek/data_stream/signature/fields/package-fields.yml @@ -0,0 +1,7 @@ +- name: zeek + type: group + fields: + - name: session_id + type: keyword + description: | + A unique identifier of the session diff --git a/packages/zeek/data_stream/signature/manifest.yml b/packages/zeek/data_stream/signature/manifest.yml new file mode 100644 index 000000000000..4f24033343eb --- /dev/null +++ b/packages/zeek/data_stream/signature/manifest.yml @@ -0,0 +1,84 @@ +type: logs +title: Zeek signature logs +streams: + - input: logfile + template_path: log.yml.hbs + title: Zeek signature.log + description: Collect Zeek signature logs + vars: + - name: filenames + type: text + title: Filename of signature log + multi: true + required: true + show_user: true + default: + - signature.log + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - zeek-signature + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + + - input: httpjson + title: Zeek signature logs via Splunk Enterprise REST API + description: Collect Zeek signature logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: "search sourcetype=\"conn-*\"" + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded + - zeek-signature + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: >- + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. diff --git a/packages/zeek/docs/README.md b/packages/zeek/docs/README.md index 023102d475a5..f0179ef07203 100644 --- a/packages/zeek/docs/README.md +++ b/packages/zeek/docs/README.md @@ -1682,6 +1682,120 @@ Zeek notices. | zeek.session_id | A unique identifier of the session | keyword | +### ntp + +The `ntp` dataset collects the Zeek ntp.log file, which contains +NTP data. + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| destination.address | Destination network address. | keyword | +| destination.as.number | Unique number allocated to the autonomous system. | long | +| destination.as.organization.name | Organization name. | keyword | +| destination.bytes | Bytes sent from the destination to the source. | long | +| destination.geo.city_name | City name. | keyword | +| destination.geo.continent_name | Name of the continent. | keyword | +| destination.geo.country_iso_code | Country ISO code. | keyword | +| destination.geo.country_name | Country name. | keyword | +| destination.geo.location | Longitude and latitude. | geo_point | +| destination.geo.name | User-defined description of a location. | keyword | +| destination.geo.region_iso_code | Region ISO code. | keyword | +| destination.geo.region_name | Region name. | keyword | +| destination.ip | IP address of the destination. | ip | +| destination.mac | MAC address of the destination. | keyword | +| destination.packets | Packets sent from the destination to the source. | long | +| destination.port | Port of the destination. | long | +| ecs.version | ECS version this event conforms to. | keyword | +| error.message | Error message. | text | +| event.category | Event category. The second categorization field in the hierarchy. | keyword | +| event.created | Time when the event was first read by an agent or by your pipeline. | date | +| event.dataset | Event dataset | constant_keyword | +| event.duration | Duration of the event in nanoseconds. | long | +| event.id | Unique ID to describe the event. | keyword | +| event.ingested | Timestamp when an event arrived in the central data store. | date | +| event.kind | The kind of the event. The highest categorization field in the hierarchy. | keyword | +| event.module | Event module | constant_keyword | +| event.type | Event type. The third categorization field in the hierarchy. | keyword | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| input.type | Type of Filebeat input. | keyword | +| log.file.path | Full path to the log file this event came from. | keyword | +| log.flags | Flags for the log file. | keyword | +| log.offset | Offset of the entry in the log file. | long | +| network.bytes | Total bytes transferred in both directions. | long | +| network.community_id | A hash of source and destination IPs and ports. | keyword | +| network.direction | Direction of the network traffic. | keyword | +| network.packets | Total packets transferred in both directions. | long | +| network.protocol | L7 Network protocol name. | keyword | +| network.transport | Protocol Name corresponding to the field `iana_number`. | keyword | +| network.type | In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc | keyword | +| related.ip | All of the IPs seen on your event. | ip | +| source.address | Source network address. | keyword | +| source.as.number | Unique number allocated to the autonomous system. | long | +| source.as.organization.name | Organization name. | keyword | +| source.bytes | Bytes sent from the source to the destination. | long | +| source.geo.city_name | City name. | keyword | +| source.geo.continent_name | Name of the continent. | keyword | +| source.geo.country_iso_code | Country ISO code. | keyword | +| source.geo.country_name | Country name. | keyword | +| source.geo.location | Longitude and latitude. | geo_point | +| source.geo.name | User-defined description of a location. | keyword | +| source.geo.region_iso_code | Region ISO code. | keyword | +| source.geo.region_name | Region name. | keyword | +| source.ip | IP address of the source. | ip | +| source.mac | MAC address of the source. | keyword | +| source.packets | Packets sent from the source to the destination. | long | +| source.port | Port of the source. | long | +| tags | List of keywords used to tag each event. | keyword | +| zeek.ntp.mode | The NTP mode being used. | integer | +| zeek.ntp.num_exts | Number of extension fields (which are not currently parsed). | integer | +| zeek.ntp.org_time | Time at the client when the request departed for the NTP server. | date | +| zeek.ntp.poll | The maximum interval between successive messages in seconds. | double | +| zeek.ntp.precision | The precision of the system clock in seconds. | double | +| zeek.ntp.rec_time | Time at the server when the request arrived from the NTP client. | date | +| zeek.ntp.ref_id | For stratum 0, 4 character string used for debugging. For stratum 1, ID assigned to the reference clock by IANA. Above stratum 1, when using IPv4, the IP address of the reference clock. Note that the NTP protocol did not originally specify a large enough field to represent IPv6 addresses, so they use the first four bytes of the MD5 hash of the reference clock’s IPv6 address (i.e. an IPv4 address here is not necessarily IPv4). | keyword | +| zeek.ntp.ref_time | Time when the system clock was last set or correct. | date | +| zeek.ntp.root_delay | Total round-trip delay to the reference clock in seconds. | double | +| zeek.ntp.root_disp | Total dispersion to the reference clock in seconds. | double | +| zeek.ntp.stratum | The stratum (primary server, secondary server, etc.). | integer | +| zeek.ntp.version | The NTP version number (1, 2, 3, 4). | integer | +| zeek.ntp.xmt_time | Time at the server when the response departed for the NTP client. | date | +| zeek.session_id | A unique identifier of the session | keyword | + + ### ntlm The `ntlm` dataset collects the Zeek ntlm.log file, which contains NT @@ -2240,6 +2354,115 @@ Remote Framebuffer (RFB) data. | zeek.session_id | A unique identifier of the session | keyword | +### signature + +The `signature` dataset collects the Zeek signature.log file, which contains +Zeek signature matches. + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| destination.address | Destination network address. | keyword | +| destination.as.number | Unique number allocated to the autonomous system. | long | +| destination.as.organization.name | Organization name. | keyword | +| destination.bytes | Bytes sent from the destination to the source. | long | +| destination.geo.city_name | City name. | keyword | +| destination.geo.continent_name | Name of the continent. | keyword | +| destination.geo.country_iso_code | Country ISO code. | keyword | +| destination.geo.country_name | Country name. | keyword | +| destination.geo.location | Longitude and latitude. | geo_point | +| destination.geo.name | User-defined description of a location. | keyword | +| destination.geo.region_iso_code | Region ISO code. | keyword | +| destination.geo.region_name | Region name. | keyword | +| destination.ip | IP address of the destination. | ip | +| destination.mac | MAC address of the destination. | keyword | +| destination.packets | Packets sent from the destination to the source. | long | +| destination.port | Port of the destination. | long | +| ecs.version | ECS version this event conforms to. | keyword | +| error.message | Error message. | text | +| event.category | Event category. The second categorization field in the hierarchy. | keyword | +| event.created | Time when the event was first read by an agent or by your pipeline. | date | +| event.dataset | Event dataset | constant_keyword | +| event.duration | Duration of the event in nanoseconds. | long | +| event.id | Unique ID to describe the event. | keyword | +| event.ingested | Timestamp when an event arrived in the central data store. | date | +| event.kind | The kind of the event. The highest categorization field in the hierarchy. | keyword | +| event.module | Event module | constant_keyword | +| event.type | Event type. The third categorization field in the hierarchy. | keyword | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| input.type | Type of Filebeat input. | keyword | +| log.file.path | Full path to the log file this event came from. | keyword | +| log.flags | Flags for the log file. | keyword | +| log.offset | Offset of the entry in the log file. | long | +| network.bytes | Total bytes transferred in both directions. | long | +| network.community_id | A hash of source and destination IPs and ports. | keyword | +| network.direction | Direction of the network traffic. | keyword | +| network.packets | Total packets transferred in both directions. | long | +| network.protocol | L7 Network protocol name. | keyword | +| network.transport | Protocol Name corresponding to the field `iana_number`. | keyword | +| network.type | In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc | keyword | +| related.ip | All of the IPs seen on your event. | ip | +| rule.description | The description of the rule generating the event. | keyword | +| rule.id | A rule ID that is unique within the scope of an agent, observer, or other entity using the rule for detection of this event. | keyword | +| source.address | Source network address. | keyword | +| source.as.number | Unique number allocated to the autonomous system. | long | +| source.as.organization.name | Organization name. | keyword | +| source.bytes | Bytes sent from the source to the destination. | long | +| source.geo.city_name | City name. | keyword | +| source.geo.continent_name | Name of the continent. | keyword | +| source.geo.country_iso_code | Country ISO code. | keyword | +| source.geo.country_name | Country name. | keyword | +| source.geo.location | Longitude and latitude. | geo_point | +| source.geo.name | User-defined description of a location. | keyword | +| source.geo.region_iso_code | Region ISO code. | keyword | +| source.geo.region_name | Region name. | keyword | +| source.ip | IP address of the source. | ip | +| source.mac | MAC address of the source. | keyword | +| source.packets | Packets sent from the source to the destination. | long | +| source.port | Port of the source. | long | +| tags | List of keywords used to tag each event. | keyword | +| zeek.session_id | A unique identifier of the session | keyword | +| zeek.signature.event_msg | A more descriptive message of the signature-matching event. | keyword | +| zeek.signature.host_count | Number of hosts, from a summary count. | integer | +| zeek.signature.note | Notice associated with signature event. | keyword | +| zeek.signature.sig_count | Number of sigs, usually from summary count. | integer | +| zeek.signature.sig_id | The name of the signature that matched. | keyword | +| zeek.signature.sub_msg | Extracted payload data or extra message. | keyword | + + ### sip The `sip` dataset collects the Zeek sip.log file, which contains SIP diff --git a/packages/zeek/manifest.yml b/packages/zeek/manifest.yml index d143a7744af6..21ef66e17cd4 100644 --- a/packages/zeek/manifest.yml +++ b/packages/zeek/manifest.yml @@ -1,6 +1,6 @@ name: zeek title: Zeek -version: 1.2.1 +version: 1.3.0 release: ga description: This Elastic integration collects logs from Zeek type: integration @@ -26,7 +26,7 @@ policy_templates: inputs: - type: logfile title: "Collect Zeek logs" - description: "Collects logs from Zeek instances. Supported logs include: capture_loss, connection, dce_rpc, dhcp, dnp3, dns, dpd, files, ftp, http, intel, irc, kerberos, modbus, mysql, notice, ntlm, ocsp, pe, radius, rdp, rfb, sip, smb_cmd, smb_files, smb_mapping, smtp, snmp, socks, ssh, ssl, stats, syslog, traceroute, tunnel, weird and x509" + description: "Collects logs from Zeek instances. Supported logs include: capture_loss, connection, dce_rpc, dhcp, dnp3, dns, dpd, files, ftp, http, intel, irc, kerberos, modbus, mysql, notice, ntlm, ntp, ocsp, pe, radius, rdp, rfb, signature, sip, smb_cmd, smb_files, smb_mapping, smtp, snmp, socks, ssh, ssl, stats, syslog, traceroute, tunnel, weird and x509" vars: - name: base_paths required: true