From a7287011b3042e9aceea4313fbb29618e4537c17 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Thu, 8 Oct 2020 16:40:40 -0400 Subject: [PATCH] Add fields to Zeek SSL log Rename `ja3s` to `tls.server.ja3s` if present (requires zeek/salesforce/ja3 package package). Rename `ja3` to `tls.client.ja3` if present (requires zeek/salesforce/ja3 package package). Rename `subject` to `tls.server.subject` (instead of deleting it after parsing). Rename `not_valid_before` to `tls.server.not_before` if present. Rename `not_valid_after` to `tls.server.not_after` if present. Fix dashboard reference for `zeek.ssl.server.name` that was using `zeek.ssl.server_name`. Add links to Zeek scripts in docs. --- CHANGELOG.next.asciidoc | 2 + .../filebeat/module/zeek/_meta/docs.asciidoc | 8 +- .../7/dashboard/Filebeat-Zeek-Overview.json | 2 +- .../filebeat/module/zeek/ssl/config/ssl.yml | 15 ++++ .../module/zeek/ssl/ingest/pipeline.yml | 15 +++- .../zeek/ssl/test/ssl-json.log-expected.json | 2 + .../ssl-with-ja3-and-custom-fields-json.log | 1 + ...3-and-custom-fields-json.log-expected.json | 78 +++++++++++++++++++ 8 files changed, 120 insertions(+), 3 deletions(-) create mode 100644 x-pack/filebeat/module/zeek/ssl/test/ssl-with-ja3-and-custom-fields-json.log create mode 100644 x-pack/filebeat/module/zeek/ssl/test/ssl-with-ja3-and-custom-fields-json.log-expected.json diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 6a211d61fd8..271b46b1133 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -290,6 +290,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add field limit check for AWS Cloudtrail flattened fields. {pull}21388[21388] {issue}21382[21382] - Fix syslog RFC 5424 parsing in the CheckPoint module. {pull}21854[21854] - Fix incorrect connection state mapping in zeek connection pipeline. {pull}22151[22151] {issue}22149[22149] +- Fix Zeek dashboard reference to `zeek.ssl.server.name` field. {pull}21696[21696] *Heartbeat* @@ -649,6 +650,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add max_number_of_messages config into s3 input. {pull}21993[21993] - Update Okta documentation for new stateful restarts. {pull}22091[22091] - Copy tag names from MISP data into events. {pull}21664[21664] +- Added TLS JA3 fingerprint, certificate not_before/not_after, certificate SHA1 hash, and certificate subject fields to Zeek SSL dataset. {pull}21696[21696] *Heartbeat* diff --git a/x-pack/filebeat/module/zeek/_meta/docs.asciidoc b/x-pack/filebeat/module/zeek/_meta/docs.asciidoc index 7784857b260..aaef3f7803e 100644 --- a/x-pack/filebeat/module/zeek/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/zeek/_meta/docs.asciidoc @@ -8,13 +8,19 @@ This is a module for https://zeek.org/[Zeek], which used to be called Bro. It parses logs that are in the Zeek JSON format. +The Zeek SSL fileset will handle fields from these scripts if they are +installed in Zeek. + +- https://github.com/salesforce/ja3/tree/master/zeek[JA3/JA3S Hashes] +- https://github.com/rocknsm/rock-scripts/blob/1abcb137c3c0cb7bc1d54248d738255d2d6eb4ba/protocols/ssl/ssl-add-cert-hash.zeek[SHA1 Certificate Hashes] + include::../include/gs-link.asciidoc[] [float] === Compatibility This module has been developed against Zeek 2.6.1, but is expected to work -with other versions of Zeek. +with newer versions of Zeek. Zeek requires a Unix-like platform, and it currently supports Linux, FreeBSD, and Mac OS X. diff --git a/x-pack/filebeat/module/zeek/_meta/kibana/7/dashboard/Filebeat-Zeek-Overview.json b/x-pack/filebeat/module/zeek/_meta/kibana/7/dashboard/Filebeat-Zeek-Overview.json index 80a85676ede..1c8e79b5fc9 100644 --- a/x-pack/filebeat/module/zeek/_meta/kibana/7/dashboard/Filebeat-Zeek-Overview.json +++ b/x-pack/filebeat/module/zeek/_meta/kibana/7/dashboard/Filebeat-Zeek-Overview.json @@ -736,7 +736,7 @@ "enabled": true, "id": "2", "params": { - "field": "zeek.ssl.server_name", + "field": "zeek.ssl.server.name", "missingBucket": false, "missingBucketLabel": "Missing", "order": "desc", diff --git a/x-pack/filebeat/module/zeek/ssl/config/ssl.yml b/x-pack/filebeat/module/zeek/ssl/config/ssl.yml index c3ecbd94be9..8aee05e4c7c 100644 --- a/x-pack/filebeat/module/zeek/ssl/config/ssl.yml +++ b/x-pack/filebeat/module/zeek/ssl/config/ssl.yml @@ -55,6 +55,21 @@ processors: - from: "zeek.ssl.validation_code" to: "zeek.ssl.validation.code" + # These JA3 fields are available when using the zeek/salesforce/ja3 package. + - from: zeek.ssl.ja3 + to: tls.client.ja3 + - from: zeek.ssl.ja3s + to: tls.server.ja3s + + - from: zeek.ssl.not_valid_before + to: tls.server.not_before + - from: zeek.ssl.not_valid_after + to: tls.server.not_after + + - from: zeek.ssl.orig_certificate_sha1 + to: tls.client.hash.sha1 + - from: zeek.ssl.resp_certificate_sha1 + to: tls.server.hash.sha1 ignore_missing: true fail_on_error: false - convert: diff --git a/x-pack/filebeat/module/zeek/ssl/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/ssl/ingest/pipeline.yml index ad8edd5392f..4a980be985a 100644 --- a/x-pack/filebeat/module/zeek/ssl/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/ssl/ingest/pipeline.yml @@ -13,6 +13,18 @@ processors: - UNIX - remove: field: zeek.ssl.ts +- date: + if: ctx.tls?.server?.not_before != null + field: tls.server.not_before + target_field: tls.server.not_before + formats: + - UNIX +- date: + if: ctx.tls?.server?.not_after != null + field: tls.server.not_after + target_field: tls.server.not_after + formats: + - UNIX - geoip: field: destination.ip target_field: destination.geo @@ -131,8 +143,9 @@ processors: value_split: '=' target_field: zeek.ssl.server.subject ignore_missing: true -- remove: +- rename: field: zeek.ssl.subject + target_field: tls.server.subject ignore_missing: true - rename: field: zeek.ssl.server.subject.C diff --git a/x-pack/filebeat/module/zeek/ssl/test/ssl-json.log-expected.json b/x-pack/filebeat/module/zeek/ssl/test/ssl-json.log-expected.json index 67817ff0a42..72ac1dc8e22 100644 --- a/x-pack/filebeat/module/zeek/ssl/test/ssl-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/ssl/test/ssl-json.log-expected.json @@ -48,6 +48,7 @@ "tls.established": true, "tls.resumed": false, "tls.server.issuer": "CN=DigiCert SHA2 Secure Server CA,O=DigiCert Inc,C=US", + "tls.server.subject": "CN=*.gcp.cloud.es.io,O=Elasticsearch Inc.,L=Mountain View,ST=California,C=US", "tls.server.x509.issuer.common_name": "DigiCert SHA2 Secure Server CA", "tls.server.x509.issuer.country": "US", "tls.server.x509.issuer.organization": "DigiCert Inc", @@ -129,6 +130,7 @@ "tls.established": true, "tls.resumed": false, "tls.server.issuer": "CN=DigiCert SHA2 Secure Server CA,O=DigiCert Inc,C=US", + "tls.server.subject": "CN=*.gcp.cloud.es.io,O=Elasticsearch Inc.,L=Mountain View,ST=California,C=US", "tls.server.x509.issuer.common_name": "DigiCert SHA2 Secure Server CA", "tls.server.x509.issuer.country": "US", "tls.server.x509.issuer.organization": "DigiCert Inc", diff --git a/x-pack/filebeat/module/zeek/ssl/test/ssl-with-ja3-and-custom-fields-json.log b/x-pack/filebeat/module/zeek/ssl/test/ssl-with-ja3-and-custom-fields-json.log new file mode 100644 index 00000000000..a41bd75f2ee --- /dev/null +++ b/x-pack/filebeat/module/zeek/ssl/test/ssl-with-ja3-and-custom-fields-json.log @@ -0,0 +1 @@ +{"ts":1602179457.352156,"uid":"CK17Dl2SB8bZOVonSl","id.orig_h":"10.0.0.1","id.orig_p":49228,"id.resp_h":"192.168.50.1","id.resp_p":443,"version":"TLSv12","cipher":"TLS_RSA_WITH_AES_128_CBC_SHA256","resumed":false,"established":true,"cert_chain_fuids":["FOLwYQ6rs70bIMSf9"],"client_cert_chain_fuids":[],"subject":"CN=foo,OU=foo@bar,O=org,L=locality,C=LO","issuer":"CN=CA,OU=CA@example.com,O=Example Corp,L=foo,C=HI","validation_status":"self signed certificate","ja3":"74927e242d6c3febf8cb9cab10a7f889","ja3s":"80b3a14bccc8598a1f3bbe83e71f735f","resp_certificate_sha1":"5dad8b55621b6b9c30679d9d61248dd132a83c94","not_valid_before":1562022421,"not_valid_after":1577748224} diff --git a/x-pack/filebeat/module/zeek/ssl/test/ssl-with-ja3-and-custom-fields-json.log-expected.json b/x-pack/filebeat/module/zeek/ssl/test/ssl-with-ja3-and-custom-fields-json.log-expected.json new file mode 100644 index 00000000000..c4b3cb7a871 --- /dev/null +++ b/x-pack/filebeat/module/zeek/ssl/test/ssl-with-ja3-and-custom-fields-json.log-expected.json @@ -0,0 +1,78 @@ +[ + { + "@timestamp": "2020-10-08T17:50:57.352Z", + "client.address": "10.0.0.1", + "destination.address": "192.168.50.1", + "destination.ip": "192.168.50.1", + "destination.port": 443, + "event.category": [ + "network" + ], + "event.dataset": "zeek.ssl", + "event.id": "CK17Dl2SB8bZOVonSl", + "event.kind": "event", + "event.module": "zeek", + "event.type": [ + "connection", + "protocol" + ], + "fileset.name": "ssl", + "input.type": "log", + "log.offset": 0, + "network.community_id": "1:qNHgoGHFvyhhK2jU7LlS3537ODc=", + "network.transport": "tcp", + "related.ip": [ + "10.0.0.1", + "192.168.50.1" + ], + "server.address": "192.168.50.1", + "service.type": "zeek", + "source.address": "10.0.0.1", + "source.ip": "10.0.0.1", + "source.port": 49228, + "tags": [ + "zeek.ssl" + ], + "tls.cipher": "TLS_RSA_WITH_AES_128_CBC_SHA256", + "tls.client.ja3": "74927e242d6c3febf8cb9cab10a7f889", + "tls.established": true, + "tls.resumed": false, + "tls.server.hash.sha1": "5dad8b55621b6b9c30679d9d61248dd132a83c94", + "tls.server.issuer": "CN=CA,OU=CA@example.com,O=Example Corp,L=foo,C=HI", + "tls.server.ja3s": "80b3a14bccc8598a1f3bbe83e71f735f", + "tls.server.not_after": "2019-12-30T23:23:44.000Z", + "tls.server.not_before": "2019-07-01T23:07:01.000Z", + "tls.server.subject": "CN=foo,OU=foo@bar,O=org,L=locality,C=LO", + "tls.server.x509.issuer.common_name": "CA", + "tls.server.x509.issuer.country": "HI", + "tls.server.x509.issuer.locality": "foo", + "tls.server.x509.issuer.organization": "Example Corp", + "tls.server.x509.issuer.organizational_unit": "CA@example.com", + "tls.server.x509.subject.common_name": "foo", + "tls.server.x509.subject.country": "LO", + "tls.server.x509.subject.locality": "locality", + "tls.server.x509.subject.organization": "org", + "tls.server.x509.subject.organizational_unit": "foo@bar", + "tls.version": "1.2", + "tls.version_protocol": "tls", + "zeek.session_id": "CK17Dl2SB8bZOVonSl", + "zeek.ssl.cipher": "TLS_RSA_WITH_AES_128_CBC_SHA256", + "zeek.ssl.established": true, + "zeek.ssl.resumed": false, + "zeek.ssl.server.cert_chain_fuids": [ + "FOLwYQ6rs70bIMSf9" + ], + "zeek.ssl.server.issuer.common_name": "CA", + "zeek.ssl.server.issuer.country": "HI", + "zeek.ssl.server.issuer.locality": "foo", + "zeek.ssl.server.issuer.organization": "Example Corp", + "zeek.ssl.server.issuer.organizational_unit": "CA@example.com", + "zeek.ssl.server.subject.common_name": "foo", + "zeek.ssl.server.subject.country": "LO", + "zeek.ssl.server.subject.locality": "locality", + "zeek.ssl.server.subject.organization": "org", + "zeek.ssl.server.subject.organizational_unit": "foo@bar", + "zeek.ssl.validation.status": "self signed certificate", + "zeek.ssl.version": "TLSv12" + } +] \ No newline at end of file