Add Zeek NTP and Signature data streams

elastic · Aug 10, 2021 · 20b4357 · 20b4357
1 parent 3d0f9e2
commit 20b4357
Show file tree

Hide file tree

Showing 40 changed files with 2,396 additions and 74 deletions.
diff --git a/packages/zeek/_dev/build/docs/README.md b/packages/zeek/_dev/build/docs/README.md
@@ -124,6 +124,13 @@ Zeek notices.
 
 {{fields "notice"}}
 
+### ntp
+
+The `ntp` dataset collects the Zeek ntp.log file, which contains
+NTP data.
+
+{{fields "ntp"}}
+
 ### ntlm
 
 The `ntlm` dataset collects the Zeek ntlm.log file, which contains NT
@@ -166,6 +173,13 @@ Remote Framebuffer (RFB) data.
 
 {{fields "rfb"}}
 
+### signature
+
+The `signature` dataset collects the Zeek signature.log file, which contains
+Zeek signature matches.
+
+{{fields "signature"}}
+
 ### sip
 
 The `sip` dataset collects the Zeek sip.log file, which contains SIP

diff --git a/packages/zeek/_dev/deploy/docker/sample_logs/ntp.log b/packages/zeek/_dev/deploy/docker/sample_logs/ntp.log
@@ -0,0 +1,2 @@
+{"ts":1602116947.977,"uid":"CqlPpF1AQVLMPgGiL5","id.orig_h":"130.118.205.62","id.orig_p":38461,"id.resp_h":"208.79.89.249","id.resp_p":123,"version":4,"mode":3,"stratum":0,"poll":1,"precision":1,"root_delay":0,"root_disp":0,"ref_id":"\\x00\\x00\\x00\\x00","ref_time":0,"org_time":0,"rec_time":0,"xmt_time":1602116947.215,"num_exts":0}
+{"ts":1602116948.081,"uid":"CqlPpF1AQVLMPgGiL5","id.orig_h":"130.118.205.62","id.orig_p":38461,"id.resp_h":"208.79.89.249","id.resp_p":123,"version":4,"mode":4,"stratum":2,"poll":8,"precision":5.960464477539063e-8,"root_delay":0.00921630859375,"root_disp":0.0212249755859375,"ref_id":"127.67.113.92","ref_time":1602116655.942,"org_time":1602116947.215,"rec_time":1602116947.964,"xmt_time":1602116947.964,"num_exts":0}
diff --git a/packages/zeek/_dev/deploy/docker/sample_logs/signature.log b/packages/zeek/_dev/deploy/docker/sample_logs/signature.log
@@ -0,0 +1 @@
+{"ts": 1611852809.869245,"uid": "CbjAXE4CBxJ8W7VoJg","src_addr": "124.51.137.154","src_port": 51617,"dst_addr": "160.218.27.63","dst_port": 445,"note": "Signatures::Sensitive_Signature","sig_id": "my-second-sig","event_msg": "124.51.137.154: TCP traffic","sub_msg": ""}
diff --git a/packages/zeek/changelog.yml b/packages/zeek/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.3.0"
+  changes:
+    - description: Add Sigature and NTP data streams
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1515
 - version: '1.2.1'
   changes:
     - description: update to ECS 1.11.0

diff --git a/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log-expected.json b/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log-expected.json
diff --git a/packages/zeek/data_stream/connection/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/connection/elasticsearch/ingest_pipeline/default.yml
@@ -322,6 +322,7 @@ processors:
         }
   - remove:
       field: 
+        - zeek.connection.id
         - zeek.connection.orig_bytes
         - zeek.connection.resp_bytes
         - zeek.connection.tunnel_parents

diff --git a/packages/zeek/data_stream/ntp/_dev/test/pipeline/test-ntp.log b/packages/zeek/data_stream/ntp/_dev/test/pipeline/test-ntp.log
@@ -0,0 +1,2 @@
+{"ts":1602116947.977,"uid":"CqlPpF1AQVLMPgGiL5","id.orig_h":"130.118.205.62","id.orig_p":38461,"id.resp_h":"208.79.89.249","id.resp_p":123,"version":4,"mode":3,"stratum":0,"poll":1,"precision":1,"root_delay":0,"root_disp":0,"ref_id":"\\x00\\x00\\x00\\x00","ref_time":0,"org_time":0,"rec_time":0,"xmt_time":1602116947.215,"num_exts":0}
+{"ts":1602116948.081,"uid":"CqlPpF1AQVLMPgGiL5","id.orig_h":"130.118.205.62","id.orig_p":38461,"id.resp_h":"208.79.89.249","id.resp_p":123,"version":4,"mode":4,"stratum":2,"poll":8,"precision":5.960464477539063e-8,"root_delay":0.00921630859375,"root_disp":0.0212249755859375,"ref_id":"127.67.113.92","ref_time":1602116655.942,"org_time":1602116947.215,"rec_time":1602116947.964,"xmt_time":1602116947.964,"num_exts":0}
diff --git a/packages/zeek/data_stream/ntp/_dev/test/pipeline/test-ntp.log-config.yml b/packages/zeek/data_stream/ntp/_dev/test/pipeline/test-ntp.log-config.yml
@@ -0,0 +1,6 @@
+dynamic_fields:
+  event.ingested: ".*"
+fields:
+  "@timestamp": "2020-04-28T11:07:58.223Z"
+  tags:
+    - preserve_original_event
diff --git a/packages/zeek/data_stream/ntp/_dev/test/pipeline/test-ntp.log-expected.json b/packages/zeek/data_stream/ntp/_dev/test/pipeline/test-ntp.log-expected.json
@@ -0,0 +1,176 @@
+{
+    "expected": [
+        {
+            "@timestamp": "2020-10-08T00:29:07.977Z",
+            "ecs": {
+                "version": "1.10.0"
+            },
+            "related": {
+                "ip": [
+                    "130.118.205.62",
+                    "208.79.89.249"
+                ]
+            },
+            "destination": {
+                "geo": {
+                    "continent_name": "North America",
+                    "country_name": "United States",
+                    "location": {
+                        "lon": -97.822,
+                        "lat": 37.751
+                    },
+                    "country_iso_code": "US"
+                },
+                "as": {
+                    "number": 25795,
+                    "organization": {
+                        "name": "ARP NETWORKS, INC."
+                    }
+                },
+                "address": "208.79.89.249",
+                "port": 123,
+                "ip": "208.79.89.249"
+            },
+            "zeek": {
+                "session_id": "CqlPpF1AQVLMPgGiL5",
+                "ntp": {
+                    "ref_id": "\\x00\\x00\\x00\\x00",
+                    "rec_time": "1970-01-01T00:00:00.000Z",
+                    "ref_time": "1970-01-01T00:00:00.000Z",
+                    "root_delay": 0,
+                    "precision": 1,
+                    "poll": 1,
+                    "version": 4,
+                    "num_exts": 0,
+                    "stratum": 0,
+                    "mode": 3,
+                    "root_disp": 0,
+                    "org_time": "1970-01-01T00:00:00.000Z",
+                    "xmt_time": "2020-10-08T00:29:07.215Z"
+                }
+            },
+            "source": {
+                "geo": {
+                    "continent_name": "North America",
+                    "country_name": "United States",
+                    "location": {
+                        "lon": -97.822,
+                        "lat": 37.751
+                    },
+                    "country_iso_code": "US"
+                },
+                "address": "130.118.205.62",
+                "port": 38461,
+                "ip": "130.118.205.62"
+            },
+            "event": {
+                "ingested": "2021-08-10T12:16:45.963639174Z",
+                "original": "{\"ts\":1602116947.977,\"uid\":\"CqlPpF1AQVLMPgGiL5\",\"id.orig_h\":\"130.118.205.62\",\"id.orig_p\":38461,\"id.resp_h\":\"208.79.89.249\",\"id.resp_p\":123,\"version\":4,\"mode\":3,\"stratum\":0,\"poll\":1,\"precision\":1,\"root_delay\":0,\"root_disp\":0,\"ref_id\":\"\\\\x00\\\\x00\\\\x00\\\\x00\",\"ref_time\":0,\"org_time\":0,\"rec_time\":0,\"xmt_time\":1602116947.215,\"num_exts\":0}",
+                "created": "2020-04-28T11:07:58.223Z",
+                "kind": "event",
+                "id": "CqlPpF1AQVLMPgGiL5",
+                "category": "network",
+                "type": [
+                    "connection",
+                    "protocol",
+                    "info"
+                ]
+            },
+            "tags": [
+                "preserve_original_event"
+            ],
+            "network": {
+                "protocol": "ntp",
+                "community_id": "1:IDiKR+C1G8mk7LQhFpp+4p1tHrk=",
+                "transport": "udp",
+                "type": "ipv4"
+            }
+        },
+        {
+            "@timestamp": "2020-10-08T00:29:08.081Z",
+            "ecs": {
+                "version": "1.10.0"
+            },
+            "related": {
+                "ip": [
+                    "130.118.205.62",
+                    "208.79.89.249"
+                ]
+            },
+            "destination": {
+                "geo": {
+                    "continent_name": "North America",
+                    "country_name": "United States",
+                    "location": {
+                        "lon": -97.822,
+                        "lat": 37.751
+                    },
+                    "country_iso_code": "US"
+                },
+                "as": {
+                    "number": 25795,
+                    "organization": {
+                        "name": "ARP NETWORKS, INC."
+                    }
+                },
+                "address": "208.79.89.249",
+                "port": 123,
+                "ip": "208.79.89.249"
+            },
+            "zeek": {
+                "session_id": "CqlPpF1AQVLMPgGiL5",
+                "ntp": {
+                    "ref_id": "127.67.113.92",
+                    "rec_time": "2020-10-08T00:29:07.964Z",
+                    "ref_time": "2020-10-08T00:24:15.942Z",
+                    "root_delay": 0.00921630859375,
+                    "precision": 5.9604644775390625E-8,
+                    "poll": 8,
+                    "version": 4,
+                    "num_exts": 0,
+                    "stratum": 2,
+                    "mode": 4,
+                    "root_disp": 0.0212249755859375,
+                    "org_time": "2020-10-08T00:29:07.215Z",
+                    "xmt_time": "2020-10-08T00:29:07.964Z"
+                }
+            },
+            "source": {
+                "geo": {
+                    "continent_name": "North America",
+                    "country_name": "United States",
+                    "location": {
+                        "lon": -97.822,
+                        "lat": 37.751
+                    },
+                    "country_iso_code": "US"
+                },
+                "address": "130.118.205.62",
+                "port": 38461,
+                "ip": "130.118.205.62"
+            },
+            "event": {
+                "ingested": "2021-08-10T12:16:45.963645447Z",
+                "original": "{\"ts\":1602116948.081,\"uid\":\"CqlPpF1AQVLMPgGiL5\",\"id.orig_h\":\"130.118.205.62\",\"id.orig_p\":38461,\"id.resp_h\":\"208.79.89.249\",\"id.resp_p\":123,\"version\":4,\"mode\":4,\"stratum\":2,\"poll\":8,\"precision\":5.960464477539063e-8,\"root_delay\":0.00921630859375,\"root_disp\":0.0212249755859375,\"ref_id\":\"127.67.113.92\",\"ref_time\":1602116655.942,\"org_time\":1602116947.215,\"rec_time\":1602116947.964,\"xmt_time\":1602116947.964,\"num_exts\":0}",
+                "created": "2020-04-28T11:07:58.223Z",
+                "kind": "event",
+                "id": "CqlPpF1AQVLMPgGiL5",
+                "category": "network",
+                "type": [
+                    "connection",
+                    "protocol",
+                    "info"
+                ]
+            },
+            "tags": [
+                "preserve_original_event"
+            ],
+            "network": {
+                "protocol": "ntp",
+                "community_id": "1:IDiKR+C1G8mk7LQhFpp+4p1tHrk=",
+                "transport": "udp",
+                "type": "ipv4"
+            }
+        }
+    ]
+}
diff --git a/packages/zeek/data_stream/ntp/_dev/test/system/test-logs-config.yml b/packages/zeek/data_stream/ntp/_dev/test/system/test-logs-config.yml
@@ -0,0 +1,6 @@
+vars:
+  base_paths:
+    - "{{SERVICE_LOGS_DIR}}"
+input: logfile
+data_stream:
+  vars: ~
diff --git a/packages/zeek/data_stream/ntp/_dev/test/system/test-splunk-config.yml b/packages/zeek/data_stream/ntp/_dev/test/system/test-splunk-config.yml
@@ -0,0 +1,9 @@
+input: httpjson
+service: splunk-mock
+vars:
+  url: http://{{Hostname}}:{{Port}}
+  username: test
+  password: test
+data_stream:
+  vars:
+    preserve_original_event: true
diff --git a/packages/zeek/data_stream/ntp/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/ntp/agent/stream/httpjson.yml.hbs
@@ -0,0 +1,63 @@
+config_version: 2
+interval: {{interval}}
+{{#unless token}}
+{{#if username}}
+{{#if password}}
+auth.basic.user: {{username}}
+auth.basic.password: {{password}}
+{{/if}}
+{{/if}}
+{{/unless}}
+cursor:
+  index_earliest:
+    value: '[[.last_event.result.max_indextime]]'
+request.url: {{url}}/services/search/jobs/export
+{{#if ssl}}
+request.ssl: {{ssl}}
+{{/if}}
+request.method: POST
+request.transforms:
+  - set:
+      target: url.params.search
+      value: {{search}} | streamstats max(_indextime) AS max_indextime
+  - set:
+      target: url.params.output_mode
+      value: "json"
+  - set:
+      target: url.params.index_earliest
+      value: '[[ .cursor.index_earliest ]]'
+      default: '[[(now (parseDuration "-{{interval}}")).Unix]]'
+  - set:
+      target: url.params.index_latest
+      value: '[[(now).Unix]]'
+  - set:
+      target: header.Content-Type
+      value: application/x-www-form-urlencoded
+{{#unless username}}
+{{#unless password}}
+{{#if token}}
+  - set:
+      target: header.Authorization
+      value: {{token}}
+{{/if}}
+{{/unless}}
+{{/unless}}
+response.decode_as: application/x-ndjson
+response.split:
+  target: body.result._raw
+  type: string
+  delimiter: "\n"
+tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
+{{#each tags as |tag i|}}
+  - {{tag}}
+{{/each}}
+{{#contains tags "forwarded"}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}
diff --git a/packages/zeek/data_stream/ntp/agent/stream/log.yml.hbs b/packages/zeek/data_stream/ntp/agent/stream/log.yml.hbs
@@ -0,0 +1,21 @@
+paths:
+{{#each base_paths}}
+  {{#each ../filenames}}
+ - {{../this}}/{{this}}
+  {{/each}}
+{{/each}}
+exclude_files: [".gz$"]
+tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
+{{#each tags as |tag i|}}
+  - {{tag}}
+{{/each}}
+{{#contains tags "forwarded"}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}