Add system tests for security data_stream

elastic · Jun 2, 2021 · 7a01309 · 7a01309
1 parent b8e3522
commit 7a01309
Show file tree

Hide file tree

Showing 87 changed files with 200 additions and 153 deletions.
diff --git a/packages/system/_dev/build/docs/README.md b/packages/system/_dev/build/docs/README.md
@@ -42,6 +42,8 @@ event log.
 The Windows `security` dataset provides events from the Windows
 `Security` event log.
 
+{{event "security"}}
+
 {{fields "security"}}
 
 ### Auth

diff --git a/packages/system/_dev/deploy/docker/docker-compose.yml b/packages/system/_dev/deploy/docker/docker-compose.yml
@@ -0,0 +1,14 @@
+version: '2.3'
+services:
+  security:
+    image: docker.elastic.co/observability/stream:v0.4.0
+    ports:
+      - 8080
+    volumes:
+      - ./sample_logs:/sample_logs:ro
+    command:
+      - log
+      - --start-signal=SIGHUP
+      - --addr=:8080
+      - -p=http-server
+      - /sample_logs/security.json.log
diff --git a/packages/system/_dev/deploy/docker/sample_logs/security.json.log b/packages/system/_dev/deploy/docker/sample_logs/security.json.log
@@ -0,0 +1 @@
+{"preview": false,"offset": 194,"lastrow": true,"result": {"_bkt": "main~0~1212176D-89E1-485D-89E6-3ADC276CCA38","_cd": "0:315","_indextime": "1622471463","_raw": "<Event xmlns='http://schemas.microsoft.com/win/2004/08/events/event'><System><Provider Name='Microsoft-Windows-Eventlog' Guid='{fc65ddd8-d6ef-4962-83d5-6e5cfe9ce148}'/><EventID>1100</EventID><Version>0</Version><Level>4</Level><Task>103</Task><Opcode>0</Opcode><Keywords>0x4020000000000000</Keywords><TimeCreated SystemTime='2019-11-07T10:37:04.226092500Z'/><EventRecordID>14257</EventRecordID><Correlation/><Execution ProcessID='1144' ThreadID='4532'/><Channel>Security</Channel><Computer>WIN-41OB2LO92CR.wlbeat.local</Computer><Security/></System><UserData><ServiceShutdown xmlns='http://manifests.microsoft.com/win/2004/08/windows/eventlog'></ServiceShutdown></UserData></Event>","_serial": "194","_si": ["69819b6ce1bd","main"],"_sourcetype": "XmlWinEventLog:Security","_time": "2021-05-25 13:11:45.000 UTC","host": "VAGRANT","index": "main","linecount": "1","max_indextime": "1622471606","source": "WinEventLog:Security","sourcetype": "XmlWinEventLog:Security","splunk_server": "69819b6ce1bd"}}
diff --git a/packages/system/changelog.yml b/packages/system/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.13.1"
+  changes:
+    - description: Add system tests for security data_stream.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1069
 - version: "0.13.0"
   changes:
     - description: Render units and metric types in exported fields table

diff --git a/packages/system/data_stream/security/_dev/test/pipeline/test-1100.json-config.yml b/packages/system/data_stream/security/_dev/test/pipeline/test-1100.json-config.yml
diff --git a/packages/system/data_stream/security/_dev/test/pipeline/test-1102.json-config.yml b/packages/system/data_stream/security/_dev/test/pipeline/test-1102.json-config.yml
diff --git a/packages/system/data_stream/security/_dev/test/pipeline/test-1104.json-config.yml b/packages/system/data_stream/security/_dev/test/pipeline/test-1104.json-config.yml
diff --git a/packages/system/data_stream/security/_dev/test/pipeline/test-1105.json-config.yml b/packages/system/data_stream/security/_dev/test/pipeline/test-1105.json-config.yml
diff --git a/...s/system/data_stream/security/_dev/test/pipeline/test-4670-windowssrv2016.json-config.yml b/...s/system/data_stream/security/_dev/test/pipeline/test-4670-windowssrv2016.json-config.yml
diff --git a/...s/system/data_stream/security/_dev/test/pipeline/test-4706-windowssrv2016.json-config.yml b/...s/system/data_stream/security/_dev/test/pipeline/test-4706-windowssrv2016.json-config.yml
diff --git a/...s/system/data_stream/security/_dev/test/pipeline/test-4707-windowssrv2016.json-config.yml b/...s/system/data_stream/security/_dev/test/pipeline/test-4707-windowssrv2016.json-config.yml
diff --git a/...s/system/data_stream/security/_dev/test/pipeline/test-4713-windowssrv2016.json-config.yml b/...s/system/data_stream/security/_dev/test/pipeline/test-4713-windowssrv2016.json-config.yml
diff --git a/...s/system/data_stream/security/_dev/test/pipeline/test-4716-windowssrv2016.json-config.yml b/...s/system/data_stream/security/_dev/test/pipeline/test-4716-windowssrv2016.json-config.yml
diff --git a/...s/system/data_stream/security/_dev/test/pipeline/test-4717-windowssrv2016.json-config.yml b/...s/system/data_stream/security/_dev/test/pipeline/test-4717-windowssrv2016.json-config.yml
diff --git a/...s/system/data_stream/security/_dev/test/pipeline/test-4718-windowssrv2016.json-config.yml b/...s/system/data_stream/security/_dev/test/pipeline/test-4718-windowssrv2016.json-config.yml
diff --git a/...s/system/data_stream/security/_dev/test/pipeline/test-4719-windowssrv2016.json-config.yml b/...s/system/data_stream/security/_dev/test/pipeline/test-4719-windowssrv2016.json-config.yml
diff --git a/packages/system/data_stream/security/_dev/test/pipeline/test-4719.json-config.yml b/packages/system/data_stream/security/_dev/test/pipeline/test-4719.json-config.yml
diff --git a/...s/system/data_stream/security/_dev/test/pipeline/test-4739-windowssrv2016.json-config.yml b/...s/system/data_stream/security/_dev/test/pipeline/test-4739-windowssrv2016.json-config.yml
diff --git a/packages/system/data_stream/security/_dev/test/pipeline/test-4743.json-config.yml b/packages/system/data_stream/security/_dev/test/pipeline/test-4743.json-config.yml
diff --git a/packages/system/data_stream/security/_dev/test/pipeline/test-4744.json-config.yml b/packages/system/data_stream/security/_dev/test/pipeline/test-4744.json-config.yml
diff --git a/packages/system/data_stream/security/_dev/test/pipeline/test-4745.json-config.yml b/packages/system/data_stream/security/_dev/test/pipeline/test-4745.json-config.yml
diff --git a/packages/system/data_stream/security/_dev/test/pipeline/test-4746.json-config.yml b/packages/system/data_stream/security/_dev/test/pipeline/test-4746.json-config.yml
diff --git a/packages/system/data_stream/security/_dev/test/pipeline/test-4747.json-config.yml b/packages/system/data_stream/security/_dev/test/pipeline/test-4747.json-config.yml
diff --git a/packages/system/data_stream/security/_dev/test/pipeline/test-4748.json-config.yml b/packages/system/data_stream/security/_dev/test/pipeline/test-4748.json-config.yml
diff --git a/packages/system/data_stream/security/_dev/test/pipeline/test-4749.json-config.yml b/packages/system/data_stream/security/_dev/test/pipeline/test-4749.json-config.yml
diff --git a/packages/system/data_stream/security/_dev/test/pipeline/test-4750.json-config.yml b/packages/system/data_stream/security/_dev/test/pipeline/test-4750.json-config.yml
diff --git a/packages/system/data_stream/security/_dev/test/pipeline/test-4751.json-config.yml b/packages/system/data_stream/security/_dev/test/pipeline/test-4751.json-config.yml
diff --git a/packages/system/data_stream/security/_dev/test/pipeline/test-4752.json-config.yml b/packages/system/data_stream/security/_dev/test/pipeline/test-4752.json-config.yml
diff --git a/packages/system/data_stream/security/_dev/test/pipeline/test-4753.json-config.yml b/packages/system/data_stream/security/_dev/test/pipeline/test-4753.json-config.yml
diff --git a/packages/system/data_stream/security/_dev/test/pipeline/test-4759.json-config.yml b/packages/system/data_stream/security/_dev/test/pipeline/test-4759.json-config.yml
diff --git a/packages/system/data_stream/security/_dev/test/pipeline/test-4760.json-config.yml b/packages/system/data_stream/security/_dev/test/pipeline/test-4760.json-config.yml
diff --git a/packages/system/data_stream/security/_dev/test/pipeline/test-4761.json-config.yml b/packages/system/data_stream/security/_dev/test/pipeline/test-4761.json-config.yml
diff --git a/packages/system/data_stream/security/_dev/test/pipeline/test-4762.json-config.yml b/packages/system/data_stream/security/_dev/test/pipeline/test-4762.json-config.yml
diff --git a/packages/system/data_stream/security/_dev/test/pipeline/test-4763.json-config.yml b/packages/system/data_stream/security/_dev/test/pipeline/test-4763.json-config.yml
diff --git a/...s/system/data_stream/security/_dev/test/pipeline/test-4817-windowssrv2016.json-config.yml b/...s/system/data_stream/security/_dev/test/pipeline/test-4817-windowssrv2016.json-config.yml
diff --git a/...s/system/data_stream/security/_dev/test/pipeline/test-4902-windowssrv2016.json-config.yml b/...s/system/data_stream/security/_dev/test/pipeline/test-4902-windowssrv2016.json-config.yml
diff --git a/...s/system/data_stream/security/_dev/test/pipeline/test-4904-windowssrv2016.json-config.yml b/...s/system/data_stream/security/_dev/test/pipeline/test-4904-windowssrv2016.json-config.yml
diff --git a/...s/system/data_stream/security/_dev/test/pipeline/test-4905-windowssrv2016.json-config.yml b/...s/system/data_stream/security/_dev/test/pipeline/test-4905-windowssrv2016.json-config.yml
diff --git a/...s/system/data_stream/security/_dev/test/pipeline/test-4906-windowssrv2016.json-config.yml b/...s/system/data_stream/security/_dev/test/pipeline/test-4906-windowssrv2016.json-config.yml
diff --git a/...s/system/data_stream/security/_dev/test/pipeline/test-4907-windowssrv2016.json-config.yml b/...s/system/data_stream/security/_dev/test/pipeline/test-4907-windowssrv2016.json-config.yml
diff --git a/packages/system/data_stream/security/_dev/test/pipeline/test-common-config.yml b/packages/system/data_stream/security/_dev/test/pipeline/test-common-config.yml
@@ -0,0 +1,2 @@
+dynamic_fields:
+  event.ingested: ".*"
diff --git a/...em/data_stream/security/_dev/test/pipeline/test-security-windows2012-4673.json-config.yml b/...em/data_stream/security/_dev/test/pipeline/test-security-windows2012-4673.json-config.yml
diff --git a/...em/data_stream/security/_dev/test/pipeline/test-security-windows2012-4697.json-config.yml b/...em/data_stream/security/_dev/test/pipeline/test-security-windows2012-4697.json-config.yml
diff --git a/...em/data_stream/security/_dev/test/pipeline/test-security-windows2012-4768.json-config.yml b/...em/data_stream/security/_dev/test/pipeline/test-security-windows2012-4768.json-config.yml
diff --git a/...em/data_stream/security/_dev/test/pipeline/test-security-windows2012-4769.json-config.yml b/...em/data_stream/security/_dev/test/pipeline/test-security-windows2012-4769.json-config.yml
diff --git a/...em/data_stream/security/_dev/test/pipeline/test-security-windows2012-4770.json-config.yml b/...em/data_stream/security/_dev/test/pipeline/test-security-windows2012-4770.json-config.yml
diff --git a/...em/data_stream/security/_dev/test/pipeline/test-security-windows2012-4771.json-config.yml b/...em/data_stream/security/_dev/test/pipeline/test-security-windows2012-4771.json-config.yml
diff --git a/...em/data_stream/security/_dev/test/pipeline/test-security-windows2012-4776.json-config.yml b/...em/data_stream/security/_dev/test/pipeline/test-security-windows2012-4776.json-config.yml
diff --git a/...em/data_stream/security/_dev/test/pipeline/test-security-windows2012-4778.json-config.yml b/...em/data_stream/security/_dev/test/pipeline/test-security-windows2012-4778.json-config.yml
diff --git a/...em/data_stream/security/_dev/test/pipeline/test-security-windows2012-4779.json-config.yml b/...em/data_stream/security/_dev/test/pipeline/test-security-windows2012-4779.json-config.yml
diff --git a/...data_stream/security/_dev/test/pipeline/test-security-windows2012r2-logon.json-config.yml b/...data_stream/security/_dev/test/pipeline/test-security-windows2012r2-logon.json-config.yml
diff --git a/...ecurity/_dev/test/pipeline/test-security-windows2016-4722-account-enabled.json-config.yml b/...ecurity/_dev/test/pipeline/test-security-windows2016-4722-account-enabled.json-config.yml
diff --git a/...ecurity/_dev/test/pipeline/test-security-windows2016-4723-password-change.json-config.yml b/...ecurity/_dev/test/pipeline/test-security-windows2016-4723-password-change.json-config.yml
diff --git a/...security/_dev/test/pipeline/test-security-windows2016-4724-password-reset.json-config.yml b/...security/_dev/test/pipeline/test-security-windows2016-4724-password-reset.json-config.yml
diff --git a/...curity/_dev/test/pipeline/test-security-windows2016-4725-account-disabled.json-config.yml b/...curity/_dev/test/pipeline/test-security-windows2016-4725-account-disabled.json-config.yml
diff --git a/...ecurity/_dev/test/pipeline/test-security-windows2016-4726-account-deleted.json-config.yml b/...ecurity/_dev/test/pipeline/test-security-windows2016-4726-account-deleted.json-config.yml
diff --git a/...em/data_stream/security/_dev/test/pipeline/test-security-windows2016-4727.json-config.yml b/...em/data_stream/security/_dev/test/pipeline/test-security-windows2016-4727.json-config.yml
diff --git a/...em/data_stream/security/_dev/test/pipeline/test-security-windows2016-4728.json-config.yml b/...em/data_stream/security/_dev/test/pipeline/test-security-windows2016-4728.json-config.yml
diff --git a/...em/data_stream/security/_dev/test/pipeline/test-security-windows2016-4729.json-config.yml b/...em/data_stream/security/_dev/test/pipeline/test-security-windows2016-4729.json-config.yml
diff --git a/...em/data_stream/security/_dev/test/pipeline/test-security-windows2016-4730.json-config.yml b/...em/data_stream/security/_dev/test/pipeline/test-security-windows2016-4730.json-config.yml
diff --git a/...em/data_stream/security/_dev/test/pipeline/test-security-windows2016-4731.json-config.yml b/...em/data_stream/security/_dev/test/pipeline/test-security-windows2016-4731.json-config.yml
diff --git a/...em/data_stream/security/_dev/test/pipeline/test-security-windows2016-4732.json-config.yml b/...em/data_stream/security/_dev/test/pipeline/test-security-windows2016-4732.json-config.yml
diff --git a/...em/data_stream/security/_dev/test/pipeline/test-security-windows2016-4733.json-config.yml b/...em/data_stream/security/_dev/test/pipeline/test-security-windows2016-4733.json-config.yml
diff --git a/...em/data_stream/security/_dev/test/pipeline/test-security-windows2016-4734.json-config.yml b/...em/data_stream/security/_dev/test/pipeline/test-security-windows2016-4734.json-config.yml
diff --git a/...em/data_stream/security/_dev/test/pipeline/test-security-windows2016-4735.json-config.yml b/...em/data_stream/security/_dev/test/pipeline/test-security-windows2016-4735.json-config.yml
diff --git a/...em/data_stream/security/_dev/test/pipeline/test-security-windows2016-4737.json-config.yml b/...em/data_stream/security/_dev/test/pipeline/test-security-windows2016-4737.json-config.yml
diff --git a/...ecurity/_dev/test/pipeline/test-security-windows2016-4738-account-changed.json-config.yml b/...ecurity/_dev/test/pipeline/test-security-windows2016-4738-account-changed.json-config.yml
diff --git a/...rity/_dev/test/pipeline/test-security-windows2016-4740-account-locked-out.json-config.yml b/...rity/_dev/test/pipeline/test-security-windows2016-4740-account-locked-out.json-config.yml
diff --git a/...em/data_stream/security/_dev/test/pipeline/test-security-windows2016-4754.json-config.yml b/...em/data_stream/security/_dev/test/pipeline/test-security-windows2016-4754.json-config.yml
diff --git a/...em/data_stream/security/_dev/test/pipeline/test-security-windows2016-4755.json-config.yml b/...em/data_stream/security/_dev/test/pipeline/test-security-windows2016-4755.json-config.yml
diff --git a/...em/data_stream/security/_dev/test/pipeline/test-security-windows2016-4756.json-config.yml b/...em/data_stream/security/_dev/test/pipeline/test-security-windows2016-4756.json-config.yml
diff --git a/...em/data_stream/security/_dev/test/pipeline/test-security-windows2016-4757.json-config.yml b/...em/data_stream/security/_dev/test/pipeline/test-security-windows2016-4757.json-config.yml
diff --git a/...em/data_stream/security/_dev/test/pipeline/test-security-windows2016-4758.json-config.yml b/...em/data_stream/security/_dev/test/pipeline/test-security-windows2016-4758.json-config.yml
diff --git a/...em/data_stream/security/_dev/test/pipeline/test-security-windows2016-4764.json-config.yml b/...em/data_stream/security/_dev/test/pipeline/test-security-windows2016-4764.json-config.yml
diff --git a/...curity/_dev/test/pipeline/test-security-windows2016-4767-account-unlocked.json-config.yml b/...curity/_dev/test/pipeline/test-security-windows2016-4767-account-unlocked.json-config.yml
diff --git a/...ecurity/_dev/test/pipeline/test-security-windows2016-4781-account-renamed.json-config.yml b/...ecurity/_dev/test/pipeline/test-security-windows2016-4781-account-renamed.json-config.yml
diff --git a/...em/data_stream/security/_dev/test/pipeline/test-security-windows2016-4798.json-config.yml b/...em/data_stream/security/_dev/test/pipeline/test-security-windows2016-4798.json-config.yml
diff --git a/...em/data_stream/security/_dev/test/pipeline/test-security-windows2016-4799.json-config.yml b/...em/data_stream/security/_dev/test/pipeline/test-security-windows2016-4799.json-config.yml
diff --git a/.../data_stream/security/_dev/test/pipeline/test-security-windows2016-logoff.json-config.yml b/.../data_stream/security/_dev/test/pipeline/test-security-windows2016-logoff.json-config.yml
diff --git a/...ecurity/_dev/test/pipeline/test-security-windows2019-4688-process-created.json-config.yml b/...ecurity/_dev/test/pipeline/test-security-windows2019-4688-process-created.json-config.yml
diff --git a/...security/_dev/test/pipeline/test-security-windows2019-4689-process-exited.json-config.yml b/...security/_dev/test/pipeline/test-security-windows2019-4689-process-exited.json-config.yml
diff --git a/packages/system/data_stream/security/_dev/test/system/test-default-config.yml b/packages/system/data_stream/security/_dev/test/system/test-default-config.yml
@@ -0,0 +1,10 @@
+input: httpjson
+service: security
+service_notify_signal: SIGHUP
+vars:
+  url: http://{{Hostname}}:{{Port}}/api/v1/logs
+  username: test
+  password: test
+  preserve_original_event: true
+data_stream:
+  vars: ~
diff --git a/packages/system/data_stream/security/fields/base-fields.yml b/packages/system/data_stream/security/fields/base-fields.yml
@@ -19,3 +19,8 @@
 - name: '@timestamp'
   type: date
   description: Event timestamp.
+- name: tags
+  description: List of keywords used to tag each event.
+  example: '["production", "env2"]'
+  ignore_above: 1024
+  type: keyword
diff --git a/packages/system/data_stream/security/fields/beats.yml b/packages/system/data_stream/security/fields/beats.yml
@@ -0,0 +1,3 @@
+- name: input.type
+  type: keyword
+  description: Type of Filebeat input.
diff --git a/packages/system/data_stream/security/sample_event.json b/packages/system/data_stream/security/sample_event.json
@@ -0,0 +1,75 @@
+{
+    "@timestamp": "2019-11-07T10:37:04.226Z",
+    "agent": {
+        "ephemeral_id": "a0a43394-02c9-45ec-b1be-07f107bcc5eb",
+        "hostname": "docker-fleet-agent",
+        "id": "ef9fa2de-d50b-435f-a12b-c84c87b1ad22",
+        "name": "docker-fleet-agent",
+        "type": "filebeat",
+        "version": "7.13.0"
+    },
+    "data_stream": {
+        "dataset": "system.security",
+        "namespace": "ep",
+        "type": "logs"
+    },
+    "ecs": {
+        "version": "1.9.0"
+    },
+    "elastic_agent": {
+        "id": "26eba643-ca27-421e-a6d9-a843188ba452",
+        "snapshot": true,
+        "version": "7.13.0"
+    },
+    "event": {
+        "action": "logging-service-shutdown",
+        "category": [
+            "process"
+        ],
+        "code": "1100",
+        "created": "2021-06-02T08:02:12.685Z",
+        "dataset": "system.security",
+        "ingested": "2021-06-02T08:02:13.706065692Z",
+        "kind": "event",
+        "original": "\u003cEvent xmlns='http://schemas.microsoft.com/win/2004/08/events/event'\u003e\u003cSystem\u003e\u003cProvider Name='Microsoft-Windows-Eventlog' Guid='{fc65ddd8-d6ef-4962-83d5-6e5cfe9ce148}'/\u003e\u003cEventID\u003e1100\u003c/EventID\u003e\u003cVersion\u003e0\u003c/Version\u003e\u003cLevel\u003e4\u003c/Level\u003e\u003cTask\u003e103\u003c/Task\u003e\u003cOpcode\u003e0\u003c/Opcode\u003e\u003cKeywords\u003e0x4020000000000000\u003c/Keywords\u003e\u003cTimeCreated SystemTime='2019-11-07T10:37:04.226092500Z'/\u003e\u003cEventRecordID\u003e14257\u003c/EventRecordID\u003e\u003cCorrelation/\u003e\u003cExecution ProcessID='1144' ThreadID='4532'/\u003e\u003cChannel\u003eSecurity\u003c/Channel\u003e\u003cComputer\u003eWIN-41OB2LO92CR.wlbeat.local\u003c/Computer\u003e\u003cSecurity/\u003e\u003c/System\u003e\u003cUserData\u003e\u003cServiceShutdown xmlns='http://manifests.microsoft.com/win/2004/08/windows/eventlog'\u003e\u003c/ServiceShutdown\u003e\u003c/UserData\u003e\u003c/Event\u003e",
+        "outcome": "success",
+        "provider": "Microsoft-Windows-Eventlog",
+        "type": [
+            "end"
+        ]
+    },
+    "host": {
+        "name": "WIN-41OB2LO92CR.wlbeat.local"
+    },
+    "input": {
+        "type": "httpjson"
+    },
+    "log": {
+        "level": "information"
+    },
+    "tags": [
+        "forwarded",
+        "preserve_original_event"
+    ],
+    "winlog": {
+        "channel": "Security",
+        "computer_name": "WIN-41OB2LO92CR.wlbeat.local",
+        "event_id": "1100",
+        "keywords": [
+            "Audit Success"
+        ],
+        "level": "information",
+        "opcode": "Info",
+        "outcome": "success",
+        "process": {
+            "pid": 1144,
+            "thread": {
+                "id": 4532
+            }
+        },
+        "provider_guid": "{fc65ddd8-d6ef-4962-83d5-6e5cfe9ce148}",
+        "provider_name": "Microsoft-Windows-Eventlog",
+        "record_id": "14257",
+        "time_created": "2019-11-07T10:37:04.226Z"
+    }
+}