Cherry-pick elastic#10029 to 6.6: Handle IPv6 zone id in IIS filebeat…

… ingest pipeline for access logs too (elastic#10030)
leweafan · Jan 14, 2019 · e05f967 · e05f967
1 parent ea3e63a
commit e05f967
Show file tree

Hide file tree

Showing 4 changed files with 61 additions and 2 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -40,7 +40,7 @@ https://github.com/elastic/beats/compare/1035569addc4a3b29ffa14f8a08c27c1ace16ef
 
 *Filebeat*
 
-- Support IPv6 addresses with zone id in IIS ingest pipeline. {issue}9836[9836] {pull}9869[9869]
+- Support IPv6 addresses with zone id in IIS ingest pipeline. {issue}9836[9836] error log: {pull}9869[9869] access log: {pull}10030[10030]
 - Support haproxy log lines without captured headers. {issue}9463[9463] {pull}9958[9958]
 
 *Heartbeat*

diff --git a/filebeat/module/iis/access/ingest/default.json b/filebeat/module/iis/access/ingest/default.json
@@ -40,10 +40,24 @@
       "target_field": "iis.access.user_agent.original"
     }
   }, {
-    "geoip": {
+   "grok": {
       "field": "iis.access.remote_ip",
+      "patterns": [
+        "%{NOZONEIP:iis.access.remote_ip_geoip}"
+      ],
+      "pattern_definitions": {
+         "NOZONEIP": "[^%]*"
+      }
+    }
+  }, {
+    "geoip": {
+      "field": "iis.access.remote_ip_geoip",
       "target_field": "iis.access.geoip"
     }
+  }, {
+    "remove": {
+      "field": "iis.access.remote_ip_geoip"
+    }
   }],
   "on_failure" : [{
     "set" : {

diff --git a/filebeat/module/iis/access/test/test-ipv6zone.log b/filebeat/module/iis/access/test/test-ipv6zone.log
@@ -0,0 +1,5 @@
+#Software: Microsoft Internet Information Services 10.0
+#Version: 1.0
+#Date: 2018-01-01 10:11:12
+#Fields: date time s-sitename s-computername s-ip cs-method cs-uri-stem cs-uri-query s-port cs-username c-ip cs-version cs(User-Agent) cs(Cookie) cs(Referer) cs-host sc-status sc-substatus sc-win32-status sc-bytes cs-bytes time-taken
+2018-01-01 10:11:12 W3SVC1 MACHINE-NAME ::1%0 GET / - 80 - ::1%0 HTTP/1.1 Mozilla/5.0+(Macintosh;+Intel+Mac+OS+X+10_14_0)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/70.0.3538.102+Safari/537.36 - - example.com 200 0 0 123 456 789
diff --git a/filebeat/module/iis/access/test/test-ipv6zone.log-expected.json b/filebeat/module/iis/access/test/test-ipv6zone.log-expected.json
@@ -0,0 +1,40 @@
+[
+    {
+        "@timestamp": "2018-01-01T10:11:12.000Z",
+        "event.dataset": "iis.access",
+        "fileset.module": "iis",
+        "fileset.name": "access",
+        "iis.access.body_received.bytes": "456",
+        "iis.access.body_sent.bytes": "123",
+        "iis.access.cookie": "-",
+        "iis.access.hostname": "example.com",
+        "iis.access.http_version": "1.1",
+        "iis.access.method": "GET",
+        "iis.access.port": "80",
+        "iis.access.query_string": "-",
+        "iis.access.referrer": "-",
+        "iis.access.remote_ip": "::1%0",
+        "iis.access.request_time_ms": "789",
+        "iis.access.response_code": "200",
+        "iis.access.server_ip": "::1%0",
+        "iis.access.server_name": "MACHINE-NAME",
+        "iis.access.site_name": "W3SVC1",
+        "iis.access.sub_status": "0",
+        "iis.access.url": "/",
+        "iis.access.user_agent.device": "Other",
+        "iis.access.user_agent.major": "70",
+        "iis.access.user_agent.minor": "0",
+        "iis.access.user_agent.name": "Chrome",
+        "iis.access.user_agent.original": "Mozilla/5.0+(Macintosh;+Intel+Mac+OS+X+10_14_0)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/70.0.3538.102+Safari/537.36",
+        "iis.access.user_agent.os": "Mac OS X 10.14.0",
+        "iis.access.user_agent.os_major": "10",
+        "iis.access.user_agent.os_minor": "14",
+        "iis.access.user_agent.os_name": "Mac OS X",
+        "iis.access.user_agent.patch": "3538",
+        "iis.access.user_name": "-",
+        "iis.access.win32_status": "0",
+        "input.type": "log",
+        "offset": 331,
+        "prospector.type": "log"
+    }
+]