From db3d4e6d93745889f5e2a76a327b8cbd8f891fac Mon Sep 17 00:00:00 2001
From: Armin Braun <me@obrown.io>
Date: Mon, 2 Jul 2018 14:36:55 +0200
Subject: [PATCH] Extend allowed characters for grok field names (#21745)
 (#31653) (#31722)

---
 .../java/org/elasticsearch/grok/Grok.java     |  6 +--
 .../org/elasticsearch/grok/GrokTests.java     | 50 ++++++++++++++++++-
 2 files changed, 51 insertions(+), 5 deletions(-)

diff --git a/libs/grok/src/main/java/org/elasticsearch/grok/Grok.java b/libs/grok/src/main/java/org/elasticsearch/grok/Grok.java
index 02388d838bc2a..6c68710c6d8bd 100644
--- a/libs/grok/src/main/java/org/elasticsearch/grok/Grok.java
+++ b/libs/grok/src/main/java/org/elasticsearch/grok/Grok.java
@@ -52,7 +52,7 @@ public final class Grok {
             "%\\{" +
             "(?<name>" +
             "(?<pattern>[A-z0-9]+)" +
-            "(?::(?<subname>[A-z0-9_:.-]+))?" +
+            "(?::(?<subname>[[:alnum:]@\\[\\]_:.-]+))?" +
             ")" +
             "(?:=(?<definition>" +
             "(?:" +
@@ -81,11 +81,11 @@ public final class Grok {
     public Grok(Map<String, String> patternBank, String grokPattern) {
         this(patternBank, grokPattern, true, ThreadWatchdog.noop());
     }
-    
+
     public Grok(Map<String, String> patternBank, String grokPattern, ThreadWatchdog threadWatchdog) {
         this(patternBank, grokPattern, true, threadWatchdog);
     }
-    
+
     Grok(Map<String, String> patternBank, String grokPattern, boolean namedCaptures) {
         this(patternBank, grokPattern, namedCaptures, ThreadWatchdog.noop());
     }
diff --git a/libs/grok/src/test/java/org/elasticsearch/grok/GrokTests.java b/libs/grok/src/test/java/org/elasticsearch/grok/GrokTests.java
index 983c84cf76b4c..7c1b473c6b3a1 100644
--- a/libs/grok/src/test/java/org/elasticsearch/grok/GrokTests.java
+++ b/libs/grok/src/test/java/org/elasticsearch/grok/GrokTests.java
@@ -379,10 +379,10 @@ public void testMultipleNamedCapturesWithSameName() {
         expected.put("num", "1");
         assertThat(grok.captures("12"), equalTo(expected));
     }
-    
+
     public void testExponentialExpressions() {
         AtomicBoolean run = new AtomicBoolean(true); // to avoid a lingering thread when test has completed
-        
+
         String grokPattern = "Bonsuche mit folgender Anfrage: Belegart->\\[%{WORD:param2},(?<param5>(\\s*%{NOTSPACE})*)\\] " +
             "Zustand->ABGESCHLOSSEN Kassennummer->%{WORD:param9} Bonnummer->%{WORD:param10} Datum->%{DATESTAMP_OTHER:param11}";
         String logLine = "Bonsuche mit folgender Anfrage: Belegart->[EINGESCHRAENKTER_VERKAUF, VERKAUF, NACHERFASSUNG] " +
@@ -406,4 +406,50 @@ public void testExponentialExpressions() {
         run.set(false);
         assertThat(e.getMessage(), equalTo("grok pattern matching was interrupted after [200] ms"));
     }
+
+    public void testAtInFieldName() {
+        assertGrokedField("@metadata");
+    }
+
+    public void assertNonAsciiLetterInFieldName() {
+        assertGrokedField("metädata");
+    }
+
+    public void assertSquareBracketInFieldName() {
+        assertGrokedField("metadat[a]");
+        assertGrokedField("metad[a]ta");
+        assertGrokedField("[m]etadata");
+    }
+
+    public void testUnderscoreInFieldName() {
+        assertGrokedField("meta_data");
+    }
+
+    public void testDotInFieldName() {
+        assertGrokedField("meta.data");
+    }
+
+    public void testMinusInFieldName() {
+        assertGrokedField("meta-data");
+    }
+
+    public void testAlphanumericFieldName() {
+        assertGrokedField(randomAlphaOfLengthBetween(1, 5));
+        assertGrokedField(randomAlphaOfLengthBetween(1, 5) + randomIntBetween(0, 100));
+        assertGrokedField(randomIntBetween(0, 100) + randomAlphaOfLengthBetween(1, 5));
+        assertGrokedField(String.valueOf(randomIntBetween(0, 100)));
+    }
+
+    public void testUnsupportedBracketsInFieldName() {
+        Grok grok = new Grok(basePatterns, "%{WORD:unsuppo(r)ted}");
+        Map<String, Object> matches = grok.captures("line");
+        assertNull(matches);
+    }
+
+    private void assertGrokedField(String fieldName) {
+        String line = "foo";
+        Grok grok = new Grok(basePatterns, "%{WORD:" + fieldName + "}");
+        Map<String, Object> matches = grok.captures(line);
+        assertEquals(line, matches.get(fieldName));
+    }
 }