Add tests for keyword script field's fielddata (#59523)

Adds tests for the `keyword` typed `script` field's fielddata implementation. One unit test which runs quickly and an integretion test for fetching and an integration test for the `terms` agg.
elastic · Jul 14, 2020 · 46fcc58 · 46fcc58
1 parent 4eb18b2
commit 46fcc58
Show file tree

Hide file tree

Showing 4 changed files with 87 additions and 5 deletions.
diff --git a/.../src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryFieldData.java b/.../src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryFieldData.java
@@ -51,7 +51,7 @@ public Builder(StringScriptFieldScript.Factory scriptFactory) {
         }
 
         @Override
-        public IndexFieldData<?> build(
+        public ScriptBinaryFieldData build(
             IndexSettings indexSettings,
             MappedFieldType fieldType,
             IndexFieldDataCache cache,
@@ -130,7 +130,7 @@ public void clear() {
 
     }
 
-    static class ScriptBinaryLeafFieldData implements LeafFieldData {
+    public static class ScriptBinaryLeafFieldData implements LeafFieldData {
         private final ScriptBinaryDocValues scriptBinaryDocValues;
 
         ScriptBinaryLeafFieldData(ScriptBinaryDocValues scriptBinaryDocValues) {

diff --git a/...main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeKeywordMappedFieldType.java b/...main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeKeywordMappedFieldType.java
@@ -8,10 +8,9 @@
 
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.xcontent.ToXContent.Params;
 import org.elasticsearch.common.lucene.BytesRefs;
+import org.elasticsearch.common.xcontent.ToXContent.Params;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.TextSearchInfo;
 import org.elasticsearch.index.query.QueryShardContext;
@@ -53,7 +52,7 @@ public String typeName() {
     }
 
     @Override
-    public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) {
+    public ScriptBinaryFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) {
         // TODO once we get SearchLookup as an argument, we can already call scriptFactory.newFactory here and pass through the result
         return new ScriptBinaryFieldData.Builder(scriptFactory);
     }

diff --git a/...java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeKeywordMappedFieldTypeTests.java b/...java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeKeywordMappedFieldTypeTests.java
@@ -8,11 +8,21 @@
 
 import org.apache.lucene.document.StoredField;
 import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.search.Collector;
 import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.LeafCollector;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.Scorable;
+import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.painless.PainlessPlugin;
@@ -25,8 +35,10 @@
 import org.elasticsearch.xpack.runtimefields.RuntimeFields;
 import org.elasticsearch.xpack.runtimefields.RuntimeFieldsPainlessExtension;
 import org.elasticsearch.xpack.runtimefields.StringScriptFieldScript;
+import org.elasticsearch.xpack.runtimefields.fielddata.ScriptBinaryFieldData;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
 
 import static java.util.Collections.emptyMap;
@@ -35,6 +47,50 @@
 import static org.mockito.Mockito.when;
 
 public class RuntimeKeywordMappedFieldTypeTests extends ESTestCase {
+    public void testDocValues() throws IOException {
+        try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) {
+            iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [1]}"))));
+            iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [2, 1]}"))));
+            List<String> results = new ArrayList<>();
+            try (DirectoryReader reader = iw.getReader()) {
+                IndexSearcher searcher = newSearcher(reader);
+                RuntimeKeywordMappedFieldType ft = build("for (def v : source.foo) {value(v.toString())}");
+                IndexMetadata imd = IndexMetadata.builder("test")
+                    .settings(Settings.builder().put("index.version.created", Version.CURRENT))
+                    .numberOfShards(1)
+                    .numberOfReplicas(1)
+                    .build();
+                ScriptBinaryFieldData ifd = ft.fielddataBuilder("test").build(new IndexSettings(imd, Settings.EMPTY), ft, null, null, null);
+                ifd.setSearchLookup(mockContext().lookup());
+                searcher.search(new MatchAllDocsQuery(), new Collector() {
+                    @Override
+                    public ScoreMode scoreMode() {
+                        return ScoreMode.COMPLETE_NO_SCORES;
+                    }
+
+                    @Override
+                    public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException {
+                        SortedBinaryDocValues dv = ifd.load(context).getBytesValues();
+                        return new LeafCollector() {
+                            @Override
+                            public void setScorer(Scorable scorer) throws IOException {}
+
+                            @Override
+                            public void collect(int doc) throws IOException {
+                                if (dv.advanceExact(doc)) {
+                                    for (int i = 0; i < dv.docValueCount(); i++) {
+                                        results.add(dv.nextValue().utf8ToString());
+                                    }
+                                }
+                            }
+                        };
+                    }
+                });
+                assertThat(results, equalTo(List.of("1", "1", "2")));
+            }
+        }
+    }
+
     public void testTermQuery() throws IOException {
         try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) {
             iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 1}"))));

diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml
@@ -40,6 +40,33 @@ setup:
           {"index":{}}
           {"timestamp": 1516297294000, "temperature": 202, "voltage": 4.0, "node": "c"}
 
+---
+"docvalue_fields":
+  - do:
+      search:
+        index: sensor
+        body:
+          sort: timestamp
+          docvalue_fields: [day_of_week]
+  - match: {hits.total.value: 6}
+  - match: {hits.hits.0.fields.day_of_week: [Thursday] }
+
+---
+"terms agg":
+  - do:
+      search:
+        index: sensor
+        body:
+          aggs:
+            dow:
+              terms:
+                field: day_of_week
+  - match: {hits.total.value: 6}
+  - match: {aggregations.dow.buckets.0.key: Friday}
+  - match: {aggregations.dow.buckets.0.doc_count: 1}
+  - match: {aggregations.dow.buckets.1.key: Monday}
+  - match: {aggregations.dow.buckets.1.doc_count: 1}
+
 ---
 "term query":
   - do: