Skip to content

Commit

Permalink
In FieldTypeLookup, factor out flat object field logic. (#52091)
Browse files Browse the repository at this point in the history
Currently, the logic for looking up `flattened` field types lives in the
top-level `FieldTypeLookup`. This PR moves it into a dedicated class
`DynamicKeyFieldTypeLookup`.
  • Loading branch information
jtibshirani committed Feb 10, 2020
1 parent 7b58ed0 commit 28a8db7
Show file tree
Hide file tree
Showing 3 changed files with 171 additions and 116 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.elasticsearch.index.mapper;

import org.elasticsearch.common.collect.CopyOnWriteHashMap;

import java.util.Collections;
import java.util.Iterator;
import java.util.Map;

/**
* A container that supports looking up field types for 'dynamic key' fields ({@link DynamicKeyFieldMapper}).
*
* Compared to standard fields, 'dynamic key' fields require special handling. Given a field name of the form
* 'path_to_field.path_to_key', the container will dynamically return a new {@link MappedFieldType} that is
* suitable for performing searches on the sub-key.
*
* Note: we anticipate that 'flattened' fields will be the only implementation {@link DynamicKeyFieldMapper}.
* Flattened object fields live in the 'mapper-flattened' module.
*/
class DynamicKeyFieldTypeLookup {
private final CopyOnWriteHashMap<String, DynamicKeyFieldMapper> mappers;
private final Map<String, String> aliasToConcreteName;

/**
* The maximum field depth of any dynamic key mapper. Allows us to stop searching for
* a dynamic key mapper as soon as we've passed the maximum possible field depth.
*/
private final int maxKeyDepth;

DynamicKeyFieldTypeLookup() {
this.mappers = new CopyOnWriteHashMap<>();
this.aliasToConcreteName = Collections.emptyMap();
this.maxKeyDepth = 0;
}

private DynamicKeyFieldTypeLookup(CopyOnWriteHashMap<String, DynamicKeyFieldMapper> mappers,
Map<String, String> aliasToConcreteName,
int maxKeyDepth) {
this.mappers = mappers;
this.aliasToConcreteName = aliasToConcreteName;
this.maxKeyDepth = maxKeyDepth;
}

DynamicKeyFieldTypeLookup copyAndAddAll(Map<String, DynamicKeyFieldMapper> newMappers,
Map<String, String> aliasToConcreteName) {
CopyOnWriteHashMap<String, DynamicKeyFieldMapper> combinedMappers = this.mappers.copyAndPutAll(newMappers);
int maxKeyDepth = getMaxKeyDepth(combinedMappers, aliasToConcreteName);
return new DynamicKeyFieldTypeLookup(combinedMappers, aliasToConcreteName, maxKeyDepth);
}

/**
* Check if the given field corresponds to a dynamic key mapper of the
* form 'path_to_field.path_to_key'. If so, returns a field type that
* can be used to perform searches on this field. Otherwise returns null.
*/
MappedFieldType get(String field) {
if (mappers.isEmpty()) {
return null;
}

int dotIndex = -1;
int fieldDepth = 0;

while (true) {
if (++fieldDepth > maxKeyDepth) {
return null;
}

dotIndex = field.indexOf('.', dotIndex + 1);
if (dotIndex < 0) {
return null;
}

String parentField = field.substring(0, dotIndex);
String concreteField = aliasToConcreteName.getOrDefault(parentField, parentField);
DynamicKeyFieldMapper mapper = mappers.get(concreteField);

if (mapper != null) {
String key = field.substring(dotIndex + 1);
return mapper.keyedFieldType(key);
}
}
}

Iterator<MappedFieldType> fieldTypes() {
return mappers.values().stream()
.<MappedFieldType>map(mapper -> mapper.keyedFieldType(""))
.iterator();
}

// Visible for testing.
static int getMaxKeyDepth(Map<String, DynamicKeyFieldMapper> dynamicKeyMappers,
Map<String, String> aliasToConcreteName) {
int maxFieldDepth = 0;
for (Map.Entry<String, String> entry : aliasToConcreteName.entrySet()) {
String aliasName = entry.getKey();
String path = entry.getValue();
if (dynamicKeyMappers.containsKey(path)) {
maxFieldDepth = Math.max(maxFieldDepth, fieldDepth(aliasName));
}
}

for (String fieldName : dynamicKeyMappers.keySet()) {
if (dynamicKeyMappers.containsKey(fieldName)) {
maxFieldDepth = Math.max(maxFieldDepth, fieldDepth(fieldName));
}
}

return maxFieldDepth;
}

/**
* Computes the total depth of this field by counting the number of parent fields
* in its path. As an example, the field 'parent1.parent2.field' has depth 3.
*/
private static int fieldDepth(String field) {
int numDots = 0;
int dotIndex = -1;
while (true) {
dotIndex = field.indexOf('.', dotIndex + 1);
if (dotIndex < 0) {
break;
}
numDots++;
}
return numDots + 1;
}
}
114 changes: 12 additions & 102 deletions server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import org.elasticsearch.common.regex.Regex;

import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
Expand All @@ -37,31 +38,21 @@ class FieldTypeLookup implements Iterable<MappedFieldType> {

final CopyOnWriteHashMap<String, MappedFieldType> fullNameToFieldType;
private final CopyOnWriteHashMap<String, String> aliasToConcreteName;
private final DynamicKeyFieldTypeLookup dynamicKeyLookup;

private final CopyOnWriteHashMap<String, DynamicKeyFieldMapper> dynamicKeyMappers;

/**
* The maximum field depth of any mapper that implements {@link DynamicKeyFieldMapper}.
* Allows us stop searching for a 'dynamic key' mapper as soon as we've passed the maximum
* possible field depth.
*/
private final int maxDynamicKeyDepth;

FieldTypeLookup() {
fullNameToFieldType = new CopyOnWriteHashMap<>();
aliasToConcreteName = new CopyOnWriteHashMap<>();
dynamicKeyMappers = new CopyOnWriteHashMap<>();
maxDynamicKeyDepth = 0;
dynamicKeyLookup = new DynamicKeyFieldTypeLookup();
}

private FieldTypeLookup(CopyOnWriteHashMap<String, MappedFieldType> fullNameToFieldType,
CopyOnWriteHashMap<String, String> aliasToConcreteName,
CopyOnWriteHashMap<String, DynamicKeyFieldMapper> dynamicKeyMappers,
int maxDynamicKeyDepth) {
DynamicKeyFieldTypeLookup dynamicKeyLookup) {
this.fullNameToFieldType = fullNameToFieldType;
this.aliasToConcreteName = aliasToConcreteName;
this.dynamicKeyMappers = dynamicKeyMappers;
this.maxDynamicKeyDepth = maxDynamicKeyDepth;
this.dynamicKeyLookup = dynamicKeyLookup;
}

/**
Expand All @@ -80,7 +71,7 @@ public FieldTypeLookup copyAndAddAll(String type,

CopyOnWriteHashMap<String, MappedFieldType> fullName = this.fullNameToFieldType;
CopyOnWriteHashMap<String, String> aliases = this.aliasToConcreteName;
CopyOnWriteHashMap<String, DynamicKeyFieldMapper> dynamicKeyMappers = this.dynamicKeyMappers;
Map<String, DynamicKeyFieldMapper> dynamicKeyMappers = new HashMap<>();

for (FieldMapper fieldMapper : fieldMappers) {
String fieldName = fieldMapper.name();
Expand All @@ -92,8 +83,7 @@ public FieldTypeLookup copyAndAddAll(String type,
}

if (fieldMapper instanceof DynamicKeyFieldMapper) {
DynamicKeyFieldMapper dynamicKeyMapper = (DynamicKeyFieldMapper) fieldMapper;
dynamicKeyMappers = dynamicKeyMappers.copyAndPut(fieldName, dynamicKeyMapper);
dynamicKeyMappers.put(fieldName, (DynamicKeyFieldMapper) fieldMapper);
}
}

Expand All @@ -103,46 +93,8 @@ public FieldTypeLookup copyAndAddAll(String type,
aliases = aliases.copyAndPut(aliasName, path);
}

int maxDynamicKeyDepth = getMaxDynamicKeyDepth(aliases, dynamicKeyMappers);

return new FieldTypeLookup(fullName, aliases, dynamicKeyMappers, maxDynamicKeyDepth);
}

private static int getMaxDynamicKeyDepth(CopyOnWriteHashMap<String, String> aliases,
CopyOnWriteHashMap<String, DynamicKeyFieldMapper> dynamicKeyMappers) {
int maxFieldDepth = 0;
for (Map.Entry<String, String> entry : aliases.entrySet()) {
String aliasName = entry.getKey();
String path = entry.getValue();
if (dynamicKeyMappers.containsKey(path)) {
maxFieldDepth = Math.max(maxFieldDepth, fieldDepth(aliasName));
}
}

for (String fieldName : dynamicKeyMappers.keySet()) {
if (dynamicKeyMappers.containsKey(fieldName)) {
maxFieldDepth = Math.max(maxFieldDepth, fieldDepth(fieldName));
}
}

return maxFieldDepth;
}

/**
* Computes the total depth of this field by counting the number of parent fields
* in its path. As an example, the field 'parent1.parent2.field' has depth 3.
*/
private static int fieldDepth(String field) {
int numDots = 0;
int dotIndex = -1;
while (true) {
dotIndex = field.indexOf('.', dotIndex + 1);
if (dotIndex < 0) {
break;
}
numDots++;
}
return numDots + 1;
DynamicKeyFieldTypeLookup newDynamicKeyLookup = this.dynamicKeyLookup.copyAndAddAll(dynamicKeyMappers, aliases);
return new FieldTypeLookup(fullName, aliases, newDynamicKeyLookup);
}

/**
Expand All @@ -157,37 +109,7 @@ public MappedFieldType get(String field) {

// If the mapping contains fields that support dynamic sub-key lookup, check
// if this could correspond to a keyed field of the form 'path_to_field.path_to_key'.
return !dynamicKeyMappers.isEmpty() ? getKeyedFieldType(field) : null;
}

/**
* Check if the given field corresponds to a dynamic lookup mapper of the
* form 'path_to_field.path_to_key'. If so, returns a field type that
* can be used to perform searches on this field.
*/
private MappedFieldType getKeyedFieldType(String field) {
int dotIndex = -1;
int fieldDepth = 0;

while (true) {
if (++fieldDepth > maxDynamicKeyDepth) {
return null;
}

dotIndex = field.indexOf('.', dotIndex + 1);
if (dotIndex < 0) {
return null;
}

String parentField = field.substring(0, dotIndex);
String concreteField = aliasToConcreteName.getOrDefault(parentField, parentField);
DynamicKeyFieldMapper mapper = dynamicKeyMappers.get(concreteField);

if (mapper != null) {
String key = field.substring(dotIndex + 1);
return mapper.keyedFieldType(key);
}
}
return dynamicKeyLookup.get(field);
}

/**
Expand All @@ -211,19 +133,7 @@ public Set<String> simpleMatchToFullName(String pattern) {
@Override
public Iterator<MappedFieldType> iterator() {
Iterator<MappedFieldType> concreteFieldTypes = fullNameToFieldType.values().iterator();

if (dynamicKeyMappers.isEmpty()) {
return concreteFieldTypes;
} else {
Iterator<MappedFieldType> keyedFieldTypes = dynamicKeyMappers.values().stream()
.<MappedFieldType>map(mapper -> mapper.keyedFieldType(""))
.iterator();
return Iterators.concat(concreteFieldTypes, keyedFieldTypes);
}
}

// Visible for testing.
int maxKeyedLookupDepth() {
return maxDynamicKeyDepth;
Iterator<MappedFieldType> keyedFieldTypes = dynamicKeyLookup.fieldTypes();
return Iterators.concat(concreteFieldTypes, keyedFieldTypes);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,9 @@
import org.elasticsearch.xpack.flattened.mapper.FlatObjectFieldMapper.KeyedFlatObjectFieldType;

import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.function.Function;

Expand Down Expand Up @@ -95,33 +97,30 @@ public void testFieldTypeLookupWithMultipleFields() {
}

public void testMaxDynamicKeyDepth() {
FieldTypeLookup lookup = new FieldTypeLookup();
assertEquals(0, lookup.maxKeyedLookupDepth());
Map<String, DynamicKeyFieldMapper> mappers = new HashMap<>();
Map<String, String> aliases = new HashMap<>();
assertEquals(0, DynamicKeyFieldTypeLookup.getMaxKeyDepth(mappers, aliases));

// Add a flattened object field.
String flatObjectName = "object1.object2.field";
FlatObjectFieldMapper flatObjectField = createFlatObjectMapper(flatObjectName);
lookup = lookup.copyAndAddAll("type", singletonList(flatObjectField), emptyList());
assertEquals(3, lookup.maxKeyedLookupDepth());
mappers.put(flatObjectName, flatObjectField);
assertEquals(3, DynamicKeyFieldTypeLookup.getMaxKeyDepth(mappers, aliases));

// Add a short alias to that field.
String aliasName = "alias";
FieldAliasMapper alias = new FieldAliasMapper(aliasName, aliasName, flatObjectName);
lookup = lookup.copyAndAddAll("type", emptyList(), singletonList(alias));
assertEquals(3, lookup.maxKeyedLookupDepth());
aliases.put(aliasName, flatObjectName);
assertEquals(3, DynamicKeyFieldTypeLookup.getMaxKeyDepth(mappers, aliases));

// Add a longer alias to that field.
String longAliasName = "object1.object2.object3.alias";
FieldAliasMapper longAlias = new FieldAliasMapper(longAliasName, longAliasName, flatObjectName);
lookup = lookup.copyAndAddAll("type", emptyList(), singletonList(longAlias));
assertEquals(4, lookup.maxKeyedLookupDepth());
aliases.put(longAliasName, flatObjectName);
assertEquals(4, DynamicKeyFieldTypeLookup.getMaxKeyDepth(mappers, aliases));

// Update the long alias to refer to a non-flattened object field.
String fieldName = "field";
MockFieldMapper field = new MockFieldMapper(fieldName);
longAlias = new FieldAliasMapper(longAliasName, longAliasName, fieldName);
lookup = lookup.copyAndAddAll("type", singletonList(field), singletonList(longAlias));
assertEquals(3, lookup.maxKeyedLookupDepth());
aliases.put(longAliasName, fieldName);
assertEquals(3, DynamicKeyFieldTypeLookup.getMaxKeyDepth(mappers, aliases));
}

public void testFieldLookupIterator() {
Expand Down

0 comments on commit 28a8db7

Please sign in to comment.