diff --git a/docs/CHANGELOG.asciidoc b/docs/CHANGELOG.asciidoc index 30018fcf42a29..31541f0dfe7af 100644 --- a/docs/CHANGELOG.asciidoc +++ b/docs/CHANGELOG.asciidoc @@ -28,8 +28,12 @@ This section summarizes the changes in each release. //[float] //=== Deprecations -//[float] -//=== New Features +[float] +=== New Features + +The new <> field allows to know which fields +got ignored at index time because of the <> +option. ({pull}30140[#29658]) [float] === Enhancements diff --git a/docs/reference/mapping/fields.asciidoc b/docs/reference/mapping/fields.asciidoc index 9361c6b26dd9d..1333d9942f74a 100644 --- a/docs/reference/mapping/fields.asciidoc +++ b/docs/reference/mapping/fields.asciidoc @@ -48,6 +48,14 @@ can be customised when a mapping type is created. All fields in the document which contain non-null values. +[float] +=== Indexing meta-fields + +<>:: + + All fields in the document that have been ignored at index time because of + <>. + [float] === Routing meta-field @@ -67,6 +75,8 @@ include::fields/all-field.asciidoc[] include::fields/field-names-field.asciidoc[] +include::fields/ignored-field.asciidoc[] + include::fields/id-field.asciidoc[] include::fields/index-field.asciidoc[] diff --git a/docs/reference/mapping/fields/ignored-field.asciidoc b/docs/reference/mapping/fields/ignored-field.asciidoc new file mode 100644 index 0000000000000..d2776ea86b26b --- /dev/null +++ b/docs/reference/mapping/fields/ignored-field.asciidoc @@ -0,0 +1,45 @@ +[[mapping-ignored-field]] +=== `_ignored` field + +added[6.4.0] + +The `_ignored` field indexes and stores the names of every field in a document +that has been ignored because it was malformed and +<> was turned on. + +This field is searchable with <>, +<> and <> +queries, and is returned as part of the search hits. + +For instance the below query matches all documents that have one or more fields +that got ignored: + +[source,js] +-------------------------------------------------- +GET _search +{ + "query": { + "exists": { + "field": "_ignored" + } + } +} +-------------------------------------------------- +// CONSOLE + +Similarly, the below query finds all documents whose `@timestamp` field was +ignored at index time: + +[source,js] +-------------------------------------------------- +GET _search +{ + "query": { + "term": { + "_ignored": "@timestamp" + } + } +} +-------------------------------------------------- +// CONSOLE + diff --git a/docs/reference/mapping/params/ignore-malformed.asciidoc b/docs/reference/mapping/params/ignore-malformed.asciidoc index 9a2fef1a23ebf..be0bdfe4ffaf8 100644 --- a/docs/reference/mapping/params/ignore-malformed.asciidoc +++ b/docs/reference/mapping/params/ignore-malformed.asciidoc @@ -85,3 +85,13 @@ PUT my_index <1> The `number_one` field inherits the index-level setting. <2> The `number_two` field overrides the index-level setting to turn off `ignore_malformed`. + +==== Dealing with malformed fields + +Malformed fields are silently ignored at indexing time when `ignore_malformed` +is turned on. Whenever possible it is recommended to keep the number of +documents that have a malformed field contained, or queries on this field will +become meaningless. Elasticsearch makes it easy to check how many documents +have malformed fields by using `exist` or `term` queries on the special +<> field. + diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/200_ignore_malformed.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/200_ignore_malformed.yml new file mode 100644 index 0000000000000..996501e7d7984 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/200_ignore_malformed.yml @@ -0,0 +1,92 @@ +--- +setup: + - skip: + version: " - 6.3.99" + reason: _ignored was added in 6.4.0 + + - do: + indices.create: + index: test + body: + mappings: + _doc: + properties: + my_date: + type: date + ignore_malformed: true + store: true + my_ip: + type: ip + ignore_malformed: true + + - do: + index: + index: test + type: _doc + id: 1 + body: { "my_date": "2018-05-11", "my_ip": ":::1" } + + - do: + index: + index: test + type: _doc + id: 2 + body: { "my_date": "bar", "my_ip": "192.168.1.42" } + + - do: + index: + index: test + type: _doc + id: 3 + body: { "my_date": "bar", "my_ip": "quux" } + + - do: + indices.refresh: {} + +--- +"Exists on _ignored": + + - do: + search: + body: { query: { exists: { "field": "_ignored" } } } + + - length: { hits.hits: 3 } + +--- +"Search on _ignored with term": + + - do: + search: + body: { query: { term: { "_ignored": "my_date" } } } + + - length: { hits.hits: 2 } + +--- +"Search on _ignored with terms": + + - do: + search: + body: { query: { terms: { "_ignored": [ "my_date", "my_ip" ] } } } + + - length: { hits.hits: 3 } + +--- +"_ignored is returned by default": + + - do: + search: + body: { query: { ids: { "values": [ "3" ] } } } + + - length: { hits.hits: 1 } + - length: { hits.hits.0._ignored: 2} + +--- +"_ignored is still returned with explicit list of stored fields": + + - do: + search: + stored_fields: [ "my_date" ] + body: { query: { ids: { "values": [ "3" ] } } } + + - length: { hits.hits: 1 } + - is_true: hits.hits.0._ignored diff --git a/server/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java b/server/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java index 13baf81f84c43..6eae8bdb1668d 100644 --- a/server/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java +++ b/server/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.index.mapper.IdFieldMapper; +import org.elasticsearch.index.mapper.IgnoredFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ParentFieldMapper; @@ -73,6 +74,12 @@ public Status needsField(FieldInfo fieldInfo) throws IOException { if (requiredFields.remove(fieldInfo.name)) { return Status.YES; } + // Always load _ignored to be explicit about ignored fields + // This works because _ignored is added as the first metadata mapper, + // so its stored fields always appear first in the list. + if (IgnoredFieldMapper.NAME.equals(fieldInfo.name)) { + return Status.YES; + } // All these fields are single-valued so we can stop when the set is // empty return requiredFields.isEmpty() diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index d7dc07fb8d313..bd08d3eadb1a3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -456,6 +456,7 @@ protected void parseCreateField(ParseContext context, List field timestamp = fieldType().parse(dateAsString); } catch (IllegalArgumentException e) { if (ignoreMalformed.value()) { + context.addIgnoredField(fieldType.name()); return; } else { throw e; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java index af44f52b6aed9..db03dfe405d3c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java @@ -306,6 +306,7 @@ public Mapper parse(ParseContext context) throws IOException { if (ignoreMalformed.value() == false) { throw e; } + context.addIgnoredField(fieldType.name()); } token = context.parser().nextToken(); } @@ -353,6 +354,7 @@ public Mapper parse(ParseContext context) throws IOException { if (ignoreMalformed.value() == false) { throw e; } + context.addIgnoredField(fieldType.name()); } } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java index d71b3c17e4e91..b4a6840050896 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java @@ -516,6 +516,7 @@ public Mapper parse(ParseContext context) throws IOException { if (ignoreMalformed.value() == false) { throw new MapperParsingException("failed to parse [" + fieldType().name() + "]", e); } + context.addIgnoredField(fieldType.name()); } return null; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java new file mode 100644 index 0000000000000..69f1e36664ef6 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java @@ -0,0 +1,154 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermRangeQuery; +import org.elasticsearch.common.lucene.Lucene; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.query.QueryShardContext; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +/** + * A field mapper that records fields that have been ignored because they were malformed. + */ +public final class IgnoredFieldMapper extends MetadataFieldMapper { + + public static final String NAME = "_ignored"; + + public static final String CONTENT_TYPE = "_ignored"; + + public static class Defaults { + public static final String NAME = IgnoredFieldMapper.NAME; + + public static final MappedFieldType FIELD_TYPE = new IgnoredFieldType(); + + static { + FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); + FIELD_TYPE.setTokenized(false); + FIELD_TYPE.setStored(true); + FIELD_TYPE.setOmitNorms(true); + FIELD_TYPE.setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); + FIELD_TYPE.setSearchAnalyzer(Lucene.KEYWORD_ANALYZER); + FIELD_TYPE.setName(NAME); + FIELD_TYPE.freeze(); + } + } + + public static class Builder extends MetadataFieldMapper.Builder { + + public Builder(MappedFieldType existing) { + super(Defaults.NAME, existing == null ? Defaults.FIELD_TYPE : existing, Defaults.FIELD_TYPE); + } + + @Override + public IgnoredFieldMapper build(BuilderContext context) { + return new IgnoredFieldMapper(context.indexSettings()); + } + } + + public static class TypeParser implements MetadataFieldMapper.TypeParser { + @Override + public MetadataFieldMapper.Builder parse(String name, Map node, + ParserContext parserContext) throws MapperParsingException { + return new Builder(parserContext.mapperService().fullName(NAME)); + } + + @Override + public MetadataFieldMapper getDefault(MappedFieldType fieldType, ParserContext context) { + final Settings indexSettings = context.mapperService().getIndexSettings().getSettings(); + return new IgnoredFieldMapper(indexSettings); + } + } + + public static final class IgnoredFieldType extends TermBasedFieldType { + + public IgnoredFieldType() { + } + + protected IgnoredFieldType(IgnoredFieldType ref) { + super(ref); + } + + @Override + public IgnoredFieldType clone() { + return new IgnoredFieldType(this); + } + + @Override + public String typeName() { + return CONTENT_TYPE; + } + + @Override + public Query existsQuery(QueryShardContext context) { + // This query is not performance sensitive, it only helps assess + // quality of the data, so we may use a slow query. It shouldn't + // be too slow in practice since the number of unique terms in this + // field is bounded by the number of fields in the mappings. + return new TermRangeQuery(name(), null, null, true, true); + } + + } + + private IgnoredFieldMapper(Settings indexSettings) { + super(NAME, Defaults.FIELD_TYPE, Defaults.FIELD_TYPE, indexSettings); + } + + @Override + public void preParse(ParseContext context) throws IOException { + } + + @Override + public void postParse(ParseContext context) throws IOException { + super.parse(context); + } + + @Override + public Mapper parse(ParseContext context) throws IOException { + // done in post-parse + return null; + } + + @Override + protected void parseCreateField(ParseContext context, List fields) throws IOException { + for (String field : context.getIgnoredFields()) { + context.doc().add(new Field(NAME, field, fieldType())); + } + } + + @Override + protected String contentType() { + return CONTENT_TYPE; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder; + } + +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index 13e1cfb75f1dd..70934f8c55ec6 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -373,6 +373,7 @@ protected void parseCreateField(ParseContext context, List field address = InetAddresses.forString(addressAsString); } catch (IllegalArgumentException e) { if (ignoreMalformed.value()) { + context.addIgnoredField(fieldType.name()); return; } else { throw e; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 71ff29aa39e68..e6ba9e8f39025 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -108,7 +108,7 @@ public enum MergeReason { //also missing, not sure if on purpose. See IndicesModule#getMetadataMappers private static ObjectHashSet META_FIELDS = ObjectHashSet.from( "_uid", "_id", "_type", "_all", "_parent", "_routing", "_index", - "_size", "_timestamp", "_ttl" + "_size", "_timestamp", "_ttl", IgnoredFieldMapper.NAME ); private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(MapperService.class)); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 188eb9d181d2e..05f1853b313e2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -996,6 +996,7 @@ protected void parseCreateField(ParseContext context, List field numericValue = fieldType().type.parse(parser, coerce.value()); } catch (IllegalArgumentException e) { if (ignoreMalformed.value()) { + context.addIgnoredField(fieldType.name()); return; } else { throw e; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ParseContext.java b/server/src/main/java/org/elasticsearch/index/mapper/ParseContext.java index 9dac1b7c70a14..8f73778f5b20d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ParseContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ParseContext.java @@ -31,9 +31,12 @@ import org.elasticsearch.common.xcontent.XContentParser; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.Set; public abstract class ParseContext implements Iterable{ @@ -293,6 +296,16 @@ public List getDynamicMappers() { public Iterator iterator() { return in.iterator(); } + + @Override + public void addIgnoredField(String field) { + in.addIgnoredField(field); + } + + @Override + public Collection getIgnoredFields() { + return in.getIgnoredFields(); + } } public static class InternalParseContext extends ParseContext { @@ -324,6 +337,8 @@ public static class InternalParseContext extends ParseContext { private boolean docsReversed = false; + private final Set ignoredFields = new HashSet<>(); + public InternalParseContext(@Nullable Settings indexSettings, DocumentMapperParser docMapperParser, DocumentMapper docMapper, SourceToParse source, XContentParser parser) { this.indexSettings = indexSettings; @@ -455,6 +470,17 @@ void postParse() { public Iterator iterator() { return documents.iterator(); } + + + @Override + public void addIgnoredField(String field) { + ignoredFields.add(field); + } + + @Override + public Collection getIgnoredFields() { + return Collections.unmodifiableCollection(ignoredFields); + } } /** @@ -463,6 +489,17 @@ public Iterator iterator() { */ public abstract Iterable nonRootDocuments(); + + /** + * Add the given {@code field} to the set of ignored fields. + */ + public abstract void addIgnoredField(String field); + + /** + * Return the collection of fields that have been ignored so far. + */ + public abstract Collection getIgnoredFields(); + public abstract DocumentMapperParser docMapperParser(); /** Return a view of this {@link ParseContext} that changes the return diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java index c3d40118fcf71..65e06de78b9d9 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java @@ -36,6 +36,7 @@ import org.elasticsearch.index.mapper.GeoPointFieldMapper; import org.elasticsearch.index.mapper.GeoShapeFieldMapper; import org.elasticsearch.index.mapper.IdFieldMapper; +import org.elasticsearch.index.mapper.IgnoredFieldMapper; import org.elasticsearch.index.mapper.IndexFieldMapper; import org.elasticsearch.index.mapper.IpFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; @@ -133,7 +134,10 @@ private static Map initBuiltInMetadataMa Map builtInMetadataMappers; // Use a LinkedHashMap for metadataMappers because iteration order matters builtInMetadataMappers = new LinkedHashMap<>(); - // UID first so it will be the first stored field to load (so will benefit from "fields: []" early termination + // _ignored first so that we always load it, even if only _id is requested + builtInMetadataMappers.put(IgnoredFieldMapper.NAME, new IgnoredFieldMapper.TypeParser()); + // UID second so it will be the first (if no ignored fields) stored field to load + // (so will benefit from "fields: []" early termination builtInMetadataMappers.put(UidFieldMapper.NAME, new UidFieldMapper.TypeParser()); builtInMetadataMappers.put(IdFieldMapper.NAME, new IdFieldMapper.TypeParser()); builtInMetadataMappers.put(RoutingFieldMapper.NAME, new RoutingFieldMapper.TypeParser()); diff --git a/server/src/main/java/org/elasticsearch/search/SearchHit.java b/server/src/main/java/org/elasticsearch/search/SearchHit.java index 96a5ebc25e2da..ba5102bf7b3fe 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchHit.java +++ b/server/src/main/java/org/elasticsearch/search/SearchHit.java @@ -43,6 +43,7 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.index.mapper.IgnoredFieldMapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.shard.ShardId; @@ -444,8 +445,13 @@ public XContentBuilder toInnerXContent(XContentBuilder builder, Params params) t builder.field(Fields._SCORE, score); } for (DocumentField field : metaFields) { - Object value = field.getValue(); - builder.field(field.getName(), value); + // _ignored is the only multi-valued meta field + // TODO: can we avoid having an exception here? + if (field.getName().equals(IgnoredFieldMapper.NAME)) { + builder.field(field.getName(), field.getValues()); + } else { + builder.field(field.getName(), field.getValue()); + } } if (source != null) { XContentHelper.writeRawField(SourceFieldMapper.NAME, source, builder, params); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java index d6b13750b3f3d..e423dd7412b6a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java @@ -192,6 +192,7 @@ public void testIgnoreMalformed() throws Exception { IndexableField[] fields = doc.rootDoc().getFields("field"); assertEquals(0, fields.length); + assertArrayEquals(new String[] { "field" }, doc.rootDoc().getValues("_ignored")); } public void testChangeFormat() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldTypeTests.java new file mode 100644 index 0000000000000..4035383893d81 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldTypeTests.java @@ -0,0 +1,29 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper; + +public class IgnoredFieldTypeTests extends FieldTypeTestCase { + + @Override + protected MappedFieldType createDefaultFieldType() { + return new IgnoredFieldMapper.IgnoredFieldType(); + } + +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java index bb80fb0b796eb..df60333c30dc8 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java @@ -194,6 +194,7 @@ public void testIgnoreMalformed() throws Exception { IndexableField[] fields = doc.rootDoc().getFields("field"); assertEquals(0, fields.length); + assertArrayEquals(new String[] { "field" }, doc.rootDoc().getValues("_ignored")); } public void testNullValue() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java index 7c1ff5d95601a..02d7cea2f4362 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java @@ -256,6 +256,7 @@ private void doTestIgnoreMalformed(String type) throws IOException { IndexableField[] fields = doc.rootDoc().getFields("field"); assertEquals(0, fields.length); + assertArrayEquals(new String[] { "field" }, doc.rootDoc().getValues("_ignored")); } public void testRejectNorms() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java index 0d2607af15763..23e5ac256e6ec 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.index.mapper.AllFieldMapper; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.IdFieldMapper; +import org.elasticsearch.index.mapper.IgnoredFieldMapper; import org.elasticsearch.index.mapper.IndexFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; @@ -85,9 +86,9 @@ public Map getMetadataMappers() { } }); - private static String[] EXPECTED_METADATA_FIELDS = new String[]{UidFieldMapper.NAME, IdFieldMapper.NAME, RoutingFieldMapper.NAME, - IndexFieldMapper.NAME, SourceFieldMapper.NAME, TypeFieldMapper.NAME, AllFieldMapper.NAME, VersionFieldMapper.NAME, - ParentFieldMapper.NAME, SeqNoFieldMapper.NAME, FieldNamesFieldMapper.NAME}; + private static String[] EXPECTED_METADATA_FIELDS = new String[]{IgnoredFieldMapper.NAME, UidFieldMapper.NAME, IdFieldMapper.NAME, + RoutingFieldMapper.NAME, IndexFieldMapper.NAME, SourceFieldMapper.NAME, TypeFieldMapper.NAME, AllFieldMapper.NAME, + VersionFieldMapper.NAME, ParentFieldMapper.NAME, SeqNoFieldMapper.NAME, FieldNamesFieldMapper.NAME}; public void testBuiltinMappers() { IndicesModule module = new IndicesModule(Collections.emptyList()); @@ -109,7 +110,7 @@ public void testBuiltinWithPlugins() { greaterThan(noPluginsModule.getMapperRegistry().getMetadataMapperParsers().size())); Map metadataMapperParsers = module.getMapperRegistry().getMetadataMapperParsers(); Iterator iterator = metadataMapperParsers.keySet().iterator(); - assertEquals(UidFieldMapper.NAME, iterator.next()); + assertEquals(IgnoredFieldMapper.NAME, iterator.next()); String last = null; while(iterator.hasNext()) { last = iterator.next();