Skip to content

Commit

Permalink
Mappings: Remove ability to disable _source field
Browse files Browse the repository at this point in the history 
Current features (eg. update API) and future features (eg. reindex API)
depend on _source. This change locks down the field so that
it can no longer be disabled. It also removes legacy settings
compress/compress_threshold.

closes elastic#8142
closes elastic#10915
  • Loading branch information
@rjernst
rjernst committed May 6, 2015
1 parent fe5af6f commit 7a7bd60
Show file tree
Hide file tree
Showing 14 changed files with 76 additions and 84 deletions.
15 changes: 3 additions & 12 deletions docs/reference/mapping/fields/source-field.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,6 @@ actual JSON that was used as the indexed document. It is not indexed
<<search-search,search>>, the `_source` field is
returned by default.

Though very handy to have around, the source field does incur storage
overhead within the index. For this reason, it can be disabled. For
example:

[source,js]
--------------------------------------------------
{
"tweet" : {
"_source" : {"enabled" : false}
}
}
--------------------------------------------------
Many APIs may use the `_source` field. For example, the
<<docs-update,Update API>>. To minimize the storage cost of
`_source`, set `index.codec: best_compression` in index settings.
9 changes: 9 additions & 0 deletions docs/reference/migration/migrate_2_0.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -272,6 +272,15 @@ to provide special features. They now have limited configuration options.
* `_field_names` configuration is limited to disabling the field.
* `_size` configuration is limited to enabling the field.

==== Source field limitations
The `_source` field could previously be disabled dynamically. Since this field
is a critical piece of many features like the Update API, it is no longer
possible to disable.

The options for `compress` and `compress_threshold` have also been removed.
The source field is already compressed. To minimize the storage cost,
set `index.codec: best_compression` in index settings.

==== Boolean fields

Boolean fields used to have a string fielddata with `F` meaning `false` and `T`
Expand Down
6 changes: 3 additions & 3 deletions src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,15 +145,15 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
Map.Entry<String, Object> entry = iterator.next();
String fieldName = Strings.toUnderscoreCase(entry.getKey());
Object fieldNode = entry.getValue();
if (fieldName.equals("enabled")) {
if (fieldName.equals("enabled") && parserContext.indexVersionCreated().before(Version.V_2_0_0)) {
builder.enabled(nodeBooleanValue(fieldNode));
iterator.remove();
} else if (fieldName.equals("compress")) {
} else if (fieldName.equals("compress") && parserContext.indexVersionCreated().before(Version.V_2_0_0)) {
if (fieldNode != null) {
builder.compress(nodeBooleanValue(fieldNode));
}
iterator.remove();
} else if (fieldName.equals("compress_threshold")) {
} else if (fieldName.equals("compress_threshold") && parserContext.indexVersionCreated().before(Version.V_2_0_0)) {
if (fieldNode != null) {
if (fieldNode instanceof Number) {
builder.compressThreshold(((Number) fieldNode).longValue());
Expand Down
2 changes: 1 addition & 1 deletion src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -310,7 +310,7 @@ public void doAfterNodes(int numNodes, Client client) throws Exception {

logger.info("--> add some metadata, additional type and template");
client.admin().indices().preparePutMapping("test").setType("type2")
.setSource(jsonBuilder().startObject().startObject("type2").startObject("_source").field("enabled", false).endObject().endObject().endObject())
.setSource(jsonBuilder().startObject().startObject("type2").endObject().endObject())
.execute().actionGet();
client.admin().indices().preparePutTemplate("template_1")
.setTemplate("te*")
Expand Down
34 changes: 15 additions & 19 deletions src/test/java/org/elasticsearch/get/GetActionTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -228,8 +228,9 @@ public void simpleMultiGetTests() throws Exception {
}

@Test
public void realtimeGetWithCompress() throws Exception {
assertAcked(prepareCreate("test").setSettings(ImmutableSettings.settingsBuilder().put("index.refresh_interval", -1))
public void realtimeGetWithCompressBackcompat() throws Exception {
assertAcked(prepareCreate("test")
.setSettings(ImmutableSettings.settingsBuilder().put("index.refresh_interval", -1).put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id))
.addMapping("type", jsonBuilder().startObject().startObject("type").startObject("_source").field("compress", true).endObject().endObject().endObject()));
ensureGreen();

Expand All @@ -249,9 +250,8 @@ public void realtimeGetWithCompress() throws Exception {
@Test
public void getFieldsWithDifferentTypes() throws Exception {
assertAcked(prepareCreate("test").setSettings(ImmutableSettings.settingsBuilder().put("index.refresh_interval", -1))
.addMapping("type1", jsonBuilder().startObject().startObject("type1").startObject("_source").field("enabled", true).endObject().endObject().endObject())
.addMapping("type1", jsonBuilder().startObject().startObject("type1").endObject().endObject())
.addMapping("type2", jsonBuilder().startObject().startObject("type2")
.startObject("_source").field("enabled", false).endObject()
.startObject("properties")
.startObject("str").field("type", "string").field("store", "yes").endObject()
.startObject("strs").field("type", "string").field("store", "yes").endObject()
Expand Down Expand Up @@ -339,7 +339,6 @@ public void testGetDocWithMultivaluedFields() throws Exception {
.startObject("properties")
.startObject("field").field("type", "string").field("store", "yes").endObject()
.endObject()
.startObject("_source").field("enabled", false).endObject()
.endObject().endObject().string();
assertAcked(prepareCreate("test")
.addMapping("type1", mapping1)
Expand Down Expand Up @@ -397,7 +396,7 @@ public void testGetDocWithMultivaluedFields() throws Exception {
}

@Test
public void testThatGetFromTranslogShouldWorkWithExclude() throws Exception {
public void testThatGetFromTranslogShouldWorkWithExcludeBackcompat() throws Exception {
String index = "test";
String type = "type1";

Expand Down Expand Up @@ -431,7 +430,7 @@ public void testThatGetFromTranslogShouldWorkWithExclude() throws Exception {
}

@Test
public void testThatGetFromTranslogShouldWorkWithInclude() throws Exception {
public void testThatGetFromTranslogShouldWorkWithIncludeBackcompat() throws Exception {
String index = "test";
String type = "type1";

Expand Down Expand Up @@ -466,7 +465,7 @@ public void testThatGetFromTranslogShouldWorkWithInclude() throws Exception {

@SuppressWarnings("unchecked")
@Test
public void testThatGetFromTranslogShouldWorkWithIncludeExcludeAndFields() throws Exception {
public void testThatGetFromTranslogShouldWorkWithIncludeExcludeAndFieldsBackcompat() throws Exception {
String index = "test";
String type = "type1";

Expand Down Expand Up @@ -925,9 +924,6 @@ public void testUngeneratedFieldsThatAreNeverStored() throws IOException {
" },\n" +
" \"mappings\": {\n" +
" \"doc\": {\n" +
" \"_source\": {\n" +
" \"enabled\": \"" + randomBoolean() + "\"\n" +
" },\n" +
" \"properties\": {\n" +
" \"suggest\": {\n" +
" \"type\": \"completion\"\n" +
Expand Down Expand Up @@ -970,9 +966,6 @@ public void testUngeneratedFieldsThatAreAlwaysStored() throws IOException {
" \"mappings\": {\n" +
" \"parentdoc\": {},\n" +
" \"doc\": {\n" +
" \"_source\": {\n" +
" \"enabled\": " + randomBoolean() + "\n" +
" },\n" +
" \"_parent\": {\n" +
" \"type\": \"parentdoc\"\n" +
" },\n" +
Expand Down Expand Up @@ -1002,7 +995,7 @@ public void testUngeneratedFieldsThatAreAlwaysStored() throws IOException {
}

@Test
public void testUngeneratedFieldsPartOfSourceUnstoredSourceDisabled() throws IOException {
public void testUngeneratedFieldsPartOfSourceUnstoredSourceDisabledBackcompat() throws IOException {
indexSingleDocumentWithUngeneratedFieldsThatArePartOf_source(false, false);
String[] fieldsList = {};
// before refresh - document is only in translog
Expand All @@ -1016,7 +1009,7 @@ public void testUngeneratedFieldsPartOfSourceUnstoredSourceDisabled() throws IOE
}

@Test
public void testUngeneratedFieldsPartOfSourceEitherStoredOrSourceEnabled() throws IOException {
public void testUngeneratedFieldsPartOfSourceEitherStoredOrSourceEnabledBackcompat() throws IOException {
boolean stored = randomBoolean();
boolean sourceEnabled = true;
if (stored) {
Expand All @@ -1039,7 +1032,8 @@ void indexSingleDocumentWithUngeneratedFieldsThatArePartOf_source(boolean stored
String createIndexSource = "{\n" +
" \"settings\": {\n" +
" \"index.translog.disable_flush\": true,\n" +
" \"refresh_interval\": \"-1\"\n" +
" \"refresh_interval\": \"-1\",\n" +
" \"" + IndexMetaData.SETTING_VERSION_CREATED + "\": " + Version.V_1_4_2.id + "\n" +
" },\n" +
" \"mappings\": {\n" +
" \"doc\": {\n" +
Expand Down Expand Up @@ -1161,7 +1155,8 @@ void indexSingleDocumentWithStringFieldsGeneratedFromText(boolean stored, boolea
String createIndexSource = "{\n" +
" \"settings\": {\n" +
" \"index.translog.disable_flush\": true,\n" +
" \"refresh_interval\": \"-1\"\n" +
" \"refresh_interval\": \"-1\",\n" +
" \"" + IndexMetaData.SETTING_VERSION_CREATED + "\": " + Version.V_1_4_2.id + "\n" +
" },\n" +
" \"mappings\": {\n" +
" \"doc\": {\n" +
Expand Down Expand Up @@ -1215,7 +1210,8 @@ void indexSingleDocumentWithNumericFieldsGeneratedFromText(boolean stored, boole
String createIndexSource = "{\n" +
" \"settings\": {\n" +
" \"index.translog.disable_flush\": true,\n" +
" \"refresh_interval\": \"-1\"\n" +
" \"refresh_interval\": \"-1\",\n" +
" \"" + IndexMetaData.SETTING_VERSION_CREATED + "\": " + Version.V_1_4_2.id + "\n" +
" },\n" +
" \"mappings\": {\n" +
" \"doc\": {\n" +
Expand Down
1 change: 0 additions & 1 deletion src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -423,7 +423,6 @@ public void testRootMappersStillWorking() {
//just pick some example from DocumentMapperParser.rootTypeParsers
rootTypes.put(SizeFieldMapper.NAME, "{\"enabled\" : true}");
rootTypes.put(IndexFieldMapper.NAME, "{\"enabled\" : true}");
rootTypes.put(SourceFieldMapper.NAME, "{\"enabled\" : true}");
rootTypes.put("include_in_all", "true");
rootTypes.put("dynamic_date_formats", "[\"yyyy-MM-dd\", \"dd-MM-yyyy\"]");
rootTypes.put("numeric_detection", "true");
Expand Down
11 changes: 8 additions & 3 deletions src/test/java/org/elasticsearch/index/mapper/source/CompressSourceMappingTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,11 @@
package org.elasticsearch.index.mapper.source;

import org.apache.lucene.util.BytesRef;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.compress.CompressorFactory;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.ParsedDocument;
Expand All @@ -33,14 +37,15 @@
*
*/
public class CompressSourceMappingTests extends ElasticsearchSingleNodeTest {
Settings settings = ImmutableSettings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build();

@Test
public void testCompressDisabled() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_source").field("compress", false).endObject()
.endObject().endObject().string();

DocumentMapper documentMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping);
DocumentMapper documentMapper = createIndex("test", settings).mapperService().documentMapperParser().parse(mapping);

ParsedDocument doc = documentMapper.parse("type", "1", XContentFactory.jsonBuilder().startObject()
.field("field1", "value1")
Expand All @@ -56,7 +61,7 @@ public void testCompressEnabled() throws Exception {
.startObject("_source").field("compress", true).endObject()
.endObject().endObject().string();

DocumentMapper documentMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping);
DocumentMapper documentMapper = createIndex("test", settings).mapperService().documentMapperParser().parse(mapping);

ParsedDocument doc = documentMapper.parse("type", "1", XContentFactory.jsonBuilder().startObject()
.field("field1", "value1")
Expand All @@ -73,7 +78,7 @@ public void testCompressThreshold() throws Exception {
.startObject("_source").field("compress_threshold", "200b").endObject()
.endObject().endObject().string();

DocumentMapper documentMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping);
DocumentMapper documentMapper = createIndex("test", settings).mapperService().documentMapperParser().parse(mapping);

ParsedDocument doc = documentMapper.parse("type", "1", XContentFactory.jsonBuilder().startObject()
.field("field1", "value1")
Expand Down
Loading

0 comments on commit 7a7bd60

Please sign in to comment.