Skip to content

Commit

Permalink
Introduce private settings (#33327)
Browse files Browse the repository at this point in the history
This commit introduces the formal notion of a private setting. This
enables us to register some settings that we had previously not
registered as fully-fledged settings to avoid them being exposed via
APIs such as the create index API. For example, we had hacks in the
codebase to allow index.version.created to be passed around inside of
settings objects, but was not registered as a setting so that if a user
tried to use the setting on any API then they would get an
exception. This prevented users from setting index.version.created on
index creation, or updating it via the index settings API. By
introducing private settings, we can continue to reject these attempts,
yet now we can represent these settings as actual settings. In this
change, we register index.version.created as an actual setting. We do
not cutover all settings that we had been treating as private in this
pull request, it is already quite large due to moving some tests around
to account for the fact that some tests need to be able to set the
index.version.created. This can be done in a follow-up change.
  • Loading branch information
jasontedor committed Sep 4, 2018
1 parent 4da9f18 commit 4ff1b1d
Show file tree
Hide file tree
Showing 116 changed files with 4,416 additions and 2,481 deletions.
1 change: 0 additions & 1 deletion buildSrc/src/main/resources/checkstyle_suppressions.xml
Original file line number Diff line number Diff line change
Expand Up @@ -570,7 +570,6 @@
<suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]DocumentMapperMergeTests.java" checks="LineLength" />
<suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]DocumentParserTests.java" checks="LineLength" />
<suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]DynamicMappingTests.java" checks="LineLength" />
<suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]ExternalFieldMapperTests.java" checks="LineLength" />
<suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]ExternalMapper.java" checks="LineLength" />
<suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]ExternalMetadataMapper.java" checks="LineLength" />
<suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]FieldNamesFieldMapperTests.java" checks="LineLength" />
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,11 @@ protected boolean legacy() {
return false;
}

@Override
protected boolean forbidPrivateIndexSettings() {
return legacy() == false;
}

protected IndexRequestBuilder createIndexRequest(String index, String type, String id, String parentId, Object... fields) {
Map<String, Object> source = new HashMap<>();
for (int i = 0; i < fields.length; i += 2) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,12 @@

package org.elasticsearch.index.reindex;

import org.elasticsearch.Version;
import org.elasticsearch.action.admin.indices.alias.Alias;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.search.sort.SortOrder;
import org.elasticsearch.test.InternalSettingsPlugin;

import java.util.ArrayList;
import java.util.Collection;
Expand All @@ -46,12 +43,6 @@
import static org.hamcrest.Matchers.hasSize;

public class DeleteByQueryBasicTests extends ReindexTestCase {
@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
List<Class<? extends Plugin>> plugins = new ArrayList<>(super.nodePlugins());
plugins.add(InternalSettingsPlugin.class);
return plugins;
}

public void testBasics() throws Exception {
indexRandom(true,
Expand Down Expand Up @@ -307,25 +298,4 @@ public void testMultipleSources() throws Exception {

}

/**
* Test delete by query support for filtering by type. This entire feature
* can and should be removed when we drop support for types index with
* multiple types from core.
*/
public void testFilterByType() throws Exception {
assertAcked(client().admin().indices().prepareCreate("test")
.setSettings(Settings.builder().put("index.version.created", Version.V_5_6_0.id))); // allows for multiple types
indexRandom(true,
client().prepareIndex("test", "test1", "1").setSource("foo", "a"),
client().prepareIndex("test", "test2", "2").setSource("foo", "a"),
client().prepareIndex("test", "test2", "3").setSource("foo", "b"));

assertHitCount(client().prepareSearch("test").setSize(0).get(), 3);

// Deletes doc of the type "type2" that also matches foo:a
DeleteByQueryRequestBuilder builder = deleteByQuery().source("test").filter(termQuery("foo", "a")).refresh(true);
builder.source().setTypes("test2");
assertThat(builder.get(), matcher().deleted(1));
assertHitCount(client().prepareSearch("test").setSize(0).get(), 2);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.elasticsearch.index.reindex;

import org.elasticsearch.Version;
import org.elasticsearch.common.settings.Settings;

import static org.elasticsearch.index.query.QueryBuilders.termQuery;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;

public class LegacyDeleteByQueryBasicTests extends ReindexTestCase {

@Override
protected boolean forbidPrivateIndexSettings() {
return false;
}

/**
* Test delete by query support for filtering by type. This entire feature
* can and should be removed when we drop support for types index with
* multiple types from core.
*/
public void testFilterByType() throws Exception {
assertAcked(client().admin().indices().prepareCreate("test")
.setSettings(Settings.builder().put("index.version.created", Version.V_5_6_0.id))); // allows for multiple types
indexRandom(true,
client().prepareIndex("test", "test1", "1").setSource("foo", "a"),
client().prepareIndex("test", "test2", "2").setSource("foo", "a"),
client().prepareIndex("test", "test2", "3").setSource("foo", "b"));

assertHitCount(client().prepareSearch("test").setSize(0).get(), 3);

// Deletes doc of the type "type2" that also matches foo:a
DeleteByQueryRequestBuilder builder = deleteByQuery().source("test").filter(termQuery("foo", "a")).refresh(true);
builder.source().setTypes("test2");
assertThat(builder.get(), matcher().deleted(1));
assertHitCount(client().prepareSearch("test").setSize(0).get(), 2);
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
import org.elasticsearch.join.ParentJoinPlugin;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.script.MockScriptPlugin;
import org.elasticsearch.test.InternalSettingsPlugin;

import java.util.ArrayList;
import java.util.Collection;
Expand Down Expand Up @@ -65,7 +64,6 @@ protected boolean ignoreExternalCluster() {
protected Collection<Class<? extends Plugin>> nodePlugins() {
final List<Class<? extends Plugin>> plugins = new ArrayList<>(super.nodePlugins());
plugins.add(ParentJoinPlugin.class);
plugins.add(InternalSettingsPlugin.class);
plugins.add(CustomScriptPlugin.class);
return Collections.unmodifiableList(plugins);
}
Expand All @@ -75,6 +73,11 @@ protected Collection<Class<? extends Plugin>> transportClientPlugins() {
return nodePlugins();
}

@Override
protected boolean forbidPrivateIndexSettings() {
return false;
}

public void testParentChild() throws Exception {
createParentChildIndex("source");
createParentChildIndex("dest");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,6 @@
*/
package org.elasticsearch.index.mapper;

import static org.hamcrest.Matchers.equalTo;

import com.ibm.icu.text.Collator;
import com.ibm.icu.text.RawCollationKey;
import com.ibm.icu.util.ULocale;
Expand All @@ -28,36 +26,36 @@
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.IndexableFieldType;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.Version;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.mapper.MapperService.MergeReason;
import org.elasticsearch.plugin.analysis.icu.AnalysisICUPlugin;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.ESSingleNodeTestCase;
import org.elasticsearch.test.InternalSettingsPlugin;
import org.junit.Before;

import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;

import static org.hamcrest.Matchers.equalTo;

public class ICUCollationKeywordFieldMapperTests extends ESSingleNodeTestCase {

private static final String FIELD_TYPE = "icu_collation_keyword";

@Override
protected Collection<Class<? extends Plugin>> getPlugins() {
return Arrays.asList(AnalysisICUPlugin.class, InternalSettingsPlugin.class);
return Collections.singletonList(AnalysisICUPlugin.class);
}

IndexService indexService;
DocumentMapperParser parser;
private IndexService indexService;
private DocumentMapperParser parser;

@Before
public void setup() {
Expand Down Expand Up @@ -106,51 +104,6 @@ public void testDefaults() throws Exception {
assertEquals(DocValuesType.SORTED_SET, fieldType.docValuesType());
}

public void testBackCompat() throws Exception {
indexService = createIndex("oldindex", Settings.builder().put("index.version.created", Version.V_5_5_0).build());
parser = indexService.mapperService().documentMapperParser();

String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_all").field("enabled", false).endObject()
.startObject("properties").startObject("field").field("type", FIELD_TYPE).endObject().endObject()
.endObject().endObject());

DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));

assertEquals(mapping, mapper.mappingSource().toString());

ParsedDocument doc = mapper.parse(SourceToParse.source("oldindex", "type", "1", BytesReference
.bytes(XContentFactory.jsonBuilder()
.startObject()
.field("field", "1234")
.endObject()),
XContentType.JSON));

IndexableField[] fields = doc.rootDoc().getFields("field");
assertEquals(2, fields.length);

Collator collator = Collator.getInstance(ULocale.ROOT);
RawCollationKey key = collator.getRawCollationKey("1234", null);
BytesRef expected = new BytesRef(key.bytes, 0, key.size);

assertEquals(expected, fields[0].binaryValue());
IndexableFieldType fieldType = fields[0].fieldType();
assertThat(fieldType.omitNorms(), equalTo(true));
assertFalse(fieldType.tokenized());
assertFalse(fieldType.stored());
assertThat(fieldType.indexOptions(), equalTo(IndexOptions.DOCS));
assertThat(fieldType.storeTermVectors(), equalTo(false));
assertThat(fieldType.storeTermVectorOffsets(), equalTo(false));
assertThat(fieldType.storeTermVectorPositions(), equalTo(false));
assertThat(fieldType.storeTermVectorPayloads(), equalTo(false));
assertEquals(DocValuesType.NONE, fieldType.docValuesType());

assertEquals(expected, fields[1].binaryValue());
fieldType = fields[1].fieldType();
assertThat(fieldType.indexOptions(), equalTo(IndexOptions.NONE));
assertEquals(DocValuesType.SORTED, fieldType.docValuesType());
}

public void testNullValue() throws IOException {
String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties").startObject("field").field("type", FIELD_TYPE).endObject().endObject()
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.elasticsearch.index.mapper;

import com.ibm.icu.text.Collator;
import com.ibm.icu.text.RawCollationKey;
import com.ibm.icu.util.ULocale;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.IndexableFieldType;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.Version;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.plugin.analysis.icu.AnalysisICUPlugin;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.ESSingleNodeTestCase;

import java.util.Collection;
import java.util.Collections;

import static org.hamcrest.Matchers.equalTo;

public class LegacyICUCollationKeywordFieldMapperTests extends ESSingleNodeTestCase {

@Override
protected Collection<Class<? extends Plugin>> getPlugins() {
return Collections.singletonList(AnalysisICUPlugin.class);
}

@Override
protected boolean forbidPrivateIndexSettings() {
return false;
}

public void testBackCompat() throws Exception {
final IndexService indexService = createIndex("oldindex", Settings.builder().put("index.version.created", Version.V_5_5_0).build());
final DocumentMapperParser parser = indexService.mapperService().documentMapperParser();

String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_all").field("enabled", false).endObject()
.startObject("properties").startObject("field").field("type", "icu_collation_keyword").endObject().endObject()
.endObject().endObject());

DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));

assertEquals(mapping, mapper.mappingSource().toString());

ParsedDocument doc = mapper.parse(SourceToParse.source("oldindex", "type", "1", BytesReference
.bytes(XContentFactory.jsonBuilder()
.startObject()
.field("field", "1234")
.endObject()),
XContentType.JSON));

IndexableField[] fields = doc.rootDoc().getFields("field");
assertEquals(2, fields.length);

Collator collator = Collator.getInstance(ULocale.ROOT);
RawCollationKey key = collator.getRawCollationKey("1234", null);
BytesRef expected = new BytesRef(key.bytes, 0, key.size);

assertEquals(expected, fields[0].binaryValue());
IndexableFieldType fieldType = fields[0].fieldType();
assertThat(fieldType.omitNorms(), equalTo(true));
assertFalse(fieldType.tokenized());
assertFalse(fieldType.stored());
assertThat(fieldType.indexOptions(), equalTo(IndexOptions.DOCS));
assertThat(fieldType.storeTermVectors(), equalTo(false));
assertThat(fieldType.storeTermVectorOffsets(), equalTo(false));
assertThat(fieldType.storeTermVectorPositions(), equalTo(false));
assertThat(fieldType.storeTermVectorPayloads(), equalTo(false));
assertEquals(DocValuesType.NONE, fieldType.docValuesType());

assertEquals(expected, fields[1].binaryValue());
fieldType = fields[1].fieldType();
assertThat(fieldType.indexOptions(), equalTo(IndexOptions.NONE));
assertEquals(DocValuesType.SORTED, fieldType.docValuesType());
}

}
Loading

0 comments on commit 4ff1b1d

Please sign in to comment.