Skip to content

Commit

Permalink
Merge remote-tracking branch 'elastic/master' into enable-bwc-retenti…
Browse files Browse the repository at this point in the history 
…on-leases-recovery

* elastic/master:
  SQL: Allow look-ahead resolution of aliases for WHERE clause (elastic#38450)
  Add API key settings documentation (elastic#38490)
  Remove support for maxRetryTimeout from low-level REST client (elastic#38085)
  Update IndexTemplateMetaData to allow unknown fields (elastic#38448)
  Mute testRetentionLeasesSyncOnRecovery (elastic#38488)
  Change the min supported version to 6.7.0 for API keys (elastic#38481)
  • Loading branch information
@jasontedor
jasontedor committed Feb 6, 2019
2 parents 31f302d + 1a02445 commit ca0106d
Show file tree
Hide file tree
Showing 33 changed files with 821 additions and 871 deletions.
142 changes: 53 additions & 89 deletions ...rest-high-level/src/main/java/org/elasticsearch/client/indices/IndexTemplateMetaData.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,27 +18,66 @@
*/
package org.elasticsearch.client.indices;

import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.cluster.metadata.AliasMetaData;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.set.Sets;
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.mapper.MapperService;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.AbstractMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;

import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;

public class IndexTemplateMetaData {

@SuppressWarnings("unchecked")
private static final ConstructingObjectParser<IndexTemplateMetaData, String> PARSER = new ConstructingObjectParser<>(
"IndexTemplateMetaData", true, (a, name) -> {
List<Map.Entry<String, AliasMetaData>> alias = (List<Map.Entry<String, AliasMetaData>>) a[5];
ImmutableOpenMap<String, AliasMetaData> aliasMap =
new ImmutableOpenMap.Builder<String, AliasMetaData>()
.putAll(alias.stream().collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)))
.build();
return new IndexTemplateMetaData(
name,
(Integer) a[0],
(Integer) a[1],
(List<String>) a[2],
(Settings) a[3],
(MappingMetaData) a[4],
aliasMap);
});

static {
PARSER.declareInt(optionalConstructorArg(), new ParseField("order"));
PARSER.declareInt(optionalConstructorArg(), new ParseField("version"));
PARSER.declareStringArray(optionalConstructorArg(), new ParseField("index_patterns"));
PARSER.declareObject(optionalConstructorArg(), (p, c) -> {
Settings.Builder templateSettingsBuilder = Settings.builder();
templateSettingsBuilder.put(Settings.fromXContent(p));
templateSettingsBuilder.normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX);
return templateSettingsBuilder.build();
}, new ParseField("settings"));
PARSER.declareObject(optionalConstructorArg(), (p, c) -> {
Map<String, Object> mapping = p.map();
if (mapping.isEmpty()) {
return null;
}
return new MappingMetaData(MapperService.SINGLE_MAPPING_NAME, mapping);
}, new ParseField("mappings"));
PARSER.declareNamedObjects(optionalConstructorArg(),
(p, c, name) -> new AbstractMap.SimpleEntry<>(name, AliasMetaData.Builder.fromXContent(p)), new ParseField("aliases"));
}

private final String name;

Expand Down Expand Up @@ -125,28 +164,23 @@ public static Builder builder(String name) {
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;

IndexTemplateMetaData that = (IndexTemplateMetaData) o;

if (order != that.order) return false;
if (!Objects.equals(mappings, that.mappings)) return false;
if (!name.equals(that.name)) return false;
if (!settings.equals(that.settings)) return false;
if (!patterns.equals(that.patterns)) return false;

return Objects.equals(version, that.version);
return order == that.order &&
Objects.equals(name, that.name) &&
Objects.equals(version, that.version) &&
Objects.equals(patterns, that.patterns) &&
Objects.equals(settings, that.settings) &&
Objects.equals(mappings, that.mappings) &&
Objects.equals(aliases, that.aliases);
}

@Override
public int hashCode() {
return Objects.hash(name, order, version, patterns, settings, mappings);
return Objects.hash(name, order, version, patterns, settings, mappings, aliases);
}

public static class Builder {

private static final Set<String> VALID_FIELDS = Sets.newHashSet(
"template", "order", "mappings", "settings", "index_patterns", "aliases", "version");

private String name;

private int order;
Expand Down Expand Up @@ -193,7 +227,6 @@ public Builder patterns(List<String> indexPatterns) {
return this;
}


public Builder settings(Settings.Builder settings) {
this.settings = settings.build();
return this;
Expand Down Expand Up @@ -225,76 +258,7 @@ public IndexTemplateMetaData build() {


public static IndexTemplateMetaData fromXContent(XContentParser parser, String templateName) throws IOException {
Builder builder = new Builder(templateName);

String currentFieldName = skipTemplateName(parser);
XContentParser.Token token;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if (token == XContentParser.Token.START_OBJECT) {
if ("settings".equals(currentFieldName)) {
Settings.Builder templateSettingsBuilder = Settings.builder();
templateSettingsBuilder.put(Settings.fromXContent(parser));
templateSettingsBuilder.normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX);
builder.settings(templateSettingsBuilder.build());
} else if ("mappings".equals(currentFieldName)) {
Map<String, Object> mapping = parser.map();
if (mapping.isEmpty() == false) {
MappingMetaData md = new MappingMetaData(MapperService.SINGLE_MAPPING_NAME, mapping);
builder.mapping(md);
}
} else if ("aliases".equals(currentFieldName)) {
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
builder.putAlias(AliasMetaData.Builder.fromXContent(parser));
}
} else {
throw new ElasticsearchParseException("unknown key [{}] for index template", currentFieldName);
}
} else if (token == XContentParser.Token.START_ARRAY) {
if ("mappings".equals(currentFieldName)) {
// The server-side IndexTemplateMetaData has toXContent impl that can return mappings
// in an array but also a comment saying this never happens with typeless APIs.
throw new ElasticsearchParseException("Invalid response format - "
+ "mappings are not expected to be returned in an array", currentFieldName);
} else if ("index_patterns".equals(currentFieldName)) {
List<String> index_patterns = new ArrayList<>();
while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
index_patterns.add(parser.text());
}
builder.patterns(index_patterns);
}
} else if (token.isValue()) {
// Prior to 5.1.0, elasticsearch only supported a single index pattern called `template` (#21009)
if("template".equals(currentFieldName)) {
builder.patterns(Collections.singletonList(parser.text()));
} else if ("order".equals(currentFieldName)) {
builder.order(parser.intValue());
} else if ("version".equals(currentFieldName)) {
builder.version(parser.intValue());
}
}
}
return builder.build();
}

private static String skipTemplateName(XContentParser parser) throws IOException {
XContentParser.Token token = parser.nextToken();
if (token == XContentParser.Token.START_OBJECT) {
token = parser.nextToken();
if (token == XContentParser.Token.FIELD_NAME) {
String currentFieldName = parser.currentName();
if (VALID_FIELDS.contains(currentFieldName)) {
return currentFieldName;
} else {
// we just hit the template name, which should be ignored and we move on
parser.nextToken();
}
}
}

return null;
return PARSER.parse(parser, templateName);
}
}

}
137 changes: 131 additions & 6 deletions ...-level/src/test/java/org/elasticsearch/client/indices/GetIndexTemplatesResponseTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,24 +22,37 @@
import org.elasticsearch.cluster.metadata.AliasMetaData;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.DeprecationHandler;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.test.ESTestCase;

import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import java.util.stream.IntStream;

import static org.elasticsearch.index.RandomCreateIndexGenerator.randomIndexSettings;
import static org.elasticsearch.index.RandomCreateIndexGenerator.randomMappingFields;
import static org.elasticsearch.test.AbstractXContentTestCase.xContentTester;
import static org.hamcrest.Matchers.equalTo;

public class GetIndexTemplatesResponseTests extends ESTestCase {

Expand All @@ -50,11 +63,88 @@ public class GetIndexTemplatesResponseTests extends ESTestCase {


public void testFromXContent() throws IOException {
xContentTester(this::createParser, GetIndexTemplatesResponseTests::createTestInstance, GetIndexTemplatesResponseTests::toXContent,
GetIndexTemplatesResponse::fromXContent).supportsUnknownFields(false)
.assertEqualsConsumer(GetIndexTemplatesResponseTests::assertEqualInstances)
.shuffleFieldsExceptions(new String[] {"aliases", "mappings", "patterns", "settings"})
.test();
xContentTester(this::createParser,
GetIndexTemplatesResponseTests::createTestInstance,
GetIndexTemplatesResponseTests::toXContent,
GetIndexTemplatesResponse::fromXContent)
.assertEqualsConsumer(GetIndexTemplatesResponseTests::assertEqualInstances)
.supportsUnknownFields(true)
.randomFieldsExcludeFilter(randomFieldsExcludeFilter())
.shuffleFieldsExceptions(new String[] {"aliases", "mappings", "patterns", "settings"})
.test();
}

public void testParsingFromEsResponse() throws IOException {
for (int runs = 0; runs < 20; runs++) {
org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse esResponse =
new org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse(new ArrayList<>());

XContentType xContentType = randomFrom(XContentType.values());
int numTemplates = randomIntBetween(0, 32);
for (int i = 0; i < numTemplates; i++) {
org.elasticsearch.cluster.metadata.IndexTemplateMetaData.Builder esIMD =
new org.elasticsearch.cluster.metadata.IndexTemplateMetaData.Builder(String.format(Locale.ROOT, "%02d ", i) +
randomAlphaOfLength(4));
esIMD.patterns(Arrays.asList(generateRandomStringArray(32, 4, false, false)));
esIMD.settings(randomIndexSettings());
esIMD.putMapping("_doc", new CompressedXContent(BytesReference.bytes(randomMapping("_doc", xContentType))));
int numAliases = randomIntBetween(0, 8);
for (int j = 0; j < numAliases; j++) {
esIMD.putAlias(randomAliasMetaData(String.format(Locale.ROOT, "%02d ", j) + randomAlphaOfLength(4)));
}
esIMD.order(randomIntBetween(0, Integer.MAX_VALUE));
esIMD.version(randomIntBetween(0, Integer.MAX_VALUE));
esResponse.getIndexTemplates().add(esIMD.build());
}

XContentBuilder xContentBuilder = XContentBuilder.builder(xContentType.xContent());
esResponse.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS);

try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY,
DeprecationHandler.THROW_UNSUPPORTED_OPERATION, BytesReference.bytes(xContentBuilder), xContentType)) {
GetIndexTemplatesResponse response = GetIndexTemplatesResponse.fromXContent(parser);
assertThat(response.getIndexTemplates().size(), equalTo(numTemplates));

response.getIndexTemplates().sort(Comparator.comparing(IndexTemplateMetaData::name));
for (int i = 0; i < numTemplates; i++) {
org.elasticsearch.cluster.metadata.IndexTemplateMetaData esIMD = esResponse.getIndexTemplates().get(i);
IndexTemplateMetaData result = response.getIndexTemplates().get(i);

assertThat(result.patterns(), equalTo(esIMD.patterns()));
assertThat(result.settings(), equalTo(esIMD.settings()));
assertThat(result.order(), equalTo(esIMD.order()));
assertThat(result.version(), equalTo(esIMD.version()));

assertThat(esIMD.mappings().size(), equalTo(1));
BytesArray mappingSource = new BytesArray(esIMD.mappings().valuesIt().next().uncompressed());
Map<String, Object> expectedMapping =
XContentHelper.convertToMap(mappingSource, true, xContentBuilder.contentType()).v2();
assertThat(result.mappings().sourceAsMap(), equalTo(expectedMapping.get("_doc")));

assertThat(result.aliases().size(), equalTo(esIMD.aliases().size()));
List<AliasMetaData> expectedAliases = Arrays.stream(esIMD.aliases().values().toArray(AliasMetaData.class))
.sorted(Comparator.comparing(AliasMetaData::alias))
.collect(Collectors.toList());
List<AliasMetaData> actualAliases = Arrays.stream(result.aliases().values().toArray(AliasMetaData.class))
.sorted(Comparator.comparing(AliasMetaData::alias))
.collect(Collectors.toList());
for (int j = 0; j < result.aliases().size(); j++) {
assertThat(actualAliases.get(j), equalTo(expectedAliases.get(j)));
}
}
}
}
}

private Predicate<String> randomFieldsExcludeFilter() {
return (field) ->
field.isEmpty()
|| field.endsWith("aliases")
|| field.endsWith("settings")
|| field.endsWith("settings.index")
|| field.endsWith("mappings") // uses parser.map()
|| field.contains("mappings.properties") // cannot have extra properties
;
}

private static void assertEqualInstances(GetIndexTemplatesResponse expectedInstance, GetIndexTemplatesResponse newInstance) {
Expand All @@ -64,7 +154,7 @@ private static void assertEqualInstances(GetIndexTemplatesResponse expectedInsta
new BytesArray(mappingString), true, XContentType.JSON).v2();
for (IndexTemplateMetaData template : newInstance.getIndexTemplates()) {
MappingMetaData mappingMD = template.mappings();
if(mappingMD!=null) {
if(mappingMD != null) {
Map<String, Object> mappingAsMap = mappingMD.sourceAsMap();
assertEquals(expectedMap, mappingAsMap);
}
Expand Down Expand Up @@ -133,4 +223,39 @@ static void toXContent(GetIndexTemplatesResponse response, XContentBuilder build
org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse(serverIndexTemplates);
serverResponse.toXContent(builder, ToXContent.EMPTY_PARAMS);
}

private static AliasMetaData randomAliasMetaData(String name) {
AliasMetaData.Builder alias = AliasMetaData.builder(name);
if (randomBoolean()) {
if (randomBoolean()) {
alias.routing(randomAlphaOfLength(5));
} else {
if (randomBoolean()) {
alias.indexRouting(randomAlphaOfLength(5));
}
if (randomBoolean()) {
alias.searchRouting(randomAlphaOfLength(5));
}
}
}

if (randomBoolean()) {
alias.filter("{\"term\":{\"year\":2016}}");
}

if (randomBoolean()) {
alias.writeIndex(randomBoolean());
}
return alias.build();
}

static XContentBuilder randomMapping(String type, XContentType xContentType) throws IOException {
XContentBuilder builder = XContentFactory.contentBuilder(xContentType);
builder.startObject().startObject(type);

randomMappingFields(builder, true);

builder.endObject().endObject();
return builder;
}
}
10 changes: 0 additions & 10 deletions client/rest/src/main/java/org/elasticsearch/client/ResponseException.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,16 +39,6 @@ public ResponseException(Response response) throws IOException {
this.response = response;
}

/**
* Wrap a {@linkplain ResponseException} with another one with the current
* stack trace. This is used during synchronous calls so that the caller
* ends up in the stack trace of the exception thrown.
*/
ResponseException(ResponseException e) throws IOException {
super(e.getMessage(), e);
this.response = e.getResponse();
}

static String buildMessage(Response response) throws IOException {
String message = String.format(Locale.ROOT,
"method [%s], host [%s], URI [%s], status line [%s]",
Expand Down
Loading

0 comments on commit ca0106d

Please sign in to comment.