Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Delegate toXContent logic from ClusterState to its member classes #48218 #52743

Merged
merged 23 commits into from
Mar 25, 2020
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
422dadb
Add serialization tests for ClusterState and MetaData toXContent methods
zacharymorn Feb 2, 2020
072e05d
Copy over ClusterState.toXContent serialization test metadata section…
zacharymorn Feb 11, 2020
734d5a7
Parameterize classes to emit different serialization based on API vs.…
zacharymorn Feb 25, 2020
b021eac
Substitue ClusterState.toXContent metadata serialization logic with M…
zacharymorn Feb 25, 2020
958d41a
Address comments
zacharymorn Mar 20, 2020
089d7a3
Address comments to place else on the same line, and pass down entire…
zacharymorn Mar 24, 2020
0c11613
Remove lenient deserialization parsing for API-serialized XContent, a…
zacharymorn Mar 24, 2020
977a428
Fix tests
ywelsch Mar 24, 2020
608d627
Merge remote-tracking branch 'elastic/master' into pr/52743
ywelsch Mar 24, 2020
ae97011
fix merge issue
ywelsch Mar 24, 2020
db90041
fix tests
ywelsch Mar 24, 2020
76564f9
undo more changes
ywelsch Mar 24, 2020
7a30a36
Fix alias filter parsing and test
ywelsch Mar 24, 2020
e99f60a
Remove transient settings
ywelsch Mar 24, 2020
b1eb05a
always print indices in API mode
ywelsch Mar 24, 2020
cc4f317
Merge remote-tracking branch 'elastic/master' into pr/52743
ywelsch Mar 24, 2020
3fe6cea
checkstyle
ywelsch Mar 24, 2020
7235ade
checkstyle
ywelsch Mar 24, 2020
af8286d
Merge remote-tracking branch 'elastic/master' into pr/52743
ywelsch Mar 25, 2020
6560116
More test fixing
ywelsch Mar 25, 2020
8c7a308
one more test to fix after merging master
ywelsch Mar 25, 2020
8567d91
more fixes yet
ywelsch Mar 25, 2020
51b31b4
another fix
ywelsch Mar 25, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
117 changes: 14 additions & 103 deletions server/src/main/java/org/elasticsearch/cluster/ClusterState.java
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@

package org.elasticsearch.cluster;

import com.carrotsearch.hppc.cursors.IntObjectCursor;
import com.carrotsearch.hppc.cursors.ObjectCursor;
import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
import org.elasticsearch.Version;
Expand All @@ -29,8 +28,6 @@
import org.elasticsearch.cluster.coordination.CoordinationMetaData.VotingConfigExclusion;
import org.elasticsearch.cluster.coordination.CoordinationMetaData.VotingConfiguration;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.node.DiscoveryNodes;
Expand All @@ -43,31 +40,31 @@
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.UUIDs;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.VersionedNamedWriteable;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.ToXContentFragment;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.discovery.Discovery;

import java.io.IOException;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.StreamSupport;

import static org.elasticsearch.cluster.metadata.MetaData.CONTEXT_MODE_API;
import static org.elasticsearch.cluster.metadata.MetaData.CONTEXT_MODE_PARAM;

/**
* Represents the current state of the cluster.
* <p>
Expand All @@ -93,6 +90,11 @@
public class ClusterState implements ToXContentFragment, Diffable<ClusterState> {

public static final ClusterState EMPTY_STATE = builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).build();
private static final Map<String, String> API_PARAMS;
static {
API_PARAMS = new HashMap<>();
API_PARAMS.put(CONTEXT_MODE_PARAM, CONTEXT_MODE_API);
}

public interface Custom extends NamedDiffable<Custom>, ToXContentFragment {

Expand Down Expand Up @@ -418,104 +420,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
builder.endObject();
}

// meta data
if (metrics.contains(Metric.METADATA)) {
builder.startObject("metadata");
builder.field("cluster_uuid", metaData().clusterUUID());

builder.startObject("cluster_coordination");
coordinationMetaData().toXContent(builder, params);
builder.endObject();

builder.startObject("templates");
for (ObjectCursor<IndexTemplateMetaData> cursor : metaData().templates().values()) {
IndexTemplateMetaData templateMetaData = cursor.value;
builder.startObject(templateMetaData.name());

builder.field("index_patterns", templateMetaData.patterns());
builder.field("order", templateMetaData.order());

builder.startObject("settings");
Settings settings = templateMetaData.settings();
settings.toXContent(builder, params);
builder.endObject();

builder.startObject("mappings");
for (ObjectObjectCursor<String, CompressedXContent> cursor1 : templateMetaData.mappings()) {
Map<String, Object> mapping = XContentHelper.convertToMap(new BytesArray(cursor1.value.uncompressed()), false).v2();
if (mapping.size() == 1 && mapping.containsKey(cursor1.key)) {
// the type name is the root value, reduce it
mapping = (Map<String, Object>) mapping.get(cursor1.key);
}
builder.field(cursor1.key);
builder.map(mapping);
}
builder.endObject();


builder.endObject();
}
builder.endObject();

builder.startObject("indices");
for (IndexMetaData indexMetaData : metaData()) {
builder.startObject(indexMetaData.getIndex().getName());

builder.field("state", indexMetaData.getState().toString().toLowerCase(Locale.ENGLISH));

builder.startObject("settings");
Settings settings = indexMetaData.getSettings();
settings.toXContent(builder, params);
builder.endObject();

builder.startObject("mappings");
MappingMetaData mmd = indexMetaData.mapping();
if (mmd != null) {
Map<String, Object> mapping = XContentHelper
.convertToMap(new BytesArray(mmd.source().uncompressed()), false).v2();
if (mapping.size() == 1 && mapping.containsKey(mmd.type())) {
// the type name is the root value, reduce it
mapping = (Map<String, Object>) mapping.get(mmd.type());
}
builder.field(mmd.type());
builder.map(mapping);
}
builder.endObject();

builder.startArray("aliases");
for (ObjectCursor<String> cursor : indexMetaData.getAliases().keys()) {
builder.value(cursor.value);
}
builder.endArray();

builder.startObject(IndexMetaData.KEY_PRIMARY_TERMS);
for (int shard = 0; shard < indexMetaData.getNumberOfShards(); shard++) {
builder.field(Integer.toString(shard), indexMetaData.primaryTerm(shard));
}
builder.endObject();

builder.startObject(IndexMetaData.KEY_IN_SYNC_ALLOCATIONS);
for (IntObjectCursor<Set<String>> cursor : indexMetaData.getInSyncAllocationIds()) {
builder.startArray(String.valueOf(cursor.key));
for (String allocationId : cursor.value) {
builder.value(allocationId);
}
builder.endArray();
}
builder.endObject();
API_PARAMS.put("flat_settings", params.param("flat_settings"));
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This mutates the global variable on every call? Certainly not what we want?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've removed API_PARAMS as suggested so this is no longer relevant.

API_PARAMS.put("reduce_mappings", params.param("reduce_mappings"));

// index metadata
builder.endObject();
}
builder.endObject();

for (ObjectObjectCursor<String, MetaData.Custom> cursor : metaData.customs()) {
builder.startObject(cursor.key);
cursor.value.toXContent(builder, params);
builder.endObject();
}

builder.endObject();
// meta data
if (metrics.contains(Metric.METADATA)) {
metaData.toXContent(builder, new ToXContent.MapParams(API_PARAMS));
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Instead of passing down a limited set of parameters here, I wonder if we should pass all of them, but additionally inject the API context.

Given that Metadata is already using API context as default, I wonder if we need to do anything here in this class. Perhaps it's sufficient to just pass down the params as is?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's a good point. The delegation refactoring ultimately do away the need for selecting params in this layer. I've removed it.

}

// routing table
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@
import java.util.Set;
import java.util.function.Function;

import static org.elasticsearch.cluster.metadata.MetaData.CONTEXT_MODE_PARAM;
import static org.elasticsearch.cluster.node.DiscoveryNodeFilters.IP_VALIDATOR;
import static org.elasticsearch.cluster.node.DiscoveryNodeFilters.OpType.AND;
import static org.elasticsearch.cluster.node.DiscoveryNodeFilters.OpType.OR;
Expand Down Expand Up @@ -1186,48 +1187,85 @@ public IndexMetaData build() {
}

public static void toXContent(IndexMetaData indexMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException {
MetaData.XContentContext context = MetaData.XContentContext.valueOf(params.param(CONTEXT_MODE_PARAM, "API"));

builder.startObject(indexMetaData.getIndex().getName());

builder.field(KEY_VERSION, indexMetaData.getVersion());
builder.field(KEY_MAPPING_VERSION, indexMetaData.getMappingVersion());
builder.field(KEY_SETTINGS_VERSION, indexMetaData.getSettingsVersion());
builder.field(KEY_ALIASES_VERSION, indexMetaData.getAliasesVersion());
builder.field(KEY_ROUTING_NUM_SHARDS, indexMetaData.getRoutingNumShards());

builder.field(KEY_STATE, indexMetaData.getState().toString().toLowerCase(Locale.ENGLISH));

boolean binary = params.paramAsBoolean("binary", false);

builder.startObject(KEY_SETTINGS);
indexMetaData.getSettings().toXContent(builder, new MapParams(Collections.singletonMap("flat_settings", "true")));
if (context != MetaData.XContentContext.API) {
indexMetaData.getSettings().toXContent(builder, new MapParams(Collections.singletonMap("flat_settings", "true")));
} else {
indexMetaData.getSettings().toXContent(builder, params);
}
builder.endObject();

builder.startArray(KEY_MAPPINGS);
MappingMetaData mmd = indexMetaData.mapping();
if (mmd != null) {
if (binary) {
builder.value(mmd.source().compressed());
} else {
builder.map(XContentHelper.convertToMap(new BytesArray(mmd.source().uncompressed()), true).v2());
if (context != MetaData.XContentContext.API) {
builder.startArray(KEY_MAPPINGS);
MappingMetaData mmd = indexMetaData.mapping();
if (mmd != null) {
if (binary) {
builder.value(mmd.source().compressed());
} else {
builder.map(XContentHelper.convertToMap(new BytesArray(mmd.source().uncompressed()), true).v2());
}
}
builder.endArray();
} else {
builder.startObject(KEY_MAPPINGS);
MappingMetaData mmd = indexMetaData.mapping();
if (mmd != null) {
Map<String, Object> mapping = XContentHelper.convertToMap(new BytesArray(mmd.source().uncompressed()), false).v2();
if (mapping.size() == 1 && mapping.containsKey(mmd.type())) {
// the type name is the root value, reduce it
mapping = (Map<String, Object>) mapping.get(mmd.type());
}
builder.field(mmd.type());
builder.map(mapping);
}
builder.endObject();
}
builder.endArray();

for (ObjectObjectCursor<String, DiffableStringMap> cursor : indexMetaData.customData) {
builder.field(cursor.key);
builder.map(cursor.value);
}

builder.startObject(KEY_ALIASES);
for (ObjectCursor<AliasMetaData> cursor : indexMetaData.getAliases().values()) {
AliasMetaData.Builder.toXContent(cursor.value, builder, params);
}
builder.endObject();
if (context != MetaData.XContentContext.API) {
builder.startObject(KEY_ALIASES);
for (ObjectCursor<AliasMetaData> cursor : indexMetaData.getAliases().values()) {
AliasMetaData.Builder.toXContent(cursor.value, builder, params);
}
builder.endObject();

builder.startArray(KEY_PRIMARY_TERMS);
for (int i = 0; i < indexMetaData.getNumberOfShards(); i++) {
builder.value(indexMetaData.primaryTerm(i));
}
builder.endArray();
} else {
builder.startArray(KEY_ALIASES);
for (ObjectCursor<String> cursor : indexMetaData.getAliases().keys()) {
builder.value(cursor.value);
}
builder.endArray();

builder.startArray(KEY_PRIMARY_TERMS);
for (int i = 0; i < indexMetaData.getNumberOfShards(); i++) {
builder.value(indexMetaData.primaryTerm(i));
builder.startObject(IndexMetaData.KEY_PRIMARY_TERMS);
for (int shard = 0; shard < indexMetaData.getNumberOfShards(); shard++) {
builder.field(Integer.toString(shard), indexMetaData.primaryTerm(shard));
}
builder.endObject();
}
builder.endArray();


builder.startObject(KEY_IN_SYNC_ALLOCATIONS);
for (IntObjectCursor<Set<String>> cursor : indexMetaData.inSyncAllocationIds) {
Expand Down Expand Up @@ -1318,7 +1356,21 @@ public static IndexMetaData fromXContent(XContentParser parser) throws IOExcepti
throw new IllegalArgumentException("Unexpected token: " + token);
}
}
} else if ("warmers".equals(currentFieldName)) {
} else if (KEY_PRIMARY_TERMS.equals(currentFieldName)) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why did you add deserialization code here?

Is it because we are testing API deserialization somewhere? If so, I would rather have the tests changed to test non-API deserialization.

Same comment of other places below.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The primary reason for adding these is, the original ClusterState.toXContent emits array serialization for aliases and object serialization for primary_terms, which are different from what MetaData.toXContent emits, which is object serialization for aliases and array serialization for primary_terms, under API context. The delta can been seen here 072e05d#diff-4385d650a8cef0592db3454ad42e4bf3L1172-R1157

In general, as the duplicated code diverges, there are conflicting serializations under API context.

So the additional deserialization here is to be able to read back in the different serialization emitted by the new code (namely, array serialization for aliases and object serialization for primary_terms)

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we should avoid being able to read back the API-serialized XContent. There is no requirement to do so, and it introduces leniency in the parsing here, with the possibility of silently hiding bugs where the wrong context is chosen for serialization.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've given this a shot and removed deserialization for both primary_terms and aliases. These ended up breaking a lot of tests as they serialized under default context (API), and deserialized for further checking. I thus removed the breaking tests (as little as I can, but more can be removed as what's left are just setup code) and isolated the removal in this commit 0c11613.

If we don't want to remove these many tests for such small changes, the other possible approach is to not emit those different serialization for aliases and primary_terms for ClusterState.toXContent. This obviously would break code relying on it.

Please let me know which way you prefer, or if you have other ideas around this.

LongArrayList list = new LongArrayList();
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
}
else if (token == XContentParser.Token.VALUE_NUMBER) {
list.add(parser.longValue());
} else {
throw new IllegalStateException("found a non-numeric value under [" + KEY_PRIMARY_TERMS + "]");
}
}
builder.primaryTerms(list.toArray());
}
else if ("warmers".equals(currentFieldName)) {
// TODO: do this in 6.0:
// throw new IllegalArgumentException("Warmers are not supported anymore - are you upgrading from 1.x?");
// ignore: warmers have been removed in 5.0 and are
Expand Down Expand Up @@ -1352,7 +1404,13 @@ public static IndexMetaData fromXContent(XContentParser parser) throws IOExcepti
}
}
builder.primaryTerms(list.toArray());
} else {
}
else if (KEY_ALIASES.equals(currentFieldName)) {
while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
builder.putAlias(AliasMetaData.builder(parser.text()).build());
}
}
else {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

move these elses up (lot's of other occurrences in this file)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've removed these changes as they are also for reading back API-serialized XContent. I did correct else placement in other places though.

throw new IllegalArgumentException("Unexpected field for an array " + currentFieldName);
}
} else if (token.isValue()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,8 @@
import java.util.Objects;
import java.util.Set;

import static org.elasticsearch.cluster.metadata.MetaData.CONTEXT_MODE_PARAM;

public class IndexTemplateMetaData extends AbstractDiffable<IndexTemplateMetaData> {

private final String name;
Expand Down Expand Up @@ -360,6 +362,7 @@ private static void toInnerXContent(IndexTemplateMetaData indexTemplateMetaData,
XContentBuilder builder,
ToXContent.Params params,
boolean includeTypeName) throws IOException {
MetaData.XContentContext context = MetaData.XContentContext.valueOf(params.param(CONTEXT_MODE_PARAM, "API"));

builder.field("order", indexTemplateMetaData.order());
if (indexTemplateMetaData.version() != null) {
Expand Down Expand Up @@ -400,12 +403,27 @@ private static void toInnerXContent(IndexTemplateMetaData indexTemplateMetaData,
builder.endObject();
}
} else {
builder.startArray("mappings");
for (ObjectObjectCursor<String, CompressedXContent> cursor : indexTemplateMetaData.mappings()) {
byte[] data = cursor.value.uncompressed();
builder.map(XContentHelper.convertToMap(new BytesArray(data), true).v2());
if (context != MetaData.XContentContext.API) {
builder.startArray("mappings");
for (ObjectObjectCursor<String, CompressedXContent> cursor : indexTemplateMetaData.mappings()) {
byte[] data = cursor.value.uncompressed();
builder.map(XContentHelper.convertToMap(new BytesArray(data), true).v2());
}
builder.endArray();
}
else {
builder.startObject("mappings");
for (ObjectObjectCursor<String, CompressedXContent> cursor1 : indexTemplateMetaData.mappings()) {
Map<String, Object> mapping = XContentHelper.convertToMap(new BytesArray(cursor1.value.uncompressed()), false).v2();
if (mapping.size() == 1 && mapping.containsKey(cursor1.key)) {
// the type name is the root value, reduce it
mapping = (Map<String, Object>) mapping.get(cursor1.key);
}
builder.field(cursor1.key);
builder.map(mapping);
}
builder.endObject();
}
builder.endArray();
}

builder.startObject("aliases");
Expand Down
Loading