Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Delegate toXContent logic from ClusterState to its member classes #48218 #52743

Merged
merged 23 commits into from
Mar 25, 2020
Merged

Delegate toXContent logic from ClusterState to its member classes #48218 #52743

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
422dadb
Add serialization tests for ClusterState and MetaData toXContent methods
zacharymorn Feb 2, 2020
072e05d
Copy over ClusterState.toXContent serialization test metadata section…
zacharymorn Feb 11, 2020
734d5a7
Parameterize classes to emit different serialization based on API vs.…
zacharymorn Feb 25, 2020
b021eac
Substitue ClusterState.toXContent metadata serialization logic with M…
zacharymorn Feb 25, 2020
958d41a
Address comments
zacharymorn Mar 20, 2020
089d7a3
Address comments to place else on the same line, and pass down entire…
zacharymorn Mar 24, 2020
0c11613
Remove lenient deserialization parsing for API-serialized XContent, a…
zacharymorn Mar 24, 2020
977a428
Fix tests
ywelsch Mar 24, 2020
608d627
Merge remote-tracking branch 'elastic/master' into pr/52743
ywelsch Mar 24, 2020
ae97011
fix merge issue
ywelsch Mar 24, 2020
db90041
fix tests
ywelsch Mar 24, 2020
76564f9
undo more changes
ywelsch Mar 24, 2020
7a30a36
Fix alias filter parsing and test
ywelsch Mar 24, 2020
e99f60a
Remove transient settings
ywelsch Mar 24, 2020
b1eb05a
always print indices in API mode
ywelsch Mar 24, 2020
cc4f317
Merge remote-tracking branch 'elastic/master' into pr/52743
ywelsch Mar 24, 2020
3fe6cea
checkstyle
ywelsch Mar 24, 2020
7235ade
checkstyle
ywelsch Mar 24, 2020
af8286d
Merge remote-tracking branch 'elastic/master' into pr/52743
ywelsch Mar 25, 2020
6560116
More test fixing
ywelsch Mar 25, 2020
8c7a308
one more test to fix after merging master
ywelsch Mar 25, 2020
8567d91
more fixes yet
ywelsch Mar 25, 2020
51b31b4
another fix
ywelsch Mar 25, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 1 addition & 79 deletions server/src/main/java/org/elasticsearch/cluster/ClusterState.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@

package org.elasticsearch.cluster;

import com.carrotsearch.hppc.cursors.IntObjectCursor;
import com.carrotsearch.hppc.cursors.ObjectCursor;
import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
import org.elasticsearch.Version;
Expand All @@ -29,8 +28,6 @@
import org.elasticsearch.cluster.coordination.CoordinationMetaData.VotingConfigExclusion;
import org.elasticsearch.cluster.coordination.CoordinationMetaData.VotingConfiguration;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.node.DiscoveryNodes;
Expand All @@ -43,7 +40,6 @@
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.UUIDs;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
Expand All @@ -55,13 +51,11 @@
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.ToXContentFragment;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.discovery.Discovery;

import java.io.IOException;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
Expand Down Expand Up @@ -419,79 +413,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws

// meta data
if (metrics.contains(Metric.METADATA)) {
builder.startObject("metadata");
builder.field("cluster_uuid", metaData().clusterUUID());

builder.startObject("cluster_coordination");
coordinationMetaData().toXContent(builder, params);
builder.endObject();

builder.startObject("templates");
for (ObjectCursor<IndexTemplateMetaData> cursor : metaData().templates().values()) {
IndexTemplateMetaData templateMetaData = cursor.value;
IndexTemplateMetaData.Builder.toXContentWithTypes(templateMetaData, builder, params);
}
builder.endObject();

builder.startObject("indices");
for (IndexMetaData indexMetaData : metaData()) {
builder.startObject(indexMetaData.getIndex().getName());

builder.field("state", indexMetaData.getState().toString().toLowerCase(Locale.ENGLISH));

builder.startObject("settings");
Settings settings = indexMetaData.getSettings();
settings.toXContent(builder, params);
builder.endObject();

builder.startObject("mappings");
MappingMetaData mmd = indexMetaData.mapping();
if (mmd != null) {
Map<String, Object> mapping = XContentHelper
.convertToMap(new BytesArray(mmd.source().uncompressed()), false).v2();
if (mapping.size() == 1 && mapping.containsKey(mmd.type())) {
// the type name is the root value, reduce it
mapping = (Map<String, Object>) mapping.get(mmd.type());
}
builder.field(mmd.type());
builder.map(mapping);
}
builder.endObject();

builder.startArray("aliases");
for (ObjectCursor<String> cursor : indexMetaData.getAliases().keys()) {
builder.value(cursor.value);
}
builder.endArray();

builder.startObject(IndexMetaData.KEY_PRIMARY_TERMS);
for (int shard = 0; shard < indexMetaData.getNumberOfShards(); shard++) {
builder.field(Integer.toString(shard), indexMetaData.primaryTerm(shard));
}
builder.endObject();

builder.startObject(IndexMetaData.KEY_IN_SYNC_ALLOCATIONS);
for (IntObjectCursor<Set<String>> cursor : indexMetaData.getInSyncAllocationIds()) {
builder.startArray(String.valueOf(cursor.key));
for (String allocationId : cursor.value) {
builder.value(allocationId);
}
builder.endArray();
}
builder.endObject();

// index metadata
builder.endObject();
}
builder.endObject();

for (ObjectObjectCursor<String, MetaData.Custom> cursor : metaData.customs()) {
builder.startObject(cursor.key);
cursor.value.toXContent(builder, params);
builder.endObject();
}

builder.endObject();
metaData.toXContent(builder, params);
}

// routing table
Expand Down
2 changes: 2 additions & 0 deletions server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -387,6 +387,8 @@ public static AliasMetaData fromXContent(XContentParser parser) throws IOExcepti
builder.indexRouting(parser.text());
} else if ("search_routing".equals(currentFieldName) || "searchRouting".equals(currentFieldName)) {
builder.searchRouting(parser.text());
} else if ("filter".equals(currentFieldName)) {
builder.filter(new CompressedXContent(parser.binaryValue()));
}
} else if (token == XContentParser.Token.START_ARRAY) {
parser.skipChildren();
Expand Down
84 changes: 65 additions & 19 deletions server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@
import java.util.Arrays;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
Expand All @@ -73,6 +74,7 @@
import java.util.Set;
import java.util.function.Function;

import static org.elasticsearch.cluster.metadata.MetaData.CONTEXT_MODE_PARAM;
import static org.elasticsearch.cluster.node.DiscoveryNodeFilters.IP_VALIDATOR;
import static org.elasticsearch.cluster.node.DiscoveryNodeFilters.OpType.AND;
import static org.elasticsearch.cluster.node.DiscoveryNodeFilters.OpType.OR;
Expand Down Expand Up @@ -1187,48 +1189,86 @@ public IndexMetaData build() {
}

public static void toXContent(IndexMetaData indexMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException {
MetaData.XContentContext context = MetaData.XContentContext.valueOf(
params.param(CONTEXT_MODE_PARAM, MetaData.CONTEXT_MODE_API));

builder.startObject(indexMetaData.getIndex().getName());

builder.field(KEY_VERSION, indexMetaData.getVersion());
builder.field(KEY_MAPPING_VERSION, indexMetaData.getMappingVersion());
builder.field(KEY_SETTINGS_VERSION, indexMetaData.getSettingsVersion());
builder.field(KEY_ALIASES_VERSION, indexMetaData.getAliasesVersion());
builder.field(KEY_ROUTING_NUM_SHARDS, indexMetaData.getRoutingNumShards());

builder.field(KEY_STATE, indexMetaData.getState().toString().toLowerCase(Locale.ENGLISH));

boolean binary = params.paramAsBoolean("binary", false);

builder.startObject(KEY_SETTINGS);
indexMetaData.getSettings().toXContent(builder, new MapParams(Collections.singletonMap("flat_settings", "true")));
if (context != MetaData.XContentContext.API) {
indexMetaData.getSettings().toXContent(builder, new MapParams(Collections.singletonMap("flat_settings", "true")));
} else {
indexMetaData.getSettings().toXContent(builder, params);
}
builder.endObject();

builder.startArray(KEY_MAPPINGS);
MappingMetaData mmd = indexMetaData.mapping();
if (mmd != null) {
if (binary) {
builder.value(mmd.source().compressed());
} else {
builder.map(XContentHelper.convertToMap(new BytesArray(mmd.source().uncompressed()), true).v2());
if (context != MetaData.XContentContext.API) {
builder.startArray(KEY_MAPPINGS);
MappingMetaData mmd = indexMetaData.mapping();
if (mmd != null) {
if (binary) {
builder.value(mmd.source().compressed());
} else {
builder.map(XContentHelper.convertToMap(new BytesArray(mmd.source().uncompressed()), true).v2());
}
}
builder.endArray();
} else {
builder.startObject(KEY_MAPPINGS);
MappingMetaData mmd = indexMetaData.mapping();
if (mmd != null) {
Map<String, Object> mapping = XContentHelper.convertToMap(new BytesArray(mmd.source().uncompressed()), false).v2();
if (mapping.size() == 1 && mapping.containsKey(mmd.type())) {
// the type name is the root value, reduce it
mapping = (Map<String, Object>) mapping.get(mmd.type());
}
builder.field(mmd.type());
builder.map(mapping);
}
builder.endObject();
}
builder.endArray();

for (ObjectObjectCursor<String, DiffableStringMap> cursor : indexMetaData.customData) {
builder.field(cursor.key);
builder.map(cursor.value);
}

builder.startObject(KEY_ALIASES);
for (ObjectCursor<AliasMetaData> cursor : indexMetaData.getAliases().values()) {
AliasMetaData.Builder.toXContent(cursor.value, builder, params);
}
builder.endObject();
if (context != MetaData.XContentContext.API) {
builder.startObject(KEY_ALIASES);
for (ObjectCursor<AliasMetaData> cursor : indexMetaData.getAliases().values()) {
AliasMetaData.Builder.toXContent(cursor.value, builder, params);
}
builder.endObject();

builder.startArray(KEY_PRIMARY_TERMS);
for (int i = 0; i < indexMetaData.getNumberOfShards(); i++) {
builder.value(indexMetaData.primaryTerm(i));
builder.startArray(KEY_PRIMARY_TERMS);
for (int i = 0; i < indexMetaData.getNumberOfShards(); i++) {
builder.value(indexMetaData.primaryTerm(i));
}
builder.endArray();
} else {
builder.startArray(KEY_ALIASES);
for (ObjectCursor<String> cursor : indexMetaData.getAliases().keys()) {
builder.value(cursor.value);
}
builder.endArray();

builder.startObject(IndexMetaData.KEY_PRIMARY_TERMS);
for (int shard = 0; shard < indexMetaData.getNumberOfShards(); shard++) {
builder.field(Integer.toString(shard), indexMetaData.primaryTerm(shard));
}
builder.endObject();
}
builder.endArray();


builder.startObject(KEY_IN_SYNC_ALLOCATIONS);
for (IntObjectCursor<Set<String>> cursor : indexMetaData.inSyncAllocationIds) {
Expand Down Expand Up @@ -1414,7 +1454,13 @@ public static Settings addHumanReadableSettings(Settings settings) {
return builder.build();
}

private static final ToXContent.Params FORMAT_PARAMS = new MapParams(Collections.singletonMap("binary", "true"));
private static final ToXContent.Params FORMAT_PARAMS;
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Didn't realize we could use this to save the tests, good to know!

static {
Map<String, String> params = new HashMap<>(2);
params.put("binary", "true");
params.put(MetaData.CONTEXT_MODE_PARAM, MetaData.CONTEXT_MODE_GATEWAY);
FORMAT_PARAMS = new MapParams(params);
}

/**
* State format for {@link IndexMetaData} to write to and load from disk
Expand Down
3 changes: 1 addition & 2 deletions server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -378,7 +378,6 @@ private static void toInnerXContent(IndexTemplateMetaData indexTemplateMetaData,
XContentBuilder builder,
ToXContent.Params params,
boolean includeTypeName) throws IOException {

builder.field("order", indexTemplateMetaData.order());
if (indexTemplateMetaData.version() != null) {
builder.field("version", indexTemplateMetaData.version());
Expand Down Expand Up @@ -412,7 +411,7 @@ private static void toInnerXContent(IndexTemplateMetaData indexTemplateMetaData,

@SuppressWarnings("unchecked")
private static Map<String, Object> reduceMapping(Map<String, Object> mapping) {
assert mapping.keySet().size() == 1;
assert mapping.keySet().size() == 1 : mapping.keySet();
return (Map<String, Object>) mapping.values().iterator().next();
}

Expand Down
32 changes: 11 additions & 21 deletions server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,10 +51,8 @@
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.ToXContentFragment;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.gateway.MetaDataStateFormat;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexNotFoundException;
Expand Down Expand Up @@ -156,6 +154,8 @@ public interface Custom extends NamedDiffable<Custom>, ToXContentFragment {

public static final String CONTEXT_MODE_GATEWAY = XContentContext.GATEWAY.toString();

public static final String CONTEXT_MODE_API = XContentContext.API.toString();

public static final String GLOBAL_STATE_FILE_PREFIX = "global-";

private static final NamedDiffableValueSerializer<Custom> CUSTOM_VALUE_SERIALIZER = new NamedDiffableValueSerializer<>(Custom.class);
Expand Down Expand Up @@ -1371,46 +1371,36 @@ private SortedMap<String, AliasOrIndex> buildAliasAndIndexLookup() {
return aliasAndIndexLookup;
}

public static String toXContent(MetaData metaData) throws IOException {
XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
builder.startObject();
toXContent(metaData, builder, ToXContent.EMPTY_PARAMS);
builder.endObject();
return Strings.toString(builder);
}

public static void toXContent(MetaData metaData, XContentBuilder builder, ToXContent.Params params) throws IOException {
XContentContext context = XContentContext.valueOf(params.param(CONTEXT_MODE_PARAM, "API"));
XContentContext context = XContentContext.valueOf(params.param(CONTEXT_MODE_PARAM, CONTEXT_MODE_API));

builder.startObject("meta-data");
if (context == XContentContext.API) {
builder.startObject("metadata");
} else {
builder.startObject("meta-data");
builder.field("version", metaData.version());
}

builder.field("version", metaData.version());
builder.field("cluster_uuid", metaData.clusterUUID);
builder.field("cluster_uuid_committed", metaData.clusterUUIDCommitted);

builder.startObject("cluster_coordination");
metaData.coordinationMetaData().toXContent(builder, params);
builder.endObject();

if (!metaData.persistentSettings().isEmpty()) {
if (context != XContentContext.API && !metaData.persistentSettings().isEmpty()) {
builder.startObject("settings");
metaData.persistentSettings().toXContent(builder, new MapParams(Collections.singletonMap("flat_settings", "true")));
builder.endObject();
}

if (context == XContentContext.API && !metaData.transientSettings().isEmpty()) {
builder.startObject("transient_settings");
metaData.transientSettings().toXContent(builder, new MapParams(Collections.singletonMap("flat_settings", "true")));
builder.endObject();
}

builder.startObject("templates");
for (ObjectCursor<IndexTemplateMetaData> cursor : metaData.templates().values()) {
IndexTemplateMetaData.Builder.toXContentWithTypes(cursor.value, builder, params);
}
builder.endObject();

if (context == XContentContext.API && !metaData.indices().isEmpty()) {
if (context == XContentContext.API) {
builder.startObject("indices");
for (IndexMetaData indexMetaData : metaData) {
IndexMetaData.Builder.toXContent(indexMetaData, builder, params);
Expand Down
Loading