Skip to content

Commit

Permalink
Merge branch 'master' into ccr
Browse files Browse the repository at this point in the history 
* master:
  Add proxy support to RemoteClusterConnection (#33062)
  TEST: Skip assertSeqNos for closed shards (#33130)
  TEST: resync operation on replica should acquire shard permit (#33103)
  Switch remaining x-pack tests to new style Requests (#33108)
  Switch remaining tests to new style Requests (#33109)
  Switch remaining ml tests to new style Requests (#33107)
  Build: Line up IDE detection logic
  Security index expands to a single replica (#33131)
  HLRC: request/response homogeneity and JavaDoc improvements (#33133)
  Checkstyle!
  [Test] Fix sporadic failure in MembershipActionTests
  Revert "Do NOT allow termvectors on nested fields (#32728)"
  [Rollup] Move toAggCap() methods out of rollup config objects (#32583)
  Fix race condition in scheduler engine test
  • Loading branch information
@dnhatn
dnhatn committed Aug 26, 2018
2 parents ef9607e + 3376922 commit 75304f4
Show file tree
Hide file tree
Showing 43 changed files with 1,194 additions and 956 deletions.
22 changes: 6 additions & 16 deletions client/rest-high-level/src/test/java/org/elasticsearch/client/StoredScriptsIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,6 @@
* under the License.
*/


import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.util.EntityUtils;
import org.elasticsearch.ElasticsearchStatusException;
import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest;
Expand All @@ -35,7 +32,6 @@

import java.util.Collections;

import static java.util.Collections.emptyMap;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.hamcrest.Matchers.equalTo;

Expand All @@ -52,12 +48,9 @@ public void testGetStoredScript() throws Exception {
final String script = Strings.toString(scriptSource.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS));
// TODO: change to HighLevel PutStoredScriptRequest when it will be ready
// so far - using low-level REST API
Response putResponse =
adminClient()
.performRequest("PUT", "/_scripts/calculate-score", emptyMap(),
new StringEntity("{\"script\":" + script + "}",
ContentType.APPLICATION_JSON));
assertEquals(putResponse.getStatusLine().getReasonPhrase(), 200, putResponse.getStatusLine().getStatusCode());
Request putRequest = new Request("PUT", "/_scripts/calculate-score");
putRequest.setJsonEntity("{\"script\":" + script + "}");
Response putResponse = adminClient().performRequest(putRequest);
assertEquals("{\"acknowledged\":true}", EntityUtils.toString(putResponse.getEntity()));

GetStoredScriptRequest getRequest = new GetStoredScriptRequest("calculate-score");
Expand All @@ -78,12 +71,9 @@ public void testDeleteStoredScript() throws Exception {
final String script = Strings.toString(scriptSource.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS));
// TODO: change to HighLevel PutStoredScriptRequest when it will be ready
// so far - using low-level REST API
Response putResponse =
adminClient()
.performRequest("PUT", "/_scripts/" + id, emptyMap(),
new StringEntity("{\"script\":" + script + "}",
ContentType.APPLICATION_JSON));
assertEquals(putResponse.getStatusLine().getReasonPhrase(), 200, putResponse.getStatusLine().getStatusCode());
Request putRequest = new Request("PUT", "/_scripts/" + id);
putRequest.setJsonEntity("{\"script\":" + script + "}");
Response putResponse = adminClient().performRequest(putRequest);
assertEquals("{\"acknowledged\":true}", EntityUtils.toString(putResponse.getEntity()));

DeleteStoredScriptRequest deleteRequest = new DeleteStoredScriptRequest(id);
Expand Down
12 changes: 4 additions & 8 deletions ...el/src/test/java/org/elasticsearch/client/documentation/StoredScriptsDocumentationIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,6 @@
* under the License.
*/

import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.util.EntityUtils;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.LatchedActionListener;
Expand All @@ -27,6 +25,7 @@
import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptResponse;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.ESRestHighLevelClientTestCase;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.RestHighLevelClient;
Expand All @@ -43,7 +42,6 @@
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;

import static java.util.Collections.emptyMap;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.hamcrest.Matchers.equalTo;

Expand Down Expand Up @@ -193,11 +191,9 @@ private void putStoredScript(String id, StoredScriptSource scriptSource) throws
final String script = Strings.toString(scriptSource.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS));
// TODO: change to HighLevel PutStoredScriptRequest when it will be ready
// so far - using low-level REST API
Response putResponse =
adminClient()
.performRequest("PUT", "/_scripts/" + id, emptyMap(),
new StringEntity("{\"script\":" + script + "}",
ContentType.APPLICATION_JSON));
Request request = new Request("PUT", "/_scripts/" + id);
request.setJsonEntity("{\"script\":" + script + "}");
Response putResponse = adminClient().performRequest(request);
assertEquals(putResponse.getStatusLine().getReasonPhrase(), 200, putResponse.getStatusLine().getStatusCode());
assertEquals("{\"acknowledged\":true}", EntityUtils.toString(putResponse.getEntity()));
}
Expand Down
2 changes: 1 addition & 1 deletion ...integ-test-zip/src/test/java/org/elasticsearch/test/rest/WaitForRefreshAndCloseTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ public void setupIndex() throws IOException {

@After
public void cleanupIndex() throws IOException {
client().performRequest("DELETE", indexName());
client().performRequest(new Request("DELETE", indexName()));
}

private String indexName() {
Expand Down
4 changes: 0 additions & 4 deletions docs/reference/docs/termvectors.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,6 @@ in similar way to the <<query-dsl-multi-match-query,multi match query>>
[WARNING]
Note that the usage of `/_termvector` is deprecated in 2.0, and replaced by `/_termvectors`.

[WARNING]
Term Vectors API doesn't work on nested fields. `/_termvectors` on a nested
field and any sub-fields of a nested field returns empty results.

[float]
=== Return values

Expand Down
49 changes: 0 additions & 49 deletions rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/50_nested.yml
View file

This file was deleted.

1 change: 1 addition & 0 deletions server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -272,6 +272,7 @@ public void apply(Settings value, Settings current, Settings previous) {
ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING,
TransportSearchAction.SHARD_COUNT_LIMIT_SETTING,
RemoteClusterAware.REMOTE_CLUSTERS_SEEDS,
RemoteClusterAware.REMOTE_CLUSTERS_PROXY,
RemoteClusterService.REMOTE_CLUSTER_SKIP_UNAVAILABLE,
RemoteClusterService.REMOTE_CONNECTIONS_PER_CLUSTER,
RemoteClusterService.REMOTE_INITIAL_CONNECTION_TIMEOUT_SETTING,
Expand Down
4 changes: 4 additions & 0 deletions server/src/main/java/org/elasticsearch/common/settings/Setting.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1009,6 +1009,10 @@ public static Setting<String> simpleString(String key, Property... properties) {
return new Setting<>(key, s -> "", Function.identity(), properties);
}

public static Setting<String> simpleString(String key, Function<String, String> parser, Property... properties) {
return new Setting<>(key, s -> "", parser, properties);
}

public static Setting<String> simpleString(String key, Setting<String> fallback, Property... properties) {
return new Setting<>(key, fallback, Function.identity(), properties);
}
Expand Down
17 changes: 3 additions & 14 deletions server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,6 @@
import org.elasticsearch.index.mapper.KeywordFieldMapper;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.ObjectMapper;
import org.elasticsearch.index.mapper.ParseContext;
import org.elasticsearch.index.mapper.ParsedDocument;
import org.elasticsearch.index.mapper.SourceFieldMapper;
Expand Down Expand Up @@ -161,7 +160,7 @@ private static void handleFieldWildcards(IndexShard indexShard, TermVectorsReque
request.selectedFields(fieldNames.toArray(Strings.EMPTY_ARRAY));
}

private static boolean isValidField(MappedFieldType fieldType, IndexShard indexShard) {
private static boolean isValidField(MappedFieldType fieldType) {
// must be a string
if (fieldType instanceof StringFieldType == false) {
return false;
Expand All @@ -170,16 +169,6 @@ private static boolean isValidField(MappedFieldType fieldType, IndexShard indexS
if (fieldType.indexOptions() == IndexOptions.NONE) {
return false;
}
// and must not be under nested field
int dotIndex = fieldType.name().indexOf('.');
while (dotIndex > -1) {
String parentField = fieldType.name().substring(0, dotIndex);
ObjectMapper mapper = indexShard.mapperService().getObjectMapper(parentField);
if (mapper != null && mapper.nested().isNested()) {
return false;
}
dotIndex = fieldType.name().indexOf('.', dotIndex + 1);
}
return true;
}

Expand All @@ -188,7 +177,7 @@ private static Fields addGeneratedTermVectors(IndexShard indexShard, Engine.GetR
Set<String> validFields = new HashSet<>();
for (String field : selectedFields) {
MappedFieldType fieldType = indexShard.mapperService().fullName(field);
if (isValidField(fieldType, indexShard) == false) {
if (!isValidField(fieldType)) {
continue;
}
// already retrieved, only if the analyzer hasn't been overridden at the field
Expand Down Expand Up @@ -295,7 +284,7 @@ private static Fields generateTermVectorsFromDoc(IndexShard indexShard, TermVect
Collection<DocumentField> documentFields = new HashSet<>();
for (IndexableField field : doc.getFields()) {
MappedFieldType fieldType = indexShard.mapperService().fullName(field.name());
if (isValidField(fieldType, indexShard) == false) {
if (!isValidField(fieldType)) {
continue;
}
if (request.selectedFields() != null && !request.selectedFields().contains(field.name())) {
Expand Down
61 changes: 51 additions & 10 deletions server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,14 @@
*/
package org.elasticsearch.transport;

import java.util.EnumSet;
import java.util.function.Supplier;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.metadata.ClusterNameExpressionResolver;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.UUIDs;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Setting;
Expand Down Expand Up @@ -66,6 +70,22 @@ public abstract class RemoteClusterAware extends AbstractComponent {
public static final char REMOTE_CLUSTER_INDEX_SEPARATOR = ':';
public static final String LOCAL_CLUSTER_GROUP_KEY = "";

/**
* A proxy address for the remote cluster.
* NOTE: this settings is undocumented until we have at last one transport that supports passing
* on the hostname via a mechanism like SNI.
*/
public static final Setting.AffixSetting<String> REMOTE_CLUSTERS_PROXY = Setting.affixKeySetting(
"search.remote.",
"proxy",
key -> Setting.simpleString(key, s -> {
if (Strings.hasLength(s)) {
parsePort(s);
}
return s;
}, Setting.Property.NodeScope, Setting.Property.Dynamic), REMOTE_CLUSTERS_SEEDS);


protected final ClusterNameExpressionResolver clusterNameResolver;

/**
Expand All @@ -77,25 +97,42 @@ protected RemoteClusterAware(Settings settings) {
this.clusterNameResolver = new ClusterNameExpressionResolver(settings);
}

protected static Map<String, List<Supplier<DiscoveryNode>>> buildRemoteClustersSeeds(Settings settings) {
/**
* Builds the dynamic per-cluster config from the given settings. This is a map keyed by the cluster alias that points to a tuple
* (ProxyAddresss, [SeedNodeSuppliers]). If a cluster is configured with a proxy address all seed nodes will point to
* {@link TransportAddress#META_ADDRESS} and their configured address will be used as the hostname for the generated discovery node.
*/
protected static Map<String, Tuple<String, List<Supplier<DiscoveryNode>>>> buildRemoteClustersDynamicConfig(Settings settings) {
Stream<Setting<List<String>>> allConcreteSettings = REMOTE_CLUSTERS_SEEDS.getAllConcreteSettings(settings);
return allConcreteSettings.collect(
Collectors.toMap(REMOTE_CLUSTERS_SEEDS::getNamespace, concreteSetting -> {
String clusterName = REMOTE_CLUSTERS_SEEDS.getNamespace(concreteSetting);
List<String> addresses = concreteSetting.get(settings);
final boolean proxyMode = REMOTE_CLUSTERS_PROXY.getConcreteSettingForNamespace(clusterName).exists(settings);
List<Supplier<DiscoveryNode>> nodes = new ArrayList<>(addresses.size());
for (String address : addresses) {
nodes.add(() -> {
TransportAddress transportAddress = new TransportAddress(RemoteClusterAware.parseSeedAddress(address));
return new DiscoveryNode(clusterName + "#" + transportAddress.toString(),
transportAddress,
Version.CURRENT.minimumCompatibilityVersion());
});
nodes.add(() -> buildSeedNode(clusterName, address, proxyMode));
}
return nodes;
return new Tuple<>(REMOTE_CLUSTERS_PROXY.getConcreteSettingForNamespace(clusterName).get(settings), nodes);
}));
}

static DiscoveryNode buildSeedNode(String clusterName, String address, boolean proxyMode) {
if (proxyMode) {
TransportAddress transportAddress = new TransportAddress(TransportAddress.META_ADDRESS, 0);
String hostName = address.substring(0, indexOfPortSeparator(address));
return new DiscoveryNode("", clusterName + "#" + address, UUIDs.randomBase64UUID(), hostName, address,
transportAddress, Collections
.emptyMap(), EnumSet.allOf(DiscoveryNode.Role.class),
Version.CURRENT.minimumCompatibilityVersion());
} else {
TransportAddress transportAddress = new TransportAddress(RemoteClusterAware.parseSeedAddress(address));
return new DiscoveryNode(clusterName + "#" + transportAddress.toString(),
transportAddress,
Version.CURRENT.minimumCompatibilityVersion());
}
}

/**
* Groups indices per cluster by splitting remote cluster-alias, index-name pairs on {@link #REMOTE_CLUSTER_INDEX_SEPARATOR}. All
* indices per cluster are collected as a list in the returned map keyed by the cluster alias. Local indices are grouped under
Expand Down Expand Up @@ -138,20 +175,24 @@ public Map<String, List<String>> groupClusterIndices(String[] requestIndices, Pr

protected abstract Set<String> getRemoteClusterNames();


/**
* Subclasses must implement this to receive information about updated cluster aliases. If the given address list is
* empty the cluster alias is unregistered and should be removed.
*/
protected abstract void updateRemoteCluster(String clusterAlias, List<String> addresses);
protected abstract void updateRemoteCluster(String clusterAlias, List<String> addresses, String proxy);

/**
* Registers this instance to listen to updates on the cluster settings.
*/
public void listenForUpdates(ClusterSettings clusterSettings) {
clusterSettings.addAffixUpdateConsumer(RemoteClusterAware.REMOTE_CLUSTERS_SEEDS, this::updateRemoteCluster,
clusterSettings.addAffixUpdateConsumer(RemoteClusterAware.REMOTE_CLUSTERS_PROXY,
RemoteClusterAware.REMOTE_CLUSTERS_SEEDS,
(key, value) -> updateRemoteCluster(key, value.v2(), value.v1()),
(namespace, value) -> {});
}


protected static InetSocketAddress parseSeedAddress(String remoteHost) {
String host = remoteHost.substring(0, indexOfPortSeparator(remoteHost));
InetAddress hostAddress;
Expand Down
Loading

0 comments on commit 75304f4

Please sign in to comment.