Skip to content

Commit

Permalink
wip
Browse files Browse the repository at this point in the history
  • Loading branch information
jasontedor committed Aug 15, 2018
1 parent aa147cc commit 80526b2
Show file tree
Hide file tree
Showing 13 changed files with 317 additions and 62 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -563,7 +563,7 @@ public void onFailedEngine(String reason, @Nullable Exception e) {
TimeValue.timeValueMinutes(5), refreshListenerList, Collections.emptyList(), indexSort, handler,
new NoneCircuitBreakerService(),
globalCheckpointSupplier == null ?
new ReplicationTracker(shardId, allocationId.getId(), indexSettings, SequenceNumbers.NO_OPS_PERFORMED, update -> {}) :
new ReplicationTracker(shardId, allocationId.getId(), indexSettings, SequenceNumbers.NO_OPS_PERFORMED, g -> {}) :
globalCheckpointSupplier, primaryTerm::get, tombstoneDocSupplier());
return config;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
import org.elasticsearch.xpack.ccr.action.CcrStatsAction;
import org.elasticsearch.xpack.ccr.action.CreateAndFollowIndexAction;
import org.elasticsearch.xpack.ccr.action.FollowIndexAction;
import org.elasticsearch.xpack.ccr.action.GlobalCheckpointPollAction;
import org.elasticsearch.xpack.ccr.action.ShardChangesAction;
import org.elasticsearch.xpack.ccr.action.ShardFollowNodeTask;
import org.elasticsearch.xpack.ccr.action.ShardFollowTask;
Expand Down Expand Up @@ -99,6 +100,9 @@ public List<PersistentTasksExecutor<?>> getPersistentTasksExecutor(ClusterServic
new ActionHandler<>(CcrStatsAction.INSTANCE, TransportCcrStatsAction.class),
new ActionHandler<>(CreateAndFollowIndexAction.INSTANCE, CreateAndFollowIndexAction.TransportAction.class),
new ActionHandler<>(FollowIndexAction.INSTANCE, FollowIndexAction.TransportAction.class),
new ActionHandler<>(
GlobalCheckpointPollAction.INSTANCE,
GlobalCheckpointPollAction.TransportGlobalCheckpointPollAction.class),
new ActionHandler<>(ShardChangesAction.INSTANCE, ShardChangesAction.TransportAction.class),
new ActionHandler<>(UnfollowIndexAction.INSTANCE, UnfollowIndexAction.TransportAction.class));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ public static class Request extends ActionRequest implements ToXContentObject {
followerIndex = (String) args[1];
}
return new Request((String) args[0], followerIndex, (Integer) args[2], (Integer) args[3], (Long) args[4],
(Integer) args[5], (Integer) args[6], (TimeValue) args[7], (TimeValue) args[8]);
(Integer) args[5], (Integer) args[6], (TimeValue) args[7]);
});

static {
Expand All @@ -99,9 +99,6 @@ public static class Request extends ActionRequest implements ToXContentObject {
PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(),
(p, c) -> TimeValue.parseTimeValue(p.text(), ShardFollowTask.RETRY_TIMEOUT.getPreferredName()),
ShardFollowTask.RETRY_TIMEOUT, ObjectParser.ValueType.STRING);
PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(),
(p, c) -> TimeValue.parseTimeValue(p.text(), ShardFollowTask.IDLE_SHARD_RETRY_DELAY.getPreferredName()),
ShardFollowTask.IDLE_SHARD_RETRY_DELAY, ObjectParser.ValueType.STRING);
}

public static Request fromXContent(XContentParser parser, String followerIndex) throws IOException {
Expand All @@ -126,11 +123,10 @@ public static Request fromXContent(XContentParser parser, String followerIndex)
private int maxConcurrentWriteBatches;
private int maxWriteBufferSize;
private TimeValue retryTimeout;
private TimeValue idleShardRetryDelay;

public Request(String leaderIndex, String followerIndex, Integer maxBatchOperationCount, Integer maxConcurrentReadBatches,
Long maxOperationSizeInBytes, Integer maxConcurrentWriteBatches, Integer maxWriteBufferSize,
TimeValue retryTimeout, TimeValue idleShardRetryDelay) {
TimeValue retryTimeout) {
if (leaderIndex == null) {
throw new IllegalArgumentException("leader_index is missing");
}
Expand All @@ -155,10 +151,6 @@ public Request(String leaderIndex, String followerIndex, Integer maxBatchOperati
if (retryTimeout == null) {
retryTimeout = ShardFollowNodeTask.DEFAULT_RETRY_TIMEOUT;
}
if (idleShardRetryDelay == null) {
idleShardRetryDelay = ShardFollowNodeTask.DEFAULT_IDLE_SHARD_RETRY_DELAY;
}

if (maxBatchOperationCount < 1) {
throw new IllegalArgumentException("maxBatchOperationCount must be larger than 0");
}
Expand All @@ -183,7 +175,6 @@ public Request(String leaderIndex, String followerIndex, Integer maxBatchOperati
this.maxConcurrentWriteBatches = maxConcurrentWriteBatches;
this.maxWriteBufferSize = maxWriteBufferSize;
this.retryTimeout = retryTimeout;
this.idleShardRetryDelay = idleShardRetryDelay;
}

Request() {
Expand Down Expand Up @@ -217,7 +208,6 @@ public void readFrom(StreamInput in) throws IOException {
maxConcurrentWriteBatches = in.readVInt();
maxWriteBufferSize = in.readVInt();
retryTimeout = in.readOptionalTimeValue();
idleShardRetryDelay = in.readOptionalTimeValue();
}

@Override
Expand All @@ -231,7 +221,6 @@ public void writeTo(StreamOutput out) throws IOException {
out.writeVInt(maxConcurrentWriteBatches);
out.writeVInt(maxWriteBufferSize);
out.writeOptionalTimeValue(retryTimeout);
out.writeOptionalTimeValue(idleShardRetryDelay);
}

@Override
Expand All @@ -246,7 +235,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
builder.field(ShardFollowTask.MAX_CONCURRENT_READ_BATCHES.getPreferredName(), maxConcurrentReadBatches);
builder.field(ShardFollowTask.MAX_CONCURRENT_WRITE_BATCHES.getPreferredName(), maxConcurrentWriteBatches);
builder.field(ShardFollowTask.RETRY_TIMEOUT.getPreferredName(), retryTimeout.getStringRep());
builder.field(ShardFollowTask.IDLE_SHARD_RETRY_DELAY.getPreferredName(), idleShardRetryDelay.getStringRep());
}
builder.endObject();
return builder;
Expand All @@ -263,15 +251,14 @@ public boolean equals(Object o) {
maxConcurrentWriteBatches == request.maxConcurrentWriteBatches &&
maxWriteBufferSize == request.maxWriteBufferSize &&
Objects.equals(retryTimeout, request.retryTimeout) &&
Objects.equals(idleShardRetryDelay, request.idleShardRetryDelay) &&
Objects.equals(leaderIndex, request.leaderIndex) &&
Objects.equals(followerIndex, request.followerIndex);
}

@Override
public int hashCode() {
return Objects.hash(leaderIndex, followerIndex, maxBatchOperationCount, maxConcurrentReadBatches, maxOperationSizeInBytes,
maxConcurrentWriteBatches, maxWriteBufferSize, retryTimeout, idleShardRetryDelay);
maxConcurrentWriteBatches, maxWriteBufferSize, retryTimeout);
}
}

Expand Down Expand Up @@ -372,7 +359,7 @@ void start(Request request, String clusterNameAlias, IndexMetaData leaderIndexMe
new ShardId(leaderIndexMetadata.getIndex(), shardId),
request.maxBatchOperationCount, request.maxConcurrentReadBatches, request.maxOperationSizeInBytes,
request.maxConcurrentWriteBatches, request.maxWriteBufferSize, request.retryTimeout,
request.idleShardRetryDelay, filteredHeaders);
filteredHeaders);
persistentTasksService.sendStartRequest(taskId, ShardFollowTask.NAME, shardFollowTask,
new ActionListener<PersistentTasksCustomMetaData.PersistentTask<ShardFollowTask>>() {
@Override
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,199 @@
package org.elasticsearch.xpack.ccr.action;

import org.elasticsearch.action.Action;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.action.support.ActionFilters;
import org.elasticsearch.action.support.single.shard.SingleShardRequest;
import org.elasticsearch.action.support.single.shard.TransportSingleShardAction;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.routing.ShardsIterator;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.shard.IndexShard;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportService;

import java.io.IOException;

import static org.elasticsearch.action.ValidateActions.addValidationError;
import static org.elasticsearch.index.seqno.SequenceNumbers.NO_OPS_PERFORMED;
import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO;

public class GlobalCheckpointPollAction extends Action<GlobalCheckpointPollAction.Response> {

public static final GlobalCheckpointPollAction INSTANCE = new GlobalCheckpointPollAction();
public static final String NAME = "indices:data/read/xpack/ccr/global_checkpoint_poll";

protected GlobalCheckpointPollAction() {
super(NAME);
}

public static class Request extends SingleShardRequest<Request> {

private long globalCheckpoint = UNASSIGNED_SEQ_NO;

public long getGlobalCheckpoint() {
return globalCheckpoint;
}

public void setGlobalCheckpoint(long globalCheckpoint) {
this.globalCheckpoint = globalCheckpoint;
}

private ShardId shardId;

public ShardId getShardId() {
return shardId;
}

public Request() {

}

public Request(final ShardId shardId) {
super(shardId.getIndexName());
this.shardId = shardId;
}

@Override
public ActionRequestValidationException validate() {
ActionRequestValidationException e = null;
if (globalCheckpoint < NO_OPS_PERFORMED) {
e = addValidationError("global checkpoint [" + globalCheckpoint + "] must be at least -1", e);
}
if (shardId == null) {
e = addValidationError("shard ID must be non-null", e);
}
return e;
}

@Override
public void readFrom(final StreamInput in) throws IOException {
super.readFrom(in);
globalCheckpoint = in.readZLong();
shardId = ShardId.readShardId(in);
}

@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeZLong(globalCheckpoint);
shardId.writeTo(out);
}

}

public static class Response extends ActionResponse {

private long globalCheckpoint;

public long getGlobalCheckpoint() {
return globalCheckpoint;
}

public Response() {

}

public Response(final long globalCheckpoint) {
this.globalCheckpoint = globalCheckpoint;
}

@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
globalCheckpoint = in.readZLong();
}

@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeZLong(globalCheckpoint);
}

}

@Override
public Response newResponse() {
return new Response();
}

public static class TransportGlobalCheckpointPollAction extends TransportSingleShardAction<Request, Response> {

private final IndicesService indicesService;

@Inject
public TransportGlobalCheckpointPollAction(
final Settings settings,
final ThreadPool threadPool,
final ClusterService clusterService,
final TransportService transportService,
final ActionFilters actionFilters,
final IndexNameExpressionResolver indexNameExpressionResolver,
final IndicesService indicesService) {
super(
settings,
NAME,
threadPool,
clusterService,
transportService,
actionFilters,
indexNameExpressionResolver,
Request::new,
ThreadPool.Names.MANAGEMENT);
this.indicesService = indicesService;
}

@Override
protected void asyncShardOperation(
final Request request, final ShardId shardId, final ActionListener<Response> listener) throws IOException {
final IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
final IndexShard indexShard = indexService.getShard(shardId.id());
indexShard.addGlobalCheckpointListener(
request.getGlobalCheckpoint(),
(g, e) -> {
if (g != UNASSIGNED_SEQ_NO) {
assert g >= NO_OPS_PERFORMED && e == null;
listener.onResponse(new Response(g));
} else {
assert e != null;
listener.onFailure(e);
}
});
}

@Override
protected Response shardOperation(final Request request, final ShardId shardId) throws IOException {
throw new UnsupportedOperationException();
}

@Override
protected Response newResponse() {
return new Response();
}

@Override
protected boolean resolveIndex(final Request request) {
return false;
}

@Override
protected ShardsIterator shards(final ClusterState state, final InternalRequest request) {
return state
.routingTable()
.shardRoutingTable(request.concreteIndex(), request.request().getShardId().id())
.activeInitializingShardsRandomIt();
}

}

}
Loading

0 comments on commit 80526b2

Please sign in to comment.