Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add index commit id to searcher #63963

Merged
merged 18 commits into from
Dec 12, 2020
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,11 @@
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.util.iterable.Iterables;
import org.elasticsearch.common.util.set.Sets;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.engine.Engine;
import org.elasticsearch.index.shard.IndexShard;
import org.elasticsearch.indices.IndexClosedException;
import org.elasticsearch.indices.IndicesService;
Expand Down Expand Up @@ -472,6 +474,34 @@ public void testResyncPropagatePrimaryTerm() throws Exception {
}
}

public void testCommitIdInSearcher() throws Exception {
final String indexName = "test_commit_id";
createIndex(indexName, Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
.build());
indexRandom(randomBoolean(), randomBoolean(), randomBoolean(), IntStream.range(0, randomIntBetween(0, 50))
.mapToObj(n -> client().prepareIndex(indexName).setSource("num", n)).collect(toList()));
ensureGreen(indexName);
assertAcked(client().admin().indices().prepareClose(indexName));
assertIndexIsClosed(indexName);
ensureGreen(indexName);
final String nodeWithPrimary = Iterables.get(internalCluster().nodesInclude(indexName), 0);
IndexShard shard = internalCluster().getInstance(IndicesService.class, nodeWithPrimary)
.indexService(resolveIndex(indexName)).getShard(0);
final String commitId;
try (Engine.SearcherSupplier searcherSupplier = shard.acquireSearcherSupplier(randomFrom(Engine.SearcherScope.values()))) {
assertNotNull(searcherSupplier.getCommitId());
commitId = searcherSupplier.getCommitId();
}
internalCluster().restartNode(nodeWithPrimary);
ensureGreen(indexName);
shard = internalCluster().getInstance(IndicesService.class, nodeWithPrimary).indexService(resolveIndex(indexName)).getShard(0);
try (Engine.SearcherSupplier searcherSupplier = shard.acquireSearcherSupplier(randomFrom(Engine.SearcherScope.values()))) {
assertThat(searcherSupplier.getCommitId(), equalTo(commitId));
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could we also set replicas=1 and then check that the recovered copy has the same commit id?

}
}

private static void closeIndices(final String... indices) {
closeIndices(client().admin().indices().prepareClose(indices));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1205,6 +1205,15 @@ public final void close() {
protected abstract void doClose();

protected abstract Searcher acquireSearcherInternal(String source);

/**
* Returns a commit id associated with this searcher if it's opened from an index commit; otherwise, return null.
* Two searcher with the same commit id must have identical content at the Lucene document level.
dnhatn marked this conversation as resolved.
Show resolved Hide resolved
*/
@Nullable
public String getCommitId() {
return null;
}
}

public static final class Searcher extends IndexSearcher implements Releasable {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import org.apache.lucene.search.ReferenceManager;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.Lock;
import org.apache.lucene.util.StringHelper;
import org.elasticsearch.Version;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
Expand Down Expand Up @@ -76,6 +77,7 @@ public class ReadOnlyEngine extends Engine {
private final boolean requireCompleteHistory;

protected volatile TranslogStats translogStats;
protected final String commitId;

/**
* Creates a new ReadOnlyEngine. This ctor can also be used to open a read-only engine on top of an already opened
Expand Down Expand Up @@ -107,6 +109,7 @@ public ReadOnlyEngine(EngineConfig config, SeqNoStats seqNoStats, TranslogStats
// yet this makes sure nobody else does. including some testing tools that try to be messy
indexWriterLock = obtainLock ? directory.obtainLock(IndexWriter.WRITE_LOCK_NAME) : null;
this.lastCommittedSegmentInfos = Lucene.readSegmentInfos(directory);
this.commitId = StringHelper.idToString(this.lastCommittedSegmentInfos.getId());
dnhatn marked this conversation as resolved.
Show resolved Hide resolved
if (seqNoStats == null) {
seqNoStats = buildSeqNoStats(config, lastCommittedSegmentInfos);
ensureMaxSeqNoEqualsToGlobalCheckpoint(seqNoStats);
Expand Down Expand Up @@ -497,4 +500,25 @@ protected static DirectoryReader openDirectory(Directory directory) throws IOExc
public CompletionStats completionStats(String... fieldNamePatterns) {
return completionStatsCache.get(fieldNamePatterns);
}

@Override
public SearcherSupplier acquireSearcherSupplier(Function<Searcher, Searcher> wrapper, SearcherScope scope) throws EngineException {
final SearcherSupplier delegate = super.acquireSearcherSupplier(wrapper, scope);
return new SearcherSupplier(Function.identity()) {
@Override
protected void doClose() {
delegate.close();
}

@Override
protected Searcher acquireSearcherInternal(String source) {
return delegate.acquireSearcherInternal(source);
}

@Override
public String getCommitId() {
return commitId;
}
};
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,11 @@ public Searcher acquireSearcherInternal(String source) {
protected void doClose() {
store.decRef();
}

@Override
public String getCommitId() {
return commitId;
}
};
}

Expand Down