Skip to content

Commit

Permalink
Add min_count and max_count as SLM retention predicates
Browse files Browse the repository at this point in the history 
This adds the configuration options for `min_count` and `max_count` as
well as the logic for determining whether a snapshot meets this criteria
to SLM's retention feature.

These options are optional and one, two, or all three can be specified
in an SLM policy.

Relates to elastic#43663
  • Loading branch information
@dakrone
dakrone committed Jul 26, 2019
1 parent 73c5287 commit 5e7f1c5
Show file tree
Hide file tree
Showing 7 changed files with 305 additions and 29 deletions.
50 changes: 44 additions & 6 deletions .../main/java/org/elasticsearch/client/snapshotlifecycle/SnapshotRetentionConfiguration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

package org.elasticsearch.client.snapshotlifecycle;

import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.unit.TimeValue;
Expand All @@ -32,25 +33,46 @@

public class SnapshotRetentionConfiguration implements ToXContentObject {

public static final SnapshotRetentionConfiguration EMPTY = new SnapshotRetentionConfiguration((TimeValue) null);
public static final SnapshotRetentionConfiguration EMPTY = new SnapshotRetentionConfiguration(null, null, null);

private static final ParseField EXPIRE_AFTER = new ParseField("expire_after");
private static final ParseField MINIMUM_SNAPSHOT_COUNT = new ParseField("min_count");
private static final ParseField MAXIMUM_SNAPSHOT_COUNT = new ParseField("max_count");

private static final ConstructingObjectParser<SnapshotRetentionConfiguration, Void> PARSER =
new ConstructingObjectParser<>("snapshot_retention", true, a -> {
TimeValue expireAfter = a[0] == null ? null : TimeValue.parseTimeValue((String) a[0], EXPIRE_AFTER.getPreferredName());
return new SnapshotRetentionConfiguration(expireAfter);
Integer minCount = (Integer) a[1];
Integer maxCount = (Integer) a[2];
return new SnapshotRetentionConfiguration(expireAfter, minCount, maxCount);
});

static {
PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), EXPIRE_AFTER);
PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), MINIMUM_SNAPSHOT_COUNT);
PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), MAXIMUM_SNAPSHOT_COUNT);
}

// TODO: add the rest of the configuration values
private final TimeValue expireAfter;
private final Integer minimumSnapshotCount;
private final Integer maximumSnapshotCount;

public SnapshotRetentionConfiguration(TimeValue expireAfter) {
public SnapshotRetentionConfiguration(@Nullable TimeValue expireAfter,
@Nullable Integer minimumSnapshotCount,
@Nullable Integer maximumSnapshotCount) {
this.expireAfter = expireAfter;
this.minimumSnapshotCount = minimumSnapshotCount;
this.maximumSnapshotCount = maximumSnapshotCount;
if (this.minimumSnapshotCount != null && this.minimumSnapshotCount < 1) {
throw new IllegalArgumentException("minimum snapshot count must be at least 1, but was: " + this.minimumSnapshotCount);
}
if (this.maximumSnapshotCount != null && this.maximumSnapshotCount < 1) {
throw new IllegalArgumentException("maximum snapshot count must be at least 1, but was: " + this.maximumSnapshotCount);
}
if ((maximumSnapshotCount != null && minimumSnapshotCount != null) && this.minimumSnapshotCount > this.maximumSnapshotCount) {
throw new IllegalArgumentException("minimum snapshot count " + this.minimumSnapshotCount +
" cannot be larger than maximum snapshot count " + this.maximumSnapshotCount);
}
}

public static SnapshotRetentionConfiguration parse(XContentParser parser, String name) {
Expand All @@ -61,19 +83,33 @@ public TimeValue getExpireAfter() {
return this.expireAfter;
}

public Integer getMinimumSnapshotCount() {
return this.minimumSnapshotCount;
}

public Integer getMaximumSnapshotCount() {
return this.maximumSnapshotCount;
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
if (expireAfter != null) {
builder.field(EXPIRE_AFTER.getPreferredName(), expireAfter.getStringRep());
}
if (minimumSnapshotCount != null) {
builder.field(MINIMUM_SNAPSHOT_COUNT.getPreferredName(), minimumSnapshotCount);
}
if (maximumSnapshotCount != null) {
builder.field(MAXIMUM_SNAPSHOT_COUNT.getPreferredName(), maximumSnapshotCount);
}
builder.endObject();
return builder;
}

@Override
public int hashCode() {
return Objects.hash(expireAfter);
return Objects.hash(expireAfter, minimumSnapshotCount, maximumSnapshotCount);
}

@Override
Expand All @@ -85,7 +121,9 @@ public boolean equals(Object obj) {
return false;
}
SnapshotRetentionConfiguration other = (SnapshotRetentionConfiguration) obj;
return Objects.equals(this.expireAfter, other.expireAfter);
return Objects.equals(this.expireAfter, other.expireAfter) &&
Objects.equals(minimumSnapshotCount, other.minimumSnapshotCount) &&
Objects.equals(maximumSnapshotCount, other.maximumSnapshotCount);
}

@Override
Expand Down
2 changes: 1 addition & 1 deletion ...t-high-level/src/test/java/org/elasticsearch/client/documentation/ILMDocumentationIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -775,7 +775,7 @@ public void testAddSnapshotLifecyclePolicy() throws Exception {
Map<String, Object> config = new HashMap<>();
config.put("indices", Collections.singletonList("idx"));
SnapshotRetentionConfiguration retention =
new SnapshotRetentionConfiguration(TimeValue.timeValueDays(30));
new SnapshotRetentionConfiguration(TimeValue.timeValueDays(30), 2, 10);
SnapshotLifecyclePolicy policy = new SnapshotLifecyclePolicy(
"policy_id", "name", "1 2 3 * * ?",
"my_repository", config, retention);
Expand Down
157 changes: 142 additions & 15 deletions ...n/java/org/elasticsearch/xpack/core/snapshotlifecycle/SnapshotRetentionConfiguration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@

package org.elasticsearch.xpack.core.snapshotlifecycle;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.Strings;
Expand All @@ -20,35 +22,73 @@
import org.elasticsearch.snapshots.SnapshotInfo;

import java.io.IOException;
import java.util.Comparator;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.function.LongSupplier;
import java.util.function.Predicate;
import java.util.stream.Collectors;

public class SnapshotRetentionConfiguration implements ToXContentObject, Writeable {

public static final SnapshotRetentionConfiguration EMPTY = new SnapshotRetentionConfiguration((TimeValue) null);
public static final SnapshotRetentionConfiguration EMPTY = new SnapshotRetentionConfiguration(null, null, null);

private static final ParseField EXPIRE_AFTER = new ParseField("expire_after");
private static final ParseField MINIMUM_SNAPSHOT_COUNT = new ParseField("min_count");
private static final ParseField MAXIMUM_SNAPSHOT_COUNT = new ParseField("max_count");
private static final Logger logger = LogManager.getLogger(SnapshotRetentionConfiguration.class);

private static final ConstructingObjectParser<SnapshotRetentionConfiguration, Void> PARSER =
new ConstructingObjectParser<>("snapshot_retention", true, a -> {
TimeValue expireAfter = a[0] == null ? null : TimeValue.parseTimeValue((String) a[0], EXPIRE_AFTER.getPreferredName());
return new SnapshotRetentionConfiguration(expireAfter);
Integer minCount = (Integer) a[1];
Integer maxCount = (Integer) a[2];
return new SnapshotRetentionConfiguration(expireAfter, minCount, maxCount);
});

static {
PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), EXPIRE_AFTER);
PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), MINIMUM_SNAPSHOT_COUNT);
PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), MAXIMUM_SNAPSHOT_COUNT);
}

// TODO: add the rest of the configuration values
private final LongSupplier nowSupplier;
private final TimeValue expireAfter;

public SnapshotRetentionConfiguration(@Nullable TimeValue expireAfter) {
this.expireAfter = expireAfter;
}
private final Integer minimumSnapshotCount;
private final Integer maximumSnapshotCount;

SnapshotRetentionConfiguration(StreamInput in) throws IOException {
nowSupplier = System::currentTimeMillis;
this.expireAfter = in.readOptionalTimeValue();
this.minimumSnapshotCount = in.readOptionalVInt();
this.maximumSnapshotCount = in.readOptionalVInt();
}

public SnapshotRetentionConfiguration(@Nullable TimeValue expireAfter,
@Nullable Integer minimumSnapshotCount,
@Nullable Integer maximumSnapshotCount) {
this(System::currentTimeMillis, expireAfter, minimumSnapshotCount, maximumSnapshotCount);
}

public SnapshotRetentionConfiguration(LongSupplier nowSupplier,
@Nullable TimeValue expireAfter,
@Nullable Integer minimumSnapshotCount,
@Nullable Integer maximumSnapshotCount) {
this.nowSupplier = nowSupplier;
this.expireAfter = expireAfter;
this.minimumSnapshotCount = minimumSnapshotCount;
this.maximumSnapshotCount = maximumSnapshotCount;
if (this.minimumSnapshotCount != null && this.minimumSnapshotCount < 1) {
throw new IllegalArgumentException("minimum snapshot count must be at least 1, but was: " + this.minimumSnapshotCount);
}
if (this.maximumSnapshotCount != null && this.maximumSnapshotCount < 1) {
throw new IllegalArgumentException("maximum snapshot count must be at least 1, but was: " + this.maximumSnapshotCount);
}
if ((maximumSnapshotCount != null && minimumSnapshotCount != null) && this.minimumSnapshotCount > this.maximumSnapshotCount) {
throw new IllegalArgumentException("minimum snapshot count " + this.minimumSnapshotCount +
" cannot be larger than maximum snapshot count " + this.maximumSnapshotCount);
}
}

public static SnapshotRetentionConfiguration parse(XContentParser parser, String name) {
Expand All @@ -59,44 +99,129 @@ public TimeValue getExpireAfter() {
return this.expireAfter;
}

public Integer getMinimumSnapshotCount() {
return this.minimumSnapshotCount;
}

public Integer getMaximumSnapshotCount() {
return this.maximumSnapshotCount;
}

/**
* Return a predicate by which a SnapshotInfo can be tested to see
* whether it should be deleted according to this retention policy.
* @param allSnapshots a list of all snapshot pertaining to this SLM policy and repository
*/
public Predicate<SnapshotInfo> getSnapshotDeletionPredicate(final List<SnapshotInfo> allSnapshots) {
final int snapCount = allSnapshots.size();
List<SnapshotInfo> sortedSnapshots = allSnapshots.stream()
.sorted(Comparator.comparingLong(SnapshotInfo::startTime))
.collect(Collectors.toList());

return si -> {
final String snapName = si.snapshotId().getName();

// First, enforce the maximum count, if the size is over the maximum number of
// snapshots, then allow the oldest N (where N is the number over the maximum snapshot
// count) snapshots to be eligible for deletion
if (this.maximumSnapshotCount != null) {
if (allSnapshots.size() > this.maximumSnapshotCount) {
int snapsToDelete = allSnapshots.size() - this.maximumSnapshotCount;
boolean eligible = sortedSnapshots.stream()
.limit(snapsToDelete)
.anyMatch(s -> s.equals(si));

if (eligible) {
logger.trace("[{}]: ELIGIBLE as it is one of the {} oldest snapshots with " +
"{} total snapshots, over the limit of {} maximum snapshots",
snapName, snapsToDelete, snapCount, this.maximumSnapshotCount);
return true;
} else {
logger.trace("[{}]: INELIGIBLE as it is not one of the {} oldest snapshots with " +
"{} total snapshots, over the limit of {} maximum snapshots",
snapName, snapsToDelete, snapCount, this.maximumSnapshotCount);
return false;
}
}
}

// Next check the minimum count, since that is a blanket requirement regardless of time,
// if we haven't hit the minimum then we need to keep the snapshot regardless of
// expiration time
if (this.minimumSnapshotCount != null) {
if (allSnapshots.size() <= this.minimumSnapshotCount) {
logger.trace("[{}]: INELIGIBLE as there are {} snapshots and {} minimum snapshots needed",
snapName, snapCount, this.minimumSnapshotCount);
return false;
}
}

// Finally, check the expiration time of the snapshot, if it is past, then it is
// eligible for deletion
if (this.expireAfter != null) {
TimeValue snapshotAge = new TimeValue(System.currentTimeMillis() - si.startTime());
TimeValue snapshotAge = new TimeValue(nowSupplier.getAsLong() - si.startTime());

if (this.minimumSnapshotCount != null) {
int eligibleForExpiration = snapCount - minimumSnapshotCount;

// Only the oldest N snapshots are actually eligible, since if we went below this we
// would fall below the configured minimum number of snapshots to keep
Set<SnapshotInfo> snapsEligibleForExpiration = sortedSnapshots.stream()
.limit(eligibleForExpiration)
.collect(Collectors.toSet());

if (snapsEligibleForExpiration.contains(si) == false) {
// This snapshot is *not* one of the N oldest snapshots, so even if it were
// old enough, the other snapshots would be deleted before it
logger.trace("[{}]: INELIGIBLE as snapshot expiration would pass the " +
"minimum number of configured snapshots ({}) to keep, regardless of age",
snapName, this.minimumSnapshotCount);
return false;
}
}

if (snapshotAge.compareTo(this.expireAfter) > 0) {
logger.trace("[{}]: ELIGIBLE as snapshot age of {} is older than {}",
snapName, snapshotAge.toHumanReadableString(3), this.expireAfter.toHumanReadableString(3));
return true;
} else {
logger.trace("[{}]: INELIGIBLE as snapshot age of {} is newer than {}",
snapName, snapshotAge.toHumanReadableString(3), this.expireAfter.toHumanReadableString(3));
return false;
}
}
// If nothing matched, the snapshot is not eligible for deletion
logger.trace("[{}]: INELIGIBLE as no retention predicates matched", snapName);
return false;
};
}

@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeOptionalTimeValue(this.expireAfter);
out.writeOptionalVInt(this.minimumSnapshotCount);
out.writeOptionalVInt(this.maximumSnapshotCount);
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
if (expireAfter != null) {
builder.field(EXPIRE_AFTER.getPreferredName(), expireAfter.getStringRep());
}
if (minimumSnapshotCount != null) {
builder.field(MINIMUM_SNAPSHOT_COUNT.getPreferredName(), minimumSnapshotCount);
}
if (maximumSnapshotCount != null) {
builder.field(MAXIMUM_SNAPSHOT_COUNT.getPreferredName(), maximumSnapshotCount);
}
builder.endObject();
return builder;
}

@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeOptionalTimeValue(this.expireAfter);
}

@Override
public int hashCode() {
return Objects.hash(expireAfter);
return Objects.hash(expireAfter, minimumSnapshotCount, maximumSnapshotCount);
}

@Override
Expand All @@ -108,7 +233,9 @@ public boolean equals(Object obj) {
return false;
}
SnapshotRetentionConfiguration other = (SnapshotRetentionConfiguration) obj;
return Objects.equals(this.expireAfter, other.expireAfter);
return Objects.equals(this.expireAfter, other.expireAfter) &&
Objects.equals(minimumSnapshotCount, other.minimumSnapshotCount) &&
Objects.equals(maximumSnapshotCount, other.maximumSnapshotCount);
}

@Override
Expand Down
6 changes: 4 additions & 2 deletions .../org/elasticsearch/xpack/core/snapshotlifecycle/SnapshotLifecyclePolicyMetadataTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,8 +111,10 @@ public static SnapshotLifecyclePolicy randomSnapshotLifecyclePolicy(String polic
}

public static SnapshotRetentionConfiguration randomRetention() {
return new SnapshotRetentionConfiguration(rarely() ? null :
TimeValue.parseTimeValue(randomTimeValue(), "random retention generation"));
return new SnapshotRetentionConfiguration(
rarely() ? null : TimeValue.parseTimeValue(randomTimeValue(), "random retention generation"),
rarely() ? null : randomIntBetween(1, 10),
rarely() ? null : randomIntBetween(15, 30));
}

public static String randomSchedule() {
Expand Down
Loading

0 comments on commit 5e7f1c5

Please sign in to comment.