Skip to content

Commit

Permalink
Add SLM support to xpack usage and info APIs (#48096)
Browse files Browse the repository at this point in the history
* Add SLM support to xpack usage and info APIs

This adds the missing xpack usage and info information into the
`/_xpack` and `/_xpack/usage` APIs. The output now looks like:

```
GET /_xpack/usage
{
  ...
  "slm" : {
    "available" : true,
    "enabled" : true,
    "policy_count" : 1,
    "policy_stats" : {
      "retention_runs" : 0,
      ...
    }
  }
```

and

```
GET /_xpack
{
  ...
  "features" : {
    ...
    "slm" : {
      "available" : true,
      "enabled" : true
    },
    ...
  }
}
```

Relates to #43663

* Fix test expectation

* Fix docs test
  • Loading branch information
dakrone authored Oct 16, 2019
1 parent 1aaea69 commit f9511f4
Show file tree
Hide file tree
Showing 10 changed files with 270 additions and 15 deletions.
4 changes: 4 additions & 0 deletions docs/reference/rest-api/info.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,10 @@ Example response:
"available" : true,
"enabled" : false
},
"slm" : {
"available" : true,
"enabled" : true
},
"spatial" : {
"available" : true,
"enabled" : true
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,9 @@
import org.elasticsearch.xpack.ccr.CCRInfoTransportAction;
import org.elasticsearch.xpack.core.action.XPackInfoAction;
import org.elasticsearch.xpack.core.action.XPackUsageAction;
import org.elasticsearch.xpack.core.analytics.AnalyticsFeatureSetUsage;
import org.elasticsearch.xpack.core.beats.BeatsFeatureSetUsage;
import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata;
import org.elasticsearch.xpack.core.analytics.AnalyticsFeatureSetUsage;
import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction;
import org.elasticsearch.xpack.core.flattened.FlattenedFeatureSetUsage;
import org.elasticsearch.xpack.core.frozen.FrozenIndicesFeatureSetUsage;
Expand Down Expand Up @@ -139,12 +139,12 @@
import org.elasticsearch.xpack.core.ml.dataframe.evaluation.softclassification.Recall;
import org.elasticsearch.xpack.core.ml.dataframe.evaluation.softclassification.ScoreByThresholdResult;
import org.elasticsearch.xpack.core.ml.dataframe.evaluation.softclassification.SoftClassificationMetric;
import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TrainedModel;
import org.elasticsearch.xpack.core.ml.inference.trainedmodel.tree.Tree;
import org.elasticsearch.xpack.core.ml.inference.preprocessing.FrequencyEncoding;
import org.elasticsearch.xpack.core.ml.inference.preprocessing.OneHotEncoding;
import org.elasticsearch.xpack.core.ml.inference.preprocessing.PreProcessor;
import org.elasticsearch.xpack.core.ml.inference.preprocessing.TargetMeanEncoding;
import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TrainedModel;
import org.elasticsearch.xpack.core.ml.inference.trainedmodel.tree.Tree;
import org.elasticsearch.xpack.core.ml.job.config.JobTaskState;
import org.elasticsearch.xpack.core.monitoring.MonitoringFeatureSetUsage;
import org.elasticsearch.xpack.core.rollup.RollupFeatureSetUsage;
Expand Down Expand Up @@ -188,6 +188,7 @@
import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.RoleMapperExpression;
import org.elasticsearch.xpack.core.security.authz.privilege.ConfigurableClusterPrivilege;
import org.elasticsearch.xpack.core.security.authz.privilege.ConfigurableClusterPrivileges;
import org.elasticsearch.xpack.core.slm.SLMFeatureSetUsage;
import org.elasticsearch.xpack.core.slm.SnapshotLifecycleMetadata;
import org.elasticsearch.xpack.core.slm.action.DeleteSnapshotLifecycleAction;
import org.elasticsearch.xpack.core.slm.action.ExecuteSnapshotLifecycleAction;
Expand All @@ -206,10 +207,10 @@
import org.elasticsearch.xpack.core.transform.action.PutTransformAction;
import org.elasticsearch.xpack.core.transform.action.StartTransformAction;
import org.elasticsearch.xpack.core.transform.action.StopTransformAction;
import org.elasticsearch.xpack.core.transform.transforms.TransformTaskParams;
import org.elasticsearch.xpack.core.transform.transforms.TransformState;
import org.elasticsearch.xpack.core.transform.transforms.SyncConfig;
import org.elasticsearch.xpack.core.transform.transforms.TimeSyncConfig;
import org.elasticsearch.xpack.core.transform.transforms.TransformState;
import org.elasticsearch.xpack.core.transform.transforms.TransformTaskParams;
import org.elasticsearch.xpack.core.upgrade.actions.IndexUpgradeAction;
import org.elasticsearch.xpack.core.upgrade.actions.IndexUpgradeInfoAction;
import org.elasticsearch.xpack.core.vectors.VectorsFeatureSetUsage;
Expand Down Expand Up @@ -490,6 +491,9 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
// ILM
new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.INDEX_LIFECYCLE,
IndexLifecycleFeatureSetUsage::new),
// SLM
new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.SNAPSHOT_LIFECYCLE,
SLMFeatureSetUsage::new),
// ILM - Custom Metadata
new NamedWriteableRegistry.Entry(MetaData.Custom.class, IndexLifecycleMetadata.TYPE, IndexLifecycleMetadata::new),
new NamedWriteableRegistry.Entry(NamedDiff.class, IndexLifecycleMetadata.TYPE,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ public final class XPackField {
public static final String ROLLUP = "rollup";
/** Name constant for the index lifecycle feature. */
public static final String INDEX_LIFECYCLE = "ilm";
/** Name constant for the snapshot lifecycle management feature. */
public static final String SNAPSHOT_LIFECYCLE = "slm";
/** Name constant for the CCR feature. */
public static final String CCR = "ccr";
/** Name constant for the transform feature. */
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ public class XPackInfoFeatureAction extends ActionType<XPackInfoFeatureResponse>
public static final XPackInfoFeatureAction SQL = new XPackInfoFeatureAction(XPackField.SQL);
public static final XPackInfoFeatureAction ROLLUP = new XPackInfoFeatureAction(XPackField.ROLLUP);
public static final XPackInfoFeatureAction INDEX_LIFECYCLE = new XPackInfoFeatureAction(XPackField.INDEX_LIFECYCLE);
public static final XPackInfoFeatureAction SNAPSHOT_LIFECYCLE = new XPackInfoFeatureAction(XPackField.SNAPSHOT_LIFECYCLE);
public static final XPackInfoFeatureAction CCR = new XPackInfoFeatureAction(XPackField.CCR);
public static final XPackInfoFeatureAction TRANSFORM = new XPackInfoFeatureAction(XPackField.TRANSFORM);
public static final XPackInfoFeatureAction FLATTENED = new XPackInfoFeatureAction(XPackField.FLATTENED);
Expand All @@ -41,8 +42,8 @@ public class XPackInfoFeatureAction extends ActionType<XPackInfoFeatureResponse>
public static final XPackInfoFeatureAction ANALYTICS = new XPackInfoFeatureAction(XPackField.ANALYTICS);

public static final List<XPackInfoFeatureAction> ALL = Arrays.asList(
SECURITY, MONITORING, WATCHER, GRAPH, MACHINE_LEARNING, LOGSTASH, SQL, ROLLUP, INDEX_LIFECYCLE, CCR, TRANSFORM, FLATTENED,
VECTORS, VOTING_ONLY, FROZEN_INDICES, SPATIAL, ANALYTICS
SECURITY, MONITORING, WATCHER, GRAPH, MACHINE_LEARNING, LOGSTASH, SQL, ROLLUP, INDEX_LIFECYCLE, SNAPSHOT_LIFECYCLE, CCR,
TRANSFORM, FLATTENED, VECTORS, VOTING_ONLY, FROZEN_INDICES, SPATIAL, ANALYTICS
);

private XPackInfoFeatureAction(String name) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ public class XPackUsageFeatureAction extends ActionType<XPackUsageFeatureRespons
public static final XPackUsageFeatureAction SQL = new XPackUsageFeatureAction(XPackField.SQL);
public static final XPackUsageFeatureAction ROLLUP = new XPackUsageFeatureAction(XPackField.ROLLUP);
public static final XPackUsageFeatureAction INDEX_LIFECYCLE = new XPackUsageFeatureAction(XPackField.INDEX_LIFECYCLE);
public static final XPackUsageFeatureAction SNAPSHOT_LIFECYCLE = new XPackUsageFeatureAction(XPackField.SNAPSHOT_LIFECYCLE);
public static final XPackUsageFeatureAction CCR = new XPackUsageFeatureAction(XPackField.CCR);
public static final XPackUsageFeatureAction TRANSFORM = new XPackUsageFeatureAction(XPackField.TRANSFORM);
public static final XPackUsageFeatureAction FLATTENED = new XPackUsageFeatureAction(XPackField.FLATTENED);
Expand All @@ -41,8 +42,8 @@ public class XPackUsageFeatureAction extends ActionType<XPackUsageFeatureRespons
public static final XPackUsageFeatureAction ANALYTICS = new XPackUsageFeatureAction(XPackField.ANALYTICS);

public static final List<XPackUsageFeatureAction> ALL = Arrays.asList(
SECURITY, MONITORING, WATCHER, GRAPH, MACHINE_LEARNING, LOGSTASH, SQL, ROLLUP, INDEX_LIFECYCLE, CCR, TRANSFORM, FLATTENED,
VECTORS, VOTING_ONLY, FROZEN_INDICES, SPATIAL, ANALYTICS
SECURITY, MONITORING, WATCHER, GRAPH, MACHINE_LEARNING, LOGSTASH, SQL, ROLLUP, INDEX_LIFECYCLE, SNAPSHOT_LIFECYCLE, CCR,
TRANSFORM, FLATTENED, VECTORS, VOTING_ONLY, FROZEN_INDICES, SPATIAL, ANALYTICS
);

private XPackUsageFeatureAction(String name) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

package org.elasticsearch.xpack.core.slm;

import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.xpack.core.XPackFeatureSet;
import org.elasticsearch.xpack.core.XPackField;
import org.elasticsearch.xpack.slm.SnapshotLifecycleStats;

import java.io.IOException;
import java.util.Objects;

public class SLMFeatureSetUsage extends XPackFeatureSet.Usage {
@Nullable
private final SnapshotLifecycleStats slmStats;

public SLMFeatureSetUsage(StreamInput in) throws IOException {
super(in);
this.slmStats = in.readOptionalWriteable(SnapshotLifecycleStats::new);
}

@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeOptionalWriteable(this.slmStats);
}

public SLMFeatureSetUsage(boolean available, boolean enabled, @Nullable SnapshotLifecycleStats slmStats) {
super(XPackField.SNAPSHOT_LIFECYCLE, available, enabled);
this.slmStats = slmStats;
}

public SnapshotLifecycleStats getStats() {
return this.slmStats;
}

@Override
protected void innerXContent(XContentBuilder builder, Params params) throws IOException {
super.innerXContent(builder, params);
if (slmStats != null) {
builder.field("policy_count", slmStats.getMetrics().size());
builder.field("policy_stats", slmStats);
}
}

@Override
public int hashCode() {
return Objects.hash(available, enabled, slmStats);
}

@Override
public boolean equals(Object obj) {
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
SLMFeatureSetUsage other = (SLMFeatureSetUsage) obj;
return Objects.equals(available, other.available) &&
Objects.equals(enabled, other.enabled) &&
Objects.equals(slmStats, other.slmStats);
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@
import static org.elasticsearch.xpack.core.slm.history.SnapshotHistoryItem.DELETE_OPERATION;
import static org.elasticsearch.xpack.core.slm.history.SnapshotHistoryStore.SLM_HISTORY_INDEX_PREFIX;
import static org.elasticsearch.xpack.ilm.TimeSeriesLifecycleActionsIT.getStepKeyForIndex;
import static org.hamcrest.Matchers.anyOf;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
Expand Down Expand Up @@ -425,6 +426,76 @@ public void testBasicTimeBasedRetenion() throws Exception {
}
}

@SuppressWarnings("unchecked")
public void testSLMXpackInfo() {
Map<String, Object> features = (Map<String, Object>) getLocation("/_xpack").get("features");
assertNotNull(features);
Map<String, Object> slm = (Map<String, Object>) features.get("slm");
assertNotNull(slm);
assertTrue((boolean) slm.get("available"));
assertTrue((boolean) slm.get("enabled"));
}

@SuppressWarnings("unchecked")
public void testSLMXpackUsage() throws Exception {
Map<String, Object> slm = (Map<String, Object>) getLocation("/_xpack/usage").get("slm");
assertNotNull(slm);
assertTrue((boolean) slm.get("available"));
assertTrue((boolean) slm.get("enabled"));
assertThat(slm.get("policy_count"), anyOf(equalTo(null), equalTo(0)));

// Create a snapshot repo
initializeRepo("repo");
// Create a policy with a retention period of 1 millisecond
createSnapshotPolicy("policy", "snap", "1 2 3 4 5 ?", "repo", "*", true,
new SnapshotRetentionConfiguration(TimeValue.timeValueMillis(1), null, null));
final String snapshotName = executePolicy("policy");

// Check that the executed snapshot is created
assertBusy(() -> {
try {
logger.info("--> checking for snapshot creation...");
Response response = client().performRequest(new Request("GET", "/_snapshot/repo/" + snapshotName));
Map<String, Object> snapshotResponseMap;
try (InputStream is = response.getEntity().getContent()) {
snapshotResponseMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), is, true);
}
assertThat(snapshotResponseMap.size(), greaterThan(0));
} catch (ResponseException e) {
fail("expected snapshot to exist but it does not: " + EntityUtils.toString(e.getResponse().getEntity()));
}
});

// Wait for stats to be updated
assertBusy(() -> {
logger.info("--> checking for stats to be updated...");
Map<String, Object> stats = getSLMStats();
Map<String, Object> policyStats = policyStatsAsMap(stats);
Map<String, Object> policyIdStats = (Map<String, Object>) policyStats.get("policy");
assertNotNull(policyIdStats);
});

slm = (Map<String, Object>) getLocation("/_xpack/usage").get("slm");
assertNotNull(slm);
assertTrue((boolean) slm.get("available"));
assertTrue((boolean) slm.get("enabled"));
assertThat("got: " + slm, slm.get("policy_count"), equalTo(1));
assertNotNull(slm.get("policy_stats"));
}

public Map<String, Object> getLocation(String path) {
try {
Response executeRepsonse = client().performRequest(new Request("GET", path));
try (XContentParser parser = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY,
DeprecationHandler.THROW_UNSUPPORTED_OPERATION, EntityUtils.toByteArray(executeRepsonse.getEntity()))) {
return parser.map();
}
} catch (Exception e) {
fail("failed to execute GET request to " + path + " - got: " + e);
throw new RuntimeException(e);
}
}

/**
* Execute the given policy and return the generated snapshot name
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,8 @@
import org.elasticsearch.xpack.ilm.action.TransportRetryAction;
import org.elasticsearch.xpack.ilm.action.TransportStartILMAction;
import org.elasticsearch.xpack.ilm.action.TransportStopILMAction;
import org.elasticsearch.xpack.slm.SLMInfoTransportAction;
import org.elasticsearch.xpack.slm.SLMUsageTransportAction;
import org.elasticsearch.xpack.slm.SnapshotLifecycleService;
import org.elasticsearch.xpack.slm.SnapshotLifecycleTask;
import org.elasticsearch.xpack.slm.SnapshotRetentionService;
Expand Down Expand Up @@ -254,13 +256,15 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC

@Override
public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
var usageAction =
new ActionHandler<>(XPackUsageFeatureAction.INDEX_LIFECYCLE, IndexLifecycleUsageTransportAction.class);
var infoAction =
new ActionHandler<>(XPackInfoFeatureAction.INDEX_LIFECYCLE, IndexLifecycleInfoTransportAction.class);
var ilmUsageAction = new ActionHandler<>(XPackUsageFeatureAction.INDEX_LIFECYCLE, IndexLifecycleUsageTransportAction.class);
var ilmInfoAction = new ActionHandler<>(XPackInfoFeatureAction.INDEX_LIFECYCLE, IndexLifecycleInfoTransportAction.class);
var slmUsageAction = new ActionHandler<>(XPackUsageFeatureAction.SNAPSHOT_LIFECYCLE, SLMUsageTransportAction.class);
var slmInfoAction = new ActionHandler<>(XPackInfoFeatureAction.SNAPSHOT_LIFECYCLE, SLMInfoTransportAction.class);
List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> actions = new ArrayList<>();
actions.add(usageAction);
actions.add(infoAction);
actions.add(ilmUsageAction);
actions.add(ilmInfoAction);
actions.add(slmUsageAction);
actions.add(slmInfoAction);
if (ilmEnabled) {
actions.addAll(Arrays.asList(
new ActionHandler<>(PutLifecycleAction.INSTANCE, TransportPutLifecycleAction.class),
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

package org.elasticsearch.xpack.slm;

import org.elasticsearch.action.support.ActionFilters;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.license.XPackLicenseState;
import org.elasticsearch.transport.TransportService;
import org.elasticsearch.xpack.core.XPackField;
import org.elasticsearch.xpack.core.XPackSettings;
import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
import org.elasticsearch.xpack.core.action.XPackInfoFeatureTransportAction;

public class SLMInfoTransportAction extends XPackInfoFeatureTransportAction {
private final boolean enabled;
private final XPackLicenseState licenseState;

@Inject
public SLMInfoTransportAction(TransportService transportService, ActionFilters actionFilters,
Settings settings, XPackLicenseState licenseState) {
super(XPackInfoFeatureAction.SNAPSHOT_LIFECYCLE.name(), transportService, actionFilters);
this.enabled = XPackSettings.SNAPSHOT_LIFECYCLE_ENABLED.get(settings);
this.licenseState = licenseState;
}

@Override
public String name() {
return XPackField.SNAPSHOT_LIFECYCLE;
}

@Override
public boolean available() {
return licenseState.isIndexLifecycleAllowed();
}

@Override
public boolean enabled() {
return enabled;
}
}
Loading

0 comments on commit f9511f4

Please sign in to comment.