Implement GET API for System Feature Upgrades

client/rest-high-level/src/test/java/org/elasticsearch/client/MigrationIT.java

@@ -8,6 +8,7 @@
 
 package org.elasticsearch.client;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.client.migration.DeprecationInfoRequest;
 import org.elasticsearch.client.migration.DeprecationInfoResponse;
 import org.elasticsearch.client.migration.GetFeatureUpgradeStatusRequest;
@@ -18,8 +19,11 @@
 
 import java.io.IOException;
 import java.util.Collections;
+import java.util.Optional;
 
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.nullValue;
 
 public class MigrationIT extends ESRestHighLevelClientTestCase {
@@ -38,19 +42,24 @@ public void testGetDeprecationInfo() throws IOException {
     public void testGetFeatureUpgradeStatus() throws IOException {
         GetFeatureUpgradeStatusRequest request = new GetFeatureUpgradeStatusRequest();
         GetFeatureUpgradeStatusResponse response = highLevelClient().migration().getFeatureUpgradeStatus(request, RequestOptions.DEFAULT);
-        assertThat(response.getUpgradeStatus(), equalTo("UPGRADE_NEEDED"));
-        assertThat(response.getFeatureUpgradeStatuses().size(), equalTo(1));
-        GetFeatureUpgradeStatusResponse.FeatureUpgradeStatus status = response.getFeatureUpgradeStatuses().get(0);
-        assertThat(status.getUpgradeStatus(), equalTo("UPGRADE_NEEDED"));
-        assertThat(status.getMinimumIndexVersion(), equalTo("7.1.1"));
-        assertThat(status.getFeatureName(), equalTo("security"));
-        assertThat(status.getIndexVersions().size(), equalTo(1));
+        assertThat(response.getUpgradeStatus(), equalTo("NO_UPGRADE_NEEDED"));
+        assertThat(response.getFeatureUpgradeStatuses().size(), greaterThanOrEqualTo(1));
+        Optional<GetFeatureUpgradeStatusResponse.FeatureUpgradeStatus> optionalTasksStatus = response.getFeatureUpgradeStatuses().stream()
+            .filter(status -> "tasks".equals(status.getFeatureName()))
+            .findFirst();
+
+        assertThat(optionalTasksStatus.isPresent(), is(true));
+
+        GetFeatureUpgradeStatusResponse.FeatureUpgradeStatus tasksStatus = optionalTasksStatus.get();
+
+        assertThat(tasksStatus.getUpgradeStatus(), equalTo("NO_UPGRADE_NEEDED"));
+        assertThat(tasksStatus.getMinimumIndexVersion(), equalTo(Version.CURRENT.toString()));
+        assertThat(tasksStatus.getFeatureName(), equalTo("tasks"));
     }
 
     public void testPostFeatureUpgradeStatus() throws IOException {
         PostFeatureUpgradeRequest request = new PostFeatureUpgradeRequest();
         PostFeatureUpgradeResponse response = highLevelClient().migration().postFeatureUpgrade(request, RequestOptions.DEFAULT);
-        // a test like this cannot test actual deprecations
         assertThat(response.isAccepted(), equalTo(true));
         assertThat(response.getFeatures().size(), equalTo(1));
         PostFeatureUpgradeResponse.Feature feature = response.getFeatures().get(0);

client/rest-high-level/src/test/java/org/elasticsearch/client/migration/GetFeatureUpgradeStatusResponseTests.java

@@ -8,6 +8,7 @@
 
 package org.elasticsearch.client.migration;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.client.AbstractResponseTestCase;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
@@ -37,14 +38,14 @@ protected org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStat
             randomList(5,
                 () -> new org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.FeatureUpgradeStatus(
                     randomAlphaOfLengthBetween(3, 20),
-                    randomAlphaOfLengthBetween(5, 9),
-                    randomAlphaOfLengthBetween(4, 16),
+                    randomFrom(Version.CURRENT, Version.CURRENT.minimumCompatibilityVersion()),
+                    randomFrom(org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.values()),
                     randomList(4,
                         () -> new org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.IndexVersion(
                             randomAlphaOfLengthBetween(3, 20),
-                            randomAlphaOfLengthBetween(5, 9)))
+                            randomFrom(Version.CURRENT, Version.CURRENT.minimumCompatibilityVersion())))
                 )),
-            randomAlphaOfLength(5)
+            randomFrom(org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.values())
         );
     }
 
@@ -58,7 +59,7 @@ protected void assertInstances(
         org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse serverTestInstance,
         GetFeatureUpgradeStatusResponse clientInstance) {
 
-        assertThat(clientInstance.getUpgradeStatus(), equalTo(serverTestInstance.getUpgradeStatus()));
+        assertThat(clientInstance.getUpgradeStatus(), equalTo(serverTestInstance.getUpgradeStatus().toString()));
 
         assertNotNull(serverTestInstance.getFeatureUpgradeStatuses());
         assertNotNull(clientInstance.getFeatureUpgradeStatuses());
@@ -71,8 +72,8 @@ protected void assertInstances(
             GetFeatureUpgradeStatusResponse.FeatureUpgradeStatus clientStatus = clientInstance.getFeatureUpgradeStatuses().get(i);
 
             assertThat(clientStatus.getFeatureName(), equalTo(serverTestStatus.getFeatureName()));
-            assertThat(clientStatus.getMinimumIndexVersion(), equalTo(serverTestStatus.getMinimumIndexVersion()));
-            assertThat(clientStatus.getUpgradeStatus(), equalTo(serverTestStatus.getUpgradeStatus()));
+            assertThat(clientStatus.getMinimumIndexVersion(), equalTo(serverTestStatus.getMinimumIndexVersion().toString()));
+            assertThat(clientStatus.getUpgradeStatus(), equalTo(serverTestStatus.getUpgradeStatus().toString()));
 
             assertThat(clientStatus.getIndexVersions(), hasSize(serverTestStatus.getIndexVersions().size()));
 
@@ -82,7 +83,7 @@ protected void assertInstances(
                 GetFeatureUpgradeStatusResponse.IndexVersion clientIndexVersion = clientStatus.getIndexVersions().get(j);
 
                 assertThat(clientIndexVersion.getIndexName(), equalTo(serverIndexVersion.getIndexName()));
-                assertThat(clientIndexVersion.getVersion(), equalTo(serverIndexVersion.getVersion()));
+                assertThat(clientIndexVersion.getVersion(), equalTo(serverIndexVersion.getVersion().toString()));
             }
         }
     }

docs/reference/migration/apis/feature_upgrade.asciidoc

@@ -43,19 +43,80 @@ Example response:
 --------------------------------------------------
 {
   "features" : [
+    {
+      "feature_name" : "async_search",
+      "minimum_index_version" : "8.0.0",
+      "upgrade_status" : "NO_UPGRADE_NEEDED",
+      "indices" : [ ]
+    },
+    {
+      "feature_name" : "enrich",
+      "minimum_index_version" : "8.0.0",
+      "upgrade_status" : "NO_UPGRADE_NEEDED",
+      "indices" : [ ]
+    },
+    {
+      "feature_name" : "fleet",
+      "minimum_index_version" : "8.0.0",
+      "upgrade_status" : "NO_UPGRADE_NEEDED",
+      "indices" : [ ]
+    },
+    {
+      "feature_name" : "geoip",
+      "minimum_index_version" : "8.0.0",
+      "upgrade_status" : "NO_UPGRADE_NEEDED",
+      "indices" : [ ]
+    },
+    {
+      "feature_name" : "kibana",
+      "minimum_index_version" : "8.0.0",
+      "upgrade_status" : "NO_UPGRADE_NEEDED",
+      "indices" : [ ]
+    },
+    {
+      "feature_name" : "logstash_management",
+      "minimum_index_version" : "8.0.0",
+      "upgrade_status" : "NO_UPGRADE_NEEDED",
+      "indices" : [ ]
+    },
+    {
+      "feature_name" : "machine_learning",
+      "minimum_index_version" : "8.0.0",
+      "upgrade_status" : "NO_UPGRADE_NEEDED",
+      "indices" : [ ]
+    },
+    {
+      "feature_name" : "searchable_snapshots",
+      "minimum_index_version" : "8.0.0",
+      "upgrade_status" : "NO_UPGRADE_NEEDED",
+      "indices" : [ ]
+    },
     {
       "feature_name" : "security",
-      "minimum_index_version" : "7.1.1",
-      "upgrade_status" : "UPGRADE_NEEDED",
-      "indices" : [
-        {
-          "index" : ".security-7",
-          "version" : "7.1.1"
-        }
-      ]
+      "minimum_index_version" : "8.0.0",
+      "upgrade_status" : "NO_UPGRADE_NEEDED",
+      "indices" : [ ]
+    },
+    {
+      "feature_name" : "tasks",
+      "minimum_index_version" : "8.0.0",
+      "upgrade_status" : "NO_UPGRADE_NEEDED",
+      "indices" : [ ]
+    },
+    {
+      "feature_name" : "transform",
+      "minimum_index_version" : "8.0.0",
+      "upgrade_status" : "NO_UPGRADE_NEEDED",
+      "indices" : [ ]
+    },
+    {
+      "feature_name" : "watcher",
+      "minimum_index_version" : "8.0.0",
+      "upgrade_status" : "NO_UPGRADE_NEEDED",
+      "indices" : [ ]
     }
   ],
-  "upgrade_status" : "UPGRADE_NEEDED"
+  "upgrade_status" : "NO_UPGRADE_NEEDED"
 }
 --------------------------------------------------
 

qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/FeatureUpgradeIT.java

@@ -0,0 +1,103 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.upgrades;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.test.XContentTestUtils;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+
+public class FeatureUpgradeIT extends AbstractRollingTestCase {
+
+    @SuppressWarnings("unchecked")
+    public void testGetFeatureUpgradeStatus() throws Exception {
+
+        final String systemIndexWarning = "this request accesses system indices: [.tasks], but in a future major version, direct " +
+            "access to system indices will be prevented by default";
+        if (CLUSTER_TYPE == ClusterType.OLD) {
+            // setup - put something in the tasks index
+            // create index
+            Request createTestIndex = new Request("PUT", "/feature_test_index_old");
+            createTestIndex.setJsonEntity("{\"settings\": {\"index.number_of_replicas\": 0}}");
+            client().performRequest(createTestIndex);
+
+            Request bulk = new Request("POST", "/_bulk");
+            bulk.addParameter("refresh", "true");
+            bulk.setJsonEntity("{\"index\": {\"_index\": \"feature_test_index_old\"}}\n" +
+                "{\"f1\": \"v1\", \"f2\": \"v2\"}\n");
+            client().performRequest(bulk);
+
+            // start a async reindex job
+            Request reindex = new Request("POST", "/_reindex");
+            reindex.setJsonEntity(
+                "{\n" +
+                    "  \"source\":{\n" +
+                    "    \"index\":\"feature_test_index_old\"\n" +
+                    "  },\n" +
+                    "  \"dest\":{\n" +
+                    "    \"index\":\"feature_test_index_reindex\"\n" +
+                    "  }\n" +
+                    "}");
+            reindex.addParameter("wait_for_completion", "false");
+            Map<String, Object> response = entityAsMap(client().performRequest(reindex));
+            String taskId = (String) response.get("task");
+
+            // wait for task
+            Request getTask = new Request("GET", "/_tasks/" + taskId);
+            getTask.addParameter("wait_for_completion", "true");
+            client().performRequest(getTask);
+
+            // make sure .tasks index exists
+            Request getTasksIndex = new Request("GET", "/.tasks");
+            getTasksIndex.setOptions(expectVersionSpecificWarnings(v -> {
+                v.current(systemIndexWarning);
+                v.compatible(systemIndexWarning);
+            }));
+            getTasksIndex.addParameter("allow_no_indices", "false");
+
+            assertBusy(() -> {
+                try {
+                    assertThat(client().performRequest(getTasksIndex).getStatusLine().getStatusCode(), is(200));
+                } catch (ResponseException e) {
+                    throw new AssertionError(".tasks index does not exist yet");
+                }
+            });
+
+        } else if (CLUSTER_TYPE == ClusterType.UPGRADED) {
+            // check results
+            assertBusy(() -> {
+                Request clusterStateRequest = new Request("GET", "/_migration/system_features");
+                XContentTestUtils.JsonMapView view = new XContentTestUtils.JsonMapView(
+                    entityAsMap(client().performRequest(clusterStateRequest)));
+
+                List<Map<String, Object>> features = view.get("features");
+                Map<String, Object> feature = features.stream()
+                    .filter(e -> "tasks".equals(e.get("feature_name")))
+                    .findFirst()
+                    .orElse(Collections.emptyMap());
+
+                assertThat(feature.size(), equalTo(4));
+                assertThat(feature.get("minimum_index_version"), equalTo(UPGRADE_FROM_VERSION.toString()));
+                if (UPGRADE_FROM_VERSION.before(Version.CURRENT.minimumIndexCompatibilityVersion())) {
+                    assertThat(feature.get("upgrade_status"), equalTo("UPGRADE_NEEDED"));
+                } else {
+                    assertThat(feature.get("upgrade_status"), equalTo("NO_UPGRADE_NEEDED"));
+                }
+            });
+        }
+    }
+
+}

qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/FeatureUpgradeApiIT.java

@@ -0,0 +1,70 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.system.indices;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.test.XContentTestUtils;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.junit.After;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
+
+public class FeatureUpgradeApiIT extends ESRestTestCase {
+
+    static final String BASIC_AUTH_VALUE = basicAuthHeaderValue("rest_user", new SecureString("rest-user-password".toCharArray()));
+
+    @After
+    public void resetFeatures() throws Exception {
+        client().performRequest(new Request("POST", "/_features/_reset"));
+    }
+
+    @Override
+    protected Settings restClientSettings() {
+        return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", BASIC_AUTH_VALUE).build();
+    }
+
+    public void testCreatingSystemIndex() throws Exception {
+        Response response = client().performRequest(new Request("PUT", "/_net_new_sys_index/_create"));
+        assertThat(response.getStatusLine().getStatusCode(), is(200));
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testGetFeatureUpgradedStatuses() throws Exception {
+        client().performRequest(new Request("PUT", "/_net_new_sys_index/_create"));
+        Response response = client().performRequest(new Request("GET", "/_migration/system_features"));
+        assertThat(response.getStatusLine().getStatusCode(), is(200));
+        XContentTestUtils.JsonMapView view = XContentTestUtils.createJsonMapView(response.getEntity().getContent());
+        String upgradeStatus = view.get("upgrade_status");
+        assertThat(upgradeStatus, equalTo("NO_UPGRADE_NEEDED"));
+        List<Map<String, Object>> features = view.get("features");
+        Map<String, Object> testFeature = features.stream()
+            .filter(feature -> "system indices qa".equals(feature.get("feature_name")))
+            .findFirst()
+            .orElse(Collections.emptyMap());
+
+        assertThat(testFeature.size(), equalTo(4));
+        assertThat(testFeature.get("minimum_index_version"), equalTo(Version.CURRENT.toString()));
+        assertThat(testFeature.get("upgrade_status"), equalTo("NO_UPGRADE_NEEDED"));
+        assertThat(testFeature.get("indices"), instanceOf(List.class));
+
+        assertThat((List<Object>) testFeature.get("indices"), hasSize(1));
+    }
+}