[Transform] add transform upgrade endpoint (elastic#77566)

Add an _upgrade endpoint to bulk upgrade transforms. _upgrade rewrites all transforms and its
artifacts into the latest format to the latest storage(index). If all transforms are upgraded old
indices and outdated documents get deleted. Using the dry_run option it is possible to check if
upgrades are necessary without applying changes.

client/rest-high-level/src/main/java/org/elasticsearch/client/transform/UpgradeTransformsRequest.java

@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.client.transform;
+
+import org.elasticsearch.client.Validatable;
+
+import java.util.Objects;
+
+public class UpgradeTransformsRequest implements Validatable {
+
+    private Boolean dryRun;
+
+    public UpgradeTransformsRequest() {}
+
+    public Boolean isDryRun() {
+        return dryRun;
+    }
+
+    /**
+     * Whether to only check for an upgrade without taking action
+     *
+     * @param dryRun {@code true} will only check for upgrades
+     */
+    public void setDryRun(boolean dryRun) {
+        this.dryRun = dryRun;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(dryRun);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (getClass() != obj.getClass()) {
+            return false;
+        }
+        UpgradeTransformsRequest other = (UpgradeTransformsRequest) obj;
+        return Objects.equals(dryRun, other.dryRun);
+    }
+
+}

client/rest-high-level/src/main/java/org/elasticsearch/client/transform/UpgradeTransformsResponse.java

@@ -0,0 +1,87 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.client.transform;
+
+import org.elasticsearch.xcontent.ConstructingObjectParser;
+import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.xcontent.XContentParser;
+
+import java.util.Objects;
+
+import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
+
+public class UpgradeTransformsResponse {
+
+    public static final ParseField NO_ACTION = new ParseField("no_action");
+    public static final ParseField UPDATED = new ParseField("updated");
+    public static final ParseField NEEDS_UPDATE = new ParseField("needs_update");
+
+    private static final ConstructingObjectParser<UpgradeTransformsResponse, Void> PARSER = new ConstructingObjectParser<>(
+        "upgrade_transform",
+        true,
+        args -> {
+            long updated = args[0] == null ? 0L : (Long) args[0];
+            long noAction = args[1] == null ? 0L : (Long) args[1];
+            long needsUpdate = args[2] == null ? 0L : (Long) args[2];
+
+            return new UpgradeTransformsResponse(updated, noAction, needsUpdate);
+        }
+    );
+
+    static {
+        PARSER.declareLong(optionalConstructorArg(), UPDATED);
+        PARSER.declareLong(optionalConstructorArg(), NO_ACTION);
+        PARSER.declareLong(optionalConstructorArg(), NEEDS_UPDATE);
+    }
+
+    public static UpgradeTransformsResponse fromXContent(final XContentParser parser) {
+        return UpgradeTransformsResponse.PARSER.apply(parser, null);
+    }
+
+    private final long updated;
+    private final long noAction;
+    private final long needsUpdate;
+
+    public UpgradeTransformsResponse(long updated, long noAction, long needsUpdate) {
+        this.updated = updated;
+        this.noAction = noAction;
+        this.needsUpdate = needsUpdate;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(updated, noAction, needsUpdate);
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (this == other) {
+            return true;
+        }
+
+        if (other == null || getClass() != other.getClass()) {
+            return false;
+        }
+
+        final UpgradeTransformsResponse that = (UpgradeTransformsResponse) other;
+        return this.updated == that.updated && this.noAction == that.noAction && this.needsUpdate == that.needsUpdate;
+    }
+
+    public long getUpdated() {
+        return updated;
+    }
+
+    public long getNoAction() {
+        return noAction;
+    }
+
+    public long getNeedsUpdate() {
+        return needsUpdate;
+    }
+}

client/rest-high-level/src/test/java/org/elasticsearch/client/transform/UpgradeTransformsResponseTests.java

@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.client.transform;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xcontent.NamedXContentRegistry;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+
+import static org.elasticsearch.test.AbstractXContentTestCase.xContentTester;
+
+public class UpgradeTransformsResponseTests extends ESTestCase {
+
+    public void testXContentParser() throws IOException {
+        xContentTester(
+            this::createParser,
+            UpgradeTransformsResponseTests::createTestInstance,
+            UpgradeTransformsResponseTests::toXContent,
+            UpgradeTransformsResponse::fromXContent
+        ).assertToXContentEquivalence(false).supportsUnknownFields(false).test();
+    }
+
+    private static UpgradeTransformsResponse createTestInstance() {
+        return new UpgradeTransformsResponse(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong());
+    }
+
+    private static void toXContent(UpgradeTransformsResponse response, XContentBuilder builder) throws IOException {
+        builder.startObject();
+        if (response.getUpdated() != 0) {
+            builder.field("updated", response.getUpdated());
+        }
+        if (response.getNoAction() != 0) {
+            builder.field("no_action", response.getNoAction());
+        }
+        if (response.getNeedsUpdate() != 0) {
+            builder.field("needs_update", response.getNeedsUpdate());
+        }
+        builder.endObject();
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
+        List<NamedXContentRegistry.Entry> namedXContents = searchModule.getNamedXContents();
+        namedXContents.addAll(new TransformNamedXContentProvider().getNamedXContentParsers());
+
+        return new NamedXContentRegistry(namedXContents);
+    }
+
+}

client/rest-high-level/src/test/java/org/elasticsearch/client/transform/hlrc/UpgradeTransformsResponseTests.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.client.transform.hlrc;
+
+import org.elasticsearch.client.AbstractResponseTestCase;
+import org.elasticsearch.client.transform.UpgradeTransformsResponse;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.core.transform.action.UpgradeTransformsAction.Response;
+
+import java.io.IOException;
+
+public class UpgradeTransformsResponseTests extends AbstractResponseTestCase<
+    Response,
+    org.elasticsearch.client.transform.UpgradeTransformsResponse> {
+
+    public static Response randomUpgradeResponse() {
+        return new Response(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong());
+    }
+
+    @Override
+    protected Response createServerTestInstance(XContentType xContentType) {
+        return randomUpgradeResponse();
+    }
+
+    @Override
+    protected UpgradeTransformsResponse doParseToClientInstance(XContentParser parser) throws IOException {
+        return org.elasticsearch.client.transform.UpgradeTransformsResponse.fromXContent(parser);
+    }
+
+    @Override
+    protected void assertInstances(Response serverTestInstance, UpgradeTransformsResponse clientInstance) {
+        assertEquals(serverTestInstance.getNeedsUpdate(), clientInstance.getNeedsUpdate());
+        assertEquals(serverTestInstance.getNoAction(), clientInstance.getNoAction());
+        assertEquals(serverTestInstance.getUpdated(), clientInstance.getUpdated());
+    }
+
+}

docs/reference/transform/apis/upgrade-transforms.asciidoc

@@ -0,0 +1,57 @@
+[role="xpack"]
+[testenv="basic"]
+[[upgrade-transforms]]
+= Upgrade {transform} API
+
+[subs="attributes"]
+++++
+<titleabbrev>Upgrade {transform}</titleabbrev>
+++++
+
+Upgrades all {transform}s.
+
+[[upgrade-transforms-request]]
+== {api-request-title}
+
+`POST _transform/_upgrade`
+
+[[upgrade-transforms-prereqs]]
+== {api-prereq-title}
+
+Requires the following privileges:
+
+* cluster: `manage_transform` (the `transform_admin` built-in role grants this
+  privilege)
+* source indices: `read`, `view_index_metadata`
+* destination index: `read`, `index`.
+
+
+[[upgrade-transforms-desc]]
+== {api-description-title}
+
+This API upgrades all existing {transform}s.
+
+[[upgrade-transforms-query-parms]]
+== {api-query-parms-title}
+
+`dry_run`::
+  (Optional, Boolean) When `true`, only checks for updates but does not execute them.
+
+[[upgrade-transforms-example]]
+== {api-examples-title}
+
+[source,console]
+--------------------------------------------------
+POST _transform/_upgrade
+--------------------------------------------------
+// TEST[setup:simple_kibana_continuous_pivot]
+
+When all {transform}s are upgraded, you receive a summary:
+
+[source,console-result]
+----
+{
+  "no_action": 1
+}
+----
+// TESTRESPONSE[s/"no_action" : 1/"no_action" : $body.no_action/]

rest-api-spec/src/main/resources/rest-api-spec/api/transform.upgrade_transforms.json

@@ -0,0 +1,31 @@
+{
+  "transform.upgrade_transforms":{
+    "documentation":{
+      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/current/upgrade-transforms.html",
+      "description":"Upgrades all transforms."
+    },
+    "stability":"stable",
+    "visibility":"public",
+    "headers":{
+      "accept":["application/json"],
+      "content_type":["application/json"]
+    },
+    "url":{
+      "paths":[
+        {
+          "path":"/_transform/_upgrade",
+          "methods":[
+            "POST"
+          ]
+        }
+      ]
+    },
+    "params":{
+      "dry_run":{
+        "type":"boolean",
+        "required":false,
+        "description":"Whether to only check for updates but don't execute"
+      }
+    }
+  }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java

@@ -44,6 +44,7 @@ public final class TransformField {
     public static final ParseField DELAY = new ParseField("delay");
     // TODO: Rename to "defer_data_validation" or similar to emphasize that not all validation is deferred
     public static final ParseField DEFER_VALIDATION = new ParseField("defer_validation");
+    public static final ParseField DRY_RUN = new ParseField("dry_run");
     public static final ParseField RETENTION_POLICY = new ParseField("retention_policy");
     public static final ParseField MAX_AGE = new ParseField("max_age");
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformMessages.java

@@ -55,6 +55,9 @@ public class TransformMessages {
     public static final String TRANSFORM_FAILED_TO_CREATE_COMPOSITE_AGGREGATION =
         "Failed to create composite aggregation from {0} function";
     public static final String TRANSFORM_CONFIGURATION_INVALID = "Transform configuration [{0}] has invalid elements: [{1}]";
+    public static final String TRANSFORM_CONFIGURATION_DEPRECATED = "Transform configuration is at version [{0}]. Use [{1}] or ["
+        + TransformField.REST_BASE_PATH_TRANSFORMS
+        + "_upgrade] to update.";
     public static final String UNABLE_TO_GATHER_FIELD_MAPPINGS = "Failed to gather field mappings for index [{0}]";
     public static final String TRANSFORM_UPDATE_CANNOT_CHANGE_SYNC_METHOD =
         "Cannot change the current sync configuration of transform [{0}] from [{1}] to [{2}]";