Merge branch 'master' into ccr

* master:
  Fix global checkpoint listeners test
  HLRC: adding machine learning open job (#32860)
  [ML] Add log structure finder functionality (#32788)
  INGEST: Add Configuration Except. Data to Metdata (#32322)

client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java

@@ -19,6 +19,8 @@
 package org.elasticsearch.client;
 
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.protocol.xpack.ml.OpenJobRequest;
+import org.elasticsearch.protocol.xpack.ml.OpenJobResponse;
 import org.elasticsearch.protocol.xpack.ml.PutJobRequest;
 import org.elasticsearch.protocol.xpack.ml.PutJobResponse;
 
@@ -77,4 +79,51 @@ public void putJobAsync(PutJobRequest request, RequestOptions options, ActionLis
             listener,
             Collections.emptySet());
     }
+
+    /**
+     * Opens a Machine Learning Job.
+     * When you open a new job, it starts with an empty model.
+     *
+     * When you open an existing job, the most recent model state is automatically loaded.
+     * The job is ready to resume its analysis from where it left off, once new data is received.
+     *
+     * <p>
+     *     For additional info
+     *     see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ml-open-job.html"></a>
+     * </p>
+     * @param request request containing job_id and additional optional options
+     * @param options  Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return response containing if the job was successfully opened or not.
+     * @throws IOException when there is a serialization issue sending the request or receiving the response
+     */
+    public OpenJobResponse openJob(OpenJobRequest request, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request,
+            RequestConverters::machineLearningOpenJob,
+            options,
+            OpenJobResponse::fromXContent,
+            Collections.emptySet());
+    }
+
+    /**
+     * Opens a Machine Learning Job asynchronously, notifies listener on completion.
+     * When you open a new job, it starts with an empty model.
+     *
+     * When you open an existing job, the most recent model state is automatically loaded.
+     * The job is ready to resume its analysis from where it left off, once new data is received.
+     * <p>
+     *     For additional info
+     *     see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ml-open-job.html"></a>
+     * </p>
+     * @param request request containing job_id and additional optional options
+     * @param options  Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener Listener to be notified upon request completion
+     */
+    public void openJobAsync(OpenJobRequest request, RequestOptions options, ActionListener<OpenJobResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(request,
+            RequestConverters::machineLearningOpenJob,
+            options,
+            OpenJobResponse::fromXContent,
+            listener,
+            Collections.emptySet());
+    }
 }

client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java

@@ -112,6 +112,7 @@
 import org.elasticsearch.protocol.xpack.license.GetLicenseRequest;
 import org.elasticsearch.protocol.xpack.license.PutLicenseRequest;
 import org.elasticsearch.protocol.xpack.migration.IndexUpgradeInfoRequest;
+import org.elasticsearch.protocol.xpack.ml.OpenJobRequest;
 import org.elasticsearch.protocol.xpack.ml.PutJobRequest;
 import org.elasticsearch.protocol.xpack.watcher.DeleteWatchRequest;
 import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest;
@@ -1210,6 +1211,19 @@ static Request putMachineLearningJob(PutJobRequest putJobRequest) throws IOExcep
         return request;
     }
 
+    static Request machineLearningOpenJob(OpenJobRequest openJobRequest) throws IOException {
+        String endpoint = new EndpointBuilder()
+            .addPathPartAsIs("_xpack")
+            .addPathPartAsIs("ml")
+            .addPathPartAsIs("anomaly_detectors")
+            .addPathPart(openJobRequest.getJobId())
+            .addPathPartAsIs("_open")
+            .build();
+        Request request = new Request(HttpPost.METHOD_NAME, endpoint);
+        request.setJsonEntity(openJobRequest.toString());
+        return request;
+    }
+
     static Request getMigrationAssistance(IndexUpgradeInfoRequest indexUpgradeInfoRequest) {
         EndpointBuilder endpointBuilder = new EndpointBuilder()
             .addPathPartAsIs("_xpack/migration/assistance")

client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java

@@ -20,6 +20,8 @@
 
 import com.carrotsearch.randomizedtesting.generators.CodepointSetGenerator;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.protocol.xpack.ml.OpenJobRequest;
+import org.elasticsearch.protocol.xpack.ml.OpenJobResponse;
 import org.elasticsearch.protocol.xpack.ml.PutJobRequest;
 import org.elasticsearch.protocol.xpack.ml.PutJobResponse;
 import org.elasticsearch.protocol.xpack.ml.job.config.AnalysisConfig;
@@ -46,12 +48,24 @@ public void testPutJob() throws Exception {
         assertThat(createdJob.getJobType(), is(Job.ANOMALY_DETECTOR_JOB_TYPE));
     }
 
+    public void testOpenJob() throws Exception {
+        String jobId = randomValidJobId();
+        Job job = buildJob(jobId);
+        MachineLearningClient machineLearningClient = highLevelClient().machineLearning();
+
+        machineLearningClient.putJob(new PutJobRequest(job), RequestOptions.DEFAULT);
+
+        OpenJobResponse response = execute(new OpenJobRequest(jobId), machineLearningClient::openJob, machineLearningClient::openJobAsync);
+
+        assertTrue(response.isOpened());
+    }
+
     public static String randomValidJobId() {
         CodepointSetGenerator generator = new CodepointSetGenerator("abcdefghijklmnopqrstuvwxyz0123456789".toCharArray());
         return generator.ofCodePointsLength(random(), 10, 10);
     }
 
-    private static Job buildJob(String jobId) {
+    public static Job buildJob(String jobId) {
         Job.Builder builder = new Job.Builder(jobId);
         builder.setDescription(randomAlphaOfLength(10));
 

client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java

@@ -127,6 +127,7 @@
 import org.elasticsearch.index.rankeval.RestRankEvalAction;
 import org.elasticsearch.protocol.xpack.XPackInfoRequest;
 import org.elasticsearch.protocol.xpack.migration.IndexUpgradeInfoRequest;
+import org.elasticsearch.protocol.xpack.ml.OpenJobRequest;
 import org.elasticsearch.protocol.xpack.watcher.DeleteWatchRequest;
 import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest;
 import org.elasticsearch.repositories.fs.FsRepository;
@@ -2610,6 +2611,19 @@ public void testXPackDeleteWatch() {
         assertThat(request.getEntity(), nullValue());
     }
 
+    public void testPostMachineLearningOpenJob() throws Exception {
+        String jobId = "some-job-id";
+        OpenJobRequest openJobRequest = new OpenJobRequest(jobId);
+        openJobRequest.setTimeout(TimeValue.timeValueMinutes(10));
+
+        Request request = RequestConverters.machineLearningOpenJob(openJobRequest);
+        assertEquals(HttpPost.METHOD_NAME, request.getMethod());
+        assertEquals("/_xpack/ml/anomaly_detectors/" + jobId + "/_open", request.getEndpoint());
+        ByteArrayOutputStream bos = new ByteArrayOutputStream();
+        request.getEntity().writeTo(bos);
+        assertEquals(bos.toString("UTF-8"), "{\"job_id\":\""+ jobId +"\",\"timeout\":\"10m\"}");
+    }
+
     /**
      * Randomize the {@link FetchSourceContext} request parameters.
      */

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java

@@ -21,9 +21,12 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.LatchedActionListener;
 import org.elasticsearch.client.ESRestHighLevelClientTestCase;
+import org.elasticsearch.client.MachineLearningIT;
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.RestHighLevelClient;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.protocol.xpack.ml.OpenJobRequest;
+import org.elasticsearch.protocol.xpack.ml.OpenJobResponse;
 import org.elasticsearch.protocol.xpack.ml.PutJobRequest;
 import org.elasticsearch.protocol.xpack.ml.PutJobResponse;
 import org.elasticsearch.protocol.xpack.ml.job.config.AnalysisConfig;
@@ -118,4 +121,54 @@ public void onFailure(Exception e) {
             assertTrue(latch.await(30L, TimeUnit.SECONDS));
         }
     }
+
+    public void testOpenJob() throws Exception {
+        RestHighLevelClient client = highLevelClient();
+
+        Job job = MachineLearningIT.buildJob("opening-my-first-machine-learning-job");
+        client.machineLearning().putJob(new PutJobRequest(job), RequestOptions.DEFAULT);
+
+        Job secondJob = MachineLearningIT.buildJob("opening-my-second-machine-learning-job");
+        client.machineLearning().putJob(new PutJobRequest(secondJob), RequestOptions.DEFAULT);
+
+        {
+            //tag::x-pack-ml-open-job-request
+            OpenJobRequest openJobRequest = new OpenJobRequest("opening-my-first-machine-learning-job"); //<1>
+            openJobRequest.setTimeout(TimeValue.timeValueMinutes(10)); //<2>
+            //end::x-pack-ml-open-job-request
+
+            //tag::x-pack-ml-open-job-execute
+            OpenJobResponse openJobResponse = client.machineLearning().openJob(openJobRequest, RequestOptions.DEFAULT);
+            boolean isOpened = openJobResponse.isOpened(); //<1>
+            //end::x-pack-ml-open-job-execute
+
+        }
+
+        {
+            //tag::x-pack-ml-open-job-listener
+            ActionListener<OpenJobResponse> listener = new ActionListener<OpenJobResponse>() {
+                @Override
+                public void onResponse(OpenJobResponse openJobResponse) {
+                    //<1>
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    //<2>
+                }
+            };
+            //end::x-pack-ml-open-job-listener
+            OpenJobRequest openJobRequest = new OpenJobRequest("opening-my-second-machine-learning-job");
+            // Replace the empty listener by a blocking listener in test
+            final CountDownLatch latch = new CountDownLatch(1);
+            listener = new LatchedActionListener<>(listener, latch);
+
+            // tag::x-pack-ml-open-job-execute-async
+            client.machineLearning().openJobAsync(openJobRequest, RequestOptions.DEFAULT, listener); //<1>
+            // end::x-pack-ml-open-job-execute-async
+
+            assertTrue(latch.await(30L, TimeUnit.SECONDS));
+        }
+
+    }
 }

docs/java-rest/high-level/ml/open-job.asciidoc

@@ -0,0 +1,55 @@
+[[java-rest-high-x-pack-ml-open-job]]
+=== Open Job API
+
+The Open Job API provides the ability to open {ml} jobs in the cluster.
+It accepts a `OpenJobRequest` object and responds
+with a `OpenJobResponse` object.
+
+[[java-rest-high-x-pack-ml-open-job-request]]
+==== Open Job Request
+
+An `OpenJobRequest` object gets created with an existing non-null `jobId`.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-open-job-request]
+--------------------------------------------------
+<1> Constructing a new request referencing an existing `jobId`
+<2> Optionally setting the `timeout` value for how long the
+execution should wait for the job to be opened.
+
+[[java-rest-high-x-pack-ml-open-job-execution]]
+==== Execution
+
+The request can be executed through the `MachineLearningClient` contained
+in the `RestHighLevelClient` object, accessed via the `machineLearningClient()` method.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-open-job-execute]
+--------------------------------------------------
+<1> `isOpened()` from the `OpenJobResponse` indicates if the job was successfully
+opened or not.
+
+[[java-rest-high-x-pack-ml-open-job-execution-async]]
+==== Asynchronous Execution
+
+The request can also be executed asynchronously:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-open-job-execute-async]
+--------------------------------------------------
+<1> The `OpenJobRequest` to execute and the `ActionListener` to use when
+the execution completes
+
+The method does not block and returns immediately. The passed `ActionListener` is used
+to notify the caller of completion. A typical `ActionListner` for `OpenJobResponse` may
+look like
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-open-job-listener]
+--------------------------------------------------
+<1> `onResponse` is called back when the action is completed successfully
+<2> `onFailure` is called back when some unexpected error occurs

docs/java-rest/high-level/supported-apis.asciidoc

@@ -205,8 +205,10 @@ include::licensing/delete-license.asciidoc[]
 The Java High Level REST Client supports the following Machine Learning APIs:
 
 * <<java-rest-high-x-pack-ml-put-job>>
+* <<java-rest-high-x-pack-ml-open-job>>
 
-include::ml/put_job.asciidoc[]
+include::ml/put-job.asciidoc[]
+include::ml/open-job.asciidoc[]
 
 == Migration APIs
 

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorFactoryTests.java

@@ -100,6 +100,6 @@ public void testInvalidMustacheTemplate() throws Exception {
         String processorTag = randomAlphaOfLength(10);
         ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, config));
         assertThat(exception.getMessage(), equalTo("java.lang.RuntimeException: could not compile script"));
-        assertThat(exception.getHeader("processor_tag").get(0), equalTo(processorTag));
+        assertThat(exception.getMetadata("es.processor_tag").get(0), equalTo(processorTag));
     }
 }

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorFactoryTests.java

@@ -58,9 +58,9 @@ public void testCreateUnsupportedType() throws Exception {
             fail("factory create should have failed");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), Matchers.equalTo("[type] type [" + type + "] not supported, cannot convert field."));
-            assertThat(e.getHeader("processor_type").get(0), equalTo(ConvertProcessor.TYPE));
-            assertThat(e.getHeader("property_name").get(0), equalTo("type"));
-            assertThat(e.getHeader("processor_tag"), nullValue());
+            assertThat(e.getMetadata("es.processor_type").get(0), equalTo(ConvertProcessor.TYPE));
+            assertThat(e.getMetadata("es.property_name").get(0), equalTo("type"));
+            assertThat(e.getMetadata("es.processor_tag"), nullValue());
         }
     }
 

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorFactoryTests.java

@@ -66,6 +66,6 @@ public void testInvalidMustacheTemplate() throws Exception {
         String processorTag = randomAlphaOfLength(10);
         ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, config));
         assertThat(exception.getMessage(), equalTo("java.lang.RuntimeException: could not compile script"));
-        assertThat(exception.getHeader("processor_tag").get(0), equalTo(processorTag));
+        assertThat(exception.getMetadata("es.processor_tag").get(0), equalTo(processorTag));
     }
 }

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorFactoryTests.java

@@ -79,6 +79,6 @@ public void testInvalidMustacheTemplate() throws Exception {
         String processorTag = randomAlphaOfLength(10);
         ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, config));
         assertThat(exception.getMessage(), equalTo("java.lang.RuntimeException: could not compile script"));
-        assertThat(exception.getHeader("processor_tag").get(0), equalTo(processorTag));
+        assertThat(exception.getMetadata("es.processor_tag").get(0), equalTo(processorTag));
     }
 }

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorFactoryTests.java

@@ -108,7 +108,7 @@ public void testInvalidMustacheTemplate() throws Exception {
         String processorTag = randomAlphaOfLength(10);
         ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, config));
         assertThat(exception.getMessage(), equalTo("java.lang.RuntimeException: could not compile script"));
-        assertThat(exception.getHeader("processor_tag").get(0), equalTo(processorTag));
+        assertThat(exception.getMetadata("es.processor_tag").get(0), equalTo(processorTag));
     }
 
 }

modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/20_crud.yml

@@ -158,9 +158,9 @@ teardown:
           }
   - match: { error.root_cause.0.type: "parse_exception" }
   - match: { error.root_cause.0.reason: "[field] required property is missing" }
-  - match: { error.root_cause.0.header.processor_tag: "fritag" }
-  - match: { error.root_cause.0.header.processor_type: "set" }
-  - match: { error.root_cause.0.header.property_name: "field" }
+  - match: { error.root_cause.0.processor_tag: "fritag" }
+  - match: { error.root_cause.0.processor_type: "set" }
+  - match: { error.root_cause.0.property_name: "field" }
 
 ---
 "Test basic pipeline with on_failure in processor":

modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/50_on_failure.yml

@@ -148,9 +148,9 @@ teardown:
           }
   - match: { error.root_cause.0.type: "parse_exception" }
   - match: { error.root_cause.0.reason: "[on_failure] processors list cannot be empty" }
-  - match: { error.root_cause.0.header.processor_type: "fail" }
-  - match: { error.root_cause.0.header.processor_tag: "emptyfail" }
-  - match: { error.root_cause.0.header.property_name: "on_failure" }
+  - match: { error.root_cause.0.processor_type: "fail" }
+  - match: { error.root_cause.0.processor_tag: "emptyfail" }
+  - match: { error.root_cause.0.property_name: "on_failure" }
 
 ---
 "Test pipeline with empty on_failure in pipeline":

modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/90_simulate.yml

@@ -107,9 +107,9 @@ teardown:
           }
   - match: { error.root_cause.0.type: "parse_exception" }
   - match: { error.root_cause.0.reason: "[field] required property is missing" }
-  - match: { error.root_cause.0.header.processor_tag: "fails" }
-  - match: { error.root_cause.0.header.processor_type: "set" }
-  - match: { error.root_cause.0.header.property_name: "field" }
+  - match: { error.root_cause.0.processor_tag: "fails" }
+  - match: { error.root_cause.0.processor_type: "set" }
+  - match: { error.root_cause.0.property_name: "field" }
 
 ---
 "Test simulate without index type and id":
@@ -198,9 +198,9 @@ teardown:
               }
             ]
           }
-  - is_false: error.root_cause.0.header.processor_type
-  - is_false: error.root_cause.0.header.processor_tag
-  - match: { error.root_cause.0.header.property_name: "pipeline" }
+  - is_false: error.root_cause.0.processor_type
+  - is_false: error.root_cause.0.processor_tag
+  - match: { error.root_cause.0.property_name: "pipeline" }
   - match: { error.reason: "[pipeline] required property is missing" }
 
 ---
@@ -233,9 +233,9 @@ teardown:
           }
   - match: { error.root_cause.0.type: "parse_exception" }
   - match: { error.root_cause.0.reason: "[value] required property is missing" }
-  - match: { error.root_cause.0.header.processor_type: "set" }
-  - match: { error.root_cause.0.header.property_name: "value" }
-  - is_false: error.root_cause.0.header.processor_tag
+  - match: { error.root_cause.0.processor_type: "set" }
+  - match: { error.root_cause.0.property_name: "value" }
+  - is_false: error.root_cause.0.processor_tag
 
 ---
 "Test simulate with verbose flag":