Merge branch 'master' into follow-stats-structure

* master: Rename CCR stats implementation (elastic#34300) Add max_children limit to nested sort (elastic#33587) MINOR: Remove Dead Code from Netty4Transport (elastic#34134) Rename clsuterformation -> testclusters (elastic#34299) [Build] make sure there are no duplicate classes in third party audit (elastic#34213) BWC Build: Read CI properties to determine java version (elastic#34295) [DOCS] Fix typo and add [float] Allow User/Password realms to disable authc (elastic#34033) Enable security automaton caching (elastic#34028) Preserve thread context during authentication. (elastic#34290) [ML] Allow asynchronous job deletion (elastic#34058)
jasontedor · Oct 5, 2018 · cdd96aa · cdd96aa
2 parents 11f4565 + 7478167
commit cdd96aa
Show file tree

Hide file tree

Showing 72 changed files with 1,248 additions and 747 deletions.
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/JdkJarHellCheck.java b/buildSrc/src/main/java/org/elasticsearch/gradle/JdkJarHellCheck.java
@@ -43,7 +43,7 @@ private void scanForJDKJarHell(Path root) throws IOException {
             @Override
             public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
                 String entry = root.relativize(file).toString().replace('\\', '/');
-                if (entry.endsWith(".class")) {
+                if (entry.endsWith(".class") && entry.endsWith("module-info.class") == false) {
                     if (ext.getResource(entry) != null) {
                         detected.add(
                             entry

diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java
@@ -20,12 +20,12 @@
 
 import org.apache.commons.io.output.NullOutputStream;
 import org.elasticsearch.gradle.JdkJarHellCheck;
-import org.elasticsearch.test.NamingConventionsCheck;
 import org.gradle.api.DefaultTask;
 import org.gradle.api.GradleException;
 import org.gradle.api.JavaVersion;
 import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.file.FileCollection;
+import org.gradle.api.file.FileTree;
 import org.gradle.api.tasks.Input;
 import org.gradle.api.tasks.InputFile;
 import org.gradle.api.tasks.InputFiles;
@@ -47,6 +47,7 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 
 public class ThirdPartyAuditTask extends DefaultTask {
 
@@ -171,19 +172,38 @@ private void extractJars(FileCollection jars) {
         File jarExpandDir = getJarExpandDir();
         // We need to clean up to make sure old dependencies don't linger
         getProject().delete(jarExpandDir);
-        jars.forEach(jar ->
+
+        jars.forEach(jar -> {
+            FileTree jarFiles = getProject().zipTree(jar);
             getProject().copy(spec -> {
+                spec.from(jarFiles);
+                spec.into(jarExpandDir);
+                // exclude classes from multi release jars
+                spec.exclude("META-INF/versions/**");
+            });
+            // Deal with multi release jars:
+            // The order is important, we iterate here so we don't depend on the order in which Gradle executes the spec
+            // We extract multi release jar classes ( if these exist ) going from 9 - the first to support them, to the
+            // current `targetCompatibility` version.
+            // Each extract will overwrite the top level classes that existed before it, the result is that we end up
+            // with a single version of the class in `jarExpandDir`.
+            // This will be the closes version to `targetCompatibility`, the same class that would be loaded in a JVM
+            // that has `targetCompatibility` version.
+            // This means we only scan classes that would be loaded into `targetCompatibility`, and don't look at any
+            // pther version specific implementation of said classes.
+            IntStream.rangeClosed(
+                Integer.parseInt(JavaVersion.VERSION_1_9.getMajorVersion()),
+                Integer.parseInt(targetCompatibility.getMajorVersion())
+            ).forEach(majorVersion -> getProject().copy(spec -> {
                 spec.from(getProject().zipTree(jar));
                 spec.into(jarExpandDir);
-                // Exclude classes for multi release jars above target
-                for (int i = Integer.parseInt(targetCompatibility.getMajorVersion()) + 1;
-                     i <= Integer.parseInt(JavaVersion.VERSION_HIGHER.getMajorVersion());
-                     i++
-                ) {
-                  spec.exclude("META-INF/versions/" + i + "/**");
-                }
-            })
-        );
+                String metaInfPrefix = "META-INF/versions/" + majorVersion;
+                spec.include(metaInfPrefix + "/**");
+                // Drop the version specific prefix
+                spec.eachFile(details -> details.setPath(details.getPath().replace(metaInfPrefix, "")));
+                spec.setIncludeEmptyDirs(false);
+            }));
+        });
     }
 
     private void assertNoJarHell(Set<String> jdkJarHellClasses) {
@@ -276,9 +296,9 @@ private String formatClassList(Set<String> classList) {
     private Set<String> runJdkJarHellCheck() throws IOException {
         ByteArrayOutputStream standardOut = new ByteArrayOutputStream();
         ExecResult execResult = getProject().javaexec(spec -> {
-            URL location = NamingConventionsCheck.class.getProtectionDomain().getCodeSource().getLocation();
+            URL location = JdkJarHellCheck.class.getProtectionDomain().getCodeSource().getLocation();
             if (location.getProtocol().equals("file") == false) {
-                throw new GradleException("Unexpected location for NamingConventionCheck class: " + location);
+                throw new GradleException("Unexpected location for JdkJarHellCheck class: " + location);
             }
             try {
                 spec.classpath(

diff --git a/...formation/ElasticsearchConfiguration.java → ...tclusters/ElasticsearchConfiguration.java b/...formation/ElasticsearchConfiguration.java → ...tclusters/ElasticsearchConfiguration.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.gradle.clusterformation;
+package org.elasticsearch.gradle.testclusters;
 
 import org.elasticsearch.gradle.Distribution;
 import org.elasticsearch.gradle.Version;

diff --git a/...e/clusterformation/ElasticsearchNode.java → ...radle/testclusters/ElasticsearchNode.java b/...e/clusterformation/ElasticsearchNode.java → ...radle/testclusters/ElasticsearchNode.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.gradle.clusterformation;
+package org.elasticsearch.gradle.testclusters;
 
 import org.elasticsearch.GradleServicesAdapter;
 import org.elasticsearch.gradle.Distribution;

diff --git a/...sterformation/ClusterformationPlugin.java → ...adle/testclusters/TestClustersPlugin.java b/...sterformation/ClusterformationPlugin.java → ...adle/testclusters/TestClustersPlugin.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.gradle.clusterformation;
+package org.elasticsearch.gradle.testclusters;
 
 import groovy.lang.Closure;
 import org.elasticsearch.GradleServicesAdapter;
@@ -37,12 +37,12 @@
 import java.util.List;
 import java.util.Map;
 
-public class ClusterformationPlugin implements Plugin<Project> {
+public class TestClustersPlugin implements Plugin<Project> {
 
     public static final String LIST_TASK_NAME = "listElasticSearchClusters";
     public static final String EXTENSION_NAME = "elasticSearchClusters";
 
-    private final Logger logger =  Logging.getLogger(ClusterformationPlugin.class);
+    private final Logger logger =  Logging.getLogger(TestClustersPlugin.class);
 
     @Override
     public void apply(Project project) {

diff --git a/...dSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.clusterformation.properties b/...dSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.clusterformation.properties
diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.testclusters.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.testclusters.properties
@@ -0,0 +1 @@
+implementation-class=org.elasticsearch.gradle.testclusters.TestClustersPlugin
diff --git a/...erformation/ClusterformationPluginIT.java → ...le/testclusters/TestClustersPluginIT.java b/...erformation/ClusterformationPluginIT.java → ...le/testclusters/TestClustersPluginIT.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.gradle.clusterformation;
+package org.elasticsearch.gradle.testclusters;
 
 import org.elasticsearch.gradle.test.GradleIntegrationTestCase;
 import org.gradle.testkit.runner.BuildResult;
@@ -26,11 +26,11 @@
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
 
-public class ClusterformationPluginIT extends GradleIntegrationTestCase {
+public class TestClustersPluginIT extends GradleIntegrationTestCase {
 
     public void testListClusters() {
         BuildResult result = GradleRunner.create()
-            .withProjectDir(getProjectDir("clusterformation"))
+            .withProjectDir(getProjectDir("testclusters"))
             .withArguments("listElasticSearchClusters", "-s")
             .withPluginClasspath()
             .build();
@@ -45,7 +45,7 @@ public void testListClusters() {
 
     public void testUseClusterByOne() {
         BuildResult result = GradleRunner.create()
-            .withProjectDir(getProjectDir("clusterformation"))
+            .withProjectDir(getProjectDir("testclusters"))
             .withArguments("user1", "-s")
             .withPluginClasspath()
             .build();
@@ -60,7 +60,7 @@ public void testUseClusterByOne() {
 
     public void testUseClusterByOneWithDryRun() {
         BuildResult result = GradleRunner.create()
-            .withProjectDir(getProjectDir("clusterformation"))
+            .withProjectDir(getProjectDir("testclusters"))
             .withArguments("user1", "-s", "--dry-run")
             .withPluginClasspath()
             .build();
@@ -75,7 +75,7 @@ public void testUseClusterByOneWithDryRun() {
 
     public void testUseClusterByTwo() {
         BuildResult result = GradleRunner.create()
-            .withProjectDir(getProjectDir("clusterformation"))
+            .withProjectDir(getProjectDir("testclusters"))
             .withArguments("user1", "user2", "-s")
             .withPluginClasspath()
             .build();
@@ -92,7 +92,7 @@ public void testUseClusterByTwo() {
 
     public void testUseClusterByUpToDateTask() {
         BuildResult result = GradleRunner.create()
-            .withProjectDir(getProjectDir("clusterformation"))
+            .withProjectDir(getProjectDir("testclusters"))
             .withArguments("upToDate1", "upToDate2", "-s")
             .withPluginClasspath()
             .build();
@@ -109,7 +109,7 @@ public void testUseClusterByUpToDateTask() {
 
     public void testUseClusterBySkippedTask() {
         BuildResult result = GradleRunner.create()
-            .withProjectDir(getProjectDir("clusterformation"))
+            .withProjectDir(getProjectDir("testclusters"))
             .withArguments("skipped1", "skipped2", "-s")
             .withPluginClasspath()
             .build();
@@ -126,7 +126,7 @@ public void testUseClusterBySkippedTask() {
 
     public void tetUseClusterBySkippedAndWorkingTask() {
         BuildResult result = GradleRunner.create()
-            .withProjectDir(getProjectDir("clusterformation"))
+            .withProjectDir(getProjectDir("testclusters"))
             .withArguments("skipped1", "user1", "-s")
             .withPluginClasspath()
             .build();

diff --git a/...src/testKit/clusterformation/build.gradle → ...Src/src/testKit/testclusters/build.gradle b/...src/testKit/clusterformation/build.gradle → ...Src/src/testKit/testclusters/build.gradle
@@ -1,5 +1,5 @@
 plugins {
-    id 'elasticsearch.clusterformation'
+    id 'elasticsearch.testclusters'
 }
 
 elasticSearchClusters {

diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java
@@ -148,7 +148,12 @@ static Request deleteJob(DeleteJobRequest deleteJobRequest) {
         Request request = new Request(HttpDelete.METHOD_NAME, endpoint);
 
         RequestConverters.Params params = new RequestConverters.Params(request);
-        params.putParam("force", Boolean.toString(deleteJobRequest.isForce()));
+        if (deleteJobRequest.getForce() != null) {
+            params.putParam("force", Boolean.toString(deleteJobRequest.getForce()));
+        }
+        if (deleteJobRequest.getWaitForCompletion() != null) {
+            params.putParam("wait_for_completion", Boolean.toString(deleteJobRequest.getWaitForCompletion()));
+        }
 
         return request;
     }

diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.client.ml.DeleteDatafeedRequest;
 import org.elasticsearch.client.ml.DeleteForecastRequest;
 import org.elasticsearch.client.ml.DeleteJobRequest;
+import org.elasticsearch.client.ml.DeleteJobResponse;
 import org.elasticsearch.client.ml.FlushJobRequest;
 import org.elasticsearch.client.ml.FlushJobResponse;
 import org.elasticsearch.client.ml.ForecastJobRequest;
@@ -211,14 +212,15 @@ public void getJobStatsAsync(GetJobStatsRequest request, RequestOptions options,
      *
      * @param request The request to delete the job
      * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
-     * @return action acknowledgement
+     * @return The action response which contains the acknowledgement or the task id depending on whether the action was set to wait for
+     * completion
      * @throws IOException when there is a serialization issue sending the request or receiving the response
      */
-    public AcknowledgedResponse deleteJob(DeleteJobRequest request, RequestOptions options) throws IOException {
+    public DeleteJobResponse deleteJob(DeleteJobRequest request, RequestOptions options) throws IOException {
         return restHighLevelClient.performRequestAndParseEntity(request,
             MLRequestConverters::deleteJob,
             options,
-            AcknowledgedResponse::fromXContent,
+            DeleteJobResponse::fromXContent,
             Collections.emptySet());
     }
 
@@ -232,11 +234,11 @@ public AcknowledgedResponse deleteJob(DeleteJobRequest request, RequestOptions o
      * @param options  Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
      * @param listener Listener to be notified upon request completion
      */
-    public void deleteJobAsync(DeleteJobRequest request, RequestOptions options, ActionListener<AcknowledgedResponse> listener) {
+    public void deleteJobAsync(DeleteJobRequest request, RequestOptions options, ActionListener<DeleteJobResponse> listener) {
         restHighLevelClient.performRequestAsyncAndParseEntity(request,
             MLRequestConverters::deleteJob,
             options,
-            AcknowledgedResponse::fromXContent,
+            DeleteJobResponse::fromXContent,
             listener,
             Collections.emptySet());
     }

diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/DeleteJobRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/DeleteJobRequest.java
@@ -29,7 +29,8 @@
 public class DeleteJobRequest extends ActionRequest {
 
     private String jobId;
-    private boolean force;
+    private Boolean force;
+    private Boolean waitForCompletion;
 
     public DeleteJobRequest(String jobId) {
         this.jobId = Objects.requireNonNull(jobId, "[job_id] must not be null");
@@ -47,7 +48,7 @@ public void setJobId(String jobId) {
         this.jobId = Objects.requireNonNull(jobId, "[job_id] must not be null");
     }
 
-    public boolean isForce() {
+    public Boolean getForce() {
         return force;
     }
 
@@ -57,10 +58,24 @@ public boolean isForce() {
      *
      * @param force When {@code true} forcefully delete an opened job. Defaults to {@code false}
      */
-    public void setForce(boolean force) {
+    public void setForce(Boolean force) {
         this.force = force;
     }
 
+    public Boolean getWaitForCompletion() {
+        return waitForCompletion;
+    }
+
+    /**
+     * Set whether this request should wait until the operation has completed before returning
+     * @param waitForCompletion When {@code true} the call will wait for the job deletion to complete.
+     *                          Otherwise, the deletion will be executed asynchronously and the response
+     *                          will contain the task id.
+     */
+    public void setWaitForCompletion(Boolean waitForCompletion) {
+        this.waitForCompletion = waitForCompletion;
+    }
+
     @Override
     public ActionRequestValidationException validate() {
        return null;