Merge branch 'master' into systemd_service_conf

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -140,16 +140,22 @@ class BuildPlugin implements Plugin<Project> {
 
             final GradleVersion minGradle = GradleVersion.version('4.3')
             if (currentGradleVersion < minGradle) {
-                throw new GradleException("${minGradle} or above is required to build elasticsearch")
+                throw new GradleException("${minGradle} or above is required to build Elasticsearch")
             }
 
             // enforce Java version
             if (compilerJavaVersionEnum < minimumCompilerVersion) {
-                throw new GradleException("Java ${minimumCompilerVersion} or above is required to build Elasticsearch")
+                final String message =
+                        "the environment variable JAVA_HOME must be set to a JDK installation directory for Java ${minimumCompilerVersion}" +
+                                " but is [${compilerJavaHome}] corresponding to [${compilerJavaVersionEnum}]"
+                throw new GradleException(message)
             }
 
             if (runtimeJavaVersionEnum < minimumRuntimeVersion) {
-                throw new GradleException("Java ${minimumRuntimeVersion} or above is required to run Elasticsearch")
+                final String message =
+                        "the environment variable RUNTIME_JAVA_HOME must be set to a JDK installation directory for Java ${minimumRuntimeVersion}" +
+                                " but is [${runtimeJavaHome}] corresponding to [${runtimeJavaVersionEnum}]"
+                throw new GradleException(message)
             }
 
             project.rootProject.ext.compilerJavaHome = compilerJavaHome

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.gradle.doc
 
+import groovy.transform.PackageScope
 import org.elasticsearch.gradle.doc.SnippetsTask.Snippet
 import org.gradle.api.InvalidUserDataException
 import org.gradle.api.tasks.Input
@@ -99,6 +100,43 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
         return snippet.language == 'js' || snippet.curl
     }
 
+    /**
+     * Converts Kibana's block quoted strings into standard JSON. These
+     * {@code """} delimited strings can be embedded in CONSOLE and can
+     * contain newlines and {@code "} without the normal JSON escaping.
+     * This has to add it.
+     */
+    @PackageScope
+    static String replaceBlockQuote(String body) {
+        int start = body.indexOf('"""');
+        if (start < 0) {
+            return body
+        }
+        /*
+         * 1.3 is a fairly wild guess of the extra space needed to hold
+         * the escaped string.
+         */
+        StringBuilder result = new StringBuilder((int) (body.length() * 1.3));
+        int startOfNormal = 0;
+        while (start >= 0) {
+            int end = body.indexOf('"""', start + 3);
+            if (end < 0) {
+                throw new InvalidUserDataException(
+                    "Invalid block quote starting at $start in:\n$body")
+            }
+            result.append(body.substring(startOfNormal, start));
+            result.append('"');
+            result.append(body.substring(start + 3, end)
+                .replace('"', '\\"')
+                .replace("\n", "\\n"));
+            result.append('"');
+            startOfNormal = end + 3;
+            start = body.indexOf('"""', startOfNormal);
+        }
+        result.append(body.substring(startOfNormal));
+        return result.toString();
+    }
+
     private class TestBuilder {
         private static final String SYNTAX = {
             String method = /(?<method>GET|PUT|POST|HEAD|OPTIONS|DELETE)/
@@ -259,6 +297,8 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
             if (body != null) {
                 // Throw out the leading newline we get from parsing the body
                 body = body.substring(1)
+                // Replace """ quoted strings with valid json ones
+                body = replaceBlockQuote(body)
                 current.println("        body: |")
                 body.eachLine { current.println("          $it") }
             }

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/MetaPluginBuildPlugin.groovy

@@ -37,19 +37,28 @@ class MetaPluginBuildPlugin implements Plugin<Project> {
         project.plugins.apply(RestTestPlugin)
 
         createBundleTask(project)
+        boolean isModule = project.path.startsWith(':modules:')
 
         project.integTestCluster {
             dependsOn(project.bundlePlugin)
-            plugin(project.path)
         }
         BuildPlugin.configurePomGeneration(project)
         project.afterEvaluate {
             PluginBuildPlugin.addZipPomGeneration(project)
+            if (isModule) {
+                if (project.integTestCluster.distribution == 'integ-test-zip') {
+                    project.integTestCluster.module(project)
+                }
+             } else {
+                project.integTestCluster.plugin(project.path)
+             }
         }
 
         RunTask run = project.tasks.create('run', RunTask)
         run.dependsOn(project.bundlePlugin)
-        run.clusterConfig.plugin(project.path)
+        if (isModule == false) {
+            run.clusterConfig.plugin(project.path)
+        }
     }
 
     private static void createBundleTask(Project project) {
@@ -79,12 +88,13 @@ class MetaPluginBuildPlugin implements Plugin<Project> {
             buildProperties.extension.plugins.each { String bundledPluginProjectName ->
                 Project bundledPluginProject = project.project(bundledPluginProjectName)
                 bundledPluginProject.afterEvaluate {
+                    String bundledPluginName = bundledPluginProject.esplugin.name
                     bundle.configure {
                         dependsOn bundledPluginProject.bundlePlugin
                         from(project.zipTree(bundledPluginProject.bundlePlugin.outputs.files.singleFile)) {
                             eachFile { FileCopyDetails details ->
                                 // we want each path to have the plugin name interjected
-                                details.relativePath = new RelativePath(true, bundledPluginProjectName, details.relativePath.toString())
+                                details.relativePath = new RelativePath(true, bundledPluginName, details.relativePath.toString())
                             }
                         }
                     }

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy

@@ -20,10 +20,15 @@ package org.elasticsearch.gradle.test
 
 import com.carrotsearch.gradle.junit4.RandomizedTestingTask
 import org.elasticsearch.gradle.BuildPlugin
+import org.elasticsearch.gradle.VersionProperties
 import org.gradle.api.DefaultTask
+import org.gradle.api.Project
 import org.gradle.api.Task
 import org.gradle.api.execution.TaskExecutionAdapter
 import org.gradle.api.internal.tasks.options.Option
+import org.gradle.api.provider.Property
+import org.gradle.api.provider.Provider
+import org.gradle.api.tasks.Copy
 import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.TaskState
 
@@ -47,7 +52,7 @@ public class RestIntegTestTask extends DefaultTask {
 
     /** Flag indicating whether the rest tests in the rest spec should be run. */
     @Input
-    boolean includePackaged = false
+    Property<Boolean> includePackaged = project.objects.property(Boolean)
 
     public RestIntegTestTask() {
         runner = project.tasks.create("${name}Runner", RandomizedTestingTask.class)
@@ -92,10 +97,9 @@ public class RestIntegTestTask extends DefaultTask {
         }
 
         // copy the rest spec/tests into the test resources
-        RestSpecHack.configureDependencies(project)
-        project.afterEvaluate {
-            runner.dependsOn(RestSpecHack.configureTask(project, includePackaged))
-        }
+        Task copyRestSpec = createCopyRestSpecTask(project, includePackaged)
+        runner.dependsOn(copyRestSpec)
+
         // this must run after all projects have been configured, so we know any project
         // references can be accessed as a fully configured
         project.gradle.projectsEvaluated {
@@ -109,6 +113,11 @@ public class RestIntegTestTask extends DefaultTask {
         }
     }
 
+    /** Sets the includePackaged property */
+    public void includePackaged(boolean include) {
+        includePackaged.set(include)
+    }
+
     @Option(
         option = "debug-jvm",
         description = "Enable debugging configuration, to allow attaching a debugger to elasticsearch."
@@ -184,4 +193,47 @@ public class RestIntegTestTask extends DefaultTask {
         println('=========================================')
 
     }
+
+    /**
+     * Creates a task (if necessary) to copy the rest spec files.
+     *
+     * @param project The project to add the copy task to
+     * @param includePackagedTests true if the packaged tests should be copied, false otherwise
+     */
+    private static Task createCopyRestSpecTask(Project project, Provider<Boolean> includePackagedTests) {
+        project.configurations {
+            restSpec
+        }
+        project.dependencies {
+            restSpec "org.elasticsearch:rest-api-spec:${VersionProperties.elasticsearch}"
+        }
+        Task copyRestSpec = project.tasks.findByName('copyRestSpec')
+        if (copyRestSpec != null) {
+            return copyRestSpec
+        }
+        Map copyRestSpecProps = [
+                name     : 'copyRestSpec',
+                type     : Copy,
+                dependsOn: [project.configurations.restSpec, 'processTestResources']
+        ]
+        copyRestSpec = project.tasks.create(copyRestSpecProps) {
+            into project.sourceSets.test.output.resourcesDir
+        }
+        project.afterEvaluate {
+            copyRestSpec.from({ project.zipTree(project.configurations.restSpec.singleFile) }) {
+                include 'rest-api-spec/api/**'
+                if (includePackagedTests.get()) {
+                    include 'rest-api-spec/test/**'
+                }
+            }
+        }
+        project.idea {
+            module {
+                if (scopes.TEST != null) {
+                    scopes.TEST.plus.add(project.configurations.restSpec)
+                }
+            }
+        }
+        return copyRestSpec
+    }
 }

buildSrc/src/test/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTaskTest.groovy

@@ -0,0 +1,50 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle.doc
+
+import org.elasticsearch.gradle.doc.SnippetsTask.Snippet
+import org.gradle.api.InvalidUserDataException
+
+import static org.elasticsearch.gradle.doc.RestTestsFromSnippetsTask.replaceBlockQuote
+
+class RestTestFromSnippetsTaskTest extends GroovyTestCase {
+    void testInvalidBlockQuote() {
+        String input = "\"foo\": \"\"\"bar\"";
+        String message = shouldFail({ replaceBlockQuote(input) });
+        assertEquals("Invalid block quote starting at 7 in:\n$input", message);
+    }
+
+    void testSimpleBlockQuote() {
+        assertEquals("\"foo\": \"bort baz\"",
+            replaceBlockQuote("\"foo\": \"\"\"bort baz\"\"\""));
+    }
+
+    void testMultipleBlockQuotes() {
+        assertEquals("\"foo\": \"bort baz\", \"bar\": \"other\"",
+            replaceBlockQuote("\"foo\": \"\"\"bort baz\"\"\", \"bar\": \"\"\"other\"\"\""));
+    }
+
+    void testEscapingInBlockQuote() {
+        assertEquals("\"foo\": \"bort\\\" baz\"",
+            replaceBlockQuote("\"foo\": \"\"\"bort\" baz\"\"\""));
+        assertEquals("\"foo\": \"bort\\n baz\"",
+            replaceBlockQuote("\"foo\": \"\"\"bort\n baz\"\"\""));
+    }
+}

client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java

@@ -82,7 +82,6 @@
 import java.nio.charset.Charset;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
@@ -328,7 +327,7 @@ static Request bulk(BulkRequest bulkRequest) throws IOException {
                 }
                 metadata.endObject();
 
-                BytesRef metadataSource = metadata.bytes().toBytesRef();
+                BytesRef metadataSource = BytesReference.bytes(metadata).toBytesRef();
                 content.write(metadataSource.bytes, metadataSource.offset, metadataSource.length);
                 content.write(separator);
             }
@@ -343,7 +342,7 @@ static Request bulk(BulkRequest bulkRequest) throws IOException {
                     LoggingDeprecationHandler.INSTANCE, indexSource, indexXContentType)) {
                     try (XContentBuilder builder = XContentBuilder.builder(bulkContentType.xContent())) {
                         builder.copyCurrentStructure(parser);
-                        source = builder.bytes().toBytesRef();
+                        source = BytesReference.bytes(builder).toBytesRef();
                     }
                 }
             } else if (opType == DocWriteRequest.OpType.UPDATE) {
@@ -517,9 +516,7 @@ static Request existsAlias(GetAliasesRequest getAliasesRequest) {
     }
 
     static Request rankEval(RankEvalRequest rankEvalRequest) throws IOException {
-        // TODO maybe indices should be property of RankEvalRequest and not of the spec
-        List<String> indices = rankEvalRequest.getRankEvalSpec().getIndices();
-        String endpoint = endpoint(indices.toArray(new String[indices.size()]), Strings.EMPTY_ARRAY, "_rank_eval");
+        String endpoint = endpoint(rankEvalRequest.getIndices(), Strings.EMPTY_ARRAY, "_rank_eval");
         HttpEntity entity = createEntity(rankEvalRequest.getRankEvalSpec(), REQUEST_BODY_CONTENT_TYPE);
         return new Request(HttpGet.METHOD_NAME, endpoint, Collections.emptyMap(), entity);
     }