Merge branch '6.x' into ccr-6.x

* 6.x: (41 commits)
  [TEST] Fix `GeoShapeQueryTests#testPointsOnly` failure
  Transition transport apis to use void listeners (#27440)
  AwaitsFix GeoShapeQueryTests#testPointsOnly #27454
  Ensure nested documents have consistent version and seq_ids (#27455)
  Tests: Add Fedora-27 to packaging tests
  #26800: Fix docs rendering
  Move the CLI into its own subproject (#27114)
  Correct usage of "an" to "a" in getting started docs
  Avoid NPE when getting build information
  Remove manual tracking of registered channels (#27445)
  Standardize underscore requirements in parameters (#27414)
  Remove parameters on HandshakeResponseHandler (#27444)
  [GEO] fix pointsOnly bug for MULTIPOINT
  peanut butter hamburgers
  Uses TransportMasterNodeAction to update shard snapshot status (#27165)
  Log primary-replica resync failures
  Add limits for ngram and shingle settings (#27411)
  Enforce a minimum task execution and service time of 1 nanosecond
  Fix place-holder in allocation decider messages (#27436)
  Remove newline from log message (#27425)
  ...

README.textile

@@ -208,8 +208,7 @@ In order to create a distribution, simply run the @gradle assemble@ command in t
 
 The distribution for each project will be created under the @build/distributions@ directory in that project.
 
-See the "TESTING":TESTING.asciidoc file for more information about
-running the Elasticsearch test suite.
+See the "TESTING":TESTING.asciidoc file for more information about running the Elasticsearch test suite.
 
 h3. Upgrading from Elasticsearch 1.x?
 

TESTING.asciidoc

@@ -351,8 +351,8 @@ These are the linux flavors the Vagrantfile currently supports:
 * debian-9 aka stretch, the current debian stable distribution
 * centos-6
 * centos-7
-* fedora-25
 * fedora-26
+* fedora-27
 * oel-6 aka Oracle Enterprise Linux 6
 * oel-7 aka Oracle Enterprise Linux 7
 * sles-12
@@ -428,23 +428,23 @@ sudo -E bats $BATS_TESTS/*.bats
 You can also use Gradle to prepare the test environment and then starts a single VM:
 
 -------------------------------------------------
-gradle vagrantFedora25#up
+gradle vagrantFedora27#up
 -------------------------------------------------
 
 Or any of vagrantCentos6#up, vagrantCentos7#up, vagrantDebian8#up,
-vagrantFedora25#up, vagrantOel6#up, vagrantOel7#up, vagrantOpensuse13#up,
-vagrantSles12#up, vagrantUbuntu1404#up, vagrantUbuntu1604#up.
+vagrantDebian9#up, vagrantFedora26#up, vagrantFedora27#up, vagrantOel6#up, vagrantOel7#up,
+vagrantOpensuse42#up,vagrantSles12#up, vagrantUbuntu1404#up, vagrantUbuntu1604#up.
 
 Once up, you can then connect to the VM using SSH from the elasticsearch directory:
 
 -------------------------------------------------
-vagrant ssh fedora-25
+vagrant ssh fedora-27
 -------------------------------------------------
 
 Or from another directory:
 
 -------------------------------------------------
-VAGRANT_CWD=/path/to/elasticsearch vagrant ssh fedora-25
+VAGRANT_CWD=/path/to/elasticsearch vagrant ssh fedora-27
 -------------------------------------------------
 
 Note: Starting vagrant VM outside of the elasticsearch folder requires to

Vagrantfile

@@ -60,14 +60,14 @@ Vagrant.configure(2) do |config|
     config.vm.box = "elastic/oraclelinux-7-x86_64"
     rpm_common config
   end
-  config.vm.define "fedora-25" do |config|
-    config.vm.box = "elastic/fedora-25-x86_64"
-    dnf_common config
-  end
   config.vm.define "fedora-26" do |config|
     config.vm.box = "elastic/fedora-26-x86_64"
     dnf_common config
   end
+  config.vm.define "fedora-27" do |config|
+    config.vm.box = "elastic/fedora-27-x86_64"
+    dnf_common config
+  end
   config.vm.define "opensuse-42" do |config|
     config.vm.box = "elastic/opensuse-42-x86_64"
     opensuse_common config

build.gradle

@@ -81,6 +81,7 @@ List<Version> versions = []
 // keep track of the previous major version's last minor, so we know where wire compat begins
 int prevMinorIndex = -1 // index in the versions list of the last minor from the prev major
 int lastPrevMinor = -1 // the minor version number from the prev major we most recently seen
+int prevBugfixIndex = -1 // index in the versions list of the last bugfix release from the prev major
 for (String line : versionLines) {
   /* Note that this skips alphas and betas which is fine because they aren't
    * compatible with anything. */
@@ -97,12 +98,19 @@ for (String line : versionLines) {
       prevMinorIndex = versions.size() - 1
       lastPrevMinor = minor
     }
+    if (major == prevMajor) {
+      prevBugfixIndex = versions.size() - 1
+    }
   }
 }
 if (versions.toSorted { it.id } != versions) {
   println "Versions: ${versions}"
   throw new GradleException("Versions.java contains out of order version constants")
 }
+if (prevBugfixIndex != -1) {
+  versions[prevBugfixIndex] = new Version(
+      versions[prevBugfixIndex].major, versions[prevBugfixIndex].minor, versions[prevBugfixIndex].bugfix, true)
+}
 if (currentVersion.bugfix == 0) {
   // If on a release branch, after the initial release of that branch, the bugfix version will
   // be bumped, and will be != 0. On master and N.x branches, we want to test against the
@@ -212,6 +220,7 @@ subprojects {
     "org.elasticsearch.gradle:build-tools:${version}": ':build-tools',
     "org.elasticsearch:rest-api-spec:${version}": ':rest-api-spec',
     "org.elasticsearch:elasticsearch:${version}": ':core',
+    "org.elasticsearch:elasticsearch-cli:${version}": ':core:cli',
     "org.elasticsearch.client:elasticsearch-rest-client:${version}": ':client:rest',
     "org.elasticsearch.client:elasticsearch-rest-client-sniffer:${version}": ':client:sniffer',
     "org.elasticsearch.client:elasticsearch-rest-high-level-client:${version}": ':client:rest-high-level',
@@ -231,6 +240,7 @@ subprojects {
     "org.elasticsearch.plugin:parent-join-client:${version}": ':modules:parent-join',
     "org.elasticsearch.plugin:aggs-matrix-stats-client:${version}": ':modules:aggs-matrix-stats',
     "org.elasticsearch.plugin:percolator-client:${version}": ':modules:percolator',
+    "org.elasticsearch.plugin:aggs-composite-client:${version}": ':modules:aggs-composite',
   ]
   if (indexCompatVersions[-1].snapshot) {
     /* The last and second to last versions can be snapshots. Rather than use
@@ -248,6 +258,11 @@ subprojects {
       ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch:${indexCompatVersions[-1]}"] = ':distribution:bwc-release-snapshot'
       ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${indexCompatVersions[-1]}"] = ':distribution:bwc-release-snapshot'
     }
+  } else if (indexCompatVersions[-2].snapshot) {
+    /* This is a terrible hack for the bump to 6.0.1 which will be fixed by #27397 */
+    ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch:${indexCompatVersions[-2]}"] = ':distribution:bwc-release-snapshot'
+    ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch:${indexCompatVersions[-2]}"] = ':distribution:bwc-release-snapshot'
+    ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${indexCompatVersions[-2]}"] = ':distribution:bwc-release-snapshot'
   }
   project.afterEvaluate {
     configurations.all {

buildSrc/build.gradle

@@ -19,6 +19,8 @@
 
 import java.nio.file.Files
 
+import org.gradle.util.GradleVersion
+
 apply plugin: 'groovy'
 
 group = 'org.elasticsearch.gradle'
@@ -99,9 +101,11 @@ dependencies {
 
 // Gradle 2.14+ removed ProgressLogger(-Factory) classes from the public APIs
 // Use logging dependency instead
+// Gradle 4.3.1 stopped releasing the logging jars to jcenter, just use the last available one
+GradleVersion logVersion = GradleVersion.current() > GradleVersion.version('4.3') ? GradleVersion.version('4.3') : GradleVersion.current()
 
 dependencies {
-  compileOnly "org.gradle:gradle-logging:${GradleVersion.current().getVersion()}"
+  compileOnly "org.gradle:gradle-logging:${logVersion.getVersion()}"
   compile 'ru.vyarus:gradle-animalsniffer-plugin:1.2.0' // Gradle 2.14 requires a version > 1.0.1
 }
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy

@@ -19,8 +19,8 @@ class VagrantTestPlugin implements Plugin<Project> {
             'centos-7',
             'debian-8',
             'debian-9',
-            'fedora-25',
             'fedora-26',
+            'fedora-27',
             'oel-6',
             'oel-7',
             'opensuse-42',

client/rest-high-level/build.gradle

@@ -39,7 +39,8 @@ dependencies {
   compile "org.elasticsearch.client:elasticsearch-rest-client:${version}"
   compile "org.elasticsearch.plugin:parent-join-client:${version}"
   compile "org.elasticsearch.plugin:aggs-matrix-stats-client:${version}"
-
+  compile "org.elasticsearch.plugin:aggs-composite-client:${version}"
+
   testCompile "org.elasticsearch.client:test:${version}"
   testCompile "org.elasticsearch.test:framework:${version}"
   testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"

client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java

@@ -191,23 +191,23 @@ static Request bulk(BulkRequest bulkRequest) throws IOException {
                         metadata.field("_id", request.id());
                     }
                     if (Strings.hasLength(request.routing())) {
-                        metadata.field("_routing", request.routing());
+                        metadata.field("routing", request.routing());
                     }
                     if (Strings.hasLength(request.parent())) {
-                        metadata.field("_parent", request.parent());
+                        metadata.field("parent", request.parent());
                     }
                     if (request.version() != Versions.MATCH_ANY) {
-                        metadata.field("_version", request.version());
+                        metadata.field("version", request.version());
                     }
 
                     VersionType versionType = request.versionType();
                     if (versionType != VersionType.INTERNAL) {
                         if (versionType == VersionType.EXTERNAL) {
-                            metadata.field("_version_type", "external");
+                            metadata.field("version_type", "external");
                         } else if (versionType == VersionType.EXTERNAL_GTE) {
-                            metadata.field("_version_type", "external_gte");
+                            metadata.field("version_type", "external_gte");
                         } else if (versionType == VersionType.FORCE) {
-                            metadata.field("_version_type", "force");
+                            metadata.field("version_type", "force");
                         }
                     }
 
@@ -219,7 +219,7 @@ static Request bulk(BulkRequest bulkRequest) throws IOException {
                     } else if (opType == DocWriteRequest.OpType.UPDATE) {
                         UpdateRequest updateRequest = (UpdateRequest) request;
                         if (updateRequest.retryOnConflict() > 0) {
-                            metadata.field("_retry_on_conflict", updateRequest.retryOnConflict());
+                            metadata.field("retry_on_conflict", updateRequest.retryOnConflict());
                         }
                         if (updateRequest.fetchSource() != null) {
                             metadata.field("_source", updateRequest.fetchSource());

client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java

@@ -62,6 +62,7 @@
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.aggregations.Aggregation;
 import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.aggregations.composite.CompositeAggregationBuilder;
 import org.elasticsearch.search.aggregations.matrix.stats.MatrixStatsAggregationBuilder;
 import org.elasticsearch.search.suggest.Suggest;
 import org.elasticsearch.test.ESTestCase;
@@ -647,7 +648,7 @@ public void testDefaultNamedXContents() {
 
     public void testProvidedNamedXContents() {
         List<NamedXContentRegistry.Entry> namedXContents = RestHighLevelClient.getProvidedNamedXContents();
-        assertEquals(2, namedXContents.size());
+        assertEquals(3, namedXContents.size());
         Map<Class<?>, Integer> categories = new HashMap<>();
         List<String> names = new ArrayList<>();
         for (NamedXContentRegistry.Entry namedXContent : namedXContents) {
@@ -658,9 +659,10 @@ public void testProvidedNamedXContents() {
             }
         }
         assertEquals(1, categories.size());
-        assertEquals(Integer.valueOf(2), categories.get(Aggregation.class));
+        assertEquals(Integer.valueOf(3), categories.get(Aggregation.class));
         assertTrue(names.contains(ChildrenAggregationBuilder.NAME));
         assertTrue(names.contains(MatrixStatsAggregationBuilder.NAME));
+        assertTrue(names.contains(CompositeAggregationBuilder.NAME));
     }
 
     private static class TrackingActionListener implements ActionListener<Integer> {

client/transport/build.gradle

@@ -32,6 +32,7 @@ dependencies {
   compile "org.elasticsearch.plugin:lang-mustache-client:${version}"
   compile "org.elasticsearch.plugin:percolator-client:${version}"
   compile "org.elasticsearch.plugin:parent-join-client:${version}"
+  compile "org.elasticsearch.plugin:aggs-composite-client:${version}"
   testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
   testCompile "junit:junit:${versions.junit}"
   testCompile "org.hamcrest:hamcrest-all:${versions.hamcrest}"

client/transport/src/main/java/org/elasticsearch/transport/client/PreBuiltTransportClient.java

@@ -30,6 +30,7 @@
 import org.elasticsearch.percolator.PercolatorPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.mustache.MustachePlugin;
+import org.elasticsearch.search.aggregations.composite.CompositeAggregationPlugin;
 import org.elasticsearch.transport.Netty4Plugin;
 
 import java.util.Arrays;
@@ -44,6 +45,7 @@
  * {@link PercolatorPlugin},
  * {@link MustachePlugin},
  * {@link ParentJoinPlugin}
+ * {@link CompositeAggregationPlugin}
  * plugins for the client. These plugins are all the required modules for Elasticsearch.
  */
 @SuppressWarnings({"unchecked","varargs"})
@@ -88,7 +90,8 @@ private static void setSystemPropertyIfUnset(final String key, final String valu
                 ReindexPlugin.class,
                 PercolatorPlugin.class,
                 MustachePlugin.class,
-                ParentJoinPlugin.class));
+                ParentJoinPlugin.class,
+                CompositeAggregationPlugin.class));
 
     /**
      * Creates a new transport client with pre-installed plugins.

client/transport/src/test/java/org/elasticsearch/transport/client/PreBuiltTransportClientTests.java

@@ -30,6 +30,7 @@
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.mustache.MustachePlugin;
 import org.elasticsearch.transport.Netty4Plugin;
+import org.elasticsearch.search.aggregations.composite.CompositeAggregationPlugin;
 import org.junit.Test;
 
 import java.util.Arrays;
@@ -52,7 +53,8 @@ public void testPluginInstalled() {
     @Test
     public void testInstallPluginTwice() {
         for (Class<? extends Plugin> plugin :
-                Arrays.asList(ParentJoinPlugin.class, ReindexPlugin.class, PercolatorPlugin.class, MustachePlugin.class)) {
+                Arrays.asList(ParentJoinPlugin.class, ReindexPlugin.class, PercolatorPlugin.class,
+                    MustachePlugin.class, CompositeAggregationPlugin.class)) {
             try {
                 new PreBuiltTransportClient(Settings.EMPTY, plugin);
                 fail("exception expected");

core/build.gradle

@@ -58,7 +58,7 @@ dependencies {
   compile 'org.elasticsearch:securesm:1.1'
 
   // utilities
-  compile 'net.sf.jopt-simple:jopt-simple:5.0.2'
+  compile "org.elasticsearch:elasticsearch-cli:${version}"
   compile 'com.carrotsearch:hppc:0.7.1'
 
   // time handling, remove with java 8 time
@@ -265,6 +265,12 @@ if (JavaVersion.current() > JavaVersion.VERSION_1_8) {
 dependencyLicenses {
   mapping from: /lucene-.*/, to: 'lucene'
   mapping from: /jackson-.*/, to: 'jackson'
+  dependencies = project.configurations.runtime.fileCollection {
+    it.group.startsWith('org.elasticsearch') == false ||
+            // keep the following org.elasticsearch jars in
+            (it.name == 'jna' ||
+             it.name == 'securesm')
+  }
 }
 
 if (isEclipse == false || project.path == ":core-tests") {

core/cli/build.gradle

@@ -0,0 +1,36 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.elasticsearch.gradle.precommit.PrecommitTasks
+
+apply plugin: 'elasticsearch.build'
+
+archivesBaseName = 'elasticsearch-cli'
+
+dependencies {
+    compile 'net.sf.jopt-simple:jopt-simple:5.0.2'
+}
+
+test.enabled = false
+// Since CLI does not depend on :core, it cannot run the jarHell task
+jarHell.enabled = false
+
+forbiddenApisMain {
+    signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
+}