Merge branch 'master' into pr/27409

* master: (41 commits) [Test] Fix AggregationsTests#testFromXContentWithRandomFields [DOC] Fix mathematical representation on interval (range) (elastic#27450) Update version check for CCS optional remote clusters Bump BWC version to 6.1.0 for elastic#27469 Adapt rest test BWC version after backport Fix dynamic mapping update generation. (elastic#27467) Use the primary_term field to identify parent documents (elastic#27469) Move composite aggregation to core (elastic#27474) Fix test BWC version after backport Protect shard splitting from illegal target shards (elastic#27468) Cross Cluster Search: make remote clusters optional (elastic#27182) [Docs] Fix broken bulleted lists (elastic#27470) Move resync request serialization assertion Fix resync request serialization Fix issue where pages aren't released (elastic#27459) Add YAML REST tests for filters bucket agg (elastic#27128) Remove tcp profile from low level nio channel (elastic#27441) [TEST] Fix `GeoShapeQueryTests#testPointsOnly` failure Transition transport apis to use void listeners (elastic#27440) AwaitsFix GeoShapeQueryTests#testPointsOnly elastic#27454 ...
olcbean · Nov 21, 2017 · 02703f0 · 02703f0
2 parents 3f281a7 + 3427062
commit 02703f0
Show file tree

Hide file tree

Showing 208 changed files with 3,351 additions and 1,399 deletions.
diff --git a/TESTING.asciidoc b/TESTING.asciidoc
@@ -351,8 +351,8 @@ These are the linux flavors the Vagrantfile currently supports:
 * debian-9 aka stretch, the current debian stable distribution
 * centos-6
 * centos-7
-* fedora-25
 * fedora-26
+* fedora-27
 * oel-6 aka Oracle Enterprise Linux 6
 * oel-7 aka Oracle Enterprise Linux 7
 * sles-12
@@ -428,23 +428,23 @@ sudo -E bats $BATS_TESTS/*.bats
 You can also use Gradle to prepare the test environment and then starts a single VM:
 
 -------------------------------------------------
-gradle vagrantFedora25#up
+gradle vagrantFedora27#up
 -------------------------------------------------
 
 Or any of vagrantCentos6#up, vagrantCentos7#up, vagrantDebian8#up,
-vagrantFedora25#up, vagrantOel6#up, vagrantOel7#up, vagrantOpensuse13#up,
-vagrantSles12#up, vagrantUbuntu1404#up, vagrantUbuntu1604#up.
+vagrantDebian9#up, vagrantFedora26#up, vagrantFedora27#up, vagrantOel6#up, vagrantOel7#up,
+vagrantOpensuse42#up,vagrantSles12#up, vagrantUbuntu1404#up, vagrantUbuntu1604#up.
 
 Once up, you can then connect to the VM using SSH from the elasticsearch directory:
 
 -------------------------------------------------
-vagrant ssh fedora-25
+vagrant ssh fedora-27
 -------------------------------------------------
 
 Or from another directory:
 
 -------------------------------------------------
-VAGRANT_CWD=/path/to/elasticsearch vagrant ssh fedora-25
+VAGRANT_CWD=/path/to/elasticsearch vagrant ssh fedora-27
 -------------------------------------------------
 
 Note: Starting vagrant VM outside of the elasticsearch folder requires to

diff --git a/Vagrantfile b/Vagrantfile
@@ -60,14 +60,14 @@ Vagrant.configure(2) do |config|
     config.vm.box = "elastic/oraclelinux-7-x86_64"
     rpm_common config
   end
-  config.vm.define "fedora-25" do |config|
-    config.vm.box = "elastic/fedora-25-x86_64"
-    dnf_common config
-  end
   config.vm.define "fedora-26" do |config|
     config.vm.box = "elastic/fedora-26-x86_64"
     dnf_common config
   end
+  config.vm.define "fedora-27" do |config|
+    config.vm.box = "elastic/fedora-27-x86_64"
+    dnf_common config
+  end
   config.vm.define "opensuse-42" do |config|
     config.vm.box = "elastic/opensuse-42-x86_64"
     opensuse_common config

diff --git a/build.gradle b/build.gradle
@@ -231,6 +231,7 @@ subprojects {
     "org.elasticsearch.gradle:build-tools:${version}": ':build-tools',
     "org.elasticsearch:rest-api-spec:${version}": ':rest-api-spec',
     "org.elasticsearch:elasticsearch:${version}": ':core',
+    "org.elasticsearch:elasticsearch-cli:${version}": ':core:cli',
     "org.elasticsearch.client:elasticsearch-rest-client:${version}": ':client:rest',
     "org.elasticsearch.client:elasticsearch-rest-client-sniffer:${version}": ':client:sniffer',
     "org.elasticsearch.client:elasticsearch-rest-high-level-client:${version}": ':client:rest-high-level',
@@ -250,7 +251,6 @@ subprojects {
     "org.elasticsearch.plugin:parent-join-client:${version}": ':modules:parent-join',
     "org.elasticsearch.plugin:aggs-matrix-stats-client:${version}": ':modules:aggs-matrix-stats',
     "org.elasticsearch.plugin:percolator-client:${version}": ':modules:percolator',
-    "org.elasticsearch.plugin:aggs-composite-client:${version}": ':modules:aggs-composite',
   ]
   if (indexCompatVersions[-1].snapshot) {
     /* The last and second to last versions can be snapshots. Rather than use

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy
@@ -19,8 +19,8 @@ class VagrantTestPlugin implements Plugin<Project> {
             'centos-7',
             'debian-8',
             'debian-9',
-            'fedora-25',
             'fedora-26',
+            'fedora-27',
             'oel-6',
             'oel-7',
             'opensuse-42',

diff --git a/.../src/main/java/org/elasticsearch/plugin/noop/action/search/TransportNoopSearchAction.java b/.../src/main/java/org/elasticsearch/plugin/noop/action/search/TransportNoopSearchAction.java
@@ -53,6 +53,7 @@ protected void doExecute(SearchRequest request, ActionListener<SearchResponse> l
                 new SearchHit[0], 0L, 0.0f),
             new InternalAggregations(Collections.emptyList()),
             new Suggest(Collections.emptyList()),
-            new SearchProfileShardResults(Collections.emptyMap()), false, false, 1), "", 1, 1, 0, 0, new ShardSearchFailure[0]));
+            new SearchProfileShardResults(Collections.emptyMap()), false, false, 1), "", 1, 1, 0, 0, ShardSearchFailure.EMPTY_ARRAY,
+                SearchResponse.Clusters.EMPTY));
     }
 }
diff --git a/client/rest-high-level/build.gradle b/client/rest-high-level/build.gradle
@@ -39,8 +39,7 @@ dependencies {
   compile "org.elasticsearch.client:elasticsearch-rest-client:${version}"
   compile "org.elasticsearch.plugin:parent-join-client:${version}"
   compile "org.elasticsearch.plugin:aggs-matrix-stats-client:${version}"
-  compile "org.elasticsearch.plugin:aggs-composite-client:${version}"
-
+
   testCompile "org.elasticsearch.client:test:${version}"
   testCompile "org.elasticsearch.test:framework:${version}"
   testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"

diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java
@@ -191,23 +191,23 @@ static Request bulk(BulkRequest bulkRequest) throws IOException {
                         metadata.field("_id", request.id());
                     }
                     if (Strings.hasLength(request.routing())) {
-                        metadata.field("_routing", request.routing());
+                        metadata.field("routing", request.routing());
                     }
                     if (Strings.hasLength(request.parent())) {
-                        metadata.field("_parent", request.parent());
+                        metadata.field("parent", request.parent());
                     }
                     if (request.version() != Versions.MATCH_ANY) {
-                        metadata.field("_version", request.version());
+                        metadata.field("version", request.version());
                     }
 
                     VersionType versionType = request.versionType();
                     if (versionType != VersionType.INTERNAL) {
                         if (versionType == VersionType.EXTERNAL) {
-                            metadata.field("_version_type", "external");
+                            metadata.field("version_type", "external");
                         } else if (versionType == VersionType.EXTERNAL_GTE) {
-                            metadata.field("_version_type", "external_gte");
+                            metadata.field("version_type", "external_gte");
                         } else if (versionType == VersionType.FORCE) {
-                            metadata.field("_version_type", "force");
+                            metadata.field("version_type", "force");
                         }
                     }
 
@@ -219,7 +219,7 @@ static Request bulk(BulkRequest bulkRequest) throws IOException {
                     } else if (opType == DocWriteRequest.OpType.UPDATE) {
                         UpdateRequest updateRequest = (UpdateRequest) request;
                         if (updateRequest.retryOnConflict() > 0) {
-                            metadata.field("_retry_on_conflict", updateRequest.retryOnConflict());
+                            metadata.field("retry_on_conflict", updateRequest.retryOnConflict());
                         }
                         if (updateRequest.fetchSource() != null) {
                             metadata.field("_source", updateRequest.fetchSource());

diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java
@@ -56,6 +56,8 @@
 import org.elasticsearch.search.aggregations.Aggregation;
 import org.elasticsearch.search.aggregations.bucket.adjacency.AdjacencyMatrixAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.adjacency.ParsedAdjacencyMatrix;
+import org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.composite.ParsedComposite;
 import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.filter.FiltersAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.filter.ParsedFilter;
@@ -621,6 +623,7 @@ static List<NamedXContentRegistry.Entry> getDefaultNamedXContents() {
         map.put(ScriptedMetricAggregationBuilder.NAME, (p, c) -> ParsedScriptedMetric.fromXContent(p, (String) c));
         map.put(IpRangeAggregationBuilder.NAME, (p, c) -> ParsedBinaryRange.fromXContent(p, (String) c));
         map.put(TopHitsAggregationBuilder.NAME, (p, c) -> ParsedTopHits.fromXContent(p, (String) c));
+        map.put(CompositeAggregationBuilder.NAME, (p, c) -> ParsedComposite.fromXContent(p, (String) c));
         List<NamedXContentRegistry.Entry> entries = map.entrySet().stream()
                 .map(entry -> new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(entry.getKey()), entry.getValue()))
                 .collect(Collectors.toList());

diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
@@ -62,7 +62,6 @@
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.aggregations.Aggregation;
 import org.elasticsearch.search.aggregations.InternalAggregations;
-import org.elasticsearch.search.aggregations.composite.CompositeAggregationBuilder;
 import org.elasticsearch.search.aggregations.matrix.stats.MatrixStatsAggregationBuilder;
 import org.elasticsearch.search.suggest.Suggest;
 import org.elasticsearch.test.ESTestCase;
@@ -166,7 +165,8 @@ public void testInfo() throws IOException {
     public void testSearchScroll() throws IOException {
         Header[] headers = randomHeaders(random(), "Header");
         SearchResponse mockSearchResponse = new SearchResponse(new SearchResponseSections(SearchHits.empty(), InternalAggregations.EMPTY,
-                null, false, false, null, 1), randomAlphaOfLengthBetween(5, 10), 5, 5, 0, 100, new ShardSearchFailure[0]);
+                null, false, false, null, 1), randomAlphaOfLengthBetween(5, 10), 5, 5, 0, 100, ShardSearchFailure.EMPTY_ARRAY,
+                SearchResponse.Clusters.EMPTY);
         mockResponse(mockSearchResponse);
         SearchResponse searchResponse = restHighLevelClient.searchScroll(new SearchScrollRequest(randomAlphaOfLengthBetween(5, 10)),
                 headers);
@@ -648,7 +648,7 @@ public void testDefaultNamedXContents() {
 
     public void testProvidedNamedXContents() {
         List<NamedXContentRegistry.Entry> namedXContents = RestHighLevelClient.getProvidedNamedXContents();
-        assertEquals(3, namedXContents.size());
+        assertEquals(2, namedXContents.size());
         Map<Class<?>, Integer> categories = new HashMap<>();
         List<String> names = new ArrayList<>();
         for (NamedXContentRegistry.Entry namedXContent : namedXContents) {
@@ -659,10 +659,9 @@ public void testProvidedNamedXContents() {
             }
         }
         assertEquals(1, categories.size());
-        assertEquals(Integer.valueOf(3), categories.get(Aggregation.class));
+        assertEquals(Integer.valueOf(2), categories.get(Aggregation.class));
         assertTrue(names.contains(ChildrenAggregationBuilder.NAME));
         assertTrue(names.contains(MatrixStatsAggregationBuilder.NAME));
-        assertTrue(names.contains(CompositeAggregationBuilder.NAME));
     }
 
     private static class TrackingActionListener implements ActionListener<Integer> {

diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java
@@ -470,5 +470,6 @@ private static void assertSearchHeader(SearchResponse searchResponse) {
         assertThat(searchResponse.getTotalShards(), greaterThan(0));
         assertEquals(searchResponse.getTotalShards(), searchResponse.getSuccessfulShards());
         assertEquals(0, searchResponse.getShardFailures().length);
+        assertEquals(SearchResponse.Clusters.EMPTY, searchResponse.getClusters());
     }
 }
diff --git a/client/transport/build.gradle b/client/transport/build.gradle
@@ -32,7 +32,6 @@ dependencies {
   compile "org.elasticsearch.plugin:lang-mustache-client:${version}"
   compile "org.elasticsearch.plugin:percolator-client:${version}"
   compile "org.elasticsearch.plugin:parent-join-client:${version}"
-  compile "org.elasticsearch.plugin:aggs-composite-client:${version}"
   testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
   testCompile "junit:junit:${versions.junit}"
   testCompile "org.hamcrest:hamcrest-all:${versions.hamcrest}"

diff --git a/...t/transport/src/main/java/org/elasticsearch/transport/client/PreBuiltTransportClient.java b/...t/transport/src/main/java/org/elasticsearch/transport/client/PreBuiltTransportClient.java
@@ -30,7 +30,6 @@
 import org.elasticsearch.percolator.PercolatorPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.mustache.MustachePlugin;
-import org.elasticsearch.search.aggregations.composite.CompositeAggregationPlugin;
 import org.elasticsearch.transport.Netty4Plugin;
 
 import java.util.Arrays;
@@ -45,7 +44,6 @@
  * {@link PercolatorPlugin},
  * {@link MustachePlugin},
  * {@link ParentJoinPlugin}
- * {@link CompositeAggregationPlugin}
  * plugins for the client. These plugins are all the required modules for Elasticsearch.
  */
 @SuppressWarnings({"unchecked","varargs"})
@@ -90,8 +88,7 @@ private static void setSystemPropertyIfUnset(final String key, final String valu
                 ReindexPlugin.class,
                 PercolatorPlugin.class,
                 MustachePlugin.class,
-                ParentJoinPlugin.class,
-                CompositeAggregationPlugin.class));
+                ParentJoinPlugin.class));
 
     /**
      * Creates a new transport client with pre-installed plugins.

diff --git a/...nsport/src/test/java/org/elasticsearch/transport/client/PreBuiltTransportClientTests.java b/...nsport/src/test/java/org/elasticsearch/transport/client/PreBuiltTransportClientTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.transport.client;
 
 import com.carrotsearch.randomizedtesting.RandomizedTest;
-import org.apache.lucene.util.Constants;
 import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
@@ -30,7 +29,6 @@
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.mustache.MustachePlugin;
 import org.elasticsearch.transport.Netty4Plugin;
-import org.elasticsearch.search.aggregations.composite.CompositeAggregationPlugin;
 import org.junit.Test;
 
 import java.util.Arrays;
@@ -54,7 +52,7 @@ public void testPluginInstalled() {
     public void testInstallPluginTwice() {
         for (Class<? extends Plugin> plugin :
                 Arrays.asList(ParentJoinPlugin.class, ReindexPlugin.class, PercolatorPlugin.class,
-                    MustachePlugin.class, CompositeAggregationPlugin.class)) {
+                    MustachePlugin.class)) {
             try {
                 new PreBuiltTransportClient(Settings.EMPTY, plugin);
                 fail("exception expected");

diff --git a/core/build.gradle b/core/build.gradle
@@ -58,7 +58,7 @@ dependencies {
   compile 'org.elasticsearch:securesm:1.1'
 
   // utilities
-  compile 'net.sf.jopt-simple:jopt-simple:5.0.2'
+  compile "org.elasticsearch:elasticsearch-cli:${version}"
   compile 'com.carrotsearch:hppc:0.7.1'
 
   // time handling, remove with java 8 time
@@ -265,6 +265,12 @@ if (JavaVersion.current() > JavaVersion.VERSION_1_8) {
 dependencyLicenses {
   mapping from: /lucene-.*/, to: 'lucene'
   mapping from: /jackson-.*/, to: 'jackson'
+  dependencies = project.configurations.runtime.fileCollection {
+    it.group.startsWith('org.elasticsearch') == false ||
+            // keep the following org.elasticsearch jars in
+            (it.name == 'jna' ||
+             it.name == 'securesm')
+  }
 }
 
 if (isEclipse == false || project.path == ":core-tests") {

diff --git a/...iteAggregationsClientYamlTestSuiteIT.java → core/cli/build.gradle b/...iteAggregationsClientYamlTestSuiteIT.java → core/cli/build.gradle
@@ -16,20 +16,21 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.search.aggregations.composite;
 
-import com.carrotsearch.randomizedtesting.annotations.Name;
-import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
-import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
-import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+import org.elasticsearch.gradle.precommit.PrecommitTasks
 
-public class CompositeAggregationsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
-    public CompositeAggregationsClientYamlTestSuiteIT(@Name("yaml")ClientYamlTestCandidate testCandidate) {
-        super(testCandidate);
-    }
+apply plugin: 'elasticsearch.build'
 
-    @ParametersFactory
-    public static Iterable<Object[]> parameters() throws Exception {
-        return ESClientYamlSuiteTestCase.createParameters();
-    }
+archivesBaseName = 'elasticsearch-cli'
+
+dependencies {
+    compile 'net.sf.jopt-simple:jopt-simple:5.0.2'
+}
+
+test.enabled = false
+// Since CLI does not depend on :core, it cannot run the jarHell task
+jarHell.enabled = false
+
+forbiddenApisMain {
+    signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
 }
diff --git a/core/licenses/jopt-simple-5.0.2.jar.sha1 → core/cli/licenses/jopt-simple-5.0.2.jar.sha1 b/core/licenses/jopt-simple-5.0.2.jar.sha1 → core/cli/licenses/jopt-simple-5.0.2.jar.sha1
diff --git a/core/licenses/jopt-simple-LICENSE.txt → core/cli/licenses/jopt-simple-LICENSE.txt b/core/licenses/jopt-simple-LICENSE.txt → core/cli/licenses/jopt-simple-LICENSE.txt
diff --git a/core/licenses/jopt-simple-NOTICE.txt → core/cli/licenses/jopt-simple-NOTICE.txt b/core/licenses/jopt-simple-NOTICE.txt → core/cli/licenses/jopt-simple-NOTICE.txt
diff --git a/...n/java/org/elasticsearch/cli/Command.java → ...n/java/org/elasticsearch/cli/Command.java b/...n/java/org/elasticsearch/cli/Command.java → ...n/java/org/elasticsearch/cli/Command.java
@@ -23,11 +23,6 @@
 import joptsimple.OptionParser;
 import joptsimple.OptionSet;
 import joptsimple.OptionSpec;
-import org.apache.logging.log4j.Level;
-import org.apache.lucene.util.SetOnce;
-import org.elasticsearch.common.SuppressForbidden;
-import org.elasticsearch.common.logging.LogConfigurator;
-import org.elasticsearch.common.settings.Settings;
 
 import java.io.Closeable;
 import java.io.IOException;
@@ -55,12 +50,13 @@ public Command(String description) {
         this.description = description;
     }
 
-    final SetOnce<Thread> shutdownHookThread = new SetOnce<>();
+    private Thread shutdownHookThread;
 
     /** Parses options for this command from args and executes it. */
     public final int main(String[] args, Terminal terminal) throws Exception {
         if (addShutdownHook()) {
-            shutdownHookThread.set(new Thread(() -> {
+
+            shutdownHookThread = new Thread(() -> {
                 try {
                     this.close();
                 } catch (final IOException e) {
@@ -75,16 +71,11 @@ public final int main(String[] args, Terminal terminal) throws Exception {
                         throw new AssertionError(impossible);
                     }
                 }
-            }));
-            Runtime.getRuntime().addShutdownHook(shutdownHookThread.get());
+            });
+            Runtime.getRuntime().addShutdownHook(shutdownHookThread);
         }
 
-        if (shouldConfigureLoggingWithoutConfig()) {
-            // initialize default for es.logger.level because we will not read the log4j2.properties
-            final String loggerLevel = System.getProperty("es.logger.level", Level.INFO.name());
-            final Settings settings = Settings.builder().put("logger.level", loggerLevel).build();
-            LogConfigurator.configureWithoutConfig(settings);
-        }
+        beforeExecute();
 
         try {
             mainWithoutErrorHandling(args, terminal);
@@ -103,14 +94,10 @@ public final int main(String[] args, Terminal terminal) throws Exception {
     }
 
     /**
-     * Indicate whether or not logging should be configured without reading a log4j2.properties. Most commands should do this because we do
-     * not configure logging for CLI tools. Only commands that configure logging on their own should not do this.
-     *
-     * @return true if logging should be configured without reading a log4j2.properties file
+     * Setup method to be executed before parsing or execution of the command being run. Any exceptions thrown by the
+     * method will not be cleanly caught by the parser.
      */
-    protected boolean shouldConfigureLoggingWithoutConfig() {
-        return true;
-    }
+    protected void beforeExecute() {}
 
     /**
      * Executes the command, but all errors are thrown.
@@ -166,6 +153,11 @@ protected boolean addShutdownHook() {
         return true;
     }
 
+    /** Gets the shutdown hook thread if it exists **/
+    Thread getShutdownHookThread() {
+        return shutdownHookThread;
+    }
+
     @Override
     public void close() throws IOException {
 

diff --git a/...java/org/elasticsearch/cli/ExitCodes.java → ...java/org/elasticsearch/cli/ExitCodes.java b/...java/org/elasticsearch/cli/ExitCodes.java → ...java/org/elasticsearch/cli/ExitCodes.java
diff --git a/...a/org/elasticsearch/cli/MultiCommand.java → ...a/org/elasticsearch/cli/MultiCommand.java b/...a/org/elasticsearch/cli/MultiCommand.java → ...a/org/elasticsearch/cli/MultiCommand.java