Merge branch 'master' into pr/34912

* master:
  Introduce cross-cluster replication API docs (elastic#34726)
  Responses can use Writeable.Reader interface (elastic#34655)
  SQL: Provide null-safe scripts for Not and Neg (elastic#34877)
  Fix put/resume follow request parsing (elastic#34913)
  Fix line length for org.elasticsearch.common.* files (elastic#34888)
  [ML] Extract common native process base class (elastic#34856)
  Refactor children aggregator into a generic ParentJoinAggregator (elastic#34845)
  [Style] Fix line lengths in action.admin.indices (elastic#34890)
  HLRC - add support for source exists API (elastic#34519)

client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java

@@ -261,6 +261,18 @@ private static Request getStyleRequest(String method, GetRequest getRequest) {
 
         return request;
     }
+
+    static Request sourceExists(GetRequest getRequest) {
+        Request request = new Request(HttpHead.METHOD_NAME, endpoint(getRequest.index(), getRequest.type(), getRequest.id(), "_source"));
+
+        Params parameters = new Params(request);
+        parameters.withPreference(getRequest.preference());
+        parameters.withRouting(getRequest.routing());
+        parameters.withRefresh(getRequest.refresh());
+        parameters.withRealtime(getRequest.realtime());
+        // Version params are not currently supported by the source exists API so are not passed
+        return request;
+    }    
 
     static Request multiGet(MultiGetRequest multiGetRequest) throws IOException {
         Request request = new Request(HttpPost.METHOD_NAME, "/_mget");

client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java

@@ -727,6 +727,32 @@ public final void existsAsync(GetRequest getRequest, RequestOptions options, Act
                 emptySet());
     }
 
+    /**
+     * Checks for the existence of a document with a "_source" field. Returns true if it exists, false otherwise.
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-get.html#_source">Source exists API 
+     * on elastic.co</a>
+     * @param getRequest the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return <code>true</code> if the document and _source field exists, <code>false</code> otherwise
+     * @throws IOException in case there is a problem sending the request
+     */
+    public boolean existsSource(GetRequest getRequest, RequestOptions options) throws IOException {
+        return performRequest(getRequest, RequestConverters::sourceExists, options, RestHighLevelClient::convertExistsResponse, emptySet());
+    }     
+
+    /**
+     * Asynchronously checks for the existence of a document with a "_source" field. Returns true if it exists, false otherwise.
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-get.html#_source">Source exists API 
+     * on elastic.co</a>
+     * @param getRequest the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public final void existsSourceAsync(GetRequest getRequest, RequestOptions options, ActionListener<Boolean> listener) {
+        performRequestAsync(getRequest, RequestConverters::sourceExists, options, RestHighLevelClient::convertExistsResponse, listener,
+                emptySet());
+    }    
+
     /**
      * Index a document using the Index API.
      * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html">Index API on elastic.co</a>

client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java

@@ -194,6 +194,61 @@ public void testExists() throws IOException {
             assertFalse(execute(getRequest, highLevelClient()::exists, highLevelClient()::existsAsync));
         }
     }
+
+    public void testSourceExists() throws IOException {     
+        {
+            GetRequest getRequest = new GetRequest("index", "type", "id");
+            assertFalse(execute(getRequest, highLevelClient()::existsSource, highLevelClient()::existsSourceAsync));
+        }
+        IndexRequest index = new IndexRequest("index", "type", "id");
+        index.source("{\"field1\":\"value1\",\"field2\":\"value2\"}", XContentType.JSON);
+        index.setRefreshPolicy(RefreshPolicy.IMMEDIATE);
+        highLevelClient().index(index, RequestOptions.DEFAULT);
+        {
+            GetRequest getRequest = new GetRequest("index", "type", "id");
+            assertTrue(execute(getRequest, highLevelClient()::existsSource, highLevelClient()::existsSourceAsync));
+        }
+        {
+            GetRequest getRequest = new GetRequest("index", "type", "does_not_exist");
+            assertFalse(execute(getRequest, highLevelClient()::existsSource, highLevelClient()::existsSourceAsync));
+        }
+        {
+            GetRequest getRequest = new GetRequest("index", "type", "does_not_exist").version(1);
+            assertFalse(execute(getRequest, highLevelClient()::existsSource, highLevelClient()::existsSourceAsync));
+        }
+    }
+
+    public void testSourceDoesNotExist() throws IOException {     
+        final String noSourceIndex = "no_source";
+        {
+            // Prepare
+            Settings settings = Settings.builder()
+                .put("number_of_shards", 1)
+                .put("number_of_replicas", 0)
+                .build();
+            String mapping = "\"_doc\": { \"_source\": {\n" + 
+                    "        \"enabled\": false\n" + 
+                    "      }  }";
+            createIndex(noSourceIndex, settings, mapping);
+            assertEquals(
+                RestStatus.OK,
+                highLevelClient().bulk(
+                    new BulkRequest()
+                        .add(new IndexRequest(noSourceIndex, "_doc", "1")
+                            .source(Collections.singletonMap("foo", 1), XContentType.JSON))
+                        .add(new IndexRequest(noSourceIndex, "_doc", "2")
+                            .source(Collections.singletonMap("foo", 2), XContentType.JSON))
+                        .setRefreshPolicy(RefreshPolicy.IMMEDIATE),
+                    RequestOptions.DEFAULT
+                ).status()
+            );
+        }        
+        {
+            GetRequest getRequest = new GetRequest(noSourceIndex, "_doc", "1");
+            assertTrue(execute(getRequest, highLevelClient()::exists, highLevelClient()::existsAsync));
+            assertFalse(execute(getRequest, highLevelClient()::existsSource, highLevelClient()::existsSourceAsync));
+        }
+    }    
 
     public void testGet() throws IOException {
         {

client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java

@@ -650,7 +650,6 @@ public void testApiNamingConventions() throws Exception {
             "cluster.remote_info",
             "count",
             "create",
-            "exists_source",
             "get_source",
             "indices.delete_alias",
             "indices.delete_template",

docs/build.gradle

@@ -139,7 +139,6 @@ buildRestTests.setups['host'] = '''
   - do:
       nodes.info:
         metric: [ http, transport ]
-  - is_true: nodes.$master.http.publish_address
   - set: {nodes.$master.http.publish_address: host}
   - set: {nodes.$master.transport.publish_address: transport_host}
 '''
@@ -1083,4 +1082,21 @@ buildRestTests.setups['calendar_outages_addevent'] = buildRestTests.setups['cale
            ]}
 '''
 
+buildRestTests.setups['remote_cluster'] = buildRestTests.setups['host'] + '''
+  - do:
+      cluster.put_settings:
+        body:
+          persistent:
+            cluster.remote.remote_cluster.seeds: $transport_host
+'''
 
+buildRestTests.setups['remote_cluster_and_leader_index'] = buildRestTests.setups['remote_cluster'] + '''
+  - do:
+      indices.create:
+        index: leader_index
+        body:
+          settings:
+            index.number_of_replicas: 0
+            index.number_of_shards: 1
+            index.soft_deletes.enabled: true
+'''

docs/java-rest/high-level/document/exists.asciidoc

@@ -29,3 +29,10 @@ include-tagged::{doc-tests-file}[{api}-request]
 <5> Disable fetching stored fields.
 
 include::../execution.asciidoc[]
+
+
+==== Source exists request
+A variant of the exists request is `existsSource` method which has the additional check
+that the document in question has stored the `source`. If the mapping for the index has opted
+to remove support for storing JSON source in documents then this method will return false
+for documents in this index.

docs/reference/ccr/apis/auto-follow/delete-auto-follow-pattern.asciidoc

@@ -0,0 +1,68 @@
+[role="xpack"]
+[testenv="platinum"]
+[[ccr-delete-auto-follow-pattern]]
+=== Delete Auto-Follow Pattern API
+++++
+<titleabbrev>Delete Auto-Follow Pattern</titleabbrev>
+++++
+
+Delete auto-follow patterns.
+
+==== Description
+
+This API deletes a configured auto-follow pattern collection.
+
+==== Request
+
+//////////////////////////
+
+[source,js]
+--------------------------------------------------
+PUT /_ccr/auto_follow/my_auto_follow_pattern
+{
+  "remote_cluster" : "remote_cluster",
+  "leader_index_patterns" :
+  [
+    "leader_index"
+  ],
+  "follow_index_pattern" : "{{leader_index}}-follower"
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:remote_cluster]
+// TESTSETUP
+
+//////////////////////////
+
+[source,js]
+--------------------------------------------------
+DELETE /_ccr/auto_follow/<auto_follow_pattern_name>
+--------------------------------------------------
+// CONSOLE
+// TEST[s/<auto_follow_pattern_name>/my_auto_follow_pattern/]
+
+==== Path Parameters
+`auto_follow_pattern_name` (required)::
+  (string) specifies the auto-follow pattern collection to delete
+
+==== Example
+
+This example deletes an auto-follow pattern collection named
+`my_auto_follow_pattern`:
+
+[source,js]
+--------------------------------------------------
+DELETE /_ccr/auto_follow/my_auto_follow_pattern
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:remote_cluster]
+
+The API returns the following result:
+
+[source,js]
+--------------------------------------------------
+{
+  "acknowledged" : true
+}
+--------------------------------------------------
+// TESTRESPONSE

docs/reference/ccr/apis/auto-follow/get-auto-follow-pattern.asciidoc

@@ -0,0 +1,93 @@
+[role="xpack"]
+[testenv="platinum"]
+[[ccr-get-auto-follow-pattern]]
+=== Get Auto-Follow Pattern API
+++++
+<titleabbrev>Get Auto-Follow Pattern</titleabbrev>
+++++
+
+Get auto-follow patterns.
+
+==== Description
+
+This API gets configured auto-follow patterns. This API will return the
+specified auto-follow pattern collection.
+
+==== Request
+
+//////////////////////////
+
+[source,js]
+--------------------------------------------------
+PUT /_ccr/auto_follow/my_auto_follow_pattern
+{
+  "remote_cluster" : "remote_cluster",
+  "leader_index_patterns" :
+  [
+    "leader_index*"
+  ],
+  "follow_index_pattern" : "{{leader_index}}-follower"
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:remote_cluster]
+// TESTSETUP
+
+[source,js]
+--------------------------------------------------
+DELETE /_ccr/auto_follow/my_auto_follow_pattern
+--------------------------------------------------
+// CONSOLE
+// TEST
+// TEARDOWN
+
+//////////////////////////
+
+[source,js]
+--------------------------------------------------
+GET /_ccr/auto_follow/
+--------------------------------------------------
+// CONSOLE
+
+[source,js]
+--------------------------------------------------
+GET /_ccr/auto_follow/<auto_follow_pattern_name>
+--------------------------------------------------
+// CONSOLE
+// TEST[s/<auto_follow_pattern_name>/my_auto_follow_pattern/]
+
+==== Path Parameters
+`auto_follow_pattern_name`::
+  (string) specifies the auto-follow pattern collection that you want to
+  retrieve; if you do not specify a name, the API returns information for all
+  collections
+
+==== Example
+
+This example retrieves information about an auto-follow pattern collection
+named `my_auto_follow_pattern`:
+
+[source,js]
+--------------------------------------------------
+GET /_ccr/auto_follow/my_auto_follow_pattern
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:remote_cluster]
+
+The API returns the following result:
+
+[source,js]
+--------------------------------------------------
+{
+  "my_auto_follow_pattern" :
+  {
+    "remote_cluster" : "remote_cluster",
+    "leader_index_patterns" :
+    [
+      "leader_index*"
+    ],
+    "follow_index_pattern" : "{{leader_index}}-follower"
+  }
+}
+--------------------------------------------------
+// TESTRESPONSE

docs/reference/ccr/apis/auto-follow/get-auto-follow-stats.asciidoc

@@ -0,0 +1,46 @@
+[role="xpack"]
+[testenv="platinum"]
+[[ccr-get-auto-follow-stats]]
+=== Get Auto-Follow Stats API
+++++
+<titleabbrev>Get Auto-Follow Stats</titleabbrev>
+++++
+
+Get auto-follow stats.
+
+==== Description
+
+This API gets stats about auto-follow patterns.
+
+==== Request
+
+[source,js]
+--------------------------------------------------
+GET /_ccr/auto_follow/stats
+--------------------------------------------------
+// CONSOLE
+// TEST
+
+==== Example
+
+This example retrieves stats about auto-follow patterns:
+
+[source,js]
+--------------------------------------------------
+GET /_ccr/auto_follow/stats
+--------------------------------------------------
+// CONSOLE
+// TEST
+
+The API returns the following result:
+
+[source,js]
+--------------------------------------------------
+{
+  "number_of_successful_follow_indices" : 16,
+  "number_of_failed_follow_indices" : 0,
+  "number_of_failed_remote_cluster_state_requests" : 0,
+  "recent_auto_follow_errors" : [ ]
+}
+--------------------------------------------------
+// TESTRESPONSE[s/"number_of_successful_follow_indices" : 16/"number_of_successful_follow_indices" : $body.number_of_successful_follow_indices/]