diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy index c836f0bbcb167..9e1cdad04fd6c 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy @@ -22,6 +22,7 @@ import de.thetaphi.forbiddenapis.gradle.ForbiddenApisPlugin import org.gradle.api.Project import org.gradle.api.Task import org.gradle.api.plugins.JavaBasePlugin +import org.gradle.api.plugins.quality.Checkstyle /** * Validation tasks which should be run before committing. These run before tests. @@ -142,7 +143,7 @@ class PrecommitTasks { ] toolVersion = 7.5 } - for (String taskName : ['checkstyleMain', 'checkstyleTest']) { + for (String taskName : ['checkstyleMain', 'checkstyleJava9', 'checkstyleTest']) { Task task = project.tasks.findByName(taskName) if (task != null) { project.tasks['check'].dependsOn.remove(task) @@ -154,6 +155,11 @@ class PrecommitTasks { } } } + + project.tasks.withType(Checkstyle) { + dependsOn(copyCheckstyleConf) + } + return checkstyleTask } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java index b39b7801e27e5..0b366aa99e188 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java @@ -24,6 +24,8 @@ import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; +import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest; +import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheResponse; import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; import org.elasticsearch.action.admin.indices.close.CloseIndexResponse; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; @@ -259,6 +261,29 @@ public void flushAsync(FlushRequest flushRequest, ActionListener listener, emptySet(), headers); } + /** + * Clears the cache of one or more indices using the Clear Cache API + *

+ * See + * Clear Cache API on elastic.co + */ + public ClearIndicesCacheResponse clearCache(ClearIndicesCacheRequest clearIndicesCacheRequest, Header... headers) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(clearIndicesCacheRequest, Request::clearCache, + ClearIndicesCacheResponse::fromXContent, emptySet(), headers); + } + + /** + * Asynchronously clears the cache of one or more indices using the Clear Cache API + *

+ * See + * Clear Cache API on elastic.co + */ + public void clearCacheAsync(ClearIndicesCacheRequest clearIndicesCacheRequest, ActionListener listener, + Header... headers) { + restHighLevelClient.performRequestAsyncAndParseEntity(clearIndicesCacheRequest, Request::clearCache, + ClearIndicesCacheResponse::fromXContent, listener, emptySet(), headers); + } + /** * Checks if the index (indices) exists or not. *

diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java index 8a5c333dc4645..66b34da777b6a 100755 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java @@ -32,6 +32,7 @@ import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; +import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest; import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; @@ -170,13 +171,10 @@ static Request openIndex(OpenIndexRequest openIndexRequest) { static Request closeIndex(CloseIndexRequest closeIndexRequest) { String endpoint = endpoint(closeIndexRequest.indices(), "_close"); - Params parameters = Params.builder(); - parameters.withTimeout(closeIndexRequest.timeout()); parameters.withMasterTimeout(closeIndexRequest.masterNodeTimeout()); parameters.withIndicesOptions(closeIndexRequest.indicesOptions()); - return new Request(HttpPost.METHOD_NAME, endpoint, parameters.getParams(), null); } @@ -218,14 +216,16 @@ static Request putMapping(PutMappingRequest putMappingRequest) throws IOExceptio } static Request refresh(RefreshRequest refreshRequest) { - String endpoint = endpoint(refreshRequest.indices(), "_refresh"); + String[] indices = refreshRequest.indices() == null ? Strings.EMPTY_ARRAY : refreshRequest.indices(); + String endpoint = endpoint(indices, "_refresh"); Params parameters = Params.builder(); parameters.withIndicesOptions(refreshRequest.indicesOptions()); return new Request(HttpPost.METHOD_NAME, endpoint, parameters.getParams(), null); } static Request flush(FlushRequest flushRequest) { - String endpoint = endpoint(flushRequest.indices(), "_flush"); + String[] indices = flushRequest.indices() == null ? Strings.EMPTY_ARRAY : flushRequest.indices(); + String endpoint = endpoint(indices, "_flush"); Params parameters = Params.builder(); parameters.withIndicesOptions(flushRequest.indicesOptions()); parameters.putParam("wait_if_ongoing", Boolean.toString(flushRequest.waitIfOngoing())); @@ -233,6 +233,18 @@ static Request flush(FlushRequest flushRequest) { return new Request(HttpPost.METHOD_NAME, endpoint, parameters.getParams(), null); } + static Request clearCache(ClearIndicesCacheRequest clearIndicesCacheRequest) { + String[] indices = clearIndicesCacheRequest.indices() == null ? Strings.EMPTY_ARRAY :clearIndicesCacheRequest.indices(); + String endpoint = endpoint(indices, "_cache/clear"); + Params parameters = Params.builder(); + parameters.withIndicesOptions(clearIndicesCacheRequest.indicesOptions()); + parameters.putParam("query", Boolean.toString(clearIndicesCacheRequest.queryCache())); + parameters.putParam("fielddata", Boolean.toString(clearIndicesCacheRequest.fieldDataCache())); + parameters.putParam("request", Boolean.toString(clearIndicesCacheRequest.requestCache())); + parameters.putParam("fields", String.join(",", clearIndicesCacheRequest.fields())); + return new Request(HttpPost.METHOD_NAME, endpoint, parameters.getParams(), null); + } + static Request info() { return new Request(HttpGet.METHOD_NAME, "/", Collections.emptyMap(), null); } @@ -508,10 +520,13 @@ static Request existsAlias(GetAliasesRequest getAliasesRequest) { Params params = Params.builder(); params.withIndicesOptions(getAliasesRequest.indicesOptions()); params.withLocal(getAliasesRequest.local()); - if (getAliasesRequest.indices().length == 0 && getAliasesRequest.aliases().length == 0) { + if ((getAliasesRequest.indices() == null || getAliasesRequest.indices().length == 0) && + (getAliasesRequest.aliases() == null || getAliasesRequest.aliases().length == 0)) { throw new IllegalArgumentException("existsAlias requires at least an alias or an index"); } - String endpoint = endpoint(getAliasesRequest.indices(), "_alias", getAliasesRequest.aliases()); + String[] indices = getAliasesRequest.indices() == null ? Strings.EMPTY_ARRAY : getAliasesRequest.indices(); + String[] aliases = getAliasesRequest.aliases() == null ? Strings.EMPTY_ARRAY : getAliasesRequest.aliases(); + String endpoint = endpoint(indices, "_alias", aliases); return new Request(HttpHead.METHOD_NAME, endpoint, params.getParams(), null); } @@ -540,8 +555,9 @@ private static Request resize(ResizeRequest resizeRequest) throws IOException { params.withTimeout(resizeRequest.timeout()); params.withMasterTimeout(resizeRequest.masterNodeTimeout()); params.withWaitForActiveShards(resizeRequest.getTargetIndexRequest().waitForActiveShards()); - String endpoint = buildEndpoint(resizeRequest.getSourceIndex(), "_" + resizeRequest.getResizeType().name().toLowerCase(Locale.ROOT), - resizeRequest.getTargetIndexRequest().index()); + String endpoint = new EndpointBuilder().addPathPart(resizeRequest.getSourceIndex()) + .addPathPartAsIs("_" + resizeRequest.getResizeType().name().toLowerCase(Locale.ROOT)) + .addPathPart(resizeRequest.getTargetIndexRequest().index()).build(); HttpEntity entity = createEntity(resizeRequest, REQUEST_BODY_CONTENT_TYPE); return new Request(HttpPut.METHOD_NAME, endpoint, params.getParams(), entity); } @@ -551,10 +567,8 @@ static Request clusterPutSettings(ClusterUpdateSettingsRequest clusterUpdateSett parameters.withFlatSettings(clusterUpdateSettingsRequest.flatSettings()); parameters.withTimeout(clusterUpdateSettingsRequest.timeout()); parameters.withMasterTimeout(clusterUpdateSettingsRequest.masterNodeTimeout()); - - String endpoint = buildEndpoint("_cluster", "settings"); HttpEntity entity = createEntity(clusterUpdateSettingsRequest, REQUEST_BODY_CONTENT_TYPE); - return new Request(HttpPut.METHOD_NAME, endpoint, parameters.getParams(), entity); + return new Request(HttpPut.METHOD_NAME, "/_cluster/settings", parameters.getParams(), entity); } static Request rollover(RolloverRequest rolloverRequest) throws IOException { @@ -565,64 +579,60 @@ static Request rollover(RolloverRequest rolloverRequest) throws IOException { if (rolloverRequest.isDryRun()) { params.putParam("dry_run", Boolean.TRUE.toString()); } - String endpoint = buildEndpoint(rolloverRequest.getAlias(), "_rollover", rolloverRequest.getNewIndexName()); + String endpoint = new EndpointBuilder().addPathPart(rolloverRequest.getAlias()).addPathPartAsIs("_rollover") + .addPathPart(rolloverRequest.getNewIndexName()).build(); HttpEntity entity = createEntity(rolloverRequest, REQUEST_BODY_CONTENT_TYPE); return new Request(HttpPost.METHOD_NAME, endpoint, params.getParams(), entity); } + static Request indicesExist(GetIndexRequest request) { + //this can be called with no indices as argument by transport client, not via REST though + if (request.indices() == null || request.indices().length == 0) { + throw new IllegalArgumentException("indices are mandatory"); + } + String endpoint = endpoint(request.indices(), ""); + Params params = Params.builder(); + params.withLocal(request.local()); + params.withHuman(request.humanReadable()); + params.withIndicesOptions(request.indicesOptions()); + params.withFlatSettings(request.flatSettings()); + params.withIncludeDefaults(request.includeDefaults()); + return new Request(HttpHead.METHOD_NAME, endpoint, params.getParams(), null); + } + private static HttpEntity createEntity(ToXContent toXContent, XContentType xContentType) throws IOException { BytesRef source = XContentHelper.toXContent(toXContent, xContentType, false).toBytesRef(); return new ByteArrayEntity(source.bytes, source.offset, source.length, createContentType(xContentType)); } static String endpoint(String index, String type, String id) { - return buildEndpoint(index, type, id); + return new EndpointBuilder().addPathPart(index, type, id).build(); } static String endpoint(String index, String type, String id, String endpoint) { - return buildEndpoint(index, type, id, endpoint); + return new EndpointBuilder().addPathPart(index, type, id).addPathPartAsIs(endpoint).build(); } static String endpoint(String[] indices) { - return buildEndpoint(String.join(",", indices)); + return new EndpointBuilder().addCommaSeparatedPathParts(indices).build(); } static String endpoint(String[] indices, String endpoint) { - return buildEndpoint(String.join(",", indices), endpoint); + return new EndpointBuilder().addCommaSeparatedPathParts(indices).addPathPartAsIs(endpoint).build(); } static String endpoint(String[] indices, String[] types, String endpoint) { - return buildEndpoint(String.join(",", indices), String.join(",", types), endpoint); + return new EndpointBuilder().addCommaSeparatedPathParts(indices).addCommaSeparatedPathParts(types) + .addPathPartAsIs(endpoint).build(); } static String endpoint(String[] indices, String endpoint, String[] suffixes) { - return buildEndpoint(String.join(",", indices), endpoint, String.join(",", suffixes)); + return new EndpointBuilder().addCommaSeparatedPathParts(indices).addPathPartAsIs(endpoint) + .addCommaSeparatedPathParts(suffixes).build(); } static String endpoint(String[] indices, String endpoint, String type) { - return endpoint(String.join(",", indices), endpoint, type); - } - - /** - * Utility method to build request's endpoint given its parts as strings - */ - static String buildEndpoint(String... parts) { - StringJoiner joiner = new StringJoiner("/", "/", ""); - for (String part : parts) { - if (Strings.hasLength(part)) { - try { - //encode each part (e.g. index, type and id) separately before merging them into the path - //we prepend "/" to the path part to make this pate absolute, otherwise there can be issues with - //paths that start with `-` or contain `:` - URI uri = new URI(null, null, null, -1, "/" + part, null, null); - //manually encode any slash that each part may contain - joiner.add(uri.getRawPath().substring(1).replaceAll("/", "%2F")); - } catch (URISyntaxException e) { - throw new IllegalArgumentException("Path part [" + part + "] couldn't be encoded", e); - } - } - } - return joiner.toString(); + return new EndpointBuilder().addCommaSeparatedPathParts(indices).addPathPartAsIs(endpoint).addPathPart(type).build(); } /** @@ -636,17 +646,6 @@ public static ContentType createContentType(final XContentType xContentType) { return ContentType.create(xContentType.mediaTypeWithoutParameters(), (Charset) null); } - static Request indicesExist(GetIndexRequest request) { - String endpoint = endpoint(request.indices(), Strings.EMPTY_ARRAY, ""); - Params params = Params.builder(); - params.withLocal(request.local()); - params.withHuman(request.humanReadable()); - params.withIndicesOptions(request.indicesOptions()); - params.withFlatSettings(request.flatSettings()); - params.withIncludeDefaults(request.includeDefaults()); - return new Request(HttpHead.METHOD_NAME, endpoint, params.getParams(), null); - } - /** * Utility class to build request's parameters map and centralize all parameter names. */ @@ -852,4 +851,50 @@ static XContentType enforceSameContentType(IndexRequest indexRequest, @Nullable } return xContentType; } + + /** + * Utility class to build request's endpoint given its parts as strings + */ + static class EndpointBuilder { + + private final StringJoiner joiner = new StringJoiner("/", "/", ""); + + EndpointBuilder addPathPart(String... parts) { + for (String part : parts) { + if (Strings.hasLength(part)) { + joiner.add(encodePart(part)); + } + } + return this; + } + + EndpointBuilder addCommaSeparatedPathParts(String[] parts) { + addPathPart(String.join(",", parts)); + return this; + } + + EndpointBuilder addPathPartAsIs(String part) { + if (Strings.hasLength(part)) { + joiner.add(part); + } + return this; + } + + String build() { + return joiner.toString(); + } + + private static String encodePart(String pathPart) { + try { + //encode each part (e.g. index, type and id) separately before merging them into the path + //we prepend "/" to the path part to make this pate absolute, otherwise there can be issues with + //paths that start with `-` or contain `:` + URI uri = new URI(null, null, null, -1, "/" + pathPart, null, null); + //manually encode any slash that each part may contain + return uri.getRawPath().substring(1).replaceAll("/", "%2F"); + } catch (URISyntaxException e) { + throw new IllegalArgumentException("Path part [" + pathPart + "] couldn't be encoded", e); + } + } + } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java index 7f777531440d9..8a2ba44791149 100755 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java @@ -28,6 +28,8 @@ import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; +import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest; +import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheResponse; import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; import org.elasticsearch.action.admin.indices.close.CloseIndexResponse; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; @@ -431,9 +433,36 @@ public void testFlush() throws IOException { { String nonExistentIndex = "non_existent_index"; assertFalse(indexExists(nonExistentIndex)); - FlushRequest refreshRequest = new FlushRequest(nonExistentIndex); + FlushRequest flushRequest = new FlushRequest(nonExistentIndex); ElasticsearchException exception = expectThrows(ElasticsearchException.class, - () -> execute(refreshRequest, highLevelClient().indices()::flush, highLevelClient().indices()::flushAsync)); + () -> execute(flushRequest, highLevelClient().indices()::flush, highLevelClient().indices()::flushAsync)); + assertEquals(RestStatus.NOT_FOUND, exception.status()); + } + } + + public void testClearCache() throws IOException { + { + String index = "index"; + Settings settings = Settings.builder() + .put("number_of_shards", 1) + .put("number_of_replicas", 0) + .build(); + createIndex(index, settings); + ClearIndicesCacheRequest clearCacheRequest = new ClearIndicesCacheRequest(index); + ClearIndicesCacheResponse clearCacheResponse = + execute(clearCacheRequest, highLevelClient().indices()::clearCache, highLevelClient().indices()::clearCacheAsync); + assertThat(clearCacheResponse.getTotalShards(), equalTo(1)); + assertThat(clearCacheResponse.getSuccessfulShards(), equalTo(1)); + assertThat(clearCacheResponse.getFailedShards(), equalTo(0)); + assertThat(clearCacheResponse.getShardFailures(), equalTo(BroadcastResponse.EMPTY)); + } + { + String nonExistentIndex = "non_existent_index"; + assertFalse(indexExists(nonExistentIndex)); + ClearIndicesCacheRequest clearCacheRequest = new ClearIndicesCacheRequest(nonExistentIndex); + ElasticsearchException exception = expectThrows(ElasticsearchException.class, + () -> execute(clearCacheRequest, highLevelClient().indices()::clearCache, + highLevelClient().indices()::clearCacheAsync)); assertEquals(RestStatus.NOT_FOUND, exception.status()); } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java index beddadf859b1c..f79135c44f5ec 100755 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java @@ -29,11 +29,13 @@ import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; import org.apache.http.util.EntityUtils; +import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; +import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest; import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; @@ -288,6 +290,11 @@ public void testIndicesExist() { assertNull(request.getEntity()); } + public void testIndicesExistEmptyIndices() { + expectThrows(IllegalArgumentException.class, () -> Request.indicesExist(new GetIndexRequest())); + expectThrows(IllegalArgumentException.class, () -> Request.indicesExist(new GetIndexRequest().indices((String[])null))); + } + private static void getAndExistsTest(Function requestConverter, String method) { String index = randomAlphaOfLengthBetween(3, 10); String type = randomAlphaOfLengthBetween(3, 10); @@ -361,6 +368,11 @@ public void testCreateIndex() throws IOException { assertToXContentBody(createIndexRequest, request.getEntity()); } + public void testCreateIndexNullIndex() { + ActionRequestValidationException validationException = new CreateIndexRequest(null).validate(); + assertNotNull(validationException); + } + public void testUpdateAliases() throws IOException { IndicesAliasesRequest indicesAliasesRequest = new IndicesAliasesRequest(); AliasActions aliasAction = randomAliasAction(); @@ -422,6 +434,12 @@ public void testDeleteIndex() { assertNull(request.getEntity()); } + public void testDeleteIndexEmptyIndices() { + String[] indices = randomBoolean() ? null : Strings.EMPTY_ARRAY; + ActionRequestValidationException validationException = new DeleteIndexRequest(indices).validate(); + assertNotNull(validationException); + } + public void testOpenIndex() { String[] indices = randomIndicesNames(1, 5); OpenIndexRequest openIndexRequest = new OpenIndexRequest(indices); @@ -441,6 +459,12 @@ public void testOpenIndex() { assertThat(request.getEntity(), nullValue()); } + public void testOpenIndexEmptyIndices() { + String[] indices = randomBoolean() ? null : Strings.EMPTY_ARRAY; + ActionRequestValidationException validationException = new OpenIndexRequest(indices).validate(); + assertNotNull(validationException); + } + public void testCloseIndex() { String[] indices = randomIndicesNames(1, 5); CloseIndexRequest closeIndexRequest = new CloseIndexRequest(indices); @@ -458,6 +482,12 @@ public void testCloseIndex() { assertThat(request.getEntity(), nullValue()); } + public void testCloseIndexEmptyIndices() { + String[] indices = randomBoolean() ? null : Strings.EMPTY_ARRAY; + ActionRequestValidationException validationException = new CloseIndexRequest(indices).validate(); + assertNotNull(validationException); + } + public void testIndex() throws IOException { String index = randomAlphaOfLengthBetween(3, 10); String type = randomAlphaOfLengthBetween(3, 10); @@ -537,13 +567,19 @@ public void testIndex() throws IOException { } public void testRefresh() { - String[] indices = randomIndicesNames(0, 5); - RefreshRequest refreshRequest = new RefreshRequest(indices); + String[] indices = randomBoolean() ? null : randomIndicesNames(0, 5); + RefreshRequest refreshRequest; + if (randomBoolean()) { + refreshRequest = new RefreshRequest(indices); + } else { + refreshRequest = new RefreshRequest(); + refreshRequest.indices(indices); + } Map expectedParams = new HashMap<>(); setRandomIndicesOptions(refreshRequest::indicesOptions, refreshRequest::indicesOptions, expectedParams); Request request = Request.refresh(refreshRequest); StringJoiner endpoint = new StringJoiner("/", "/", ""); - if (indices.length > 0) { + if (indices != null && indices.length > 0) { endpoint.add(String.join(",", indices)); } endpoint.add("_refresh"); @@ -554,8 +590,14 @@ public void testRefresh() { } public void testFlush() { - String[] indices = randomIndicesNames(0, 5); - FlushRequest flushRequest = new FlushRequest(indices); + String[] indices = randomBoolean() ? null : randomIndicesNames(0, 5); + FlushRequest flushRequest; + if (randomBoolean()) { + flushRequest = new FlushRequest(indices); + } else { + flushRequest = new FlushRequest(); + flushRequest.indices(indices); + } Map expectedParams = new HashMap<>(); setRandomIndicesOptions(flushRequest::indicesOptions, flushRequest::indicesOptions, expectedParams); if (randomBoolean()) { @@ -569,7 +611,7 @@ public void testFlush() { Request request = Request.flush(flushRequest); StringJoiner endpoint = new StringJoiner("/", "/", ""); - if (indices.length > 0) { + if (indices != null && indices.length > 0) { endpoint.add(String.join(",", indices)); } endpoint.add("_flush"); @@ -579,6 +621,46 @@ public void testFlush() { assertThat(request.getMethod(), equalTo(HttpPost.METHOD_NAME)); } + public void testClearCache() { + String[] indices = randomBoolean() ? null : randomIndicesNames(0, 5); + ClearIndicesCacheRequest clearIndicesCacheRequest; + if (randomBoolean()) { + clearIndicesCacheRequest = new ClearIndicesCacheRequest(indices); + } else { + clearIndicesCacheRequest = new ClearIndicesCacheRequest(); + clearIndicesCacheRequest.indices(indices); + } + Map expectedParams = new HashMap<>(); + setRandomIndicesOptions(clearIndicesCacheRequest::indicesOptions, clearIndicesCacheRequest::indicesOptions, expectedParams); + if (randomBoolean()) { + clearIndicesCacheRequest.queryCache(randomBoolean()); + } + expectedParams.put("query", Boolean.toString(clearIndicesCacheRequest.queryCache())); + if (randomBoolean()) { + clearIndicesCacheRequest.fieldDataCache(randomBoolean()); + } + expectedParams.put("fielddata", Boolean.toString(clearIndicesCacheRequest.fieldDataCache())); + if (randomBoolean()) { + clearIndicesCacheRequest.requestCache(randomBoolean()); + } + expectedParams.put("request", Boolean.toString(clearIndicesCacheRequest.requestCache())); + if (randomBoolean()) { + clearIndicesCacheRequest.fields(randomIndicesNames(1, 5)); + expectedParams.put("fields", String.join(",", clearIndicesCacheRequest.fields())); + } + + Request request = Request.clearCache(clearIndicesCacheRequest); + StringJoiner endpoint = new StringJoiner("/", "/", ""); + if (indices != null && indices.length > 0) { + endpoint.add(String.join(",", indices)); + } + endpoint.add("_cache/clear"); + assertThat(request.getEndpoint(), equalTo(endpoint.toString())); + assertThat(request.getParameters(), equalTo(expectedParams)); + assertThat(request.getEntity(), nullValue()); + assertThat(request.getMethod(), equalTo(HttpPost.METHOD_NAME)); + } + public void testUpdate() throws IOException { XContentType xContentType = randomFrom(XContentType.values()); @@ -991,6 +1073,12 @@ public void testSearch() throws Exception { assertToXContentBody(searchSourceBuilder, request.getEntity()); } + public void testSearchNullIndicesAndTypes() { + expectThrows(NullPointerException.class, () -> new SearchRequest((String[]) null)); + expectThrows(NullPointerException.class, () -> new SearchRequest().indices((String[]) null)); + expectThrows(NullPointerException.class, () -> new SearchRequest().types((String[]) null)); + } + public void testMultiSearch() throws IOException { int numberOfSearchRequests = randomIntBetween(0, 32); MultiSearchRequest multiSearchRequest = new MultiSearchRequest(); @@ -1071,10 +1159,16 @@ public void testClearScroll() throws IOException { public void testExistsAlias() { GetAliasesRequest getAliasesRequest = new GetAliasesRequest(); - String[] indices = randomIndicesNames(0, 5); + String[] indices = randomBoolean() ? null : randomIndicesNames(0, 5); getAliasesRequest.indices(indices); //the HEAD endpoint requires at least an alias or an index - String[] aliases = randomIndicesNames(indices.length == 0 ? 1 : 0, 5); + boolean hasIndices = indices != null && indices.length > 0; + String[] aliases; + if (hasIndices) { + aliases = randomBoolean() ? null : randomIndicesNames(0, 5); + } else { + aliases = randomIndicesNames(1, 5); + } getAliasesRequest.aliases(aliases); Map expectedParams = new HashMap<>(); setRandomLocal(getAliasesRequest, expectedParams); @@ -1082,14 +1176,12 @@ public void testExistsAlias() { Request request = Request.existsAlias(getAliasesRequest); StringJoiner expectedEndpoint = new StringJoiner("/", "/", ""); - String index = String.join(",", indices); - if (Strings.hasLength(index)) { - expectedEndpoint.add(index); + if (indices != null && indices.length > 0) { + expectedEndpoint.add(String.join(",", indices)); } expectedEndpoint.add("_alias"); - String alias = String.join(",", aliases); - if (Strings.hasLength(alias)) { - expectedEndpoint.add(alias); + if (aliases != null && aliases.length > 0) { + expectedEndpoint.add(String.join(",", aliases)); } assertEquals(HttpHead.METHOD_NAME, request.getMethod()); assertEquals(expectedEndpoint.toString(), request.getEndpoint()); @@ -1098,9 +1190,17 @@ public void testExistsAlias() { } public void testExistsAliasNoAliasNoIndex() { - GetAliasesRequest getAliasesRequest = new GetAliasesRequest(); - IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> Request.existsAlias(getAliasesRequest)); - assertEquals("existsAlias requires at least an alias or an index", iae.getMessage()); + { + GetAliasesRequest getAliasesRequest = new GetAliasesRequest(); + IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> Request.existsAlias(getAliasesRequest)); + assertEquals("existsAlias requires at least an alias or an index", iae.getMessage()); + } + { + GetAliasesRequest getAliasesRequest = new GetAliasesRequest((String[])null); + getAliasesRequest.indices((String[])null); + IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> Request.existsAlias(getAliasesRequest)); + assertEquals("existsAlias requires at least an alias or an index", iae.getMessage()); + } } public void testRankEval() throws Exception { @@ -1259,30 +1359,99 @@ public void testParamsNoDuplicates() { assertEquals("1", requestParams.values().iterator().next()); } - public void testBuildEndpoint() { - assertEquals("/", Request.buildEndpoint()); - assertEquals("/", Request.buildEndpoint(Strings.EMPTY_ARRAY)); - assertEquals("/", Request.buildEndpoint("")); - assertEquals("/a/b", Request.buildEndpoint("a", "b")); - assertEquals("/a/b/_create", Request.buildEndpoint("a", "b", "_create")); - assertEquals("/a/b/c/_create", Request.buildEndpoint("a", "b", "c", "_create")); - assertEquals("/a/_create", Request.buildEndpoint("a", null, null, "_create")); - } - - public void testBuildEndPointEncodeParts() { - assertEquals("/-%23index1,index%232/type/id", Request.buildEndpoint("-#index1,index#2", "type", "id")); - assertEquals("/index/type%232/id", Request.buildEndpoint("index", "type#2", "id")); - assertEquals("/index/type/this%2Fis%2Fthe%2Fid", Request.buildEndpoint("index", "type", "this/is/the/id")); - assertEquals("/index/type/this%7Cis%7Cthe%7Cid", Request.buildEndpoint("index", "type", "this|is|the|id")); - assertEquals("/index/type/id%231", Request.buildEndpoint("index", "type", "id#1")); - assertEquals("/%3Clogstash-%7Bnow%2FM%7D%3E/_search", Request.buildEndpoint("", "_search")); - assertEquals("/中文", Request.buildEndpoint("中文")); - assertEquals("/foo%20bar", Request.buildEndpoint("foo bar")); - assertEquals("/foo+bar", Request.buildEndpoint("foo+bar")); - assertEquals("/foo%2Fbar", Request.buildEndpoint("foo/bar")); - assertEquals("/foo%5Ebar", Request.buildEndpoint("foo^bar")); - assertEquals("/cluster1:index1,index2/_search", Request.buildEndpoint("cluster1:index1,index2", "_search")); - assertEquals("/*", Request.buildEndpoint("*")); + public void testEndpointBuilder() { + { + Request.EndpointBuilder endpointBuilder = new Request.EndpointBuilder(); + assertEquals("/", endpointBuilder.build()); + } + { + Request.EndpointBuilder endpointBuilder = new Request.EndpointBuilder().addPathPart(Strings.EMPTY_ARRAY); + assertEquals("/", endpointBuilder.build()); + } + { + Request.EndpointBuilder endpointBuilder = new Request.EndpointBuilder().addPathPart(""); + assertEquals("/", endpointBuilder.build()); + } + { + Request.EndpointBuilder endpointBuilder = new Request.EndpointBuilder().addPathPart("a", "b"); + assertEquals("/a/b", endpointBuilder.build()); + } + { + Request.EndpointBuilder endpointBuilder = new Request.EndpointBuilder().addPathPart("a").addPathPart("b") + .addPathPartAsIs("_create"); + assertEquals("/a/b/_create", endpointBuilder.build()); + } + + { + Request.EndpointBuilder endpointBuilder = new Request.EndpointBuilder().addPathPart("a", "b", "c") + .addPathPartAsIs("_create"); + assertEquals("/a/b/c/_create", endpointBuilder.build()); + } + { + Request.EndpointBuilder endpointBuilder = new Request.EndpointBuilder().addPathPart("a").addPathPartAsIs("_create"); + assertEquals("/a/_create", endpointBuilder.build()); + } + } + + public void testEndpointBuilderEncodeParts() { + { + Request.EndpointBuilder endpointBuilder = new Request.EndpointBuilder().addPathPart("-#index1,index#2", "type", "id"); + assertEquals("/-%23index1,index%232/type/id", endpointBuilder.build()); + } + { + Request.EndpointBuilder endpointBuilder = new Request.EndpointBuilder().addPathPart("index", "type#2", "id"); + assertEquals("/index/type%232/id", endpointBuilder.build()); + } + { + Request.EndpointBuilder endpointBuilder = new Request.EndpointBuilder().addPathPart("index", "type", "this/is/the/id"); + assertEquals("/index/type/this%2Fis%2Fthe%2Fid", endpointBuilder.build()); + } + { + Request.EndpointBuilder endpointBuilder = new Request.EndpointBuilder().addPathPart("index", "type", "this|is|the|id"); + assertEquals("/index/type/this%7Cis%7Cthe%7Cid", endpointBuilder.build()); + } + { + Request.EndpointBuilder endpointBuilder = new Request.EndpointBuilder().addPathPart("index", "type", "id#1"); + assertEquals("/index/type/id%231", endpointBuilder.build()); + } + { + Request.EndpointBuilder endpointBuilder = new Request.EndpointBuilder().addPathPart("", "_search"); + assertEquals("/%3Clogstash-%7Bnow%2FM%7D%3E/_search", endpointBuilder.build()); + } + { + Request.EndpointBuilder endpointBuilder = new Request.EndpointBuilder().addPathPart("中文"); + assertEquals("/中文", endpointBuilder.build()); + } + { + Request.EndpointBuilder endpointBuilder = new Request.EndpointBuilder().addPathPart("foo bar"); + assertEquals("/foo%20bar", endpointBuilder.build()); + } + { + Request.EndpointBuilder endpointBuilder = new Request.EndpointBuilder().addPathPart("foo+bar"); + assertEquals("/foo+bar", endpointBuilder.build()); + } + { + Request.EndpointBuilder endpointBuilder = new Request.EndpointBuilder().addPathPart("foo+bar"); + assertEquals("/foo+bar", endpointBuilder.build()); + } + { + Request.EndpointBuilder endpointBuilder = new Request.EndpointBuilder().addPathPart("foo/bar"); + assertEquals("/foo%2Fbar", endpointBuilder.build()); + } + { + Request.EndpointBuilder endpointBuilder = new Request.EndpointBuilder().addPathPart("foo^bar"); + assertEquals("/foo%5Ebar", endpointBuilder.build()); + } + { + Request.EndpointBuilder endpointBuilder = new Request.EndpointBuilder().addPathPart("cluster1:index1,index2") + .addPathPartAsIs("_search"); + assertEquals("/cluster1:index1,index2/_search", endpointBuilder.build()); + } + { + Request.EndpointBuilder endpointBuilder = new Request.EndpointBuilder() + .addCommaSeparatedPathParts(new String[]{"index1", "index2"}).addPathPartAsIs("cache/clear"); + assertEquals("/index1,index2/cache/clear", endpointBuilder.build()); + } } public void testEndpoint() { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java index 4bbc00fb41111..fb9e56d222022 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java @@ -27,6 +27,8 @@ import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; +import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest; +import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheResponse; import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; import org.elasticsearch.action.admin.indices.close.CloseIndexResponse; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; @@ -769,6 +771,91 @@ public void onFailure(Exception e) { } } + public void testClearCache() throws Exception { + RestHighLevelClient client = highLevelClient(); + + { + createIndex("index1", Settings.EMPTY); + } + + { + // tag::clear-cache-request + ClearIndicesCacheRequest request = new ClearIndicesCacheRequest("index1"); // <1> + ClearIndicesCacheRequest requestMultiple = new ClearIndicesCacheRequest("index1", "index2"); // <2> + ClearIndicesCacheRequest requestAll = new ClearIndicesCacheRequest(); // <3> + // end::clear-cache-request + + // tag::clear-cache-request-indicesOptions + request.indicesOptions(IndicesOptions.lenientExpandOpen()); // <1> + // end::clear-cache-request-indicesOptions + + // tag::clear-cache-request-query + request.queryCache(true); // <1> + // end::clear-cache-request-query + + // tag::clear-cache-request-request + request.requestCache(true); // <1> + // end::clear-cache-request-request + + // tag::clear-cache-request-fielddata + request.fieldDataCache(true); // <1> + // end::clear-cache-request-fielddata + + // tag::clear-cache-request-fields + request.fields("field1", "field2", "field3"); // <1> + // end::clear-cache-request-fields + + // tag::clear-cache-execute + ClearIndicesCacheResponse clearCacheResponse = client.indices().clearCache(request); + // end::clear-cache-execute + + // tag::clear-cache-response + int totalShards = clearCacheResponse.getTotalShards(); // <1> + int successfulShards = clearCacheResponse.getSuccessfulShards(); // <2> + int failedShards = clearCacheResponse.getFailedShards(); // <3> + DefaultShardOperationFailedException[] failures = clearCacheResponse.getShardFailures(); // <4> + // end::clear-cache-response + + // tag::clear-cache-execute-listener + ActionListener listener = new ActionListener() { + @Override + public void onResponse(ClearIndicesCacheResponse clearCacheResponse) { + // <1> + } + + @Override + public void onFailure(Exception e) { + // <2> + } + }; + // end::clear-cache-execute-listener + + // Replace the empty listener by a blocking listener in test + final CountDownLatch latch = new CountDownLatch(1); + listener = new LatchedActionListener<>(listener, latch); + + // tag::clear-cache-execute-async + client.indices().clearCacheAsync(request, listener); // <1> + // end::clear-cache-execute-async + + assertTrue(latch.await(30L, TimeUnit.SECONDS)); + } + + { + // tag::clear-cache-notfound + try { + ClearIndicesCacheRequest request = new ClearIndicesCacheRequest("does_not_exist"); + client.indices().clearCache(request); + } catch (ElasticsearchException exception) { + if (exception.status() == RestStatus.NOT_FOUND) { + // <1> + } + } + // end::clear-cache-notfound + } + } + + public void testCloseIndex() throws Exception { RestHighLevelClient client = highLevelClient(); diff --git a/dev-tools/es_release_notes.pl b/dev-tools/es_release_notes.pl index 4b5112859bedd..265df91544038 100755 --- a/dev-tools/es_release_notes.pl +++ b/dev-tools/es_release_notes.pl @@ -18,7 +18,7 @@ use strict; use warnings; -use HTTP::Tiny; +use HTTP::Tiny 0.070; use IO::Socket::SSL 1.52; use utf8; diff --git a/docs/java-rest/high-level/indices/clear_cache.asciidoc b/docs/java-rest/high-level/indices/clear_cache.asciidoc new file mode 100644 index 0000000000000..9e73283abd815 --- /dev/null +++ b/docs/java-rest/high-level/indices/clear_cache.asciidoc @@ -0,0 +1,109 @@ +[[java-rest-high-clear-cache]] +=== Clear Cache API + +[[java-rest-high-clear-cache-request]] +==== Clear Cache Request + +A `ClearIndicesCacheRquest` can be applied to one or more indices, or even on +`_all` the indices: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[clear-cache-request] +-------------------------------------------------- +<1> Clears the cache of one index +<2> Clears the cache of multiple indices +<3> Clears the cache of all the indices + +==== Optional arguments + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[clear-cache-request-indicesOptions] +-------------------------------------------------- +<1> Setting `IndicesOptions` controls how unavailable indices are resolved and +how wildcard expressions are expanded + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[clear-cache-request-query] +-------------------------------------------------- +<1> Set the `query` flag to `true` + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[clear-cache-request-fielddata] +-------------------------------------------------- +<1> Set the `fielddata` flag to `true` + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[clear-cache-request-request] +-------------------------------------------------- +<1> Set the `request` flag to `true` + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[clear-cache-request-fields] +-------------------------------------------------- +<1> Set the `fields` parameter + +[[java-rest-high-clear-cache-sync]] +==== Synchronous Execution + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[clear-cache-execute] +-------------------------------------------------- + +[[java-rest-high-clear-cache-async]] +==== Asynchronous Execution + +The asynchronous execution of a clear cache request requires both the `ClearIndicesCacheRequest` +instance and an `ActionListener` instance to be passed to the asynchronous +method: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[clear-cache-execute-async] +-------------------------------------------------- +<1> The `ClearIndicesCacheRequest` to execute and the `ActionListener` to use when +the execution completes + +The asynchronous method does not block and returns immediately. Once it is +completed the `ActionListener` is called back using the `onResponse` method +if the execution successfully completed or using the `onFailure` method if +it failed. + +A typical listener for `ClearIndicesCacheResponse` looks like: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[clear-cache-execute-listener] +-------------------------------------------------- +<1> Called when the execution is successfully completed. The response is +provided as an argument +<2> Called in case of failure. The raised exception is provided as an argument + +[[java-rest-high-clear-cache-response]] +==== Clear Cache Response + +The returned `ClearIndicesCacheResponse` allows to retrieve information about the +executed operation as follows: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[clear-cache-response] +-------------------------------------------------- +<1> Total number of shards hit by the clear cache request +<2> Number of shards where the clear cache has succeeded +<3> Number of shards where the clear cache has failed +<4> A list of failures if the operation failed on one or more shards + +By default, if the indices were not found, an `ElasticsearchException` will be thrown: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[clear-cache-notfound] +-------------------------------------------------- +<1> Do something if the indices to be cleared were not found \ No newline at end of file diff --git a/docs/java-rest/high-level/supported-apis.asciidoc b/docs/java-rest/high-level/supported-apis.asciidoc index 615634b65f1fd..bea30690fe183 100644 --- a/docs/java-rest/high-level/supported-apis.asciidoc +++ b/docs/java-rest/high-level/supported-apis.asciidoc @@ -59,6 +59,7 @@ Index Management:: * <> * <> * <> +* <> * <> Mapping Management:: @@ -77,6 +78,7 @@ include::indices/shrink_index.asciidoc[] include::indices/split_index.asciidoc[] include::indices/refresh.asciidoc[] include::indices/flush.asciidoc[] +include::indices/clear_cache.asciidoc[] include::indices/rollover.asciidoc[] include::indices/put_mapping.asciidoc[] include::indices/update_aliases.asciidoc[] diff --git a/docs/reference/cluster/reroute.asciidoc b/docs/reference/cluster/reroute.asciidoc index aba51f4893f24..0bc8610e0c792 100644 --- a/docs/reference/cluster/reroute.asciidoc +++ b/docs/reference/cluster/reroute.asciidoc @@ -83,8 +83,26 @@ Reasons why a primary shard cannot be automatically allocated include the follow the cluster. To prevent data loss, the system does not automatically promote a stale shard copy to primary. -As a manual override, two commands to forcefully allocate primary shards -are available: +[float] +=== Retry failed shards + +The cluster will attempt to allocate a shard a maximum of +`index.allocation.max_retries` times in a row (defaults to `5`), before giving +up and leaving the shard unallocated. This scenario can be caused by +structural problems such as having an analyzer which refers to a stopwords +file which doesn't exist on all nodes. + +Once the problem has been corrected, allocation can be manually retried by +calling the <> API with `?retry_failed`, which +will attempt a single retry round for these shards. + +[float] +=== Forced allocation on unrecoverable errors + +The following two commands are dangerous and may result in data loss. They are +meant to be used in cases where the original data can not be recovered and the cluster +administrator accepts the loss. If you have suffered a temporary issue that has been +fixed, please see the `retry_failed` flag described above. `allocate_stale_primary`:: Allocate a primary shard to a node that holds a stale copy. Accepts the @@ -108,15 +126,3 @@ are available: this command requires the special field `accept_data_loss` to be explicitly set to `true` for it to work. -[float] -=== Retry failed shards - -The cluster will attempt to allocate a shard a maximum of -`index.allocation.max_retries` times in a row (defaults to `5`), before giving -up and leaving the shard unallocated. This scenario can be caused by -structural problems such as having an analyzer which refers to a stopwords -file which doesn't exist on all nodes. - -Once the problem has been corrected, allocation can be manually retried by -calling the <> API with `?retry_failed`, which -will attempt a single retry round for these shards. diff --git a/docs/reference/modules/scripting/fields.asciidoc b/docs/reference/modules/scripting/fields.asciidoc index 1225d16333fce..bdd6c5785d81c 100644 --- a/docs/reference/modules/scripting/fields.asciidoc +++ b/docs/reference/modules/scripting/fields.asciidoc @@ -115,6 +115,17 @@ Doc-values can only return "simple" field values like numbers, dates, geo- points, terms, etc, or arrays of these values if the field is multi-valued. It cannot return JSON objects. +[NOTE] +.Missing fields +=================================================== + +The `doc['field']` will throw an error if `field` is missing from the mappings. +In `painless`, a check can first be done with `doc.containsKey('field')` to guard +accessing the `doc` map. Unfortunately, there is no way to check for the +existence of the field in mappings in an `expression` script. + +=================================================== + [NOTE] .Doc values and `text` fields =================================================== diff --git a/docs/reference/query-dsl/percolate-query.asciidoc b/docs/reference/query-dsl/percolate-query.asciidoc index 9a97a6bd12e10..b6e465e34dfd4 100644 --- a/docs/reference/query-dsl/percolate-query.asciidoc +++ b/docs/reference/query-dsl/percolate-query.asciidoc @@ -37,7 +37,7 @@ the `percolator` query before it gets indexed into a temporary index. The `query` field is used for indexing the query documents. It will hold a json object that represents an actual Elasticsearch query. The `query` field has been configured to use the <>. This field -type understands the query dsl and stored the query in such a way that it can be +type understands the query dsl and stores the query in such a way that it can be used later on to match documents defined on the `percolate` query. Register a query in the percolator: diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java index 0c35876ada63d..d9b89ba339a0c 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java @@ -133,7 +133,7 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder { public static final Version V_6_1_2 = new Version(V_6_1_2_ID, org.apache.lucene.util.Version.LUCENE_7_1_0); public static final int V_6_1_3_ID = 6010399; public static final Version V_6_1_3 = new Version(V_6_1_3_ID, org.apache.lucene.util.Version.LUCENE_7_1_0); + public static final int V_6_1_4_ID = 6010499; + public static final Version V_6_1_4 = new Version(V_6_1_4_ID, org.apache.lucene.util.Version.LUCENE_7_1_0); + public static final int V_6_1_5_ID = 6010599; + public static final Version V_6_1_5 = new Version(V_6_1_5_ID, org.apache.lucene.util.Version.LUCENE_7_1_0); public static final int V_6_2_0_ID = 6020099; public static final Version V_6_2_0 = new Version(V_6_2_0_ID, org.apache.lucene.util.Version.LUCENE_7_2_1); public static final int V_6_2_1_ID = 6020199; @@ -155,6 +159,8 @@ public class Version implements Comparable { public static final Version V_6_2_2 = new Version(V_6_2_2_ID, org.apache.lucene.util.Version.LUCENE_7_2_1); public static final int V_6_2_3_ID = 6020399; public static final Version V_6_2_3 = new Version(V_6_2_3_ID, org.apache.lucene.util.Version.LUCENE_7_2_1); + public static final int V_6_2_4_ID = 6020499; + public static final Version V_6_2_4 = new Version(V_6_2_4_ID, org.apache.lucene.util.Version.LUCENE_7_2_1); public static final int V_6_3_0_ID = 6030099; public static final Version V_6_3_0 = new Version(V_6_3_0_ID, org.apache.lucene.util.Version.LUCENE_7_2_1); public static final int V_7_0_0_alpha1_ID = 7000001; @@ -177,6 +183,8 @@ public static Version fromId(int id) { return V_7_0_0_alpha1; case V_6_3_0_ID: return V_6_3_0; + case V_6_2_4_ID: + return V_6_2_4; case V_6_2_3_ID: return V_6_2_3; case V_6_2_2_ID: @@ -185,6 +193,10 @@ public static Version fromId(int id) { return V_6_2_1; case V_6_2_0_ID: return V_6_2_0; + case V_6_1_5_ID: + return V_6_1_5; + case V_6_1_4_ID: + return V_6_1_4; case V_6_1_3_ID: return V_6_1_3; case V_6_1_2_ID: diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponse.java index a9a2c36970ee4..74779711c73a9 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponse.java @@ -245,7 +245,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(DELAYED_UNASSIGNED_SHARDS, getDelayedUnassignedShards()); builder.field(NUMBER_OF_PENDING_TASKS, getNumberOfPendingTasks()); builder.field(NUMBER_OF_IN_FLIGHT_FETCH, getNumberOfInFlightFetch()); - builder.timeValueField(TASK_MAX_WAIT_TIME_IN_QUEUE_IN_MILLIS, TASK_MAX_WAIT_TIME_IN_QUEUE, getTaskMaxWaitingTime()); + builder.humanReadableField(TASK_MAX_WAIT_TIME_IN_QUEUE_IN_MILLIS, TASK_MAX_WAIT_TIME_IN_QUEUE, getTaskMaxWaitingTime()); builder.percentageField(ACTIVE_SHARDS_PERCENT_AS_NUMBER, ACTIVE_SHARDS_PERCENT, getActiveShardsPercent()); String level = params.param("level", "cluster"); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java index 952589766773f..09ac7e6aa13d7 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java @@ -68,7 +68,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("version", nodeInfo.getVersion()); builder.field("build_hash", nodeInfo.getBuild().shortHash()); if (nodeInfo.getTotalIndexingBuffer() != null) { - builder.byteSizeField("total_indexing_buffer", "total_indexing_buffer_in_bytes", nodeInfo.getTotalIndexingBuffer()); + builder.humanReadableField("total_indexing_buffer", "total_indexing_buffer_in_bytes", nodeInfo.getTotalIndexingBuffer()); } builder.startArray("roles"); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStats.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStats.java index 5b2bdd7c614c6..e7957e0ac0818 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStats.java @@ -22,6 +22,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Streamable; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -143,7 +144,7 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par builder.byteSizeField(Fields.TOTAL_SIZE_IN_BYTES, Fields.TOTAL_SIZE, getTotalSize()); builder.byteSizeField(Fields.PROCESSED_SIZE_IN_BYTES, Fields.PROCESSED_SIZE, getProcessedSize()); builder.field(Fields.START_TIME_IN_MILLIS, getStartTime()); - builder.timeValueField(Fields.TIME_IN_MILLIS, Fields.TIME, getTime()); + builder.humanReadableField(Fields.TIME_IN_MILLIS, Fields.TIME, new TimeValue(getTime())); builder.endObject(); return builder; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java index f6ff6a1d643d5..2efaf2245ea04 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java @@ -488,7 +488,7 @@ static final class Fields { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.timeValueField(Fields.MAX_UPTIME_IN_MILLIS, Fields.MAX_UPTIME, maxUptime); + builder.humanReadableField(Fields.MAX_UPTIME_IN_MILLIS, Fields.MAX_UPTIME, new TimeValue(maxUptime)); builder.startArray(Fields.VERSIONS); for (ObjectIntCursor v : versions) { builder.startObject(); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java index 5ab5c49d0c791..50af29bbdc369 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java @@ -125,7 +125,7 @@ public void readFrom(StreamInput in) throws IOException { if (in.getVersion().onOrAfter(Version.V_5_3_0)) { xContentType = in.readEnum(XContentType.class); } else { - xContentType = XContentFactory.xContentType(content); + xContentType = XContentHelper.xContentType(content); } if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha2)) { context = in.readOptionalString(); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/ClearIndicesCacheResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/ClearIndicesCacheResponse.java index d0f4b3cc20beb..47b516137d795 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/ClearIndicesCacheResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/ClearIndicesCacheResponse.java @@ -21,19 +21,28 @@ import org.elasticsearch.action.support.DefaultShardOperationFailedException; import org.elasticsearch.action.support.broadcast.BroadcastResponse; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; -import java.io.IOException; +import java.util.Arrays; import java.util.List; /** - * The response of a refresh action. - * - * + * The response of a clear cache action. */ public class ClearIndicesCacheResponse extends BroadcastResponse { + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("clear_cache", + true, arg -> { + BroadcastResponse response = (BroadcastResponse) arg[0]; + return new ClearIndicesCacheResponse(response.getTotalShards(), response.getSuccessfulShards(), response.getFailedShards(), + Arrays.asList(response.getShardFailures())); + }); + + static { + declareBroadcastFields(PARSER); + } + ClearIndicesCacheResponse() { } @@ -43,13 +52,7 @@ public class ClearIndicesCacheResponse extends BroadcastResponse { super(totalShards, successfulShards, failedShards, shardFailures); } - @Override - public void readFrom(StreamInput in) throws IOException { - super.readFrom(in); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); + public static ClearIndicesCacheResponse fromXContent(XContentParser parser) { + return PARSER.apply(parser, null); } } \ No newline at end of file diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java index b9296c0242fdb..e8e2f5376cd24 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java @@ -198,7 +198,7 @@ static void toXContent(XContentBuilder builder, Sort sort) throws IOException { static void toXContent(XContentBuilder builder, Accountable tree) throws IOException { builder.startObject(); builder.field(Fields.DESCRIPTION, tree.toString()); - builder.byteSizeField(Fields.SIZE_IN_BYTES, Fields.SIZE, new ByteSizeValue(tree.ramBytesUsed())); + builder.humanReadableField(Fields.SIZE_IN_BYTES, Fields.SIZE, new ByteSizeValue(tree.ramBytesUsed())); Collection children = tree.getChildResources(); if (children.isEmpty() == false) { builder.startArray(Fields.CHILDREN); diff --git a/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java b/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java index 10b19ed8cfa17..722473d64e40c 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java @@ -25,7 +25,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; import java.io.IOException; @@ -43,7 +43,7 @@ public class PutPipelineRequest extends AcknowledgedRequest */ @Deprecated public PutPipelineRequest(String id, BytesReference source) { - this(id, source, XContentFactory.xContentType(source)); + this(id, source, XContentHelper.xContentType(source)); } /** @@ -83,7 +83,7 @@ public void readFrom(StreamInput in) throws IOException { if (in.getVersion().onOrAfter(Version.V_5_3_0)) { xContentType = in.readEnum(XContentType.class); } else { - xContentType = XContentFactory.xContentType(source); + xContentType = XContentHelper.xContentType(source); } } diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java index 428df00a68e9c..aeb4b47719dd6 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java @@ -25,8 +25,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.VersionType; import org.elasticsearch.ingest.ConfigurationUtils; @@ -56,7 +55,7 @@ public class SimulatePipelineRequest extends ActionRequest { */ @Deprecated public SimulatePipelineRequest(BytesReference source) { - this(source, XContentFactory.xContentType(source)); + this(source, XContentHelper.xContentType(source)); } /** @@ -78,7 +77,7 @@ public SimulatePipelineRequest(BytesReference source, XContentType xContentType) if (in.getVersion().onOrAfter(Version.V_5_3_0)) { xContentType = in.readEnum(XContentType.class); } else { - xContentType = XContentFactory.xContentType(source); + xContentType = XContentHelper.xContentType(source); } } diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java b/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java index 68841fe71e5b4..e75f510d80c02 100644 --- a/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java @@ -35,7 +35,7 @@ import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.VersionType; @@ -265,7 +265,7 @@ public TermVectorsRequest doc(XContentBuilder documentBuilder) { */ @Deprecated public TermVectorsRequest doc(BytesReference doc, boolean generateRandomId) { - return this.doc(doc, generateRandomId, XContentFactory.xContentType(doc)); + return this.doc(doc, generateRandomId, XContentHelper.xContentType(doc)); } /** @@ -518,7 +518,7 @@ public void readFrom(StreamInput in) throws IOException { if (in.getVersion().onOrAfter(Version.V_5_3_0)) { xContentType = in.readEnum(XContentType.class); } else { - xContentType = XContentFactory.xContentType(doc); + xContentType = XContentHelper.xContentType(doc); } } routing = in.readOptionalString(); diff --git a/server/src/main/java/org/elasticsearch/cluster/SnapshotDeletionsInProgress.java b/server/src/main/java/org/elasticsearch/cluster/SnapshotDeletionsInProgress.java index 981d6128419ba..234d1ef9f17fd 100644 --- a/server/src/main/java/org/elasticsearch/cluster/SnapshotDeletionsInProgress.java +++ b/server/src/main/java/org/elasticsearch/cluster/SnapshotDeletionsInProgress.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.snapshots.Snapshot; @@ -145,7 +146,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws { builder.field("repository", entry.snapshot.getRepository()); builder.field("snapshot", entry.snapshot.getSnapshotId().getName()); - builder.timeValueField("start_time_millis", "start_time", entry.startTime); + builder.humanReadableField("start_time_millis", "start_time", new TimeValue(entry.startTime)); builder.field("repository_state_id", entry.repositoryStateId); } builder.endObject(); diff --git a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java index 2d3886aa52f12..74b748e19a7ee 100644 --- a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java +++ b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java @@ -27,6 +27,7 @@ import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.shard.ShardId; @@ -512,7 +513,7 @@ public void toXContent(Entry entry, XContentBuilder builder, ToXContent.Params p } } builder.endArray(); - builder.timeValueField(START_TIME_MILLIS, START_TIME, entry.startTime()); + builder.humanReadableField(START_TIME_MILLIS, START_TIME, new TimeValue(entry.startTime())); builder.field(REPOSITORY_STATE_ID, entry.getRepositoryStateId()); builder.startArray(SHARDS); { diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocateUnassignedDecision.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocateUnassignedDecision.java index decdafd724c7d..fc2d81b38c493 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocateUnassignedDecision.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocateUnassignedDecision.java @@ -289,8 +289,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("allocation_id", allocationId); } if (allocationStatus == AllocationStatus.DELAYED_ALLOCATION) { - builder.timeValueField("configured_delay_in_millis", "configured_delay", TimeValue.timeValueMillis(configuredDelayInMillis)); - builder.timeValueField("remaining_delay_in_millis", "remaining_delay", TimeValue.timeValueMillis(remainingDelayInMillis)); + builder.humanReadableField("configured_delay_in_millis", "configured_delay", + TimeValue.timeValueMillis(configuredDelayInMillis)); + builder.humanReadableField("remaining_delay_in_millis", "remaining_delay", + TimeValue.timeValueMillis(remainingDelayInMillis)); } nodeDecisionsToXContent(nodeDecisions, builder, params); return builder; diff --git a/server/src/main/java/org/elasticsearch/common/compress/CompressorFactory.java b/server/src/main/java/org/elasticsearch/common/compress/CompressorFactory.java index a355a12d67238..332d9024e997f 100644 --- a/server/src/main/java/org/elasticsearch/common/compress/CompressorFactory.java +++ b/server/src/main/java/org/elasticsearch/common/compress/CompressorFactory.java @@ -24,7 +24,7 @@ import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; import java.io.IOException; @@ -44,11 +44,11 @@ public static Compressor compressor(BytesReference bytes) { // bytes should be either detected as compressed or as xcontent, // if we have bytes that can be either detected as compressed or // as a xcontent, we have a problem - assert XContentFactory.xContentType(bytes) == null; + assert XContentHelper.xContentType(bytes) == null; return COMPRESSOR; } - XContentType contentType = XContentFactory.xContentType(bytes); + XContentType contentType = XContentHelper.xContentType(bytes); if (contentType == null) { if (isAncient(bytes)) { throw new IllegalStateException("unsupported compression: index was created before v2.0.0.beta1 and wasn't upgraded?"); diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java b/server/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java index b5622a9c0d26e..b51add28bf539 100644 --- a/server/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java +++ b/server/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java @@ -43,8 +43,8 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.ServiceLoader; import java.util.Set; -import java.util.concurrent.TimeUnit; /** * A utility to build XContent (ie json). @@ -85,6 +85,7 @@ public static XContentBuilder builder(XContent xContent, Set includes, S public static final DateTimeFormatter DEFAULT_DATE_PRINTER = ISODateTimeFormat.dateTime().withZone(DateTimeZone.UTC); private static final Map, Writer> WRITERS; + private static final Map, HumanReadableTransformer> HUMAN_READABLE_TRANSFORMERS; static { Map, Writer> writers = new HashMap<>(); writers.put(Boolean.class, (b, v) -> b.value((Boolean) v)); @@ -105,14 +106,43 @@ public static XContentBuilder builder(XContent xContent, Set includes, S writers.put(String.class, (b, v) -> b.value((String) v)); writers.put(String[].class, (b, v) -> b.values((String[]) v)); + + Map, HumanReadableTransformer> humanReadableTransformer = new HashMap<>(); + // These will be moved to a different class at a later time to decouple them from XContentBuilder + humanReadableTransformer.put(TimeValue.class, v -> ((TimeValue) v).millis()); + humanReadableTransformer.put(ByteSizeValue.class, v -> ((ByteSizeValue) v).getBytes()); + + // Load pluggable extensions + for (XContentBuilderExtension service : ServiceLoader.load(XContentBuilderExtension.class)) { + Map, Writer> addlWriters = service.getXContentWriters(); + Map, HumanReadableTransformer> addlTransformers = service.getXContentHumanReadableTransformers(); + + addlWriters.forEach((key, value) -> Objects.requireNonNull(value, + "invalid null xcontent writer for class " + key)); + addlTransformers.forEach((key, value) -> Objects.requireNonNull(value, + "invalid null xcontent transformer for human readable class " + key)); + + writers.putAll(addlWriters); + humanReadableTransformer.putAll(addlTransformers); + } + WRITERS = Collections.unmodifiableMap(writers); + HUMAN_READABLE_TRANSFORMERS = Collections.unmodifiableMap(humanReadableTransformer); } @FunctionalInterface - private interface Writer { + public interface Writer { void write(XContentBuilder builder, Object value) throws IOException; } + /** + * Interface for transforming complex objects into their "raw" equivalents for human-readable fields + */ + @FunctionalInterface + public interface HumanReadableTransformer { + Object rawValue(Object value) throws IOException; + } + /** * XContentGenerator used to build the XContent object */ @@ -856,33 +886,30 @@ private XContentBuilder value(Iterable values, boolean ensureNoSelfReferences } //////////////////////////////////////////////////////////////////////////// - // Misc. + // Human readable fields + // + // These are fields that have a "raw" value and a "human readable" value, + // such as time values or byte sizes. The human readable variant is only + // used if the humanReadable flag has been set ////////////////////////////////// - public XContentBuilder timeValueField(String rawFieldName, String readableFieldName, TimeValue timeValue) throws IOException { + public XContentBuilder humanReadableField(String rawFieldName, String readableFieldName, Object value) throws IOException { if (humanReadable) { - field(readableFieldName, timeValue.toString()); + field(readableFieldName, Objects.toString(value)); } - field(rawFieldName, timeValue.millis()); - return this; - } - - public XContentBuilder timeValueField(String rawFieldName, String readableFieldName, long rawTime) throws IOException { - if (humanReadable) { - field(readableFieldName, new TimeValue(rawTime).toString()); + HumanReadableTransformer transformer = HUMAN_READABLE_TRANSFORMERS.get(value.getClass()); + if (transformer != null) { + Object rawValue = transformer.rawValue(value); + field(rawFieldName, rawValue); + } else { + throw new IllegalArgumentException("no raw transformer found for class " + value.getClass()); } - field(rawFieldName, rawTime); return this; } - public XContentBuilder timeValueField(String rawFieldName, String readableFieldName, long rawTime, TimeUnit timeUnit) throws - IOException { - if (humanReadable) { - field(readableFieldName, new TimeValue(rawTime, timeUnit).toString()); - } - field(rawFieldName, rawTime); - return this; - } + //////////////////////////////////////////////////////////////////////////// + // Misc. + ////////////////////////////////// public XContentBuilder percentageField(String rawFieldName, String readableFieldName, double percentage) throws IOException { @@ -893,14 +920,6 @@ public XContentBuilder percentageField(String rawFieldName, String readableField return this; } - public XContentBuilder byteSizeField(String rawFieldName, String readableFieldName, ByteSizeValue byteSizeValue) throws IOException { - if (humanReadable) { - field(readableFieldName, byteSizeValue.toString()); - } - field(rawFieldName, byteSizeValue.getBytes()); - return this; - } - public XContentBuilder byteSizeField(String rawFieldName, String readableFieldName, long rawSize) throws IOException { if (humanReadable) { field(readableFieldName, new ByteSizeValue(rawSize).toString()); diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/XContentBuilderExtension.java b/server/src/main/java/org/elasticsearch/common/xcontent/XContentBuilderExtension.java new file mode 100644 index 0000000000000..610be4585eb9c --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/xcontent/XContentBuilderExtension.java @@ -0,0 +1,64 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.xcontent; + +import java.util.Map; + +/** + * This interface provides a way for non-JDK classes to plug in a way to serialize to xcontent. + * + * It is greatly preferred that you implement {@link ToXContentFragment} + * in the class for encoding, however, in some situations you may not own the + * class, in which case you can add an implementation here for encoding it. + */ +public interface XContentBuilderExtension { + + /** + * Used for plugging in a generic writer for a class, for example, an example implementation: + * + * 

+     * {@code
+     *     Map, XContentBuilder.Writer> addlWriters = new HashMap<>();
+     *     addlWriters.put(BytesRef.class, (builder, value) -> b.value(((BytesRef) value).utf8String()));
+     *     return addlWriters;
+     * }
+     *
+ * + * @return a map of class name to writer + */ + Map, XContentBuilder.Writer> getXContentWriters(); + + /** + * Used for plugging in a human readable version of a class's encoding. It is assumed that + * the human readable equivalent is always behind the {@code toString()} method, so + * this transformer returns the raw value to be used. + * + * An example implementation: + * + * 
+     * {@code
+     *     Map, XContentBuilder.HumanReadableTransformer> transformers = new HashMap<>();
+     *     transformers.put(ByteSizeValue.class, (value) -> ((ByteSizeValue) value).bytes());
+     * }
+     *
+ * @return a map of class name to transformer used to retrieve raw value + */ + Map, XContentBuilder.HumanReadableTransformer> getXContentHumanReadableTransformers(); +} diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/XContentFactory.java b/server/src/main/java/org/elasticsearch/common/xcontent/XContentFactory.java index dc9d1c493a3ed..f9faa6f2b0658 100644 --- a/server/src/main/java/org/elasticsearch/common/xcontent/XContentFactory.java +++ b/server/src/main/java/org/elasticsearch/common/xcontent/XContentFactory.java @@ -22,9 +22,6 @@ import com.fasterxml.jackson.dataformat.cbor.CBORConstants; import com.fasterxml.jackson.dataformat.smile.SmileConstants; import org.elasticsearch.ElasticsearchParseException; -import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.xcontent.cbor.CborXContent; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.common.xcontent.smile.SmileXContent; @@ -221,18 +218,6 @@ public static XContent xContent(byte[] data, int offset, int length) { return xContent(type); } - /** - * Guesses the content type based on the provided bytes. - * - * @deprecated the content type should not be guessed except for few cases where we effectively don't know the content type. - * The REST layer should move to reading the Content-Type header instead. There are other places where auto-detection may be needed. - * This method is deprecated to prevent usages of it from spreading further without specific reasons. - */ - @Deprecated - public static XContentType xContentType(byte[] data) { - return xContentType(data, 0, data.length); - } - /** * Guesses the content type based on the provided input stream without consuming it. * @@ -248,8 +233,15 @@ public static XContentType xContentType(InputStream si) throws IOException { si.mark(GUESS_HEADER_LENGTH); try { final byte[] firstBytes = new byte[GUESS_HEADER_LENGTH]; - final int read = Streams.readFully(si, firstBytes); - return xContentType(new BytesArray(firstBytes, 0, read)); + int read = 0; + while (read < GUESS_HEADER_LENGTH) { + final int r = si.read(firstBytes, read, GUESS_HEADER_LENGTH - read); + if (r == -1) { + break; + } + read += r; + } + return xContentType(firstBytes, 0, read); } finally { si.reset(); } @@ -263,24 +255,8 @@ public static XContentType xContentType(InputStream si) throws IOException { * This method is deprecated to prevent usages of it from spreading further without specific reasons. */ @Deprecated - public static XContentType xContentType(byte[] data, int offset, int length) { - return xContentType(new BytesArray(data, offset, length)); - } - - /** - * Guesses the content type based on the provided bytes and returns the corresponding {@link XContent} - * - * @deprecated the content type should not be guessed except for few cases where we effectively don't know the content type. - * The REST layer should move to reading the Content-Type header instead. There are other places where auto-detection may be needed. - * This method is deprecated to prevent usages of it from spreading further without specific reasons. - */ - @Deprecated - public static XContent xContent(BytesReference bytes) { - XContentType type = xContentType(bytes); - if (type == null) { - throw new ElasticsearchParseException("Failed to derive xcontent"); - } - return xContent(type); + public static XContentType xContentType(byte[] bytes) { + return xContentType(bytes, 0, bytes.length); } /** @@ -291,19 +267,21 @@ public static XContent xContent(BytesReference bytes) { * This method is deprecated to prevent usages of it from spreading further without specific reasons. */ @Deprecated - public static XContentType xContentType(BytesReference bytes) { - int length = bytes.length(); - if (length == 0) { + public static XContentType xContentType(byte[] bytes, int offset, int length) { + int totalLength = bytes.length; + if (totalLength == 0 || length == 0) { + return null; + } else if ((offset + length) > totalLength) { return null; } - byte first = bytes.get(0); + byte first = bytes[offset]; if (first == '{') { return XContentType.JSON; } - if (length > 2 && first == SmileConstants.HEADER_BYTE_1 && bytes.get(1) == SmileConstants.HEADER_BYTE_2 && bytes.get(2) == SmileConstants.HEADER_BYTE_3) { + if (length > 2 && first == SmileConstants.HEADER_BYTE_1 && bytes[offset + 1] == SmileConstants.HEADER_BYTE_2 && bytes[offset + 2] == SmileConstants.HEADER_BYTE_3) { return XContentType.SMILE; } - if (length > 2 && first == '-' && bytes.get(1) == '-' && bytes.get(2) == '-') { + if (length > 2 && first == '-' && bytes[offset + 1] == '-' && bytes[offset + 2] == '-') { return XContentType.YAML; } // CBOR logic similar to CBORFactory#hasCBORFormat @@ -312,7 +290,7 @@ public static XContentType xContentType(BytesReference bytes) { } if (CBORConstants.hasMajorType(CBORConstants.MAJOR_TYPE_TAG, first) && length > 2) { // Actually, specific "self-describe tag" is a very good indicator - if (first == (byte) 0xD9 && bytes.get(1) == (byte) 0xD9 && bytes.get(2) == (byte) 0xF7) { + if (first == (byte) 0xD9 && bytes[offset + 1] == (byte) 0xD9 && bytes[offset + 2] == (byte) 0xF7) { return XContentType.CBOR; } } @@ -324,13 +302,13 @@ public static XContentType xContentType(BytesReference bytes) { int jsonStart = 0; // JSON may be preceded by UTF-8 BOM - if (length > 3 && first == (byte) 0xEF && bytes.get(1) == (byte) 0xBB && bytes.get(2) == (byte) 0xBF) { + if (length > 3 && first == (byte) 0xEF && bytes[offset + 1] == (byte) 0xBB && bytes[offset + 2] == (byte) 0xBF) { jsonStart = 3; } // a last chance for JSON for (int i = jsonStart; i < length; i++) { - byte b = bytes.get(i); + byte b = bytes[offset + i]; if (b == '{') { return XContentType.JSON; } diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java b/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java index e392295722959..6501f899c47bf 100644 --- a/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java +++ b/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java @@ -19,6 +19,7 @@ package org.elasticsearch.common.xcontent; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; @@ -55,7 +56,7 @@ public static XContentParser createParser(NamedXContentRegistry xContentRegistry final XContentType contentType = XContentFactory.xContentType(compressedInput); return XContentFactory.xContent(contentType).createParser(xContentRegistry, deprecationHandler, compressedInput); } else { - return XContentFactory.xContent(bytes).createParser(xContentRegistry, deprecationHandler, bytes.streamInput()); + return XContentFactory.xContent(xContentType(bytes)).createParser(xContentRegistry, deprecationHandler, bytes.streamInput()); } } @@ -151,7 +152,7 @@ public static String convertToJson(BytesReference bytes, boolean reformatJson) t @Deprecated public static String convertToJson(BytesReference bytes, boolean reformatJson, boolean prettyPrint) throws IOException { - return convertToJson(bytes, reformatJson, prettyPrint, XContentFactory.xContentType(bytes)); + return convertToJson(bytes, reformatJson, prettyPrint, XContentFactory.xContentType(bytes.toBytesRef().bytes)); } public static String convertToJson(BytesReference bytes, boolean reformatJson, XContentType xContentType) throws IOException { @@ -436,4 +437,17 @@ public static BytesReference toXContent(ToXContent toXContent, XContentType xCon return BytesReference.bytes(builder); } } + + /** + * Guesses the content type based on the provided bytes. + * + * @deprecated the content type should not be guessed except for few cases where we effectively don't know the content type. + * The REST layer should move to reading the Content-Type header instead. There are other places where auto-detection may be needed. + * This method is deprecated to prevent usages of it from spreading further without specific reasons. + */ + @Deprecated + public static XContentType xContentType(BytesReference bytes) { + BytesRef br = bytes.toBytesRef(); + return XContentFactory.xContentType(br.bytes, br.offset, br.length); + } } diff --git a/server/src/main/java/org/elasticsearch/index/flush/FlushStats.java b/server/src/main/java/org/elasticsearch/index/flush/FlushStats.java index d15a62b0a734f..4b931e47372b7 100644 --- a/server/src/main/java/org/elasticsearch/index/flush/FlushStats.java +++ b/server/src/main/java/org/elasticsearch/index/flush/FlushStats.java @@ -85,7 +85,7 @@ public TimeValue getTotalTime() { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(Fields.FLUSH); builder.field(Fields.TOTAL, total); - builder.timeValueField(Fields.TOTAL_TIME_IN_MILLIS, Fields.TOTAL_TIME, totalTimeInMillis); + builder.humanReadableField(Fields.TOTAL_TIME_IN_MILLIS, Fields.TOTAL_TIME, getTotalTime()); builder.endObject(); return builder; } diff --git a/server/src/main/java/org/elasticsearch/index/get/GetStats.java b/server/src/main/java/org/elasticsearch/index/get/GetStats.java index e89f22ea85135..bff1299a348bd 100644 --- a/server/src/main/java/org/elasticsearch/index/get/GetStats.java +++ b/server/src/main/java/org/elasticsearch/index/get/GetStats.java @@ -110,11 +110,11 @@ public long current() { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(Fields.GET); builder.field(Fields.TOTAL, getCount()); - builder.timeValueField(Fields.TIME_IN_MILLIS, Fields.TIME, getTimeInMillis()); + builder.humanReadableField(Fields.TIME_IN_MILLIS, Fields.TIME, getTime()); builder.field(Fields.EXISTS_TOTAL, existsCount); - builder.timeValueField(Fields.EXISTS_TIME_IN_MILLIS, Fields.EXISTS_TIME, existsTimeInMillis); + builder.humanReadableField(Fields.EXISTS_TIME_IN_MILLIS, Fields.EXISTS_TIME, getExistsTime()); builder.field(Fields.MISSING_TOTAL, missingCount); - builder.timeValueField(Fields.MISSING_TIME_IN_MILLIS, Fields.MISSING_TIME, missingTimeInMillis); + builder.humanReadableField(Fields.MISSING_TIME_IN_MILLIS, Fields.MISSING_TIME, getMissingTime()); builder.field(Fields.CURRENT, current); builder.endObject(); return builder; diff --git a/server/src/main/java/org/elasticsearch/index/merge/MergeStats.java b/server/src/main/java/org/elasticsearch/index/merge/MergeStats.java index 94e5b31a1cd28..20329cac98ba0 100644 --- a/server/src/main/java/org/elasticsearch/index/merge/MergeStats.java +++ b/server/src/main/java/org/elasticsearch/index/merge/MergeStats.java @@ -189,11 +189,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(Fields.CURRENT_DOCS, currentNumDocs); builder.byteSizeField(Fields.CURRENT_SIZE_IN_BYTES, Fields.CURRENT_SIZE, currentSizeInBytes); builder.field(Fields.TOTAL, total); - builder.timeValueField(Fields.TOTAL_TIME_IN_MILLIS, Fields.TOTAL_TIME, totalTimeInMillis); + builder.humanReadableField(Fields.TOTAL_TIME_IN_MILLIS, Fields.TOTAL_TIME, getTotalTime()); builder.field(Fields.TOTAL_DOCS, totalNumDocs); builder.byteSizeField(Fields.TOTAL_SIZE_IN_BYTES, Fields.TOTAL_SIZE, totalSizeInBytes); - builder.timeValueField(Fields.TOTAL_STOPPED_TIME_IN_MILLIS, Fields.TOTAL_STOPPED_TIME, totalStoppedTimeInMillis); - builder.timeValueField(Fields.TOTAL_THROTTLED_TIME_IN_MILLIS, Fields.TOTAL_THROTTLED_TIME, totalThrottledTimeInMillis); + builder.humanReadableField(Fields.TOTAL_STOPPED_TIME_IN_MILLIS, Fields.TOTAL_STOPPED_TIME, getTotalStoppedTime()); + builder.humanReadableField(Fields.TOTAL_THROTTLED_TIME_IN_MILLIS, Fields.TOTAL_THROTTLED_TIME, getTotalThrottledTime()); builder.byteSizeField(Fields.TOTAL_THROTTLE_BYTES_PER_SEC_IN_BYTES, Fields.TOTAL_THROTTLE_BYTES_PER_SEC, totalBytesPerSecAutoThrottle); builder.endObject(); return builder; diff --git a/server/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java index d362a4c534aaf..d296db28ad625 100644 --- a/server/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java @@ -48,6 +48,7 @@ import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.VersionType; @@ -224,7 +225,7 @@ public Item(@Nullable String index, @Nullable String type, XContentBuilder doc) if (in.getVersion().onOrAfter(Version.V_5_3_0)) { xContentType = in.readEnum(XContentType.class); } else { - xContentType = XContentFactory.xContentType(doc); + xContentType = XContentHelper.xContentType(doc); } } else { id = in.readString(); diff --git a/server/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java b/server/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java index aa39d5f7417fa..3712040b8de03 100644 --- a/server/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java @@ -38,6 +38,7 @@ import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.fielddata.IndexGeoPointFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData; @@ -186,7 +187,7 @@ protected ScoreFunction doToFunction(QueryShardContext context) throws IOExcepti AbstractDistanceScoreFunction scoreFunction; // EMPTY is safe because parseVariable doesn't use namedObject try (InputStream stream = functionBytes.streamInput(); - XContentParser parser = XContentFactory.xContent(functionBytes) + XContentParser parser = XContentFactory.xContent(XContentHelper.xContentType(functionBytes)) .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, stream)) { scoreFunction = parseVariable(fieldName, parser, context, multiValueMode); } diff --git a/server/src/main/java/org/elasticsearch/index/recovery/RecoveryStats.java b/server/src/main/java/org/elasticsearch/index/recovery/RecoveryStats.java index 4e3d71cce6299..b86f9c55f7b20 100644 --- a/server/src/main/java/org/elasticsearch/index/recovery/RecoveryStats.java +++ b/server/src/main/java/org/elasticsearch/index/recovery/RecoveryStats.java @@ -103,7 +103,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(Fields.RECOVERY); builder.field(Fields.CURRENT_AS_SOURCE, currentAsSource()); builder.field(Fields.CURRENT_AS_TARGET, currentAsTarget()); - builder.timeValueField(Fields.THROTTLE_TIME_IN_MILLIS, Fields.THROTTLE_TIME, throttleTime()); + builder.humanReadableField(Fields.THROTTLE_TIME_IN_MILLIS, Fields.THROTTLE_TIME, throttleTime()); builder.endObject(); return builder; } diff --git a/server/src/main/java/org/elasticsearch/index/refresh/RefreshStats.java b/server/src/main/java/org/elasticsearch/index/refresh/RefreshStats.java index 1235aad885f35..9a4830368bbd6 100644 --- a/server/src/main/java/org/elasticsearch/index/refresh/RefreshStats.java +++ b/server/src/main/java/org/elasticsearch/index/refresh/RefreshStats.java @@ -96,7 +96,7 @@ public int getListeners() { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject("refresh"); builder.field("total", total); - builder.timeValueField("total_time_in_millis", "total_time", totalTimeInMillis); + builder.humanReadableField("total_time_in_millis", "total_time", getTotalTime()); builder.field("listeners", listeners); builder.endObject(); return builder; diff --git a/server/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java b/server/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java index 276484b055253..b6a4cc2fd74e2 100644 --- a/server/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java +++ b/server/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java @@ -387,12 +387,12 @@ public XContentBuilder innerXContent(XContentBuilder builder, Params params) builder.field("search", searchRetries); } builder.endObject(); - builder.timeValueField("throttled_millis", "throttled", throttled); + builder.humanReadableField("throttled_millis", "throttled", throttled); builder.field("requests_per_second", requestsPerSecond == Float.POSITIVE_INFINITY ? -1 : requestsPerSecond); if (reasonCancelled != null) { builder.field("canceled", reasonCancelled); } - builder.timeValueField("throttled_until_millis", "throttled_until", throttledUntil); + builder.humanReadableField("throttled_until_millis", "throttled_until", throttledUntil); if (false == sliceStatuses.isEmpty()) { builder.startArray("slices"); for (StatusOrException slice : sliceStatuses) { diff --git a/server/src/main/java/org/elasticsearch/index/reindex/ClientScrollableHitSource.java b/server/src/main/java/org/elasticsearch/index/reindex/ClientScrollableHitSource.java index 5124201997edb..52c2b2ae2cf8a 100644 --- a/server/src/main/java/org/elasticsearch/index/reindex/ClientScrollableHitSource.java +++ b/server/src/main/java/org/elasticsearch/index/reindex/ClientScrollableHitSource.java @@ -38,7 +38,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; -import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.mapper.ParentFieldMapper; import org.elasticsearch.index.mapper.RoutingFieldMapper; @@ -236,7 +236,7 @@ public BytesReference getSource() { @Override public XContentType getXContentType() { - return XContentFactory.xContentType(source); + return XContentHelper.xContentType(source); } @Override public long getVersion() { diff --git a/server/src/main/java/org/elasticsearch/index/search/stats/SearchStats.java b/server/src/main/java/org/elasticsearch/index/search/stats/SearchStats.java index de0a659d5f1a6..519cd9ff9ae71 100644 --- a/server/src/main/java/org/elasticsearch/index/search/stats/SearchStats.java +++ b/server/src/main/java/org/elasticsearch/index/search/stats/SearchStats.java @@ -219,19 +219,19 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.field(Fields.QUERY_TOTAL, queryCount); - builder.timeValueField(Fields.QUERY_TIME_IN_MILLIS, Fields.QUERY_TIME, queryTimeInMillis); + builder.humanReadableField(Fields.QUERY_TIME_IN_MILLIS, Fields.QUERY_TIME, getQueryTime()); builder.field(Fields.QUERY_CURRENT, queryCurrent); builder.field(Fields.FETCH_TOTAL, fetchCount); - builder.timeValueField(Fields.FETCH_TIME_IN_MILLIS, Fields.FETCH_TIME, fetchTimeInMillis); + builder.humanReadableField(Fields.FETCH_TIME_IN_MILLIS, Fields.FETCH_TIME, getFetchTime()); builder.field(Fields.FETCH_CURRENT, fetchCurrent); builder.field(Fields.SCROLL_TOTAL, scrollCount); - builder.timeValueField(Fields.SCROLL_TIME_IN_MILLIS, Fields.SCROLL_TIME, scrollTimeInMillis); + builder.humanReadableField(Fields.SCROLL_TIME_IN_MILLIS, Fields.SCROLL_TIME, getScrollTime()); builder.field(Fields.SCROLL_CURRENT, scrollCurrent); builder.field(Fields.SUGGEST_TOTAL, suggestCount); - builder.timeValueField(Fields.SUGGEST_TIME_IN_MILLIS, Fields.SUGGEST_TIME, suggestTimeInMillis); + builder.humanReadableField(Fields.SUGGEST_TIME_IN_MILLIS, Fields.SUGGEST_TIME, getSuggestTime()); builder.field(Fields.SUGGEST_CURRENT, suggestCurrent); return builder; diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java index 13708add48124..9da8642fd61e4 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -23,7 +23,6 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.lucene.index.CheckIndex; -import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexCommit; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.LeafReaderContext; @@ -37,7 +36,6 @@ import org.apache.lucene.search.UsageTrackingQueryCachingPolicy; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.core.internal.io.IOUtils; import org.apache.lucene.util.ThreadInterruptedException; import org.elasticsearch.Assertions; import org.elasticsearch.ElasticsearchException; @@ -66,7 +64,8 @@ import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.AsyncIOProcessor; -import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexNotFoundException; @@ -1238,7 +1237,8 @@ public Engine.Result applyTranslogOperation(Translog.Operation operation, Engine // autoGeneratedID docs that are coming from the primary are updated correctly. result = applyIndexOperation(index.seqNo(), index.primaryTerm(), index.version(), index.versionType().versionTypeForReplicationAndRecovery(), index.getAutoGeneratedIdTimestamp(), true, origin, - source(shardId.getIndexName(), index.type(), index.id(), index.source(), XContentFactory.xContentType(index.source())) + source(shardId.getIndexName(), index.type(), index.id(), index.source(), + XContentHelper.xContentType(index.source())) .routing(index.routing()).parent(index.parent()), onMappingUpdate); break; case DELETE: diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexingStats.java b/server/src/main/java/org/elasticsearch/index/shard/IndexingStats.java index 26eb8b469f52b..fa658c3600ea4 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexingStats.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexingStats.java @@ -170,18 +170,18 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.field(Fields.INDEX_TOTAL, indexCount); - builder.timeValueField(Fields.INDEX_TIME_IN_MILLIS, Fields.INDEX_TIME, indexTimeInMillis); + builder.humanReadableField(Fields.INDEX_TIME_IN_MILLIS, Fields.INDEX_TIME, getIndexTime()); builder.field(Fields.INDEX_CURRENT, indexCurrent); builder.field(Fields.INDEX_FAILED, indexFailedCount); builder.field(Fields.DELETE_TOTAL, deleteCount); - builder.timeValueField(Fields.DELETE_TIME_IN_MILLIS, Fields.DELETE_TIME, deleteTimeInMillis); + builder.humanReadableField(Fields.DELETE_TIME_IN_MILLIS, Fields.DELETE_TIME, getDeleteTime()); builder.field(Fields.DELETE_CURRENT, deleteCurrent); builder.field(Fields.NOOP_UPDATE_TOTAL, noopUpdateCount); builder.field(Fields.IS_THROTTLED, isThrottled); - builder.timeValueField(Fields.THROTTLED_TIME_IN_MILLIS, Fields.THROTTLED_TIME, throttleTimeInMillis); + builder.humanReadableField(Fields.THROTTLED_TIME_IN_MILLIS, Fields.THROTTLED_TIME, getThrottleTime()); return builder; } } diff --git a/server/src/main/java/org/elasticsearch/index/warmer/WarmerStats.java b/server/src/main/java/org/elasticsearch/index/warmer/WarmerStats.java index 8149b091a3148..63b0fe37a9b28 100644 --- a/server/src/main/java/org/elasticsearch/index/warmer/WarmerStats.java +++ b/server/src/main/java/org/elasticsearch/index/warmer/WarmerStats.java @@ -92,7 +92,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(Fields.WARMER); builder.field(Fields.CURRENT, current); builder.field(Fields.TOTAL, total); - builder.timeValueField(Fields.TOTAL_TIME_IN_MILLIS, Fields.TOTAL_TIME, totalTimeInMillis); + builder.humanReadableField(Fields.TOTAL_TIME_IN_MILLIS, Fields.TOTAL_TIME, totalTime()); builder.endObject(); return builder; } diff --git a/server/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java b/server/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java index 65bacd94910b6..b8b294a90d422 100644 --- a/server/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java +++ b/server/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java @@ -561,19 +561,11 @@ static final class PreSyncedFlushResponse extends TransportResponse { } boolean includeNumDocs(Version version) { - if (version.major == Version.V_5_6_8.major) { - return version.onOrAfter(Version.V_5_6_8); - } else { - return version.onOrAfter(Version.V_6_2_2); - } + return version.onOrAfter(Version.V_5_6_8); } boolean includeExistingSyncId(Version version) { - if (version.major == Version.V_5_6_9.major) { - return version.onOrAfter(Version.V_5_6_9); - } else { - return version.onOrAfter(Version.V_6_3_0); - } + return version.onOrAfter(Version.V_5_6_9); } @Override diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryState.java b/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryState.java index a89fdcacb2bc3..3eb45318d7a19 100644 --- a/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryState.java +++ b/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryState.java @@ -266,7 +266,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (timer.stopTime > 0) { builder.dateField(Fields.STOP_TIME_IN_MILLIS, Fields.STOP_TIME, timer.stopTime); } - builder.timeValueField(Fields.TOTAL_TIME_IN_MILLIS, Fields.TOTAL_TIME, timer.time()); + builder.humanReadableField(Fields.TOTAL_TIME_IN_MILLIS, Fields.TOTAL_TIME, new TimeValue(timer.time())); if (recoverySource.getType() == RecoverySource.Type.PEER) { builder.startObject(Fields.SOURCE); @@ -444,8 +444,8 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.timeValueField(Fields.CHECK_INDEX_TIME_IN_MILLIS, Fields.CHECK_INDEX_TIME, checkIndexTime); - builder.timeValueField(Fields.TOTAL_TIME_IN_MILLIS, Fields.TOTAL_TIME, time()); + builder.humanReadableField(Fields.CHECK_INDEX_TIME_IN_MILLIS, Fields.CHECK_INDEX_TIME, new TimeValue(checkIndexTime)); + builder.humanReadableField(Fields.TOTAL_TIME_IN_MILLIS, Fields.TOTAL_TIME, new TimeValue(time())); return builder; } } @@ -549,7 +549,7 @@ public synchronized XContentBuilder toXContent(XContentBuilder builder, Params p builder.field(Fields.TOTAL, total); builder.field(Fields.PERCENT, String.format(Locale.ROOT, "%1.1f%%", recoveredPercent())); builder.field(Fields.TOTAL_ON_START, totalOnStart); - builder.timeValueField(Fields.TOTAL_TIME_IN_MILLIS, Fields.TOTAL_TIME, time()); + builder.humanReadableField(Fields.TOTAL_TIME_IN_MILLIS, Fields.TOTAL_TIME, new TimeValue(time())); return builder; } } @@ -924,9 +924,9 @@ public synchronized XContentBuilder toXContent(XContentBuilder builder, Params p builder.endArray(); } builder.endObject(); - builder.timeValueField(Fields.TOTAL_TIME_IN_MILLIS, Fields.TOTAL_TIME, time()); - builder.timeValueField(Fields.SOURCE_THROTTLE_TIME_IN_MILLIS, Fields.SOURCE_THROTTLE_TIME, sourceThrottling()); - builder.timeValueField(Fields.TARGET_THROTTLE_TIME_IN_MILLIS, Fields.TARGET_THROTTLE_TIME, targetThrottling()); + builder.humanReadableField(Fields.TOTAL_TIME_IN_MILLIS, Fields.TOTAL_TIME, new TimeValue(time())); + builder.humanReadableField(Fields.SOURCE_THROTTLE_TIME_IN_MILLIS, Fields.SOURCE_THROTTLE_TIME, sourceThrottling()); + builder.humanReadableField(Fields.TARGET_THROTTLE_TIME_IN_MILLIS, Fields.TARGET_THROTTLE_TIME, targetThrottling()); return builder; } diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestStats.java b/server/src/main/java/org/elasticsearch/ingest/IngestStats.java index fd0d7e826c070..c4c1520fd19d4 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestStats.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestStats.java @@ -22,6 +22,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -157,7 +158,7 @@ public long getIngestFailedCount() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.field("count", ingestCount); - builder.timeValueField("time_in_millis", "time", ingestTimeInMillis, TimeUnit.MILLISECONDS); + builder.humanReadableField("time_in_millis", "time", new TimeValue(ingestTimeInMillis, TimeUnit.MILLISECONDS)); builder.field("current", ingestCurrent); builder.field("failed", ingestFailedCount); return builder; diff --git a/server/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java b/server/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java index 4dea9eb6b5f68..95bfea87f8b26 100644 --- a/server/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java +++ b/server/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java @@ -30,7 +30,6 @@ import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; @@ -123,7 +122,7 @@ public static PipelineConfiguration readFrom(StreamInput in) throws IOException } else { final String id = in.readString(); final BytesReference config = in.readBytesReference(); - return new PipelineConfiguration(id, config, XContentFactory.xContentType(config)); + return new PipelineConfiguration(id, config, XContentHelper.xContentType(config)); } } diff --git a/server/src/main/java/org/elasticsearch/monitor/jvm/JvmStats.java b/server/src/main/java/org/elasticsearch/monitor/jvm/JvmStats.java index 712b594b47d52..b548afadd3d88 100644 --- a/server/src/main/java/org/elasticsearch/monitor/jvm/JvmStats.java +++ b/server/src/main/java/org/elasticsearch/monitor/jvm/JvmStats.java @@ -190,7 +190,7 @@ public Classes getClasses() { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(Fields.JVM); builder.field(Fields.TIMESTAMP, timestamp); - builder.timeValueField(Fields.UPTIME_IN_MILLIS, Fields.UPTIME, uptime); + builder.humanReadableField(Fields.UPTIME_IN_MILLIS, Fields.UPTIME, new TimeValue(uptime)); builder.startObject(Fields.MEM); @@ -229,7 +229,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws for (GarbageCollector collector : gc) { builder.startObject(collector.getName()); builder.field(Fields.COLLECTION_COUNT, collector.getCollectionCount()); - builder.timeValueField(Fields.COLLECTION_TIME_IN_MILLIS, Fields.COLLECTION_TIME, collector.collectionTime); + builder.humanReadableField(Fields.COLLECTION_TIME_IN_MILLIS, Fields.COLLECTION_TIME, new TimeValue(collector.collectionTime)); builder.endObject(); } builder.endObject(); diff --git a/server/src/main/java/org/elasticsearch/monitor/os/OsInfo.java b/server/src/main/java/org/elasticsearch/monitor/os/OsInfo.java index 7046b35839098..0c81356167440 100644 --- a/server/src/main/java/org/elasticsearch/monitor/os/OsInfo.java +++ b/server/src/main/java/org/elasticsearch/monitor/os/OsInfo.java @@ -22,6 +22,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ToXContent.Params; import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -103,7 +104,7 @@ static final class Fields { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(Fields.OS); - builder.timeValueField(Fields.REFRESH_INTERVAL_IN_MILLIS, Fields.REFRESH_INTERVAL, refreshInterval); + builder.humanReadableField(Fields.REFRESH_INTERVAL_IN_MILLIS, Fields.REFRESH_INTERVAL, new TimeValue(refreshInterval)); if (name != null) { builder.field(Fields.NAME, name); } diff --git a/server/src/main/java/org/elasticsearch/monitor/os/OsStats.java b/server/src/main/java/org/elasticsearch/monitor/os/OsStats.java index 60502679c2131..637f4cf1cbe00 100644 --- a/server/src/main/java/org/elasticsearch/monitor/os/OsStats.java +++ b/server/src/main/java/org/elasticsearch/monitor/os/OsStats.java @@ -221,9 +221,9 @@ public ByteSizeValue getTotal() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(Fields.SWAP); - builder.byteSizeField(Fields.TOTAL_IN_BYTES, Fields.TOTAL, getTotal()); - builder.byteSizeField(Fields.FREE_IN_BYTES, Fields.FREE, getFree()); - builder.byteSizeField(Fields.USED_IN_BYTES, Fields.USED, getUsed()); + builder.humanReadableField(Fields.TOTAL_IN_BYTES, Fields.TOTAL, getTotal()); + builder.humanReadableField(Fields.FREE_IN_BYTES, Fields.FREE, getFree()); + builder.humanReadableField(Fields.USED_IN_BYTES, Fields.USED, getUsed()); builder.endObject(); return builder; } @@ -273,9 +273,9 @@ public short getFreePercent() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(Fields.MEM); - builder.byteSizeField(Fields.TOTAL_IN_BYTES, Fields.TOTAL, getTotal()); - builder.byteSizeField(Fields.FREE_IN_BYTES, Fields.FREE, getFree()); - builder.byteSizeField(Fields.USED_IN_BYTES, Fields.USED, getUsed()); + builder.humanReadableField(Fields.TOTAL_IN_BYTES, Fields.TOTAL, getTotal()); + builder.humanReadableField(Fields.FREE_IN_BYTES, Fields.FREE, getFree()); + builder.humanReadableField(Fields.USED_IN_BYTES, Fields.USED, getUsed()); builder.field(Fields.FREE_PERCENT, getFreePercent()); builder.field(Fields.USED_PERCENT, getUsedPercent()); builder.endObject(); diff --git a/server/src/main/java/org/elasticsearch/monitor/process/ProcessInfo.java b/server/src/main/java/org/elasticsearch/monitor/process/ProcessInfo.java index 5d74f576181d5..6bac78d2f140c 100644 --- a/server/src/main/java/org/elasticsearch/monitor/process/ProcessInfo.java +++ b/server/src/main/java/org/elasticsearch/monitor/process/ProcessInfo.java @@ -22,6 +22,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ToXContent.Params; import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -83,7 +84,7 @@ static final class Fields { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(Fields.PROCESS); - builder.timeValueField(Fields.REFRESH_INTERVAL_IN_MILLIS, Fields.REFRESH_INTERVAL, refreshInterval); + builder.humanReadableField(Fields.REFRESH_INTERVAL_IN_MILLIS, Fields.REFRESH_INTERVAL, new TimeValue(refreshInterval)); builder.field(Fields.ID, id); builder.field(Fields.MLOCKALL, mlockall); builder.endObject(); diff --git a/server/src/main/java/org/elasticsearch/monitor/process/ProcessStats.java b/server/src/main/java/org/elasticsearch/monitor/process/ProcessStats.java index f85e013bc426b..1d051aac7b0c8 100644 --- a/server/src/main/java/org/elasticsearch/monitor/process/ProcessStats.java +++ b/server/src/main/java/org/elasticsearch/monitor/process/ProcessStats.java @@ -108,7 +108,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (cpu != null) { builder.startObject(Fields.CPU); builder.field(Fields.PERCENT, cpu.percent); - builder.timeValueField(Fields.TOTAL_IN_MILLIS, Fields.TOTAL, cpu.total); + builder.humanReadableField(Fields.TOTAL_IN_MILLIS, Fields.TOTAL, new TimeValue(cpu.total)); builder.endObject(); } if (mem != null) { diff --git a/server/src/main/java/org/elasticsearch/node/AdaptiveSelectionStats.java b/server/src/main/java/org/elasticsearch/node/AdaptiveSelectionStats.java index 3deb161cc8e97..819b3365ce066 100644 --- a/server/src/main/java/org/elasticsearch/node/AdaptiveSelectionStats.java +++ b/server/src/main/java/org/elasticsearch/node/AdaptiveSelectionStats.java @@ -22,6 +22,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.common.xcontent.ToXContent.Params; import org.elasticsearch.common.xcontent.ToXContentFragment; @@ -73,8 +74,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws long outgoingSearches = clientOutgoingConnections.getOrDefault(nodeId, 0L); builder.field("outgoing_searches", outgoingSearches); builder.field("avg_queue_size", stats.queueSize); - builder.timeValueField("avg_service_time_ns", "avg_service_time", (long) stats.serviceTime, TimeUnit.NANOSECONDS); - builder.timeValueField("avg_response_time_ns", "avg_response_time", (long) stats.responseTime, TimeUnit.NANOSECONDS); + if (builder.humanReadable()) { + builder.field("avg_service_time", new TimeValue((long) stats.serviceTime, TimeUnit.NANOSECONDS).toString()); + } + builder.field("avg_service_time_ns", (long) stats.serviceTime); + if (builder.humanReadable()) { + builder.field("avg_response_time", new TimeValue((long) stats.responseTime, TimeUnit.NANOSECONDS).toString()); + } + builder.field("avg_response_time_ns", (long) stats.responseTime); builder.field("rank", String.format(Locale.ROOT, "%.1f", stats.rank(outgoingSearches))); } builder.endObject(); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStateAction.java index 5e3743637178e..6e55ef3671ba0 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStateAction.java @@ -95,7 +95,8 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC public RestResponse buildResponse(ClusterStateResponse response, XContentBuilder builder) throws Exception { builder.startObject(); builder.field(Fields.CLUSTER_NAME, response.getClusterName().value()); - builder.byteSizeField(Fields.CLUSTER_STATE_SIZE_IN_BYTES, Fields.CLUSTER_STATE_SIZE, response.getTotalCompressedSize()); + builder.humanReadableField(Fields.CLUSTER_STATE_SIZE_IN_BYTES, Fields.CLUSTER_STATE_SIZE, + response.getTotalCompressedSize()); response.getState().toXContent(builder, request); builder.endObject(); return new BytesRestResponse(RestStatus.OK, builder); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestClearIndicesCacheAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestClearIndicesCacheAction.java index f72ee8f2cb28b..a63676c1e09ed 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestClearIndicesCacheAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestClearIndicesCacheAction.java @@ -38,7 +38,6 @@ import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.rest.RestRequest.Method.POST; import static org.elasticsearch.rest.RestStatus.OK; -import static org.elasticsearch.rest.action.RestActions.buildBroadcastShardsHeader; public class RestClearIndicesCacheAction extends BaseRestHandler { @@ -67,7 +66,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC @Override public RestResponse buildResponse(ClearIndicesCacheResponse response, XContentBuilder builder) throws Exception { builder.startObject(); - buildBroadcastShardsHeader(builder, request, response); + response.toXContent(builder, request); builder.endObject(); return new BytesRestResponse(OK, builder); } @@ -86,5 +85,4 @@ public static ClearIndicesCacheRequest fromRequest(final RestRequest request, Cl clearIndicesCacheRequest.fields(request.paramAsStringArray("fields", clearIndicesCacheRequest.fields())); return clearIndicesCacheRequest; } - } diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestGetSourceAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestGetSourceAction.java index 60c7e150dc7b6..9e61885cab252 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/document/RestGetSourceAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestGetSourceAction.java @@ -26,7 +26,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.BytesRestResponse; import org.elasticsearch.rest.RestController; @@ -86,7 +86,7 @@ public RestResponse buildResponse(final GetResponse response) throws Exception { } else { final BytesReference source = response.getSourceInternal(); try (InputStream stream = source.streamInput()) { - builder.rawValue(stream, XContentFactory.xContentType(source)); + builder.rawValue(stream, XContentHelper.xContentType(source)); } return new BytesRestResponse(OK, builder); } diff --git a/server/src/main/java/org/elasticsearch/search/profile/ProfileResult.java b/server/src/main/java/org/elasticsearch/search/profile/ProfileResult.java index 13dfa712ab448..d0965bca1a9b2 100644 --- a/server/src/main/java/org/elasticsearch/search/profile/ProfileResult.java +++ b/server/src/main/java/org/elasticsearch/search/profile/ProfileResult.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; @@ -147,11 +148,14 @@ public List getProfiledChildren() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder = builder.startObject() - .field(TYPE.getPreferredName(), type) - .field(DESCRIPTION.getPreferredName(), description) - .timeValueField(NODE_TIME_RAW.getPreferredName(), NODE_TIME.getPreferredName(), getTime(), TimeUnit.NANOSECONDS) - .field(BREAKDOWN.getPreferredName(), timings); + builder.startObject(); + builder.field(TYPE.getPreferredName(), type); + builder.field(DESCRIPTION.getPreferredName(), description); + if (builder.humanReadable()) { + builder.field(NODE_TIME.getPreferredName(), new TimeValue(getTime(), TimeUnit.NANOSECONDS).toString()); + } + builder.field(NODE_TIME_RAW.getPreferredName(), getTime()); + builder.field(BREAKDOWN.getPreferredName(), timings); if (!children.isEmpty()) { builder = builder.startArray(CHILDREN.getPreferredName()); diff --git a/server/src/main/java/org/elasticsearch/search/profile/query/CollectorResult.java b/server/src/main/java/org/elasticsearch/search/profile/query/CollectorResult.java index d553e1a8a7359..19d382dd8f380 100644 --- a/server/src/main/java/org/elasticsearch/search/profile/query/CollectorResult.java +++ b/server/src/main/java/org/elasticsearch/search/profile/query/CollectorResult.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -142,10 +143,13 @@ public List getProfiledChildren() { @Override public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { - builder = builder.startObject() - .field(NAME.getPreferredName(), getName()) - .field(REASON.getPreferredName(), getReason()) - .timeValueField(TIME_NANOS.getPreferredName(), TIME.getPreferredName(), getTime(), TimeUnit.NANOSECONDS); + builder = builder.startObject(); + builder.field(NAME.getPreferredName(), getName()); + builder.field(REASON.getPreferredName(), getReason()); + if (builder.humanReadable()) { + builder.field(TIME.getPreferredName(), new TimeValue(getTime(), TimeUnit.NANOSECONDS).toString()); + } + builder.field(TIME_NANOS.getPreferredName(), getTime()); if (!children.isEmpty()) { builder = builder.startArray(CHILDREN.getPreferredName()); diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java index b75b840e8b255..073007f4225df 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java @@ -28,6 +28,7 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.joda.FormatDateTimeFormatter; import org.elasticsearch.common.joda.Joda; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; @@ -392,7 +393,7 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa if (verbose || endTime != 0) { builder.field(END_TIME, DATE_TIME_FORMATTER.printer().print(endTime)); builder.field(END_TIME_IN_MILLIS, endTime); - builder.timeValueField(DURATION_IN_MILLIS, DURATION, endTime - startTime); + builder.humanReadableField(DURATION_IN_MILLIS, DURATION, new TimeValue(endTime - startTime)); } if (verbose || !shardFailures.isEmpty()) { builder.startArray(FAILURES); diff --git a/server/src/main/java/org/elasticsearch/tasks/RawTaskStatus.java b/server/src/main/java/org/elasticsearch/tasks/RawTaskStatus.java index a6deb85c10662..e3f76892a87ad 100644 --- a/server/src/main/java/org/elasticsearch/tasks/RawTaskStatus.java +++ b/server/src/main/java/org/elasticsearch/tasks/RawTaskStatus.java @@ -24,7 +24,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; import java.io.IOException; import java.io.InputStream; @@ -60,7 +60,7 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { try (InputStream stream = status.streamInput()) { - return builder.rawValue(stream, XContentFactory.xContentType(status)); + return builder.rawValue(stream, XContentHelper.xContentType(status)); } } diff --git a/server/src/main/java/org/elasticsearch/tasks/TaskInfo.java b/server/src/main/java/org/elasticsearch/tasks/TaskInfo.java index 19e9baedd753b..bc40df2b8f0c4 100644 --- a/server/src/main/java/org/elasticsearch/tasks/TaskInfo.java +++ b/server/src/main/java/org/elasticsearch/tasks/TaskInfo.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.ToXContent.Params; import org.elasticsearch.common.xcontent.ToXContentFragment; @@ -196,7 +197,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("description", description); } builder.dateField("start_time_in_millis", "start_time", startTime); - builder.timeValueField("running_time_in_nanos", "running_time", runningTimeNanos, TimeUnit.NANOSECONDS); + if (builder.humanReadable()) { + builder.field("running_time", new TimeValue(runningTimeNanos, TimeUnit.NANOSECONDS).toString()); + } + builder.field("running_time_in_nanos", runningTimeNanos); builder.field("cancellable", cancellable); if (parentTaskId.isSet()) { builder.field("parent_task_id", parentTaskId.toString()); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/cache/clear/ClearIndicesCacheResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/cache/clear/ClearIndicesCacheResponseTests.java new file mode 100644 index 0000000000000..4b58d5fb70246 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/cache/clear/ClearIndicesCacheResponseTests.java @@ -0,0 +1,40 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.cache.clear; + +import org.elasticsearch.action.support.DefaultShardOperationFailedException; +import org.elasticsearch.action.support.broadcast.AbstractBroadcastResponseTestCase; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.util.List; + +public class ClearIndicesCacheResponseTests extends AbstractBroadcastResponseTestCase { + + @Override + protected ClearIndicesCacheResponse createTestInstance(int totalShards, int successfulShards, int failedShards, + List failures) { + return new ClearIndicesCacheResponse(totalShards, successfulShards, failedShards, failures); + } + + @Override + protected ClearIndicesCacheResponse doParseInstance(XContentParser parser) { + return ClearIndicesCacheResponse.fromXContent(parser); + } +} diff --git a/server/src/test/java/org/elasticsearch/common/xcontent/XContentFactoryTests.java b/server/src/test/java/org/elasticsearch/common/xcontent/XContentFactoryTests.java index 65489a997ac7c..a893fb63ec8cc 100644 --- a/server/src/test/java/org/elasticsearch/common/xcontent/XContentFactoryTests.java +++ b/server/src/test/java/org/elasticsearch/common/xcontent/XContentFactoryTests.java @@ -54,7 +54,7 @@ private void testGuessType(XContentType type) throws IOException { builder.field("field1", "value1"); builder.endObject(); - assertThat(XContentFactory.xContentType(BytesReference.bytes(builder)), equalTo(type)); + assertThat(XContentHelper.xContentType(BytesReference.bytes(builder)), equalTo(type)); assertThat(XContentFactory.xContentType(BytesReference.bytes(builder).streamInput()), equalTo(type)); // CBOR is binary, cannot use String diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java index e605a672c2982..06d5c3fb443c5 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java @@ -28,6 +28,7 @@ import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; @@ -61,7 +62,7 @@ public void testNoFormat() throws Exception { .endObject()), XContentType.JSON)); - assertThat(XContentFactory.xContentType(doc.source()), equalTo(XContentType.JSON)); + assertThat(XContentFactory.xContentType(doc.source().toBytesRef().bytes), equalTo(XContentType.JSON)); documentMapper = parser.parse("type", new CompressedXContent(mapping)); doc = documentMapper.parse(SourceToParse.source("test", "type", "1", BytesReference.bytes(XContentFactory.smileBuilder().startObject() @@ -69,7 +70,7 @@ public void testNoFormat() throws Exception { .endObject()), XContentType.SMILE)); - assertThat(XContentFactory.xContentType(doc.source()), equalTo(XContentType.SMILE)); + assertThat(XContentHelper.xContentType(doc.source()), equalTo(XContentType.SMILE)); } public void testIncludes() throws Exception { diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java b/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java index 4bdd9b84ec463..983ced2a6edc8 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java @@ -21,7 +21,7 @@ import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.IndexAnalyzers; @@ -121,7 +121,8 @@ private Engine.Operation convertToEngineOp(Translog.Operation operation, Engine. final String indexName = mapperService.index().getName(); final Engine.Index engineIndex = IndexShard.prepareIndex(docMapper(index.type()), mapperService.getIndexSettings().getIndexVersionCreated(), - source(indexName, index.type(), index.id(), index.source(), XContentFactory.xContentType(index.source())) + source(indexName, index.type(), index.id(), index.source(), + XContentHelper.xContentType(index.source())) .routing(index.routing()).parent(index.parent()), index.seqNo(), index.primaryTerm(), index.version(), index.versionType().versionTypeForReplicationAndRecovery(), origin, index.getAutoGeneratedIdTimestamp(), true);