From 20d6fbcd551cc9b9cf1dbb45fac062b101dfa4a8 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Tue, 8 Dec 2020 10:15:08 +0100 Subject: [PATCH] Add xpack info and usage endpoints for runtime fields (#65600) Relates to #59332 --- docs/reference/rest-api/info.asciidoc | 4 + docs/reference/rest-api/usage.asciidoc | 5 + .../xpack/core/XPackClientPlugin.java | 6 +- .../elasticsearch/xpack/core/XPackField.java | 2 + .../core/action/XPackInfoFeatureAction.java | 3 +- .../core/action/XPackUsageFeatureAction.java | 39 ++- .../RuntimeFieldsFeatureSetUsage.java | 274 ++++++++++++++++++ .../RuntimeFieldsFeatureSetUsageTests.java | 137 +++++++++ .../xpack/runtimefields/RuntimeFields.java | 15 +- .../RuntimeFieldsInfoTransportAction.java | 36 +++ .../RuntimeFieldsUsageTransportAction.java | 54 ++++ .../xpack/security/operator/Constants.java | 2 + .../200_runtime_fields_stats.yml | 205 +++++++++++++ 13 files changed, 765 insertions(+), 17 deletions(-) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/runtimefields/RuntimeFieldsFeatureSetUsage.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/runtimefields/RuntimeFieldsFeatureSetUsageTests.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFieldsInfoTransportAction.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFieldsUsageTransportAction.java create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/200_runtime_fields_stats.yml diff --git a/docs/reference/rest-api/info.asciidoc b/docs/reference/rest-api/info.asciidoc index 5f4176b44069..da7912227202 100644 --- a/docs/reference/rest-api/info.asciidoc +++ b/docs/reference/rest-api/info.asciidoc @@ -107,6 +107,10 @@ Example response: "available": true, "enabled": true }, + "runtime_fields": { + "available": true, + "enabled": true + }, "searchable_snapshots" : { "available" : true, "enabled" : true diff --git a/docs/reference/rest-api/usage.asciidoc b/docs/reference/rest-api/usage.asciidoc index 78d718d41f8c..27d29d6949a4 100644 --- a/docs/reference/rest-api/usage.asciidoc +++ b/docs/reference/rest-api/usage.asciidoc @@ -341,6 +341,11 @@ GET /_xpack/usage "aggregate_metric" : { "available" : true, "enabled" : true + }, + "runtime_fields" : { + "available" : true, + "enabled" : true, + "field_types" : [] } } ------------------------------------------------------------ diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java index e15606c902e7..1a34a4e3ba60 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java @@ -151,11 +151,12 @@ import org.elasticsearch.xpack.core.rollup.action.GetRollupJobsAction; import org.elasticsearch.xpack.core.rollup.action.PutRollupJobAction; import org.elasticsearch.xpack.core.rollup.action.RollupSearchAction; -import org.elasticsearch.xpack.core.rollup.v2.RollupAction; import org.elasticsearch.xpack.core.rollup.action.StartRollupJobAction; import org.elasticsearch.xpack.core.rollup.action.StopRollupJobAction; import org.elasticsearch.xpack.core.rollup.job.RollupJob; import org.elasticsearch.xpack.core.rollup.job.RollupJobStatus; +import org.elasticsearch.xpack.core.rollup.v2.RollupAction; +import org.elasticsearch.xpack.core.runtimefields.RuntimeFieldsFeatureSetUsage; import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction; import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchAction; import org.elasticsearch.xpack.core.searchablesnapshots.SearchableSnapshotFeatureSetUsage; @@ -522,7 +523,8 @@ public List getNamedWriteables() { // Data Streams new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.DATA_STREAMS, DataStreamFeatureSetUsage::new), // Data Tiers - new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.DATA_TIERS, DataTiersFeatureSetUsage::new) + new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.DATA_TIERS, DataTiersFeatureSetUsage::new), + new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.RUNTIME_FIELDS, RuntimeFieldsFeatureSetUsage::new) ); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java index 698547f5843f..fcdac563eaa7 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java @@ -67,6 +67,8 @@ public final class XPackField { public static final String DATA_TIERS = "data_tiers"; /** Name constant for the aggregate_metric plugin. */ public static final String AGGREGATE_METRIC = "aggregate_metric"; + /** Name constant for the runtime fields plugin. */ + public static final String RUNTIME_FIELDS = "runtime_fields"; /** Name constant for the operator privileges feature. */ public static final String OPERATOR_PRIVILEGES = "operator_privileges"; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoFeatureAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoFeatureAction.java index 08bfe678bb91..3391ba306717 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoFeatureAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoFeatureAction.java @@ -47,6 +47,7 @@ public class XPackInfoFeatureAction extends ActionType public static final XPackInfoFeatureAction DATA_STREAMS = new XPackInfoFeatureAction(XPackField.DATA_STREAMS); public static final XPackInfoFeatureAction DATA_TIERS = new XPackInfoFeatureAction(XPackField.DATA_TIERS); public static final XPackInfoFeatureAction AGGREGATE_METRIC = new XPackInfoFeatureAction(XPackField.AGGREGATE_METRIC); + public static final XPackInfoFeatureAction RUNTIME_FIELDS = new XPackInfoFeatureAction(XPackField.RUNTIME_FIELDS); public static final List ALL; static { @@ -54,7 +55,7 @@ public class XPackInfoFeatureAction extends ActionType actions.addAll(Arrays.asList( SECURITY, MONITORING, WATCHER, GRAPH, MACHINE_LEARNING, LOGSTASH, EQL, SQL, ROLLUP, INDEX_LIFECYCLE, SNAPSHOT_LIFECYCLE, CCR, TRANSFORM, VECTORS, VOTING_ONLY, FROZEN_INDICES, SPATIAL, ANALYTICS, ENRICH, DATA_STREAMS, SEARCHABLE_SNAPSHOTS, DATA_TIERS, - AGGREGATE_METRIC + AGGREGATE_METRIC, RUNTIME_FIELDS )); ALL = Collections.unmodifiableList(actions); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java index 2ed84560093b..878d5beb81b3 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java @@ -8,9 +8,6 @@ import org.elasticsearch.action.ActionType; import org.elasticsearch.xpack.core.XPackField; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; import java.util.List; /** @@ -47,17 +44,33 @@ public class XPackUsageFeatureAction extends ActionType ALL; - static { - final List actions = new ArrayList<>(); - actions.addAll(Arrays.asList( - SECURITY, MONITORING, WATCHER, GRAPH, MACHINE_LEARNING, LOGSTASH, EQL, SQL, ROLLUP, INDEX_LIFECYCLE, SNAPSHOT_LIFECYCLE, CCR, - TRANSFORM, VECTORS, VOTING_ONLY, FROZEN_INDICES, SPATIAL, ANALYTICS, DATA_STREAMS, SEARCHABLE_SNAPSHOTS, DATA_TIERS, - AGGREGATE_METRIC - )); - ALL = Collections.unmodifiableList(actions); - } + static final List ALL = List.of( + AGGREGATE_METRIC, + ANALYTICS, + CCR, + DATA_STREAMS, + DATA_TIERS, + EQL, + FROZEN_INDICES, + GRAPH, + INDEX_LIFECYCLE, + LOGSTASH, + MACHINE_LEARNING, + MONITORING, + ROLLUP, + RUNTIME_FIELDS, + SEARCHABLE_SNAPSHOTS, + SECURITY, + SNAPSHOT_LIFECYCLE, + SPATIAL, + SQL, + TRANSFORM, + VECTORS, + VOTING_ONLY, + WATCHER + ); private XPackUsageFeatureAction(String name) { super(BASE_NAME + name, XPackUsageFeatureResponse::new); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/runtimefields/RuntimeFieldsFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/runtimefields/RuntimeFieldsFeatureSetUsage.java new file mode 100644 index 000000000000..f58149f86f28 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/runtimefields/RuntimeFieldsFeatureSetUsage.java @@ -0,0 +1,274 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.runtimefields; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.MappingMetadata; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.core.XPackFeatureSet; +import org.elasticsearch.xpack.core.XPackField; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class RuntimeFieldsFeatureSetUsage extends XPackFeatureSet.Usage { + + public static RuntimeFieldsFeatureSetUsage fromMetadata(Iterable metadata) { + Map fieldTypes = new HashMap<>(); + for (IndexMetadata indexMetadata : metadata) { + if (indexMetadata.isSystem()) { + // Don't include system indices in statistics about mappings, we care about the user's indices. + continue; + } + Set indexFieldTypes = new HashSet<>(); + MappingMetadata mappingMetadata = indexMetadata.mapping(); + if (mappingMetadata != null) { + Object runtimeObject = mappingMetadata.getSourceAsMap().get("runtime"); + if (runtimeObject instanceof Map == false) { + continue; + } + Map runtimeMappings = (Map) runtimeObject; + for (Object runtimeFieldMappingObject : runtimeMappings.values()) { + if (runtimeFieldMappingObject instanceof Map == false) { + continue; + } + Map runtimeFieldMapping = (Map) runtimeFieldMappingObject; + Object typeObject = runtimeFieldMapping.get("type"); + if (typeObject == null) { + continue; + } + String type = typeObject.toString(); + RuntimeFieldStats stats = fieldTypes.computeIfAbsent(type, RuntimeFieldStats::new); + stats.count++; + if (indexFieldTypes.add(type)) { + stats.indexCount++; + } + Object scriptObject = runtimeFieldMapping.get("script"); + if (scriptObject == null) { + stats.scriptLessCount++; + } else if (scriptObject instanceof Map) { + Map script = (Map) scriptObject; + Object sourceObject = script.get("source"); + if (sourceObject != null) { + String scriptSource = sourceObject.toString(); + int chars = scriptSource.length(); + long lines = scriptSource.lines().count(); + int docUsages = countOccurrences(scriptSource, "doc[\\[\\.]"); + int sourceUsages = countOccurrences(scriptSource, "params\\._source"); + stats.update(chars, lines, sourceUsages, docUsages); + } + Object langObject = script.get("lang"); + if (langObject != null) { + stats.scriptLangs.add(langObject.toString()); + } + } + } + } + } + List runtimeFieldStats = new ArrayList<>(fieldTypes.values()); + runtimeFieldStats.sort(Comparator.comparing(RuntimeFieldStats::type)); + return new RuntimeFieldsFeatureSetUsage(Collections.unmodifiableList(runtimeFieldStats)); + } + + private final List stats; + + RuntimeFieldsFeatureSetUsage(List stats) { + super(XPackField.RUNTIME_FIELDS, true, true); + this.stats = stats; + } + + public RuntimeFieldsFeatureSetUsage(StreamInput in) throws IOException { + super(in); + this.stats = in.readList(RuntimeFieldStats::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeList(stats); + } + + List getRuntimeFieldStats() { + return stats; + } + + @Override + protected void innerXContent(XContentBuilder builder, Params params) throws IOException { + super.innerXContent(builder, params); + builder.startArray("field_types"); + for (RuntimeFieldStats stats : stats) { + stats.toXContent(builder, params); + } + builder.endArray(); + } + + @Override + public Version getMinimalSupportedVersion() { + return Version.V_7_11_0; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + RuntimeFieldsFeatureSetUsage that = (RuntimeFieldsFeatureSetUsage) o; + return stats.equals(that.stats); + } + + @Override + public int hashCode() { + return Objects.hash(stats); + } + + private static int countOccurrences(String script, String keyword) { + int occurrences = 0; + Pattern pattern = Pattern.compile(keyword); + Matcher matcher = pattern.matcher(script); + while (matcher.find()) { + occurrences++; + } + return occurrences; + } + + static final class RuntimeFieldStats implements Writeable, ToXContentObject { + private final String type; + private int count = 0; + private int indexCount = 0; + private final Set scriptLangs; + private long scriptLessCount = 0; + private long maxLines = 0; + private long totalLines = 0; + private long maxChars = 0; + private long totalChars = 0; + private long maxSourceUsages = 0; + private long totalSourceUsages = 0; + private long maxDocUsages = 0; + private long totalDocUsages = 0; + + RuntimeFieldStats(String type) { + this.type = Objects.requireNonNull(type); + this.scriptLangs = new HashSet<>(); + } + + RuntimeFieldStats(StreamInput in) throws IOException { + this.type = in.readString(); + this.count = in.readInt(); + this.indexCount = in.readInt(); + this.scriptLangs = in.readSet(StreamInput::readString); + this.scriptLessCount = in.readLong(); + this.maxLines = in.readLong(); + this.totalLines = in.readLong(); + this.maxChars = in.readLong(); + this.totalChars = in.readLong(); + this.maxSourceUsages = in.readLong(); + this.totalSourceUsages = in.readLong(); + this.maxDocUsages = in.readLong(); + this.totalDocUsages = in.readLong(); + } + + String type() { + return type; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(type); + out.writeInt(count); + out.writeInt(indexCount); + out.writeCollection(scriptLangs, StreamOutput::writeString); + out.writeLong(scriptLessCount); + out.writeLong(maxLines); + out.writeLong(totalLines); + out.writeLong(maxChars); + out.writeLong(totalChars); + out.writeLong(maxSourceUsages); + out.writeLong(totalSourceUsages); + out.writeLong(maxDocUsages); + out.writeLong(totalDocUsages); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("name", type); + builder.field("count", count); + builder.field("index_count", indexCount); + builder.field("scriptless_count", scriptLessCount); + builder.array("lang", scriptLangs.toArray(new String[0])); + builder.field("lines_max", maxLines); + builder.field("lines_total", totalLines); + builder.field("chars_max", maxChars); + builder.field("chars_total", totalChars); + builder.field("source_max", maxSourceUsages); + builder.field("source_total", totalSourceUsages); + builder.field("doc_max", maxDocUsages); + builder.field("doc_total", totalDocUsages); + builder.endObject(); + return builder; + } + + void update(int chars, long lines, int sourceUsages, int docUsages) { + this.maxChars = Math.max(this.maxChars, chars); + this.totalChars += chars; + this.maxLines = Math.max(this.maxLines, lines); + this.totalLines += lines; + this.totalSourceUsages += sourceUsages; + this.maxSourceUsages = Math.max(this.maxSourceUsages, sourceUsages); + this.totalDocUsages += docUsages; + this.maxDocUsages = Math.max(this.maxDocUsages, docUsages); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + RuntimeFieldStats that = (RuntimeFieldStats) o; + return count == that.count && + indexCount == that.indexCount && + scriptLessCount == that.scriptLessCount && + maxLines == that.maxLines && + totalLines == that.totalLines && + maxChars == that.maxChars && + totalChars == that.totalChars && + maxSourceUsages == that.maxSourceUsages && + totalSourceUsages == that.totalSourceUsages && + maxDocUsages == that.maxDocUsages && + totalDocUsages == that.totalDocUsages && + type.equals(that.type) && + scriptLangs.equals(that.scriptLangs); + } + + @Override + public int hashCode() { + return Objects.hash(type, count, indexCount, scriptLangs, scriptLessCount, maxLines, totalLines, maxChars, totalChars, + maxSourceUsages, totalSourceUsages, maxDocUsages, totalDocUsages); + } + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/runtimefields/RuntimeFieldsFeatureSetUsageTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/runtimefields/RuntimeFieldsFeatureSetUsageTests.java new file mode 100644 index 000000000000..dda7fbd32f59 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/runtimefields/RuntimeFieldsFeatureSetUsageTests.java @@ -0,0 +1,137 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.runtimefields; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.script.Script; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xpack.core.runtimefields.RuntimeFieldsFeatureSetUsage.RuntimeFieldStats; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class RuntimeFieldsFeatureSetUsageTests extends AbstractWireSerializingTestCase { + + public void testToXContent() { + Settings settings = Settings.builder() + .put("index.number_of_replicas", 0) + .put("index.number_of_shards", 1) + .put("index.version.created", Version.CURRENT) + .build(); + Script script1 = new Script("doc['field'] + doc.field + params._source.field"); + Script script2 = new Script("doc['field']"); + Script script3 = new Script("params._source.field + params._source.field \n + params._source.field"); + Script script4 = new Script("params._source.field"); + IndexMetadata meta = IndexMetadata.builder("index").settings(settings) + .putMapping("{" + + " \"runtime\" : {" + + " \"keyword1\": {" + + " \"type\": \"keyword\"," + + " \"script\": " + Strings.toString(script1) + + " }," + + " \"keyword2\": {" + + " \"type\": \"keyword\"" + + " }," + + " \"keyword3\": {" + + " \"type\": \"keyword\"," + + " \"script\": " + Strings.toString(script2) + + " }," + + " \"long\": {" + + " \"type\": \"long\"," + + " \"script\": " + Strings.toString(script3) + + " }," + + " \"long2\": {" + + " \"type\": \"long\"," + + " \"script\": " + Strings.toString(script4) + + " }" + + " }" + + "}") + .build(); + + RuntimeFieldsFeatureSetUsage featureSetUsage = RuntimeFieldsFeatureSetUsage.fromMetadata(List.of(meta, meta)); + assertEquals("{\n" + + " \"available\" : true,\n" + + " \"enabled\" : true,\n" + + " \"field_types\" : [\n" + + " {\n" + + " \"name\" : \"keyword\",\n" + + " \"count\" : 6,\n" + + " \"index_count\" : 2,\n" + + " \"scriptless_count\" : 2,\n" + + " \"lang\" : [\n" + + " \"painless\"\n" + + " ],\n" + + " \"lines_max\" : 1,\n" + + " \"lines_total\" : 4,\n" + + " \"chars_max\" : 47,\n" + + " \"chars_total\" : 118,\n" + + " \"source_max\" : 1,\n" + + " \"source_total\" : 2,\n" + + " \"doc_max\" : 2,\n" + + " \"doc_total\" : 6\n" + + " },\n" + + " {\n" + + " \"name\" : \"long\",\n" + + " \"count\" : 4,\n" + + " \"index_count\" : 2,\n" + + " \"scriptless_count\" : 0,\n" + + " \"lang\" : [\n" + + " \"painless\"\n" + + " ],\n" + + " \"lines_max\" : 2,\n" + + " \"lines_total\" : 6,\n" + + " \"chars_max\" : 68,\n" + + " \"chars_total\" : 176,\n" + + " \"source_max\" : 3,\n" + + " \"source_total\" : 8,\n" + + " \"doc_max\" : 0,\n" + + " \"doc_total\" : 0\n" + + " }\n" + + " ]\n" + + "}", Strings.toString(featureSetUsage, true, true)); + } + + @Override + protected RuntimeFieldsFeatureSetUsage createTestInstance() { + int numItems = randomIntBetween(0, 10); + List stats = new ArrayList<>(numItems); + for (int i = 0; i < numItems; i++) { + stats.add(randomRuntimeFieldStats("type" + i)); + } + return new RuntimeFieldsFeatureSetUsage(stats); + } + + private static RuntimeFieldStats randomRuntimeFieldStats(String type) { + RuntimeFieldStats stats = new RuntimeFieldStats(type); + if (randomBoolean()) { + stats.update(randomIntBetween(1, 100), randomLongBetween(100, 1000), randomIntBetween(1, 10), randomIntBetween(1, 10)); + } + return stats; + } + + @Override + protected RuntimeFieldsFeatureSetUsage mutateInstance(RuntimeFieldsFeatureSetUsage instance) throws IOException { + List runtimeFieldStats = instance.getRuntimeFieldStats(); + if (runtimeFieldStats.size() == 0) { + return new RuntimeFieldsFeatureSetUsage(Collections.singletonList(randomRuntimeFieldStats("type"))); + } + List mutated = new ArrayList<>(runtimeFieldStats); + mutated.remove(randomIntBetween(0, mutated.size() - 1)); + return new RuntimeFieldsFeatureSetUsage(mutated); + } + + @Override + protected Writeable.Reader instanceReader() { + return RuntimeFieldsFeatureSetUsage::new; + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java index 948c7c02b40a..7169469615cb 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java @@ -6,6 +6,8 @@ package org.elasticsearch.xpack.runtimefields; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; import org.elasticsearch.index.mapper.BooleanFieldMapper; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.GeoPointFieldMapper; @@ -13,10 +15,13 @@ import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.RuntimeFieldType; +import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.ScriptPlugin; import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction; +import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction; import org.elasticsearch.xpack.runtimefields.mapper.BooleanFieldScript; import org.elasticsearch.xpack.runtimefields.mapper.BooleanScriptFieldType; import org.elasticsearch.xpack.runtimefields.mapper.DateFieldScript; @@ -35,7 +40,7 @@ import java.util.List; import java.util.Map; -public final class RuntimeFields extends Plugin implements MapperPlugin, ScriptPlugin { +public final class RuntimeFields extends Plugin implements MapperPlugin, ScriptPlugin, ActionPlugin { @Override public Map getRuntimeFieldTypes() { @@ -62,4 +67,12 @@ public List> getContexts() { StringFieldScript.CONTEXT ); } + + @Override + public List> getActions() { + return List.of( + new ActionPlugin.ActionHandler<>(XPackUsageFeatureAction.RUNTIME_FIELDS, RuntimeFieldsUsageTransportAction.class), + new ActionPlugin.ActionHandler<>(XPackInfoFeatureAction.RUNTIME_FIELDS, RuntimeFieldsInfoTransportAction.class) + ); + } } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFieldsInfoTransportAction.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFieldsInfoTransportAction.java new file mode 100644 index 000000000000..b0c33620e144 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFieldsInfoTransportAction.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.XPackField; +import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction; +import org.elasticsearch.xpack.core.action.XPackInfoFeatureTransportAction; + +public class RuntimeFieldsInfoTransportAction extends XPackInfoFeatureTransportAction { + @Inject + public RuntimeFieldsInfoTransportAction(TransportService transportService, ActionFilters actionFilters) { + super(XPackInfoFeatureAction.RUNTIME_FIELDS.name(), transportService, actionFilters); + } + + @Override + protected String name() { + return XPackField.RUNTIME_FIELDS; + } + + @Override + protected boolean available() { + return true; + } + + @Override + protected boolean enabled() { + return true; + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFieldsUsageTransportAction.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFieldsUsageTransportAction.java new file mode 100644 index 000000000000..c5c68b93c631 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFieldsUsageTransportAction.java @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.protocol.xpack.XPackUsageRequest; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction; +import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse; +import org.elasticsearch.xpack.core.action.XPackUsageFeatureTransportAction; +import org.elasticsearch.xpack.core.runtimefields.RuntimeFieldsFeatureSetUsage; + +public final class RuntimeFieldsUsageTransportAction extends XPackUsageFeatureTransportAction { + + @Inject + public RuntimeFieldsUsageTransportAction( + TransportService transportService, + ClusterService clusterService, + ThreadPool threadPool, + ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver + ) { + super( + XPackUsageFeatureAction.RUNTIME_FIELDS.name(), + transportService, + clusterService, + threadPool, + actionFilters, + indexNameExpressionResolver + ); + } + + @Override + protected void masterOperation( + Task task, + XPackUsageRequest request, + ClusterState state, + ActionListener listener + ) { + RuntimeFieldsFeatureSetUsage runtimeFieldsFeatureSetUsage = RuntimeFieldsFeatureSetUsage.fromMetadata(state.metadata()); + listener.onResponse(new XPackUsageFeatureResponse(runtimeFieldsFeatureSetUsage)); + } +} diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java index 965be69866e2..7a1f63529b1a 100644 --- a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java +++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java @@ -245,6 +245,7 @@ public class Constants { "cluster:monitor/xpack/info/ml", "cluster:monitor/xpack/info/monitoring", "cluster:monitor/xpack/info/rollup", + "cluster:monitor/xpack/info/runtime_fields", "cluster:monitor/xpack/info/searchable_snapshots", "cluster:monitor/xpack/info/security", "cluster:monitor/xpack/info/slm", @@ -298,6 +299,7 @@ public class Constants { "cluster:monitor/xpack/usage/ml", "cluster:monitor/xpack/usage/monitoring", "cluster:monitor/xpack/usage/rollup", + "cluster:monitor/xpack/usage/runtime_fields", "cluster:monitor/xpack/usage/searchable_snapshots", "cluster:monitor/xpack/usage/security", "cluster:monitor/xpack/usage/slm", diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/200_runtime_fields_stats.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/200_runtime_fields_stats.yml new file mode 100644 index 000000000000..2a2369b5b52b --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/200_runtime_fields_stats.yml @@ -0,0 +1,205 @@ +--- +"Usage stats without runtime fields": + - do: + indices.create: + index: sensor + + - do: {xpack.info: {}} + - match: { features.runtime_fields.available: true } + - match: { features.runtime_fields.enabled: true } + + - do: {xpack.usage: {}} + - match: { runtime_fields.available: true } + - match: { runtime_fields.enabled: true } + - length: { runtime_fields.field_types: 0 } + +--- +"Usage stats with runtime fields": + - do: + indices.create: + index: sensor + body: + mappings: + runtime: + message_from_source: + type: keyword + day_of_week: + type: keyword + script: | + emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT)); + # Test fetching from _source + day_of_week_from_source: + type: keyword + script: | + Instant instant = Instant.ofEpochMilli(params._source.timestamp); + ZonedDateTime dt = ZonedDateTime.ofInstant(instant, ZoneId.of("UTC")); + emit(dt.dayOfWeek.getDisplayName(TextStyle.FULL, Locale.ROOT)); + millis_ago: + type: date + script: + source: | + for (def dt : doc['timestamp']) { + emit(System.currentTimeMillis() - dt.toInstant().toEpochMilli()); + } + tomorrow: + type: date + script: + source: | + for (def dt : doc['timestamp']) { + emit(dt.plus(params.days, ChronoUnit.DAYS).toEpochMilli()); + } + params: + days: 1 + voltage_times_ten: + type: long + script: + source: | + for (double v : doc['voltage']) { + emit((long)(v * params.multiplier)); + } + params: + multiplier: 10 + voltage_percent_from_source: + type: double + script: + source: | + emit(params._source.voltage / params.max); + params: + max: 5.8 + over_v: + type: boolean + script: + source: | + for (def v : doc['voltage']) { + emit(v >= params.min_v); + } + params: + min_v: 5.0 + ip: + type: ip + script: + source: | + Matcher m = /([^ ]+) .+/.matcher(doc["message"].value); + if (m.matches()) { + emit(m.group(1)); + } + location_from_source: + type: geo_point + script: + source: | + emit(params._source.location.lat, params._source.location.lon); + properties: + timestamp: + type: date + message: + type: keyword + voltage: + type: double + location: + type: geo_point + + - do: {xpack.info: {}} + - match: { features.runtime_fields.available: true } + - match: { features.runtime_fields.enabled: true } + + - do: {xpack.usage: {}} + - match: { runtime_fields.available: true } + - match: { runtime_fields.enabled: true } + - length: { runtime_fields.field_types: 7 } + - match: { runtime_fields.field_types.0.name: boolean } + - match: { runtime_fields.field_types.0.lang: [painless] } + - match: { runtime_fields.field_types.0.count: 1 } + - match: { runtime_fields.field_types.0.index_count: 1 } + - match: { runtime_fields.field_types.0.scriptless_count: 0 } + - match: { runtime_fields.field_types.0.source_max: 0 } + - match: { runtime_fields.field_types.0.source_total: 0 } + - match: { runtime_fields.field_types.0.lines_max: 3 } + - match: { runtime_fields.field_types.0.lines_total: 3 } + - is_true: runtime_fields.field_types.0.chars_max + - is_true: runtime_fields.field_types.0.chars_total + - match: { runtime_fields.field_types.0.doc_max: 1 } + - match: { runtime_fields.field_types.0.doc_total: 1 } + + - match: { runtime_fields.field_types.1.name: date } + - match: { runtime_fields.field_types.1.lang: [painless] } + - match: { runtime_fields.field_types.1.count: 2 } + - match: { runtime_fields.field_types.1.index_count: 1 } + - match: { runtime_fields.field_types.1.scriptless_count: 0 } + - match: { runtime_fields.field_types.1.source_max: 0 } + - match: { runtime_fields.field_types.1.source_total: 0 } + - match: { runtime_fields.field_types.1.lines_max: 3 } + - match: { runtime_fields.field_types.1.lines_total: 6 } + - is_true: runtime_fields.field_types.1.chars_max + - is_true: runtime_fields.field_types.1.chars_total + - match: { runtime_fields.field_types.1.doc_max: 1 } + - match: { runtime_fields.field_types.1.doc_total: 2 } + + - match: { runtime_fields.field_types.2.name: double } + - match: { runtime_fields.field_types.2.lang: [painless] } + - match: { runtime_fields.field_types.2.count: 1 } + - match: { runtime_fields.field_types.2.index_count: 1 } + - match: { runtime_fields.field_types.2.scriptless_count: 0 } + - match: { runtime_fields.field_types.2.source_max: 1 } + - match: { runtime_fields.field_types.2.source_total: 1 } + - match: { runtime_fields.field_types.2.lines_max: 1 } + - match: { runtime_fields.field_types.2.lines_total: 1 } + - is_true: runtime_fields.field_types.2.chars_max + - is_true: runtime_fields.field_types.2.chars_total + - match: { runtime_fields.field_types.2.doc_max: 0 } + - match: { runtime_fields.field_types.2.doc_total: 0 } + + - match: { runtime_fields.field_types.3.name: geo_point } + - match: { runtime_fields.field_types.3.lang: [painless] } + - match: { runtime_fields.field_types.3.count: 1 } + - match: { runtime_fields.field_types.3.index_count: 1 } + - match: { runtime_fields.field_types.3.scriptless_count: 0 } + - match: { runtime_fields.field_types.3.source_max: 2 } + - match: { runtime_fields.field_types.3.source_total: 2 } + - match: { runtime_fields.field_types.3.lines_max: 1 } + - match: { runtime_fields.field_types.3.lines_total: 1 } + - is_true: runtime_fields.field_types.3.chars_max + - is_true: runtime_fields.field_types.3.chars_total + - match: { runtime_fields.field_types.3.doc_max: 0 } + - match: { runtime_fields.field_types.3.doc_total: 0 } + + - match: { runtime_fields.field_types.4.name: ip } + - match: { runtime_fields.field_types.4.lang: [painless] } + - match: { runtime_fields.field_types.4.count: 1 } + - match: { runtime_fields.field_types.4.index_count: 1 } + - match: { runtime_fields.field_types.4.scriptless_count: 0 } + - match: { runtime_fields.field_types.4.source_max: 0 } + - match: { runtime_fields.field_types.4.source_total: 0 } + - match: { runtime_fields.field_types.4.lines_max: 4 } + - match: { runtime_fields.field_types.4.lines_total: 4 } + - is_true: runtime_fields.field_types.4.chars_max + - is_true: runtime_fields.field_types.4.chars_total + - match: { runtime_fields.field_types.4.doc_max: 1 } + - match: { runtime_fields.field_types.4.doc_total: 1 } + + - match: { runtime_fields.field_types.5.name: keyword } + - match: { runtime_fields.field_types.5.lang: [painless] } + - match: { runtime_fields.field_types.5.count: 3 } + - match: { runtime_fields.field_types.5.index_count: 1 } + - match: { runtime_fields.field_types.5.scriptless_count: 1 } + - match: { runtime_fields.field_types.5.source_max: 1 } + - match: { runtime_fields.field_types.5.source_total: 1 } + - match: { runtime_fields.field_types.5.lines_max: 3 } + - match: { runtime_fields.field_types.5.lines_total: 4 } + - is_true: runtime_fields.field_types.5.chars_max + - is_true: runtime_fields.field_types.5.chars_total + - match: { runtime_fields.field_types.5.doc_max: 1 } + - match: { runtime_fields.field_types.5.doc_total: 1 } + + - match: { runtime_fields.field_types.6.name: long } + - match: { runtime_fields.field_types.6.lang: [painless] } + - match: { runtime_fields.field_types.6.count: 1 } + - match: { runtime_fields.field_types.6.index_count: 1 } + - match: { runtime_fields.field_types.6.scriptless_count: 0 } + - match: { runtime_fields.field_types.6.source_max: 0 } + - match: { runtime_fields.field_types.6.source_total: 0 } + - match: { runtime_fields.field_types.6.lines_max: 3 } + - match: { runtime_fields.field_types.6.lines_total: 3 } + - is_true: runtime_fields.field_types.6.chars_max + - is_true: runtime_fields.field_types.6.chars_total + - match: { runtime_fields.field_types.6.doc_max: 1 } + - match: { runtime_fields.field_types.6.doc_total: 1 }