From 5dc7cccf1368061d27689f0335d3cfa17ab1f57c Mon Sep 17 00:00:00 2001 From: Mary Gouseti Date: Wed, 11 Oct 2023 12:00:52 +0300 Subject: [PATCH] Fix remaining logs tests (#100407) In this PR we convert the remaining 3 yaml logs tests to java rest tests. This should remove all the flaky tests from the data stream yaml suite that's why we are reenabling it. Closes: https://github.com/elastic/elasticsearch/issues/99911 Closes: https://github.com/elastic/elasticsearch/issues/97795 --- .../datastreams/EcsLogsDataStreamIT.java | 433 ++++++++++++++++++ .../datastreams/LogsDataStreamIT.java | 408 ++++++++++++++++- .../resources/ecs-logs/es-agent-ecs-log.json | 118 +++++ .../DataStreamsClientYamlTestSuiteIT.java | 2 - .../data_stream/230_logs_message_pipeline.yml | 114 ----- .../data_stream/240_logs_ecs_mappings.yml | 406 ---------------- .../data_stream/250_logs_no_subobjects.yml | 218 --------- .../test/data_stream/lifecycle/20_basic.yml | 3 + 8 files changed, 946 insertions(+), 756 deletions(-) create mode 100644 modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/EcsLogsDataStreamIT.java create mode 100644 modules/data-streams/src/javaRestTest/resources/ecs-logs/es-agent-ecs-log.json delete mode 100644 modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/230_logs_message_pipeline.yml delete mode 100644 modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_logs_ecs_mappings.yml delete mode 100644 modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/250_logs_no_subobjects.yml diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/EcsLogsDataStreamIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/EcsLogsDataStreamIT.java new file mode 100644 index 0000000000000..7de4ed2f2843c --- /dev/null +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/EcsLogsDataStreamIT.java @@ -0,0 +1,433 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.datastreams; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.core.PathUtils; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.datastreams.LogsDataStreamIT.createDataStream; +import static org.elasticsearch.datastreams.LogsDataStreamIT.getMappingProperties; +import static org.elasticsearch.datastreams.LogsDataStreamIT.getValueFromPath; +import static org.elasticsearch.datastreams.LogsDataStreamIT.getWriteBackingIndex; +import static org.elasticsearch.datastreams.LogsDataStreamIT.indexDoc; +import static org.elasticsearch.datastreams.LogsDataStreamIT.searchDocs; +import static org.elasticsearch.datastreams.LogsDataStreamIT.waitForLogs; +import static org.hamcrest.Matchers.is; + +public class EcsLogsDataStreamIT extends DisabledSecurityDataStreamTestCase { + + private static final String DATA_STREAM_NAME = "logs-generic-default"; + private RestClient client; + private String backingIndex; + + @Before + public void setup() throws Exception { + client = client(); + waitForLogs(client); + + { + Request request = new Request("PUT", "/_ingest/pipeline/logs@custom"); + request.setJsonEntity(""" + { + "processors": [ + { + "pipeline" : { + "name": "logs@json-message", + "description": "A pipeline that automatically parses JSON log events into top-level fields if they are such" + } + } + ] + } + """); + assertOK(client.performRequest(request)); + } + createDataStream(client, DATA_STREAM_NAME); + backingIndex = getWriteBackingIndex(client, DATA_STREAM_NAME); + } + + @After + public void cleanUp() throws IOException { + adminClient().performRequest(new Request("DELETE", "_data_stream/*")); + } + + @SuppressWarnings("unchecked") + public void testElasticAgentLogEcsMappings() throws Exception { + { + Path path = PathUtils.get(Thread.currentThread().getContextClassLoader().getResource("ecs-logs/es-agent-ecs-log.json").toURI()); + String agentLog = Files.readString(path); + indexDoc(client, DATA_STREAM_NAME, agentLog); + List results = searchDocs(client, DATA_STREAM_NAME, """ + { + "query": { + "term": { + "test": { + "value": "elastic-agent-log" + } + } + }, + "fields": ["message"] + } + """); + assertThat(results.size(), is(1)); + Map source = ((Map>) results.get(0)).get("_source"); + Map fields = ((Map>) results.get(0)).get("fields"); + + // timestamp from deserialized JSON message field should win + assertThat(source.get("@timestamp"), is("2023-05-16T13:49:40.374Z")); + assertThat( + ((Map>) source.get("kubernetes")).get("pod").get("name"), + is("elastic-agent-managed-daemonset-jwktj") + ); + // expecting the extracted message from within the original JSON-formatted message + assertThat(((List) fields.get("message")).get(0), is("Non-zero metrics in the last 30s")); + + Map properties = getMappingProperties(client, backingIndex); + assertThat(getValueFromPath(properties, List.of("@timestamp", "type")), is("date")); + assertThat(getValueFromPath(properties, List.of("message", "type")), is("match_only_text")); + assertThat( + getValueFromPath(properties, List.of("kubernetes", "properties", "pod", "properties", "name", "type")), + is("keyword") + ); + assertThat(getValueFromPath(properties, List.of("kubernetes", "properties", "pod", "properties", "ip", "type")), is("ip")); + assertThat(getValueFromPath(properties, List.of("kubernetes", "properties", "pod", "properties", "test_ip", "type")), is("ip")); + assertThat( + getValueFromPath( + properties, + List.of("kubernetes", "properties", "labels", "properties", "pod-template-generation", "type") + ), + is("keyword") + ); + assertThat(getValueFromPath(properties, List.of("log", "properties", "file", "properties", "path", "type")), is("keyword")); + assertThat( + getValueFromPath(properties, List.of("log", "properties", "file", "properties", "path", "fields", "text", "type")), + is("match_only_text") + ); + assertThat(getValueFromPath(properties, List.of("host", "properties", "os", "properties", "name", "type")), is("keyword")); + assertThat( + getValueFromPath(properties, List.of("host", "properties", "os", "properties", "name", "fields", "text", "type")), + is("match_only_text") + ); + } + } + + @SuppressWarnings("unchecked") + public void testGeneralMockupEcsMappings() throws Exception { + { + indexDoc(client, DATA_STREAM_NAME, """ + { + "start_timestamp": "not a date", + "start-timestamp": "not a date", + "timestamp.us": 1688550340718000, + "test": "mockup-ecs-log", + "registry": { + "data": { + "strings": ["C:\\\\rta\\\\red_ttp\\\\bin\\\\myapp.exe"] + } + }, + "process": { + "title": "ssh", + "executable": "/usr/bin/ssh", + "name": "ssh", + "command_line": "/usr/bin/ssh -l user 10.0.0.16", + "working_directory": "/home/ekoren", + "io": { + "text": "test" + } + }, + "url": { + "path": "/page", + "full": "https://mydomain.com/app/page", + "original": "https://mydomain.com/app/original" + }, + "email": { + "message_id": "81ce15$8r2j59@mail01.example.com" + }, + "parent": { + "url": { + "path": "/page", + "full": "https://mydomain.com/app/page", + "original": "https://mydomain.com/app/original" + }, + "body": { + "content": "Some content" + }, + "file": { + "path": "/path/to/my/file", + "target_path": "/path/to/my/file" + }, + "code_signature.timestamp": "2023-07-05", + "registry.data.strings": ["C:\\\\rta\\\\red_ttp\\\\bin\\\\myapp.exe"] + }, + "error": { + "stack_trace": "co.elastic.test.TestClass error:\\n at co.elastic.test.BaseTestClass", + "message": "Error occurred" + }, + "file": { + "path": "/path/to/my/file", + "target_path": "/path/to/my/file" + }, + "os": { + "full": "Mac OS Mojave" + }, + "user_agent": { + "original": "Mozilla/5.0 (iPhone; CPU iPhone OS 12_1 like Mac OS X) AppleWebKit/605.1.15" + }, + "user": { + "full_name": "John Doe" + }, + "vulnerability": { + "score": { + "base": 5.5, + "temporal": 5.5, + "version": "2.0" + }, + "textual_score": "bad" + }, + "host": { + "cpu": { + "usage": 0.68 + } + }, + "geo": { + "location": { + "lon": -73.614830, + "lat": 45.505918 + } + }, + "data_stream": { + "dataset": "nginx.access", + "namespace": "production", + "custom": "whatever" + }, + "structured_data": { + "key1": "value1", + "key2": ["value2", "value3"] + }, + "exports": { + "key": "value" + }, + "top_level_imports": { + "key": "value" + }, + "nested": { + "imports": { + "key": "value" + } + }, + "numeric_as_string": "42", + "socket": { + "ip": "127.0.0.1", + "remote_ip": "187.8.8.8" + } + } + """); + List results = searchDocs(client, DATA_STREAM_NAME, """ + { + "query": { + "term": { + "test": { + "value": "mockup-ecs-log" + } + } + }, + "fields": ["start-timestamp", "start_timestamp"], + "script_fields": { + "data_stream_type": { + "script": { + "source": "doc['data_stream.type'].value" + } + } + } + } + """); + assertThat(results.size(), is(1)); + Map fields = ((Map>) results.get(0)).get("fields"); + List ignored = ((Map>) results.get(0)).get("_ignored"); + Map ignoredFieldValues = ((Map>) results.get(0)).get("ignored_field_values"); + + // the ECS date dynamic template enforces mapping of "*_timestamp" fields to a date type + assertThat(ignored.size(), is(2)); + assertThat(ignored.get(0), is("start_timestamp")); + List startTimestampValues = (List) ignoredFieldValues.get("start_timestamp"); + assertThat(startTimestampValues.size(), is(1)); + assertThat(startTimestampValues.get(0), is("not a date")); + // "start-timestamp" doesn't match the ECS dynamic mapping pattern "*_timestamp" + assertThat(fields.get("start-timestamp"), is(List.of("not a date"))); + // verify that data_stream.type has the correct constant_keyword value + assertThat(fields.get("data_stream_type"), is(List.of("logs"))); + assertThat(ignored.get(1), is("vulnerability.textual_score")); + + Map properties = getMappingProperties(client, backingIndex); + assertThat(getValueFromPath(properties, List.of("error", "properties", "message", "type")), is("match_only_text")); + assertThat( + getValueFromPath(properties, List.of("registry", "properties", "data", "properties", "strings", "type")), + is("wildcard") + ); + assertThat( + getValueFromPath( + properties, + List.of("parent", "properties", "registry", "properties", "data", "properties", "strings", "type") + ), + is("wildcard") + ); + assertThat(getValueFromPath(properties, List.of("process", "properties", "io", "properties", "text", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("email", "properties", "message_id", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("url", "properties", "path", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("parent", "properties", "url", "properties", "path", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("url", "properties", "full", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("url", "properties", "full", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("parent", "properties", "url", "properties", "full", "type")), is("wildcard")); + assertThat( + getValueFromPath(properties, List.of("parent", "properties", "url", "properties", "full", "fields", "text", "type")), + is("match_only_text") + ); + assertThat(getValueFromPath(properties, List.of("url", "properties", "original", "type")), is("wildcard")); + assertThat( + getValueFromPath(properties, List.of("url", "properties", "original", "fields", "text", "type")), + is("match_only_text") + ); + assertThat( + getValueFromPath(properties, List.of("parent", "properties", "url", "properties", "original", "type")), + is("wildcard") + ); + assertThat( + getValueFromPath(properties, List.of("parent", "properties", "url", "properties", "original", "fields", "text", "type")), + is("match_only_text") + ); + assertThat( + getValueFromPath(properties, List.of("parent", "properties", "body", "properties", "content", "type")), + is("wildcard") + ); + assertThat( + getValueFromPath(properties, List.of("parent", "properties", "body", "properties", "content", "fields", "text", "type")), + is("match_only_text") + ); + assertThat(getValueFromPath(properties, List.of("process", "properties", "command_line", "type")), is("wildcard")); + assertThat( + getValueFromPath(properties, List.of("process", "properties", "command_line", "fields", "text", "type")), + is("match_only_text") + ); + assertThat(getValueFromPath(properties, List.of("error", "properties", "stack_trace", "type")), is("wildcard")); + assertThat( + getValueFromPath(properties, List.of("error", "properties", "stack_trace", "fields", "text", "type")), + is("match_only_text") + ); + assertThat(getValueFromPath(properties, List.of("file", "properties", "path", "type")), is("keyword")); + assertThat( + getValueFromPath(properties, List.of("file", "properties", "path", "fields", "text", "type")), + is("match_only_text") + ); + assertThat(getValueFromPath(properties, List.of("parent", "properties", "file", "properties", "path", "type")), is("keyword")); + assertThat( + getValueFromPath(properties, List.of("parent", "properties", "file", "properties", "path", "fields", "text", "type")), + is("match_only_text") + ); + assertThat(getValueFromPath(properties, List.of("file", "properties", "target_path", "type")), is("keyword")); + assertThat( + getValueFromPath(properties, List.of("file", "properties", "target_path", "fields", "text", "type")), + is("match_only_text") + ); + assertThat( + getValueFromPath(properties, List.of("parent", "properties", "file", "properties", "target_path", "type")), + is("keyword") + ); + assertThat( + getValueFromPath( + properties, + List.of("parent", "properties", "file", "properties", "target_path", "fields", "text", "type") + ), + is("match_only_text") + ); + assertThat(getValueFromPath(properties, List.of("os", "properties", "full", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("os", "properties", "full", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("user_agent", "properties", "original", "type")), is("keyword")); + assertThat( + getValueFromPath(properties, List.of("user_agent", "properties", "original", "fields", "text", "type")), + is("match_only_text") + ); + assertThat(getValueFromPath(properties, List.of("process", "properties", "title", "type")), is("keyword")); + assertThat( + getValueFromPath(properties, List.of("process", "properties", "title", "fields", "text", "type")), + is("match_only_text") + ); + assertThat(getValueFromPath(properties, List.of("process", "properties", "executable", "type")), is("keyword")); + assertThat( + getValueFromPath(properties, List.of("process", "properties", "executable", "fields", "text", "type")), + is("match_only_text") + ); + assertThat(getValueFromPath(properties, List.of("process", "properties", "name", "type")), is("keyword")); + assertThat( + getValueFromPath(properties, List.of("process", "properties", "name", "fields", "text", "type")), + is("match_only_text") + ); + assertThat(getValueFromPath(properties, List.of("process", "properties", "working_directory", "type")), is("keyword")); + assertThat( + getValueFromPath(properties, List.of("process", "properties", "working_directory", "fields", "text", "type")), + is("match_only_text") + ); + assertThat(getValueFromPath(properties, List.of("user", "properties", "full_name", "type")), is("keyword")); + assertThat( + getValueFromPath(properties, List.of("user", "properties", "full_name", "fields", "text", "type")), + is("match_only_text") + ); + assertThat(getValueFromPath(properties, List.of("start_timestamp", "type")), is("date")); + // testing the default mapping of string input fields to keyword if not matching any pattern + assertThat(getValueFromPath(properties, List.of("start-timestamp", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("timestamp", "properties", "us", "type")), is("long")); + assertThat( + getValueFromPath(properties, List.of("parent", "properties", "code_signature", "properties", "timestamp", "type")), + is("date") + ); + assertThat( + getValueFromPath(properties, List.of("vulnerability", "properties", "score", "properties", "base", "type")), + is("float") + ); + assertThat( + getValueFromPath(properties, List.of("vulnerability", "properties", "score", "properties", "temporal", "type")), + is("float") + ); + assertThat( + getValueFromPath(properties, List.of("vulnerability", "properties", "score", "properties", "version", "type")), + is("keyword") + ); + assertThat(getValueFromPath(properties, List.of("vulnerability", "properties", "textual_score", "type")), is("float")); + assertThat( + getValueFromPath(properties, List.of("host", "properties", "cpu", "properties", "usage", "type")), + is("scaled_float") + ); + assertThat( + getValueFromPath(properties, List.of("host", "properties", "cpu", "properties", "usage", "scaling_factor")), + is(1000.0) + ); + assertThat(getValueFromPath(properties, List.of("geo", "properties", "location", "type")), is("geo_point")); + assertThat(getValueFromPath(properties, List.of("data_stream", "properties", "dataset", "type")), is("constant_keyword")); + assertThat(getValueFromPath(properties, List.of("data_stream", "properties", "namespace", "type")), is("constant_keyword")); + assertThat(getValueFromPath(properties, List.of("data_stream", "properties", "type", "type")), is("constant_keyword")); + // not one of the three data_stream fields that are explicitly mapped to constant_keyword + assertThat(getValueFromPath(properties, List.of("data_stream", "properties", "custom", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("structured_data", "type")), is("flattened")); + assertThat(getValueFromPath(properties, List.of("exports", "type")), is("flattened")); + assertThat(getValueFromPath(properties, List.of("top_level_imports", "type")), is("flattened")); + assertThat(getValueFromPath(properties, List.of("nested", "properties", "imports", "type")), is("flattened")); + // verifying the default mapping for strings into keyword, overriding the automatic numeric string detection + assertThat(getValueFromPath(properties, List.of("numeric_as_string", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("socket", "properties", "ip", "type")), is("ip")); + assertThat(getValueFromPath(properties, List.of("socket", "properties", "remote_ip", "type")), is("ip")); + } + } +} diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java index 5bb9c8b340ee9..cc8695b9e0e5b 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java @@ -21,6 +21,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.matchesRegex; +import static org.hamcrest.Matchers.nullValue; public class LogsDataStreamIT extends DisabledSecurityDataStreamTestCase { @@ -45,8 +46,8 @@ public void testDefaultLogsSettingAndMapping() throws Exception { // Extend the mapping and verify putMapping(client, backingIndex); Map mappingProperties = getMappingProperties(client, backingIndex); - assertThat(((Map) mappingProperties.get("@timestamp")).get("ignore_malformed"), equalTo(false)); - assertThat(((Map) mappingProperties.get("numeric_field")).get("type"), equalTo("integer")); + assertThat(getValueFromPath(mappingProperties, List.of("@timestamp", "ignore_malformed")), equalTo(false)); + assertThat(getValueFromPath(mappingProperties, List.of("numeric_field", "type")), equalTo("integer")); // Insert valid doc and verify successful indexing { @@ -149,11 +150,8 @@ public void testCustomMapping() throws Exception { // Verify that the new field from the custom component template is applied putMapping(client, backingIndex); Map mappingProperties = getMappingProperties(client, backingIndex); - assertThat(((Map) mappingProperties.get("numeric_field")).get("type"), equalTo("integer")); - assertThat( - ((Map) mappingProperties.get("socket")).get("properties"), - equalTo(Map.of("ip", Map.of("type", "keyword"))) - ); + assertThat(getValueFromPath(mappingProperties, List.of("numeric_field", "type")), equalTo("integer")); + assertThat(getValueFromPath(mappingProperties, List.of("socket", "properties", "ip", "type")), is("keyword")); // Insert valid doc and verify successful indexing { @@ -227,7 +225,7 @@ public void testLogsDefaultPipeline() throws Exception { // Verify mapping from custom logs Map mappingProperties = getMappingProperties(client, backingIndex); - assertThat(((Map) mappingProperties.get("@timestamp")).get("type"), equalTo("date")); + assertThat(getValueFromPath(mappingProperties, List.of("@timestamp", "type")), equalTo("date")); // no timestamp - testing default pipeline's @timestamp set processor { @@ -284,7 +282,358 @@ public void testLogsDefaultPipeline() throws Exception { } } - private static void waitForLogs(RestClient client) throws Exception { + @SuppressWarnings("unchecked") + public void testLogsMessagePipeline() throws Exception { + RestClient client = client(); + waitForLogs(client); + + { + Request request = new Request("PUT", "/_ingest/pipeline/logs@custom"); + request.setJsonEntity(""" + { + "processors": [ + { + "pipeline" : { + "name": "logs@json-message", + "description": "A pipeline that automatically parses JSON log events into top-level fields if they are such" + } + } + ] + } + """); + assertOK(client.performRequest(request)); + } + + String dataStreamName = "logs-generic-default"; + createDataStream(client, dataStreamName); + + { + indexDoc(client, dataStreamName, """ + { + "@timestamp":"2023-05-09T16:48:34.135Z", + "message":"json", + "log.level": "INFO", + "ecs.version": "1.6.0", + "service.name":"my-app", + "event.dataset":"my-app.RollingFile", + "process.thread.name":"main", + "log.logger":"root.pkg.MyApp" + } + """); + List results = searchDocs(client, dataStreamName, """ + { + "query": { + "term": { + "message": { + "value": "json" + } + } + }, + "fields": ["message"] + } + """); + assertThat(results.size(), is(1)); + Map source = ((Map>) results.get(0)).get("_source"); + Map fields = ((Map>) results.get(0)).get("fields"); + + // root field parsed from JSON should win + assertThat(source.get("@timestamp"), is("2023-05-09T16:48:34.135Z")); + assertThat(source.get("message"), is("json")); + assertThat(((List) fields.get("message")).get(0), is("json")); + + // successful access to subfields verifies that dot expansion is part of the pipeline + assertThat(source.get("log.level"), is("INFO")); + assertThat(source.get("ecs.version"), is("1.6.0")); + assertThat(source.get("service.name"), is("my-app")); + assertThat(source.get("event.dataset"), is("my-app.RollingFile")); + assertThat(source.get("process.thread.name"), is("main")); + assertThat(source.get("log.logger"), is("root.pkg.MyApp")); + // _tmp_json_message should be removed by the pipeline + assertThat(source.get("_tmp_json_message"), is(nullValue())); + } + + // test malformed-JSON parsing - parsing error should be ignored and the document should be indexed with original message + { + indexDoc(client, dataStreamName, """ + { + "@timestamp":"2023-05-10", + "test":"malformed_json", + "message": "{\\"@timestamp\\":\\"2023-05-09T16:48:34.135Z\\", \\"message\\":\\"malformed_json\\"}}" + } + """); + List results = searchDocs(client, dataStreamName, """ + { + "query": { + "term": { + "test": { + "value": "malformed_json" + } + } + } + } + """); + assertThat(results.size(), is(1)); + Map source = ((Map>) results.get(0)).get("_source"); + + // root field parsed from JSON should win + assertThat(source.get("@timestamp"), is("2023-05-10")); + assertThat(source.get("message"), is("{\"@timestamp\":\"2023-05-09T16:48:34.135Z\", \"message\":\"malformed_json\"}}")); + assertThat(source.get("_tmp_json_message"), is(nullValue())); + } + + // test non-string message field + { + indexDoc(client, dataStreamName, """ + { + "message": 42, + "test": "numeric_message" + } + """); + List results = searchDocs(client, dataStreamName, """ + { + "query": { + "term": { + "test": { + "value": "numeric_message" + } + } + }, + "fields": ["message"] + } + """); + assertThat(results.size(), is(1)); + Map source = ((Map>) results.get(0)).get("_source"); + Map fields = ((Map>) results.get(0)).get("fields"); + + assertThat(source.get("message"), is(42)); + assertThat(((List) fields.get("message")).get(0), is("42")); + } + } + + @SuppressWarnings("unchecked") + public void testNoSubobjects() throws Exception { + RestClient client = client(); + waitForLogs(client); + { + Request request = new Request("POST", "/_component_template/logs-test-subobjects-mappings"); + request.setJsonEntity(""" + { + "template": { + "settings": { + "mapping": { + "ignore_malformed": true + } + }, + "mappings": { + "subobjects": false, + "date_detection": false, + "properties": { + "data_stream.type": { + "type": "constant_keyword", + "value": "logs" + }, + "data_stream.dataset": { + "type": "constant_keyword" + }, + "data_stream.namespace": { + "type": "constant_keyword" + } + } + } + } + } + """); + assertOK(client.performRequest(request)); + } + { + Request request = new Request("POST", "/_index_template/logs-ecs-test-template"); + request.setJsonEntity(""" + { + "priority": 200, + "data_stream": {}, + "index_patterns": ["logs-*-*"], + "composed_of": ["logs-test-subobjects-mappings", "ecs@dynamic_templates"] + } + """); + assertOK(client.performRequest(request)); + } + String dataStream = "logs-ecs-test-subobjects"; + createDataStream(client, dataStream); + String backingIndexName = getWriteBackingIndex(client, dataStream); + + indexDoc(client, dataStream, """ + { + "@timestamp": "2023-06-12", + "start_timestamp": "2023-06-08", + "location" : "POINT (-71.34 41.12)", + "test": "flattened", + "test.start_timestamp": "not a date", + "test.start-timestamp": "not a date", + "registry.data.strings": ["C:\\\\rta\\\\red_ttp\\\\bin\\\\myapp.exe"], + "process.title": "ssh", + "process.executable": "/usr/bin/ssh", + "process.name": "ssh", + "process.command_line": "/usr/bin/ssh -l user 10.0.0.16", + "process.working_directory": "/home/ekoren", + "process.io.text": "test", + "url.path": "/page", + "url.full": "https://mydomain.com/app/page", + "url.original": "https://mydomain.com/app/original", + "email.message_id": "81ce15$8r2j59@mail01.example.com", + "parent.url.path": "/page", + "parent.url.full": "https://mydomain.com/app/page", + "parent.url.original": "https://mydomain.com/app/original", + "parent.body.content": "Some content", + "parent.file.path": "/path/to/my/file", + "parent.file.target_path": "/path/to/my/file", + "parent.registry.data.strings": ["C:\\\\rta\\\\red_ttp\\\\bin\\\\myapp.exe"], + "error.stack_trace": "co.elastic.test.TestClass error:\\n at co.elastic.test.BaseTestClass", + "error.message": "Error occurred", + "file.path": "/path/to/my/file", + "file.target_path": "/path/to/my/file", + "os.full": "Mac OS Mojave", + "user_agent.original": "Mozilla/5.0 (iPhone; CPU iPhone OS 12_1 like Mac OS X) AppleWebKit/605.1.15", + "user.full_name": "John Doe", + "vulnerability.score.base": 5.5, + "vulnerability.score.temporal": 5.5, + "vulnerability.score.version": "2.0", + "vulnerability.textual_score": "bad", + "host.cpu.usage": 0.68, + "geo.location": [-73.614830, 45.505918], + "data_stream.dataset": "nginx.access", + "data_stream.namespace": "production", + "data_stream.custom": "whatever", + "structured_data": {"key1": "value1", "key2": ["value2", "value3"]}, + "exports": {"key": "value"}, + "top_level_imports": {"key": "value"}, + "nested.imports": {"key": "value"}, + "numeric_as_string": "42", + "socket.ip": "127.0.0.1", + "socket.remote_ip": "187.8.8.8" + } + """); + List hits = searchDocs(client, dataStream, """ + { + "query": { + "term": { + "test": { + "value": "flattened" + } + } + }, + "fields": [ + "data_stream.type", + "location", + "geo.location", + "test.start-timestamp", + "test.start_timestamp", + "vulnerability.textual_score" + ] + } + """); + assertThat(hits.size(), is(1)); + Map fields = ((Map>) hits.get(0)).get("fields"); + List ignored = ((Map>) hits.get(0)).get("_ignored"); + Map> ignoredFieldValues = ((Map>>) hits.get(0)).get("ignored_field_values"); + + // verify that data_stream.type has the correct constant_keyword value + assertThat(fields.get("data_stream.type"), is(List.of("logs"))); + // verify geo_point subfields evaluation + assertThat(((List>) fields.get("location")).get(0).get("type"), is("Point")); + List coordinates = ((List>>) fields.get("location")).get(0).get("coordinates"); + assertThat(coordinates.size(), is(2)); + assertThat(coordinates.get(0), equalTo(-71.34)); + assertThat(coordinates.get(1), equalTo(41.12)); + List geoLocation = (List) fields.get("geo.location"); + assertThat(((Map) geoLocation.get(0)).get("type"), is("Point")); + coordinates = ((Map>) geoLocation.get(0)).get("coordinates"); + assertThat(coordinates.size(), is(2)); + assertThat(coordinates.get(0), equalTo(-73.614830)); + assertThat(coordinates.get(1), equalTo(45.505918)); + // "start-timestamp" doesn't match the ECS dynamic mapping pattern "*_timestamp" + assertThat(fields.get("test.start-timestamp"), is(List.of("not a date"))); + assertThat(ignored.size(), is(2)); + assertThat(ignored.get(0), is("vulnerability.textual_score")); + // the ECS date dynamic template enforces mapping of "*_timestamp" fields to a date type + assertThat(ignored.get(1), is("test.start_timestamp")); + assertThat(ignoredFieldValues.get("test.start_timestamp").size(), is(1)); + assertThat(ignoredFieldValues.get("test.start_timestamp"), is(List.of("not a date"))); + assertThat(ignoredFieldValues.get("vulnerability.textual_score").size(), is(1)); + assertThat(ignoredFieldValues.get("vulnerability.textual_score").get(0), is("bad")); + + Map properties = getMappingProperties(client, backingIndexName); + assertThat(getValueFromPath(properties, List.of("error.message", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("registry.data.strings", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("parent.registry.data.strings", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("process.io.text", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("email.message_id", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("url.path", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("parent.url.path", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("url.full", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("url.full", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("parent.url.full", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("parent.url.full", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("url.original", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("url.original", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("parent.url.original", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("parent.url.original", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("parent.body.content", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("parent.body.content", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("process.command_line", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("process.command_line", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("error.stack_trace", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("error.stack_trace", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("file.path", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("file.path", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("parent.file.path", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("parent.file.path", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("file.target_path", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("file.target_path", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("parent.file.target_path", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("parent.file.target_path", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("os.full", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("os.full", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("user_agent.original", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("user_agent.original", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("process.title", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("process.title", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("process.executable", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("process.executable", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("process.name", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("process.name", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("process.working_directory", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("process.working_directory", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("user.full_name", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("user.full_name", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("start_timestamp", "type")), is("date")); + assertThat(getValueFromPath(properties, List.of("test.start_timestamp", "type")), is("date")); + // testing the default mapping of string input fields to keyword if not matching any pattern + assertThat(getValueFromPath(properties, List.of("test.start-timestamp", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("vulnerability.score.base", "type")), is("float")); + assertThat(getValueFromPath(properties, List.of("vulnerability.score.temporal", "type")), is("float")); + assertThat(getValueFromPath(properties, List.of("vulnerability.score.version", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("vulnerability.textual_score", "type")), is("float")); + assertThat(getValueFromPath(properties, List.of("host.cpu.usage", "type")), is("scaled_float")); + assertThat(getValueFromPath(properties, List.of("host.cpu.usage", "scaling_factor")), is(1000.0)); + assertThat(getValueFromPath(properties, List.of("location", "type")), is("geo_point")); + assertThat(getValueFromPath(properties, List.of("geo.location", "type")), is("geo_point")); + assertThat(getValueFromPath(properties, List.of("data_stream.dataset", "type")), is("constant_keyword")); + assertThat(getValueFromPath(properties, List.of("data_stream.namespace", "type")), is("constant_keyword")); + assertThat(getValueFromPath(properties, List.of("data_stream.type", "type")), is("constant_keyword")); + // not one of the three data_stream fields that are explicitly mapped to constant_keyword + assertThat(getValueFromPath(properties, List.of("data_stream.custom", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("structured_data", "type")), is("flattened")); + assertThat(getValueFromPath(properties, List.of("exports", "type")), is("flattened")); + assertThat(getValueFromPath(properties, List.of("top_level_imports", "type")), is("flattened")); + assertThat(getValueFromPath(properties, List.of("nested.imports", "type")), is("flattened")); + // verifying the default mapping for strings into keyword, overriding the automatic numeric string detection + assertThat(getValueFromPath(properties, List.of("numeric_as_string", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("socket.ip", "type")), is("ip")); + assertThat(getValueFromPath(properties, List.of("socket.remote_ip", "type")), is("ip")); + + } + + static void waitForLogs(RestClient client) throws Exception { assertBusy(() -> { try { Request request = new Request("GET", "_index_template/logs"); @@ -295,13 +644,13 @@ private static void waitForLogs(RestClient client) throws Exception { }); } - private static void createDataStream(RestClient client, String name) throws IOException { + static void createDataStream(RestClient client, String name) throws IOException { Request request = new Request("PUT", "_data_stream/" + name); assertOK(client.performRequest(request)); } @SuppressWarnings("unchecked") - private static String getWriteBackingIndex(RestClient client, String name) throws IOException { + static String getWriteBackingIndex(RestClient client, String name) throws IOException { Request request = new Request("GET", "_data_stream/" + name); List dataStreams = (List) entityAsMap(client.performRequest(request)).get("data_streams"); Map dataStream = (Map) dataStreams.get(0); @@ -310,12 +659,12 @@ private static String getWriteBackingIndex(RestClient client, String name) throw } @SuppressWarnings("unchecked") - private static Map getSettings(RestClient client, String indexName) throws IOException { + static Map getSettings(RestClient client, String indexName) throws IOException { Request request = new Request("GET", "/" + indexName + "/_settings?flat_settings"); return ((Map>) entityAsMap(client.performRequest(request)).get(indexName)).get("settings"); } - private static void putMapping(RestClient client, String indexName) throws IOException { + static void putMapping(RestClient client, String indexName) throws IOException { Request request = new Request("PUT", "/" + indexName + "/_mapping"); request.setJsonEntity(""" { @@ -330,24 +679,51 @@ private static void putMapping(RestClient client, String indexName) throws IOExc } @SuppressWarnings("unchecked") - private static Map getMappingProperties(RestClient client, String indexName) throws IOException { + static Map getMappingProperties(RestClient client, String indexName) throws IOException { Request request = new Request("GET", "/" + indexName + "/_mapping"); Map map = (Map) entityAsMap(client.performRequest(request)).get(indexName); Map mappings = (Map) map.get("mappings"); return (Map) mappings.get("properties"); } - private static void indexDoc(RestClient client, String dataStreamName, String doc) throws IOException { + static void indexDoc(RestClient client, String dataStreamName, String doc) throws IOException { Request request = new Request("POST", "/" + dataStreamName + "/_doc?refresh=true"); request.setJsonEntity(doc); assertOK(client.performRequest(request)); } @SuppressWarnings("unchecked") - private static List searchDocs(RestClient client, String dataStreamName, String query) throws IOException { + static List searchDocs(RestClient client, String dataStreamName, String query) throws IOException { Request request = new Request("GET", "/" + dataStreamName + "/_search"); request.setJsonEntity(query); Map hits = (Map) entityAsMap(client.performRequest(request)).get("hits"); return (List) hits.get("hits"); } + + @SuppressWarnings("unchecked") + static Object getValueFromPath(Map map, List path) { + Map current = map; + for (int i = 0; i < path.size(); i++) { + Object value = current.get(path.get(i)); + if (i == path.size() - 1) { + return value; + } + if (value == null) { + throw new IllegalStateException("Path " + String.join(".", path) + " was not found in " + map); + } + if (value instanceof Map next) { + current = (Map) next; + } else { + throw new IllegalStateException( + "Failed to reach the end of the path " + + String.join(".", path) + + " last reachable field was " + + path.get(i) + + " in " + + map + ); + } + } + return current; + } } diff --git a/modules/data-streams/src/javaRestTest/resources/ecs-logs/es-agent-ecs-log.json b/modules/data-streams/src/javaRestTest/resources/ecs-logs/es-agent-ecs-log.json new file mode 100644 index 0000000000000..29ae669e1290d --- /dev/null +++ b/modules/data-streams/src/javaRestTest/resources/ecs-logs/es-agent-ecs-log.json @@ -0,0 +1,118 @@ +{ + "@timestamp": "2023-05-16T13:49:40.377Z", + "test": "elastic-agent-log", + "container": { + "image": { + "name": "docker.elastic.co/beats/elastic-agent:8.9.0-SNAPSHOT" + }, + "runtime": "containerd", + "id": "bdabf58305b2b537d06b85764c588ff659190d875cb5470214bc16ba50ea1a4d" + }, + "kubernetes": { + "container": { + "name": "elastic-agent" + }, + "node": { + "uid": "0f4dd3b8-0b29-418e-ad7a-ebc55bc279ff", + "hostname": "multi-v1.27.1-worker", + "name": "multi-v1.27.1-worker", + "labels": { + "kubernetes_io/hostname": "multi-v1.27.1-worker", + "beta_kubernetes_io/os": "linux", + "kubernetes_io/arch": "arm64", + "kubernetes_io/os": "linux", + "beta_kubernetes_io/arch": "arm64" + } + }, + "pod": { + "uid": "c91d1354-27cf-40f3-a2d6-e2b75aa96bf2", + "ip": "172.18.0.4", + "test_ip": "172.18.0.5", + "name": "elastic-agent-managed-daemonset-jwktj" + }, + "namespace": "kube-system", + "namespace_uid": "63294aeb-b23f-429d-827c-e793ccf91024", + "daemonset": { + "name": "elastic-agent-managed-daemonset" + }, + "namespace_labels": { + "kubernetes_io/metadata_name": "kube-system" + }, + "labels": { + "controller-revision-hash": "7ff74fcd4b", + "pod-template-generation": "1", + "k8s-app": "elastic-agent" + } + }, + "agent": { + "name": "multi-v1.27.1-worker", + "id": "230358e2-6c5d-4675-9069-04feaddad64b", + "ephemeral_id": "e0934bfb-7e35-4bcc-a935-803643841213", + "type": "filebeat", + "version": "8.9.0" + }, + "log": { + "file": { + "path": "/var/log/containers/elastic-agent-managed-daemonset-jwktj_kube-system_elastic-agent-bdabf58305b2b537d06b85764c588ff659190d875cb5470214bc16ba50ea1a4d.log" + }, + "offset": 635247 + }, + "elastic_agent": { + "id": "230358e2-6c5d-4675-9069-04feaddad64b", + "version": "8.9.0", + "snapshot": true + }, + "message": "{\"log.level\":\"info\",\"@timestamp\":\"2023-05-16T13:49:40.374Z\",\"message\":\"Non-zero metrics in the last 30s\",\"component\":{\"binary\":\"metricbeat\",\"dataset\":\"elastic_agent.metricbeat\",\"id\":\"kubernetes/metrics-a92ab320-f3ed-11ed-9c8d-45656839f031\",\"type\":\"kubernetes/metrics\"},\"log\":{\"source\":\"kubernetes/metrics-a92ab320-f3ed-11ed-9c8d-45656839f031\"},\"log.logger\":\"monitoring\",\"log.origin\":{\"file.line\":187,\"file.name\":\"log/log.go\"},\"service.name\":\"metricbeat\",\"ecs.version\":\"1.6.0\"}", + "orchestrator": { + "cluster": { + "name": "multi-v1.27.1", + "url": "multi-v1.27.1-control-plane:6443" + } + }, + "input": { + "type": "filestream" + }, + "ecs": { + "version": "8.0.0" + }, + "stream": "stderr", + "data_stream": { + "namespace": "default", + "dataset": "kubernetes.container_logs" + }, + "host": { + "hostname": "multi-v1.27.1-worker", + "os": { + "kernel": "5.15.49-linuxkit", + "codename": "focal", + "name": "Ubuntu", + "type": "linux", + "family": "debian", + "version": "20.04.6 LTS (Focal Fossa)", + "platform": "ubuntu" + }, + "ip": [ + "10.244.2.1", + "10.244.2.1", + "172.18.0.4", + "fc00:f853:ccd:e793::4", + "fe80::42:acff:fe12:4", + "172.21.0.9" + ], + "containerized": false, + "name": "multi-v1.27.1-worker", + "id": "b2c527655d7746328f0686e25d3c413a", + "mac": [ + "02-42-AC-12-00-04", + "02-42-AC-15-00-09", + "32-7E-AA-73-39-04", + "EA-F3-80-1D-88-E3" + ], + "architecture": "aarch64" + }, + "event": { + "agent_id_status": "verified", + "ingested": "2023-05-16T13:49:47Z", + "dataset": "kubernetes.container_logs" + } +} \ No newline at end of file diff --git a/modules/data-streams/src/yamlRestTest/java/org/elasticsearch/datastreams/DataStreamsClientYamlTestSuiteIT.java b/modules/data-streams/src/yamlRestTest/java/org/elasticsearch/datastreams/DataStreamsClientYamlTestSuiteIT.java index 43438bfe9e5fb..fa7b4ca1a80c0 100644 --- a/modules/data-streams/src/yamlRestTest/java/org/elasticsearch/datastreams/DataStreamsClientYamlTestSuiteIT.java +++ b/modules/data-streams/src/yamlRestTest/java/org/elasticsearch/datastreams/DataStreamsClientYamlTestSuiteIT.java @@ -9,7 +9,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.apache.lucene.tests.util.LuceneTestCase; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; @@ -20,7 +19,6 @@ import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; import org.junit.ClassRule; -@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/99764") public class DataStreamsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { public DataStreamsClientYamlTestSuiteIT(final ClientYamlTestCandidate testCandidate) { diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/230_logs_message_pipeline.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/230_logs_message_pipeline.yml deleted file mode 100644 index 6fd6f24a4ea14..0000000000000 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/230_logs_message_pipeline.yml +++ /dev/null @@ -1,114 +0,0 @@ ---- -Test log message JSON-parsing pipeline: - - do: - ingest.put_pipeline: - # opting in to use the JSON parsing pipeline for message field - id: "logs@custom" - body: > - { - "processors": [ - { - "pipeline" : { - "name": "logs@json-message", - "description": "A pipeline that automatically parses JSON log events into top-level fields if they are such" - } - } - ] - } - - - do: - indices.create_data_stream: - name: logs-generic-default - - is_true: acknowledged - - - do: - index: - index: logs-generic-default - refresh: true - body: - '@timestamp': '2023-05-10' - message: |- - { - "@timestamp":"2023-05-09T16:48:34.135Z", - "message":"json", - "log.level": "INFO", - "ecs.version": "1.6.0", - "service.name":"my-app", - "event.dataset":"my-app.RollingFile", - "process.thread.name":"main", - "log.logger":"root.pkg.MyApp" - } - - match: {result: "created"} - - - do: - search: - index: logs-generic-default - body: - query: - term: - message: - value: 'json' - fields: - - field: 'message' - - length: { hits.hits: 1 } - # root field parsed from JSON should win - - match: { hits.hits.0._source.@timestamp: '2023-05-09T16:48:34.135Z' } - - match: { hits.hits.0._source.message: 'json' } - - match: { hits.hits.0.fields.message.0: 'json' } - # successful access to subfields verifies that dot expansion is part of the pipeline - - match: { hits.hits.0._source.log.level: 'INFO' } - - match: { hits.hits.0._source.ecs.version: '1.6.0' } - - match: { hits.hits.0._source.service.name: 'my-app' } - - match: { hits.hits.0._source.event.dataset: 'my-app.RollingFile' } - - match: { hits.hits.0._source.process.thread.name: 'main' } - - match: { hits.hits.0._source.log.logger: 'root.pkg.MyApp' } - # _tmp_json_message should be removed by the pipeline - - match: { hits.hits.0._source._tmp_json_message: null } - - # test malformed-JSON parsing - parsing error should be ignored and the document should be indexed with original message - - do: - index: - index: logs-generic-default - refresh: true - body: - '@timestamp': '2023-05-10' - test: 'malformed_json' - message: '{"@timestamp":"2023-05-09T16:48:34.135Z", "message":"malformed_json"}}' - - match: {result: "created"} - - - do: - search: - index: logs-generic-default - body: - query: - term: - test: - value: 'malformed_json' - - length: { hits.hits: 1 } - - match: { hits.hits.0._source.@timestamp: '2023-05-10' } - - match: { hits.hits.0._source.message: '{"@timestamp":"2023-05-09T16:48:34.135Z", "message":"malformed_json"}}' } - - match: { hits.hits.0._source._tmp_json_message: null } - - # test non-string message field - - do: - index: - index: logs-generic-default - refresh: true - body: - test: 'numeric_message' - message: 42 - - match: {result: "created"} - - - do: - search: - index: logs-generic-default - body: - query: - term: - test: - value: 'numeric_message' - fields: - - field: 'message' - - length: { hits.hits: 1 } - - match: { hits.hits.0._source.message: 42 } - - match: { hits.hits.0.fields.message.0: '42' } diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_logs_ecs_mappings.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_logs_ecs_mappings.yml deleted file mode 100644 index 538e362ed9ec0..0000000000000 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_logs_ecs_mappings.yml +++ /dev/null @@ -1,406 +0,0 @@ -setup: - - do: - ingest.put_pipeline: - # opting in to use the JSON parsing pipeline for message field - id: "logs@custom" - body: > - { - "processors": [ - { - "pipeline" : { - "name": "logs@json-message", - "description": "A pipeline that automatically parses JSON log events into top-level fields if they are such" - } - } - ] - } - - - do: - indices.create_data_stream: - name: logs-generic-default - ---- -Test Elastic Agent log ECS mappings: - - skip: - version: all - reason: https://github.com/elastic/elasticsearch/issues/97795 - - do: - indices.get_data_stream: - name: logs-generic-default - - set: { data_streams.0.indices.0.index_name: idx0name } - - - do: - index: - index: logs-generic-default - refresh: true - body: > - { - "@timestamp": "2023-05-16T13:49:40.377Z", - "test": "elastic-agent-log", - "container": { - "image": { - "name": "docker.elastic.co/beats/elastic-agent:8.9.0-SNAPSHOT" - }, - "runtime": "containerd", - "id": "bdabf58305b2b537d06b85764c588ff659190d875cb5470214bc16ba50ea1a4d" - }, - "kubernetes": { - "container": { - "name": "elastic-agent" - }, - "node": { - "uid": "0f4dd3b8-0b29-418e-ad7a-ebc55bc279ff", - "hostname": "multi-v1.27.1-worker", - "name": "multi-v1.27.1-worker", - "labels": { - "kubernetes_io/hostname": "multi-v1.27.1-worker", - "beta_kubernetes_io/os": "linux", - "kubernetes_io/arch": "arm64", - "kubernetes_io/os": "linux", - "beta_kubernetes_io/arch": "arm64" - } - }, - "pod": { - "uid": "c91d1354-27cf-40f3-a2d6-e2b75aa96bf2", - "ip": "172.18.0.4", - "test_ip": "172.18.0.5", - "name": "elastic-agent-managed-daemonset-jwktj" - }, - "namespace": "kube-system", - "namespace_uid": "63294aeb-b23f-429d-827c-e793ccf91024", - "daemonset": { - "name": "elastic-agent-managed-daemonset" - }, - "namespace_labels": { - "kubernetes_io/metadata_name": "kube-system" - }, - "labels": { - "controller-revision-hash": "7ff74fcd4b", - "pod-template-generation": "1", - "k8s-app": "elastic-agent" - } - }, - "agent": { - "name": "multi-v1.27.1-worker", - "id": "230358e2-6c5d-4675-9069-04feaddad64b", - "ephemeral_id": "e0934bfb-7e35-4bcc-a935-803643841213", - "type": "filebeat", - "version": "8.9.0" - }, - "log": { - "file": { - "path": "/var/log/containers/elastic-agent-managed-daemonset-jwktj_kube-system_elastic-agent-bdabf58305b2b537d06b85764c588ff659190d875cb5470214bc16ba50ea1a4d.log" - }, - "offset": 635247 - }, - "elastic_agent": { - "id": "230358e2-6c5d-4675-9069-04feaddad64b", - "version": "8.9.0", - "snapshot": true - }, - "message": "{\"log.level\":\"info\",\"@timestamp\":\"2023-05-16T13:49:40.374Z\",\"message\":\"Non-zero metrics in the last 30s\",\"component\":{\"binary\":\"metricbeat\",\"dataset\":\"elastic_agent.metricbeat\",\"id\":\"kubernetes/metrics-a92ab320-f3ed-11ed-9c8d-45656839f031\",\"type\":\"kubernetes/metrics\"},\"log\":{\"source\":\"kubernetes/metrics-a92ab320-f3ed-11ed-9c8d-45656839f031\"},\"log.logger\":\"monitoring\",\"log.origin\":{\"file.line\":187,\"file.name\":\"log/log.go\"},\"service.name\":\"metricbeat\",\"ecs.version\":\"1.6.0\"}", - "orchestrator": { - "cluster": { - "name": "multi-v1.27.1", - "url": "multi-v1.27.1-control-plane:6443" - } - }, - "input": { - "type": "filestream" - }, - "ecs": { - "version": "8.0.0" - }, - "stream": "stderr", - "data_stream": { - "namespace": "default", - "dataset": "kubernetes.container_logs" - }, - "host": { - "hostname": "multi-v1.27.1-worker", - "os": { - "kernel": "5.15.49-linuxkit", - "codename": "focal", - "name": "Ubuntu", - "type": "linux", - "family": "debian", - "version": "20.04.6 LTS (Focal Fossa)", - "platform": "ubuntu" - }, - "ip": [ - "10.244.2.1", - "10.244.2.1", - "172.18.0.4", - "fc00:f853:ccd:e793::4", - "fe80::42:acff:fe12:4", - "172.21.0.9" - ], - "containerized": false, - "name": "multi-v1.27.1-worker", - "id": "b2c527655d7746328f0686e25d3c413a", - "mac": [ - "02-42-AC-12-00-04", - "02-42-AC-15-00-09", - "32-7E-AA-73-39-04", - "EA-F3-80-1D-88-E3" - ], - "architecture": "aarch64" - }, - "event": { - "agent_id_status": "verified", - "ingested": "2023-05-16T13:49:47Z", - "dataset": "kubernetes.container_logs" - } - } - - match: {result: "created"} - - - do: - search: - index: logs-generic-default - body: - query: - term: - test: - value: 'elastic-agent-log' - fields: - - field: 'message' - - length: { hits.hits: 1 } - # timestamp from deserialized JSON message field should win - - match: { hits.hits.0._source.@timestamp: '2023-05-16T13:49:40.374Z' } - - match: { hits.hits.0._source.kubernetes.pod.name: 'elastic-agent-managed-daemonset-jwktj' } - # expecting the extracted message from within the original JSON-formatted message - - match: { hits.hits.0.fields.message.0: 'Non-zero metrics in the last 30s' } - - - do: - indices.get_mapping: - index: logs-generic-default - - match: { .$idx0name.mappings.properties.@timestamp.type: "date" } - - match: { .$idx0name.mappings.properties.message.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.kubernetes.properties.pod.properties.name.type: "keyword" } - - match: { .$idx0name.mappings.properties.kubernetes.properties.pod.properties.ip.type: "ip" } - - match: { .$idx0name.mappings.properties.kubernetes.properties.pod.properties.test_ip.type: "ip" } - - match: { .$idx0name.mappings.properties.kubernetes.properties.labels.properties.pod-template-generation.type: "keyword" } - - match: { .$idx0name.mappings.properties.log.properties.file.properties.path.type: "keyword" } - - match: { .$idx0name.mappings.properties.log.properties.file.properties.path.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.host.properties.os.properties.name.type: "keyword" } - - match: { .$idx0name.mappings.properties.host.properties.os.properties.name.fields.text.type: "match_only_text" } - ---- -Test general mockup ECS mappings: - - do: - indices.get_data_stream: - name: logs-generic-default - - set: { data_streams.0.indices.0.index_name: idx0name } - - - do: - index: - index: logs-generic-default - refresh: true - body: > - { - "start_timestamp": "not a date", - "start-timestamp": "not a date", - "timestamp.us": 1688550340718000, - "test": "mockup-ecs-log", - "registry": { - "data": { - "strings": ["C:\\rta\\red_ttp\\bin\\myapp.exe"] - } - }, - "process": { - "title": "ssh", - "executable": "/usr/bin/ssh", - "name": "ssh", - "command_line": "/usr/bin/ssh -l user 10.0.0.16", - "working_directory": "/home/ekoren", - "io": { - "text": "test" - } - }, - "url": { - "path": "/page", - "full": "https://mydomain.com/app/page", - "original": "https://mydomain.com/app/original" - }, - "email": { - "message_id": "81ce15$8r2j59@mail01.example.com" - }, - "parent": { - "url": { - "path": "/page", - "full": "https://mydomain.com/app/page", - "original": "https://mydomain.com/app/original" - }, - "body": { - "content": "Some content" - }, - "file": { - "path": "/path/to/my/file", - "target_path": "/path/to/my/file" - }, - "code_signature.timestamp": "2023-07-05", - "registry.data.strings": ["C:\\rta\\red_ttp\\bin\\myapp.exe"] - }, - "error": { - "stack_trace": "co.elastic.test.TestClass error:\n at co.elastic.test.BaseTestClass", - "message": "Error occurred" - }, - "file": { - "path": "/path/to/my/file", - "target_path": "/path/to/my/file" - }, - "os": { - "full": "Mac OS Mojave" - }, - "user_agent": { - "original": "Mozilla/5.0 (iPhone; CPU iPhone OS 12_1 like Mac OS X) AppleWebKit/605.1.15" - }, - "user": { - "full_name": "John Doe" - }, - "vulnerability": { - "score": { - "base": 5.5, - "temporal": 5.5, - "version": "2.0" - }, - "textual_score": "bad" - }, - "host": { - "cpu": { - "usage": 0.68 - } - }, - "geo": { - "location": { - "lon": -73.614830, - "lat": 45.505918 - } - }, - "data_stream": { - "dataset": "nginx.access", - "namespace": "production", - "custom": "whatever" - }, - "structured_data": { - "key1": "value1", - "key2": ["value2", "value3"] - }, - "exports": { - "key": "value" - }, - "top_level_imports": { - "key": "value" - }, - "nested": { - "imports": { - "key": "value" - } - }, - "numeric_as_string": "42", - "socket": { - "ip": "127.0.0.1", - "remote_ip": "187.8.8.8" - } - } - - match: {result: "created"} - - - do: - search: - index: logs-generic-default - body: - query: - term: - test: - value: 'mockup-ecs-log' - fields: - - field: 'start_timestamp' - - field: 'start-timestamp' - script_fields: - data_stream_type: - script: - source: "doc['data_stream.type'].value" - - length: { hits.hits: 1 } - # the ECS date dynamic template enforces mapping of "*_timestamp" fields to a date type - - length: { hits.hits.0._ignored: 2 } - - match: { hits.hits.0._ignored.0: 'start_timestamp' } - - length: { hits.hits.0.ignored_field_values.start_timestamp: 1 } - - match: { hits.hits.0.ignored_field_values.start_timestamp.0: 'not a date' } - # "start-timestamp" doesn't match the ECS dynamic mapping pattern "*_timestamp" - - match: { hits.hits.0.fields.start-timestamp.0: 'not a date' } - # verify that data_stream.type has the correct constant_keyword value - - match: { hits.hits.0.fields.data_stream_type.0: 'logs' } - - match: { hits.hits.0._ignored.1: 'vulnerability.textual_score' } - - - do: - indices.get_mapping: - index: logs-generic-default - - match: { .$idx0name.mappings.properties.error.properties.message.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.registry.properties.data.properties.strings.type: "wildcard" } - - match: { .$idx0name.mappings.properties.parent.properties.registry.properties.data.properties.strings.type: "wildcard" } - - match: { .$idx0name.mappings.properties.process.properties.io.properties.text.type: "wildcard" } - - match: { .$idx0name.mappings.properties.email.properties.message_id.type: "wildcard" } - - match: { .$idx0name.mappings.properties.url.properties.path.type: "wildcard" } - - match: { .$idx0name.mappings.properties.parent.properties.url.properties.path.type: "wildcard" } - - match: { .$idx0name.mappings.properties.url.properties.full.type: "wildcard" } - - match: { .$idx0name.mappings.properties.url.properties.full.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.parent.properties.url.properties.full.type: "wildcard" } - - match: { .$idx0name.mappings.properties.parent.properties.url.properties.full.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.url.properties.original.type: "wildcard" } - - match: { .$idx0name.mappings.properties.url.properties.original.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.parent.properties.url.properties.original.type: "wildcard" } - - match: { .$idx0name.mappings.properties.parent.properties.url.properties.original.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.parent.properties.body.properties.content.type: "wildcard" } - - match: { .$idx0name.mappings.properties.parent.properties.body.properties.content.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.process.properties.command_line.type: "wildcard" } - - match: { .$idx0name.mappings.properties.process.properties.command_line.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.error.properties.stack_trace.type: "wildcard" } - - match: { .$idx0name.mappings.properties.error.properties.stack_trace.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.file.properties.path.type: "keyword" } - - match: { .$idx0name.mappings.properties.file.properties.path.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.parent.properties.file.properties.path.type: "keyword" } - - match: { .$idx0name.mappings.properties.parent.properties.file.properties.path.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.file.properties.target_path.type: "keyword" } - - match: { .$idx0name.mappings.properties.file.properties.target_path.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.parent.properties.file.properties.target_path.type: "keyword" } - - match: { .$idx0name.mappings.properties.parent.properties.file.properties.target_path.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.os.properties.full.type: "keyword" } - - match: { .$idx0name.mappings.properties.os.properties.full.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.user_agent.properties.original.type: "keyword" } - - match: { .$idx0name.mappings.properties.user_agent.properties.original.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.process.properties.title.type: "keyword" } - - match: { .$idx0name.mappings.properties.process.properties.title.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.process.properties.executable.type: "keyword" } - - match: { .$idx0name.mappings.properties.process.properties.executable.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.process.properties.name.type: "keyword" } - - match: { .$idx0name.mappings.properties.process.properties.name.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.process.properties.working_directory.type: "keyword" } - - match: { .$idx0name.mappings.properties.process.properties.working_directory.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.user.properties.full_name.type: "keyword" } - - match: { .$idx0name.mappings.properties.user.properties.full_name.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.start_timestamp.type: "date" } - # testing the default mapping of string input fields to keyword if not matching any pattern - - match: { .$idx0name.mappings.properties.start-timestamp.type: "keyword" } - - match: { .$idx0name.mappings.properties.timestamp.properties.us.type: "long" } - - match: { .$idx0name.mappings.properties.parent.properties.code_signature.properties.timestamp.type: "date" } - - match: { .$idx0name.mappings.properties.vulnerability.properties.score.properties.base.type: "float" } - - match: { .$idx0name.mappings.properties.vulnerability.properties.score.properties.temporal.type: "float" } - - match: { .$idx0name.mappings.properties.vulnerability.properties.score.properties.version.type: "keyword" } - - match: { .$idx0name.mappings.properties.vulnerability.properties.textual_score.type: "float" } - - match: { .$idx0name.mappings.properties.host.properties.cpu.properties.usage.type: "scaled_float" } - - match: { .$idx0name.mappings.properties.host.properties.cpu.properties.usage.scaling_factor: 1000 } - - match: { .$idx0name.mappings.properties.geo.properties.location.type: "geo_point" } - - match: { .$idx0name.mappings.properties.data_stream.properties.dataset.type: "constant_keyword" } - - match: { .$idx0name.mappings.properties.data_stream.properties.namespace.type: "constant_keyword" } - - match: { .$idx0name.mappings.properties.data_stream.properties.type.type: "constant_keyword" } - # not one of the three data_stream fields that are explicitly mapped to constant_keyword - - match: { .$idx0name.mappings.properties.data_stream.properties.custom.type: "keyword" } - - match: { .$idx0name.mappings.properties.structured_data.type: "flattened" } - - match: { .$idx0name.mappings.properties.exports.type: "flattened" } - - match: { .$idx0name.mappings.properties.top_level_imports.type: "flattened" } - - match: { .$idx0name.mappings.properties.nested.properties.imports.type: "flattened" } - # verifying the default mapping for strings into keyword, overriding the automatic numeric string detection - - match: { .$idx0name.mappings.properties.numeric_as_string.type: "keyword" } - - match: { .$idx0name.mappings.properties.socket.properties.ip.type: "ip" } - - match: { .$idx0name.mappings.properties.socket.properties.remote_ip.type: "ip" } - diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/250_logs_no_subobjects.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/250_logs_no_subobjects.yml deleted file mode 100644 index 607693e9f9955..0000000000000 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/250_logs_no_subobjects.yml +++ /dev/null @@ -1,218 +0,0 @@ ---- -Test flattened document with subobjects-false: -# NOTE: this doesn't work. In order to run this test set "subobjects: false" through logs-mappings.json - - skip: - features: allowed_warnings - - - do: - cluster.put_component_template: - name: logs-test-subobjects-mappings - body: - template: - settings: - mapping: - ignore_malformed: true - mappings: - subobjects: false - date_detection: false - properties: - data_stream.type: - type: constant_keyword - value: logs - data_stream.dataset: - type: constant_keyword - data_stream.namespace: - type: constant_keyword - - - do: - allowed_warnings: - - "index template [logs-ecs-test-template] has index patterns [logs-*-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logs-ecs-test-template] will take precedence during new index creation" - indices.put_index_template: - name: logs-ecs-test-template - body: - priority: 200 - data_stream: {} - index_patterns: - - logs-*-* - composed_of: - - logs-test-subobjects-mappings - - ecs@dynamic_templates - - - do: - indices.create_data_stream: - name: logs-ecs-test-subobjects - - is_true: acknowledged - - - do: - indices.get_data_stream: - name: logs-ecs-test-subobjects - - set: { data_streams.0.indices.0.index_name: idx0name } - - - do: - index: - index: logs-ecs-test-subobjects - refresh: true - body: > - { - "@timestamp": "2023-06-12", - "start_timestamp": "2023-06-08", - "location" : "POINT (-71.34 41.12)", - "test": "flattened", - "test.start_timestamp": "not a date", - "test.start-timestamp": "not a date", - "registry.data.strings": ["C:\\rta\\red_ttp\\bin\\myapp.exe"], - "process.title": "ssh", - "process.executable": "/usr/bin/ssh", - "process.name": "ssh", - "process.command_line": "/usr/bin/ssh -l user 10.0.0.16", - "process.working_directory": "/home/ekoren", - "process.io.text": "test", - "url.path": "/page", - "url.full": "https://mydomain.com/app/page", - "url.original": "https://mydomain.com/app/original", - "email.message_id": "81ce15$8r2j59@mail01.example.com", - "parent.url.path": "/page", - "parent.url.full": "https://mydomain.com/app/page", - "parent.url.original": "https://mydomain.com/app/original", - "parent.body.content": "Some content", - "parent.file.path": "/path/to/my/file", - "parent.file.target_path": "/path/to/my/file", - "parent.registry.data.strings": ["C:\\rta\\red_ttp\\bin\\myapp.exe"], - "error.stack_trace": "co.elastic.test.TestClass error:\n at co.elastic.test.BaseTestClass", - "error.message": "Error occurred", - "file.path": "/path/to/my/file", - "file.target_path": "/path/to/my/file", - "os.full": "Mac OS Mojave", - "user_agent.original": "Mozilla/5.0 (iPhone; CPU iPhone OS 12_1 like Mac OS X) AppleWebKit/605.1.15", - "user.full_name": "John Doe", - "vulnerability.score.base": 5.5, - "vulnerability.score.temporal": 5.5, - "vulnerability.score.version": "2.0", - "vulnerability.textual_score": "bad", - "host.cpu.usage": 0.68, - "geo.location": [-73.614830, 45.505918], - "data_stream.dataset": "nginx.access", - "data_stream.namespace": "production", - "data_stream.custom": "whatever", - "structured_data": {"key1": "value1", "key2": ["value2", "value3"]}, - "exports": {"key": "value"}, - "top_level_imports": {"key": "value"}, - "nested.imports": {"key": "value"}, - "numeric_as_string": "42", - "socket.ip": "127.0.0.1", - "socket.remote_ip": "187.8.8.8" - } - - match: {result: "created"} - - - do: - search: - index: logs-ecs-test-subobjects - body: - query: - term: - test: - value: 'flattened' - fields: - - field: 'data_stream.type' - - field: 'location' - - field: 'geo.location' - - field: 'test.start-timestamp' - - field: 'test.start_timestamp' - - field: 'vulnerability.textual_score' - - length: { hits.hits: 1 } - # verify that data_stream.type has the correct constant_keyword value - - match: { hits.hits.0.fields.data_stream\.type.0: 'logs' } - # verify geo_point subfields evaluation - - match: { hits.hits.0.fields.location.0.type: 'Point' } - - length: { hits.hits.0.fields.location.0.coordinates: 2 } - - match: { hits.hits.0.fields.location.0.coordinates.0: -71.34 } - - match: { hits.hits.0.fields.location.0.coordinates.1: 41.12 } - - match: { hits.hits.0.fields.geo\.location.0.type: 'Point' } - - length: { hits.hits.0.fields.geo\.location.0.coordinates: 2 } - - match: { hits.hits.0.fields.geo\.location.0.coordinates.0: -73.614830 } - - match: { hits.hits.0.fields.geo\.location.0.coordinates.1: 45.505918 } - # "start-timestamp" doesn't match the ECS dynamic mapping pattern "*_timestamp" - # TODO: uncomment once https://github.com/elastic/elasticsearch/issues/96700 gets resolved - # - match: { hits.hits.0.fields.test\.start-timestamp.0: 'not a date' } - - length: { hits.hits.0._ignored: 2 } - - match: { hits.hits.0._ignored.0: 'vulnerability.textual_score' } - # the ECS date dynamic template enforces mapping of "*_timestamp" fields to a date type - - match: { hits.hits.0._ignored.1: 'test.start_timestamp' } - - length: { hits.hits.0.ignored_field_values.test\.start_timestamp: 1 } - # TODO: uncomment once https://github.com/elastic/elasticsearch/issues/96700 gets resolved - # - match: { hits.hits.0.ignored_field_values.test\.start_timestamp.0: 'not a date' } - - length: { hits.hits.0.ignored_field_values.vulnerability\.textual_score: 1 } - - match: { hits.hits.0.ignored_field_values.vulnerability\.textual_score.0: 'bad' } - - - do: - indices.get_mapping: - index: logs-ecs-test-subobjects - - match: { .$idx0name.mappings.properties.error\.message.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.registry\.data\.strings.type: "wildcard" } - - match: { .$idx0name.mappings.properties.parent\.registry\.data\.strings.type: "wildcard" } - - match: { .$idx0name.mappings.properties.process\.io\.text.type: "wildcard" } - - match: { .$idx0name.mappings.properties.email\.message_id.type: "wildcard" } - - match: { .$idx0name.mappings.properties.url\.path.type: "wildcard" } - - match: { .$idx0name.mappings.properties.parent\.url\.path.type: "wildcard" } - - match: { .$idx0name.mappings.properties.url\.full.type: "wildcard" } - - match: { .$idx0name.mappings.properties.url\.full.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.parent\.url\.full.type: "wildcard" } - - match: { .$idx0name.mappings.properties.parent\.url\.full.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.url\.original.type: "wildcard" } - - match: { .$idx0name.mappings.properties.url\.original.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.parent\.url\.original.type: "wildcard" } - - match: { .$idx0name.mappings.properties.parent\.url\.original.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.parent\.body\.content.type: "wildcard" } - - match: { .$idx0name.mappings.properties.parent\.body\.content.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.process\.command_line.type: "wildcard" } - - match: { .$idx0name.mappings.properties.process\.command_line.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.error\.stack_trace.type: "wildcard" } - - match: { .$idx0name.mappings.properties.error\.stack_trace.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.file\.path.type: "keyword" } - - match: { .$idx0name.mappings.properties.file\.path.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.parent\.file\.path.type: "keyword" } - - match: { .$idx0name.mappings.properties.parent\.file\.path.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.file\.target_path.type: "keyword" } - - match: { .$idx0name.mappings.properties.file\.target_path.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.parent\.file\.target_path.type: "keyword" } - - match: { .$idx0name.mappings.properties.parent\.file\.target_path.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.os\.full.type: "keyword" } - - match: { .$idx0name.mappings.properties.os\.full.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.user_agent\.original.type: "keyword" } - - match: { .$idx0name.mappings.properties.user_agent\.original.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.process\.title.type: "keyword" } - - match: { .$idx0name.mappings.properties.process\.title.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.process\.executable.type: "keyword" } - - match: { .$idx0name.mappings.properties.process\.executable.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.process\.name.type: "keyword" } - - match: { .$idx0name.mappings.properties.process\.name.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.process\.working_directory.type: "keyword" } - - match: { .$idx0name.mappings.properties.process\.working_directory.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.user\.full_name.type: "keyword" } - - match: { .$idx0name.mappings.properties.user\.full_name.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.start_timestamp.type: "date" } - - match: { .$idx0name.mappings.properties.test\.start_timestamp.type: "date" } - # testing the default mapping of string input fields to keyword if not matching any pattern - - match: { .$idx0name.mappings.properties.test\.start-timestamp.type: "keyword" } - - match: { .$idx0name.mappings.properties.vulnerability\.score\.base.type: "float" } - - match: { .$idx0name.mappings.properties.vulnerability\.score\.temporal.type: "float" } - - match: { .$idx0name.mappings.properties.vulnerability\.score\.version.type: "keyword" } - - match: { .$idx0name.mappings.properties.vulnerability\.textual_score.type: "float" } - - match: { .$idx0name.mappings.properties.host\.cpu\.usage.type: "scaled_float" } - - match: { .$idx0name.mappings.properties.host\.cpu\.usage.scaling_factor: 1000 } - - match: { .$idx0name.mappings.properties.location.type: "geo_point" } - - match: { .$idx0name.mappings.properties.geo\.location.type: "geo_point" } - - match: { .$idx0name.mappings.properties.data_stream\.dataset.type: "constant_keyword" } - - match: { .$idx0name.mappings.properties.data_stream\.namespace.type: "constant_keyword" } - - match: { .$idx0name.mappings.properties.data_stream\.type.type: "constant_keyword" } - # not one of the three data_stream fields that are explicitly mapped to constant_keyword - - match: { .$idx0name.mappings.properties.data_stream\.custom.type: "keyword" } - - match: { .$idx0name.mappings.properties.structured_data.type: "flattened" } - - match: { .$idx0name.mappings.properties.exports.type: "flattened" } - - match: { .$idx0name.mappings.properties.top_level_imports.type: "flattened" } - - match: { .$idx0name.mappings.properties.nested\.imports.type: "flattened" } - # verifying the default mapping for strings into keyword, overriding the automatic numeric string detection - - match: { .$idx0name.mappings.properties.numeric_as_string.type: "keyword" } - - match: { .$idx0name.mappings.properties.socket\.ip.type: "ip" } - - match: { .$idx0name.mappings.properties.socket\.remote_ip.type: "ip" } - diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/20_basic.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/20_basic.yml index 296c692fa2d49..1ea39087211dd 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/20_basic.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/20_basic.yml @@ -51,6 +51,9 @@ setup: --- "Get data stream with default lifecycle": + - skip: + version: all + reason: https://github.com/elastic/elasticsearch/pull/100187 - do: indices.get_data_lifecycle: