Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

INGEST: Add Configuration Except. Data to Metdata #32322

Merged
merged 2 commits into from
Aug 15, 2018
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,6 @@ public void testInvalidMustacheTemplate() throws Exception {
String processorTag = randomAlphaOfLength(10);
ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, config));
assertThat(exception.getMessage(), equalTo("java.lang.RuntimeException: could not compile script"));
assertThat(exception.getHeader("processor_tag").get(0), equalTo(processorTag));
assertThat(exception.getMetadata("es.processor_tag").get(0), equalTo(processorTag));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -58,9 +58,9 @@ public void testCreateUnsupportedType() throws Exception {
fail("factory create should have failed");
} catch (ElasticsearchParseException e) {
assertThat(e.getMessage(), Matchers.equalTo("[type] type [" + type + "] not supported, cannot convert field."));
assertThat(e.getHeader("processor_type").get(0), equalTo(ConvertProcessor.TYPE));
assertThat(e.getHeader("property_name").get(0), equalTo("type"));
assertThat(e.getHeader("processor_tag"), nullValue());
assertThat(e.getMetadata("es.processor_type").get(0), equalTo(ConvertProcessor.TYPE));
assertThat(e.getMetadata("es.property_name").get(0), equalTo("type"));
assertThat(e.getMetadata("es.processor_tag"), nullValue());
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,6 @@ public void testInvalidMustacheTemplate() throws Exception {
String processorTag = randomAlphaOfLength(10);
ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, config));
assertThat(exception.getMessage(), equalTo("java.lang.RuntimeException: could not compile script"));
assertThat(exception.getHeader("processor_tag").get(0), equalTo(processorTag));
assertThat(exception.getMetadata("es.processor_tag").get(0), equalTo(processorTag));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,6 @@ public void testInvalidMustacheTemplate() throws Exception {
String processorTag = randomAlphaOfLength(10);
ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, config));
assertThat(exception.getMessage(), equalTo("java.lang.RuntimeException: could not compile script"));
assertThat(exception.getHeader("processor_tag").get(0), equalTo(processorTag));
assertThat(exception.getMetadata("es.processor_tag").get(0), equalTo(processorTag));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ public void testInvalidMustacheTemplate() throws Exception {
String processorTag = randomAlphaOfLength(10);
ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, config));
assertThat(exception.getMessage(), equalTo("java.lang.RuntimeException: could not compile script"));
assertThat(exception.getHeader("processor_tag").get(0), equalTo(processorTag));
assertThat(exception.getMetadata("es.processor_tag").get(0), equalTo(processorTag));
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -158,9 +158,9 @@ teardown:
}
- match: { error.root_cause.0.type: "parse_exception" }
- match: { error.root_cause.0.reason: "[field] required property is missing" }
- match: { error.root_cause.0.header.processor_tag: "fritag" }
- match: { error.root_cause.0.header.processor_type: "set" }
- match: { error.root_cause.0.header.property_name: "field" }
- match: { error.root_cause.0.processor_tag: "fritag" }
- match: { error.root_cause.0.processor_type: "set" }
- match: { error.root_cause.0.property_name: "field" }

---
"Test basic pipeline with on_failure in processor":
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -148,9 +148,9 @@ teardown:
}
- match: { error.root_cause.0.type: "parse_exception" }
- match: { error.root_cause.0.reason: "[on_failure] processors list cannot be empty" }
- match: { error.root_cause.0.header.processor_type: "fail" }
- match: { error.root_cause.0.header.processor_tag: "emptyfail" }
- match: { error.root_cause.0.header.property_name: "on_failure" }
- match: { error.root_cause.0.processor_type: "fail" }
- match: { error.root_cause.0.processor_tag: "emptyfail" }
- match: { error.root_cause.0.property_name: "on_failure" }

---
"Test pipeline with empty on_failure in pipeline":
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -107,9 +107,9 @@ teardown:
}
- match: { error.root_cause.0.type: "parse_exception" }
- match: { error.root_cause.0.reason: "[field] required property is missing" }
- match: { error.root_cause.0.header.processor_tag: "fails" }
- match: { error.root_cause.0.header.processor_type: "set" }
- match: { error.root_cause.0.header.property_name: "field" }
- match: { error.root_cause.0.processor_tag: "fails" }
- match: { error.root_cause.0.processor_type: "set" }
- match: { error.root_cause.0.property_name: "field" }

---
"Test simulate without index type and id":
Expand Down Expand Up @@ -198,9 +198,9 @@ teardown:
}
]
}
- is_false: error.root_cause.0.header.processor_type
- is_false: error.root_cause.0.header.processor_tag
- match: { error.root_cause.0.header.property_name: "pipeline" }
- is_false: error.root_cause.0.processor_type
- is_false: error.root_cause.0.processor_tag
- match: { error.root_cause.0.property_name: "pipeline" }
- match: { error.reason: "[pipeline] required property is missing" }

---
Expand Down Expand Up @@ -233,9 +233,9 @@ teardown:
}
- match: { error.root_cause.0.type: "parse_exception" }
- match: { error.root_cause.0.reason: "[value] required property is missing" }
- match: { error.root_cause.0.header.processor_type: "set" }
- match: { error.root_cause.0.header.property_name: "value" }
- is_false: error.root_cause.0.header.processor_tag
- match: { error.root_cause.0.processor_type: "set" }
- match: { error.root_cause.0.property_name: "value" }
- is_false: error.root_cause.0.processor_tag

---
"Test simulate with verbose flag":
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -347,6 +347,6 @@
}
]
}
- match: { error.header.processor_type: "set" }
- match: { error.processor_type: "set" }
- match: { error.type: "script_exception" }
- match: { error.reason: "Mustache function [join] must contain one and only one identifier" }
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@
}
]
}
- match: { error.header.processor_type: "script" }
- match: { error.processor_type: "script" }
- match: { error.type: "script_exception" }
- match: { error.reason: "compile error" }

Original file line number Diff line number Diff line change
Expand Up @@ -284,14 +284,14 @@ public static ElasticsearchException newConfigurationException(String processorT
msg = "[" + propertyName + "] " + reason;
}
ElasticsearchParseException exception = new ElasticsearchParseException(msg);
addHeadersToException(exception, processorType, processorTag, propertyName);
addMetadataToException(exception, processorType, processorTag, propertyName);
return exception;
}

public static ElasticsearchException newConfigurationException(String processorType, String processorTag,
String propertyName, Exception cause) {
ElasticsearchException exception = ExceptionsHelper.convertToElastic(cause);
addHeadersToException(exception, processorType, processorTag, propertyName);
addMetadataToException(exception, processorType, processorTag, propertyName);
return exception;
}

Expand Down Expand Up @@ -341,16 +341,16 @@ public String execute() {
}
}

private static void addHeadersToException(ElasticsearchException exception, String processorType,
String processorTag, String propertyName) {
private static void addMetadataToException(ElasticsearchException exception, String processorType,
String processorTag, String propertyName) {
if (processorType != null) {
exception.addHeader("processor_type", processorType);
exception.addMetadata("es.processor_type", processorType);
}
if (processorTag != null) {
exception.addHeader("processor_tag", processorTag);
exception.addMetadata("es.processor_tag", processorTag);
}
if (propertyName != null) {
exception.addHeader("property_name", propertyName);
exception.addMetadata("es.property_name", propertyName);
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -131,9 +131,9 @@ public void testReadProcessors() throws Exception {
ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class,
() -> ConfigurationUtils.readProcessorConfigs(config, registry));
assertThat(e.getMessage(), equalTo("No processor type exists with name [unknown_processor]"));
assertThat(e.getHeader("processor_tag"), equalTo(Collections.singletonList("my_unknown")));
assertThat(e.getHeader("processor_type"), equalTo(Collections.singletonList("unknown_processor")));
assertThat(e.getHeader("property_name"), is(nullValue()));
assertThat(e.getMetadata("es.processor_tag"), equalTo(Collections.singletonList("my_unknown")));
assertThat(e.getMetadata("es.processor_type"), equalTo(Collections.singletonList("unknown_processor")));
assertThat(e.getMetadata("es.property_name"), is(nullValue()));

List<Map<String, Object>> config2 = new ArrayList<>();
unknownTaggedConfig = new HashMap<>();
Expand All @@ -144,17 +144,17 @@ public void testReadProcessors() throws Exception {
config2.add(Collections.singletonMap("second_unknown_processor", secondUnknonwTaggedConfig));
e = expectThrows(ElasticsearchParseException.class, () -> ConfigurationUtils.readProcessorConfigs(config2, registry));
assertThat(e.getMessage(), equalTo("No processor type exists with name [unknown_processor]"));
assertThat(e.getHeader("processor_tag"), equalTo(Collections.singletonList("my_unknown")));
assertThat(e.getHeader("processor_type"), equalTo(Collections.singletonList("unknown_processor")));
assertThat(e.getHeader("property_name"), is(nullValue()));
assertThat(e.getMetadata("es.processor_tag"), equalTo(Collections.singletonList("my_unknown")));
assertThat(e.getMetadata("es.processor_type"), equalTo(Collections.singletonList("unknown_processor")));
assertThat(e.getMetadata("es.property_name"), is(nullValue()));

assertThat(e.getSuppressed().length, equalTo(1));
assertThat(e.getSuppressed()[0], instanceOf(ElasticsearchParseException.class));
ElasticsearchParseException e2 = (ElasticsearchParseException) e.getSuppressed()[0];
assertThat(e2.getMessage(), equalTo("No processor type exists with name [second_unknown_processor]"));
assertThat(e2.getHeader("processor_tag"), equalTo(Collections.singletonList("my_second_unknown")));
assertThat(e2.getHeader("processor_type"), equalTo(Collections.singletonList("second_unknown_processor")));
assertThat(e2.getHeader("property_name"), is(nullValue()));
assertThat(e2.getMetadata("es.processor_tag"), equalTo(Collections.singletonList("my_second_unknown")));
assertThat(e2.getMetadata("es.processor_type"), equalTo(Collections.singletonList("second_unknown_processor")));
assertThat(e2.getMetadata("es.property_name"), is(nullValue()));
}

public void testReadProcessorFromObjectOrMap() throws Exception {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -356,8 +356,8 @@ public void testValidate() throws Exception {
ElasticsearchParseException e =
expectThrows(ElasticsearchParseException.class, () -> store.validatePipeline(ingestInfos, putRequest));
assertEquals("Processor type [remove] is not installed on node [" + node2 + "]", e.getMessage());
assertEquals("remove", e.getHeader("processor_type").get(0));
assertEquals("tag2", e.getHeader("processor_tag").get(0));
assertEquals("remove", e.getMetadata("es.processor_type").get(0));
assertEquals("tag2", e.getMetadata("es.processor_tag").get(0));

ingestInfos.put(node2, new IngestInfo(Arrays.asList(new ProcessorInfo("set"), new ProcessorInfo("remove"))));
store.validatePipeline(ingestInfos, putRequest);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,9 @@ public void testProcessor_noField() throws Exception {
SetSecurityUserProcessor.Factory factory = new SetSecurityUserProcessor.Factory(null);
Map<String, Object> config = new HashMap<>();
ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", config));
assertThat(e.getHeader("property_name").get(0), equalTo("field"));
assertThat(e.getHeader("processor_type").get(0), equalTo(SetSecurityUserProcessor.TYPE));
assertThat(e.getHeader("processor_tag").get(0), equalTo("_tag"));
assertThat(e.getMetadata("es.property_name").get(0), equalTo("field"));
assertThat(e.getMetadata("es.processor_type").get(0), equalTo(SetSecurityUserProcessor.TYPE));
assertThat(e.getMetadata("es.processor_tag").get(0), equalTo("_tag"));
}

public void testProcessor_validProperties() throws Exception {
Expand All @@ -52,9 +52,9 @@ public void testProcessor_invalidProperties() throws Exception {
config.put("field", "_field");
config.put("properties", Arrays.asList("invalid"));
ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", config));
assertThat(e.getHeader("property_name").get(0), equalTo("properties"));
assertThat(e.getHeader("processor_type").get(0), equalTo(SetSecurityUserProcessor.TYPE));
assertThat(e.getHeader("processor_tag").get(0), equalTo("_tag"));
assertThat(e.getMetadata("es.property_name").get(0), equalTo("properties"));
assertThat(e.getMetadata("es.processor_type").get(0), equalTo(SetSecurityUserProcessor.TYPE));
assertThat(e.getMetadata("es.processor_tag").get(0), equalTo("_tag"));
}

}