From 642db0d49bb2ac9581c1c58602d9417cec1eb9d8 Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Fri, 2 Aug 2024 14:44:32 -0500 Subject: [PATCH] ADD: data prepper plugin schema generation (#4777) * ADD: data-prepper-plugin-schema Signed-off-by: George Chen --- config/checkstyle/checkstyle-suppressions.xml | 2 + .../model/configuration/PipelineModel.java | 15 +- data-prepper-plugin-schema-cli/README.md | 12 ++ data-prepper-plugin-schema-cli/build.gradle | 29 ++++ .../DataPrepperPluginSchemaExecute.java | 74 ++++++++++ .../schemas/JsonSchemaConverter.java | 52 +++++++ .../PluginConfigsJsonSchemaConverter.java | 135 ++++++++++++++++++ .../schemas/module/CustomJacksonModule.java | 31 ++++ .../schemas/JsonSchemaConverterTest.java | 60 ++++++++ .../PluginConfigsJsonSchemaConverterIT.java | 80 +++++++++++ .../PluginConfigsJsonSchemaConverterTest.java | 110 ++++++++++++++ settings.gradle | 1 + 12 files changed, 596 insertions(+), 5 deletions(-) create mode 100644 data-prepper-plugin-schema-cli/README.md create mode 100644 data-prepper-plugin-schema-cli/build.gradle create mode 100644 data-prepper-plugin-schema-cli/src/main/java/org/opensearch/dataprepper/schemas/DataPrepperPluginSchemaExecute.java create mode 100644 data-prepper-plugin-schema-cli/src/main/java/org/opensearch/dataprepper/schemas/JsonSchemaConverter.java create mode 100644 data-prepper-plugin-schema-cli/src/main/java/org/opensearch/dataprepper/schemas/PluginConfigsJsonSchemaConverter.java create mode 100644 data-prepper-plugin-schema-cli/src/main/java/org/opensearch/dataprepper/schemas/module/CustomJacksonModule.java create mode 100644 data-prepper-plugin-schema-cli/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterTest.java create mode 100644 data-prepper-plugin-schema-cli/src/test/java/org/opensearch/dataprepper/schemas/PluginConfigsJsonSchemaConverterIT.java create mode 100644 data-prepper-plugin-schema-cli/src/test/java/org/opensearch/dataprepper/schemas/PluginConfigsJsonSchemaConverterTest.java diff --git a/config/checkstyle/checkstyle-suppressions.xml b/config/checkstyle/checkstyle-suppressions.xml index 42c37e7dd5..ab3ba001a9 100644 --- a/config/checkstyle/checkstyle-suppressions.xml +++ b/config/checkstyle/checkstyle-suppressions.xml @@ -14,4 +14,6 @@ + + diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/configuration/PipelineModel.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/configuration/PipelineModel.java index 1c8221f899..7af56175a0 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/configuration/PipelineModel.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/configuration/PipelineModel.java @@ -24,25 +24,30 @@ * @since 1.2 */ public class PipelineModel { + public static final String SOURCE_PLUGIN_TYPE = "source"; + public static final String PROCESSOR_PLUGIN_TYPE = "processor"; + public static final String BUFFER_PLUGIN_TYPE = "buffer"; + public static final String ROUTE_PLUGIN_TYPE = "route"; + public static final String SINK_PLUGIN_TYPE = "sink"; private static final Logger LOG = LoggerFactory.getLogger(PipelineModel.class); - @JsonProperty("source") + @JsonProperty(SOURCE_PLUGIN_TYPE) private final PluginModel source; - @JsonProperty("processor") + @JsonProperty(PROCESSOR_PLUGIN_TYPE) @JsonInclude(JsonInclude.Include.NON_NULL) private final List processors; - @JsonProperty("buffer") + @JsonProperty(BUFFER_PLUGIN_TYPE) @JsonInclude(JsonInclude.Include.NON_NULL) private final PluginModel buffer; @JsonProperty("routes") - @JsonAlias("route") + @JsonAlias(ROUTE_PLUGIN_TYPE) @JsonInclude(JsonInclude.Include.NON_EMPTY) private final List routes; - @JsonProperty("sink") + @JsonProperty(SINK_PLUGIN_TYPE) private final List sinks; @JsonProperty("workers") diff --git a/data-prepper-plugin-schema-cli/README.md b/data-prepper-plugin-schema-cli/README.md new file mode 100644 index 0000000000..7a4d9bc11b --- /dev/null +++ b/data-prepper-plugin-schema-cli/README.md @@ -0,0 +1,12 @@ +# Data Prepper Plugin Schema CLI + +This module includes the SDK and CLI for generating schemas for Data Prepper pipeline plugins. + +## CLI Usage + +``` +./gradlew :data-prepper-plugin-schema-cli:run --args='--plugin_type=processor --plugin_names=grok' +``` + +* plugin_type: A required parameter specifies type of processor. Valid options are `source`, `buffer`, `processor`, `route`, `sink`. +* plugin_names: An optional parameter filters the result by plugin names separated by `,`, e.g. `grok,date`. diff --git a/data-prepper-plugin-schema-cli/build.gradle b/data-prepper-plugin-schema-cli/build.gradle new file mode 100644 index 0000000000..2c2db93ee6 --- /dev/null +++ b/data-prepper-plugin-schema-cli/build.gradle @@ -0,0 +1,29 @@ +plugins { + id 'data-prepper.publish' + id 'application' +} + +application { + mainClass = 'org.opensearch.dataprepper.schemas.DataPrepperPluginSchemaExecute' +} + +dependencies { + implementation project(':data-prepper-plugins') + implementation project(':data-prepper-plugin-framework') + implementation 'com.fasterxml.jackson.core:jackson-databind' + implementation 'org.reflections:reflections:0.10.2' + implementation 'com.github.victools:jsonschema-maven-plugin:4.35.0' + implementation 'com.github.victools:jsonschema-generator:4.35.0' + implementation 'com.github.victools:jsonschema-module-jackson:4.35.0' + implementation 'com.github.victools:jsonschema-module-jakarta-validation:4.35.0' + implementation 'javax.inject:javax.inject:1' + implementation 'info.picocli:picocli:4.6.1' + implementation(libs.spring.core) { + exclude group: 'commons-logging', module: 'commons-logging' + } + implementation(libs.spring.context) { + exclude group: 'commons-logging', module: 'commons-logging' + } + testImplementation(platform("org.junit:junit-bom:5.9.1")) + testImplementation("org.junit.jupiter:junit-jupiter") +} \ No newline at end of file diff --git a/data-prepper-plugin-schema-cli/src/main/java/org/opensearch/dataprepper/schemas/DataPrepperPluginSchemaExecute.java b/data-prepper-plugin-schema-cli/src/main/java/org/opensearch/dataprepper/schemas/DataPrepperPluginSchemaExecute.java new file mode 100644 index 0000000000..a1a76c0510 --- /dev/null +++ b/data-prepper-plugin-schema-cli/src/main/java/org/opensearch/dataprepper/schemas/DataPrepperPluginSchemaExecute.java @@ -0,0 +1,74 @@ +package org.opensearch.dataprepper.schemas; + +import com.github.victools.jsonschema.generator.Module; +import com.github.victools.jsonschema.generator.OptionPreset; +import com.github.victools.jsonschema.generator.SchemaVersion; +import com.github.victools.jsonschema.module.jakarta.validation.JakartaValidationModule; +import com.github.victools.jsonschema.module.jakarta.validation.JakartaValidationOption; +import org.opensearch.dataprepper.schemas.module.CustomJacksonModule; +import org.reflections.Reflections; +import org.reflections.scanners.Scanners; +import org.reflections.util.ClasspathHelper; +import org.reflections.util.ConfigurationBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import picocli.CommandLine; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static com.github.victools.jsonschema.module.jackson.JacksonOption.RESPECT_JSONPROPERTY_REQUIRED; + +public class DataPrepperPluginSchemaExecute implements Runnable { + private static final Logger LOG = LoggerFactory.getLogger(DataPrepperPluginSchemaExecute.class); + static final String DEFAULT_PLUGINS_CLASSPATH = "org.opensearch.dataprepper.plugins"; + + @CommandLine.Option(names = {"--plugin_type"}, required = true) + private String pluginTypeName; + + @CommandLine.Option(names = {"--plugin_names"}) + private String pluginNames; + + @CommandLine.Option(names = {"--site.url"}, defaultValue = "https://opensearch.org") + private String siteUrl; + @CommandLine.Option(names = {"--site.baseurl"}, defaultValue = "/docs/latest") + private String siteBaseUrl; + + public static void main(String[] args) { + final int exitCode = new CommandLine(new DataPrepperPluginSchemaExecute()).execute(args); + System.exit(exitCode); + } + + @Override + public void run() { + final List modules = List.of( + new CustomJacksonModule(RESPECT_JSONPROPERTY_REQUIRED), + new JakartaValidationModule(JakartaValidationOption.NOT_NULLABLE_FIELD_IS_REQUIRED, + JakartaValidationOption.INCLUDE_PATTERN_EXPRESSIONS) + ); + final Reflections reflections = new Reflections(new ConfigurationBuilder() + .setUrls(ClasspathHelper.forPackage(DEFAULT_PLUGINS_CLASSPATH)) + .setScanners(Scanners.TypesAnnotated, Scanners.SubTypes)); + final PluginConfigsJsonSchemaConverter pluginConfigsJsonSchemaConverter = new PluginConfigsJsonSchemaConverter( + reflections, new JsonSchemaConverter(modules), siteUrl, siteBaseUrl); + final Class pluginType = pluginConfigsJsonSchemaConverter.pluginTypeNameToPluginType(pluginTypeName); + final Map pluginNameToJsonSchemaMap = pluginConfigsJsonSchemaConverter.convertPluginConfigsIntoJsonSchemas( + SchemaVersion.DRAFT_2020_12, OptionPreset.PLAIN_JSON, pluginType); + if (pluginNames == null) { + pluginNameToJsonSchemaMap.values().forEach(System.out::println); + } else { + final Set pluginNamesSet = Set.of(pluginNames.split(",")); + final List result = pluginNamesSet.stream().flatMap(name -> { + if (!pluginNameToJsonSchemaMap.containsKey(name)) { + LOG.error("plugin name: {} not found", name); + return Stream.empty(); + } + return Stream.of(pluginNameToJsonSchemaMap.get(name)); + }).collect(Collectors.toList()); + result.forEach(System.out::println); + } + } +} diff --git a/data-prepper-plugin-schema-cli/src/main/java/org/opensearch/dataprepper/schemas/JsonSchemaConverter.java b/data-prepper-plugin-schema-cli/src/main/java/org/opensearch/dataprepper/schemas/JsonSchemaConverter.java new file mode 100644 index 0000000000..fe08825af4 --- /dev/null +++ b/data-prepper-plugin-schema-cli/src/main/java/org/opensearch/dataprepper/schemas/JsonSchemaConverter.java @@ -0,0 +1,52 @@ +package org.opensearch.dataprepper.schemas; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.github.victools.jsonschema.generator.FieldScope; +import com.github.victools.jsonschema.generator.Module; +import com.github.victools.jsonschema.generator.OptionPreset; +import com.github.victools.jsonschema.generator.SchemaGenerator; +import com.github.victools.jsonschema.generator.SchemaGeneratorConfig; +import com.github.victools.jsonschema.generator.SchemaGeneratorConfigBuilder; +import com.github.victools.jsonschema.generator.SchemaGeneratorConfigPart; +import com.github.victools.jsonschema.generator.SchemaVersion; + +import java.util.List; + +public class JsonSchemaConverter { + static final String DEPRECATED_SINCE_KEY = "deprecated"; + private final List jsonSchemaGeneratorModules; + + public JsonSchemaConverter(final List jsonSchemaGeneratorModules) { + this.jsonSchemaGeneratorModules = jsonSchemaGeneratorModules; + } + + public ObjectNode convertIntoJsonSchema( + final SchemaVersion schemaVersion, final OptionPreset optionPreset, final Class clazz) + throws JsonProcessingException { + final SchemaGeneratorConfigBuilder configBuilder = new SchemaGeneratorConfigBuilder( + schemaVersion, optionPreset); + loadJsonSchemaGeneratorModules(configBuilder); + final SchemaGeneratorConfigPart scopeSchemaGeneratorConfigPart = configBuilder.forFields(); + overrideInstanceAttributeWithDeprecated(scopeSchemaGeneratorConfigPart); + + final SchemaGeneratorConfig config = configBuilder.build(); + final SchemaGenerator generator = new SchemaGenerator(config); + return generator.generateSchema(clazz); + } + + private void loadJsonSchemaGeneratorModules(final SchemaGeneratorConfigBuilder configBuilder) { + jsonSchemaGeneratorModules.forEach(configBuilder::with); + } + + private void overrideInstanceAttributeWithDeprecated( + final SchemaGeneratorConfigPart scopeSchemaGeneratorConfigPart) { + scopeSchemaGeneratorConfigPart.withInstanceAttributeOverride((node, field, context) -> { + final Deprecated deprecatedAnnotation = field.getAnnotationConsideringFieldAndGetter( + Deprecated.class); + if (deprecatedAnnotation != null) { + node.put(DEPRECATED_SINCE_KEY, deprecatedAnnotation.since()); + } + }); + } +} diff --git a/data-prepper-plugin-schema-cli/src/main/java/org/opensearch/dataprepper/schemas/PluginConfigsJsonSchemaConverter.java b/data-prepper-plugin-schema-cli/src/main/java/org/opensearch/dataprepper/schemas/PluginConfigsJsonSchemaConverter.java new file mode 100644 index 0000000000..b7f4c1a531 --- /dev/null +++ b/data-prepper-plugin-schema-cli/src/main/java/org/opensearch/dataprepper/schemas/PluginConfigsJsonSchemaConverter.java @@ -0,0 +1,135 @@ +package org.opensearch.dataprepper.schemas; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.github.victools.jsonschema.generator.OptionPreset; +import com.github.victools.jsonschema.generator.SchemaVersion; +import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; +import org.opensearch.dataprepper.model.buffer.Buffer; +import org.opensearch.dataprepper.model.configuration.ConditionalRoute; +import org.opensearch.dataprepper.model.processor.Processor; +import org.opensearch.dataprepper.model.sink.Sink; +import org.opensearch.dataprepper.model.source.Source; +import org.reflections.Reflections; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.opensearch.dataprepper.model.configuration.PipelineModel.BUFFER_PLUGIN_TYPE; +import static org.opensearch.dataprepper.model.configuration.PipelineModel.PROCESSOR_PLUGIN_TYPE; +import static org.opensearch.dataprepper.model.configuration.PipelineModel.ROUTE_PLUGIN_TYPE; +import static org.opensearch.dataprepper.model.configuration.PipelineModel.SINK_PLUGIN_TYPE; +import static org.opensearch.dataprepper.model.configuration.PipelineModel.SOURCE_PLUGIN_TYPE; + +public class PluginConfigsJsonSchemaConverter { + private static final Logger LOG = LoggerFactory.getLogger(PluginConfigsJsonSchemaConverter.class); + static final String SITE_URL_PLACEHOLDER = "{{site.url}}"; + static final String SITE_BASE_URL_PLACEHOLDER = "{{site.baseurl}}"; + static final String DOCUMENTATION_LINK_KEY = "documentation"; + static final String PLUGIN_NAME_KEY = "name"; + static final String PLUGIN_DOCUMENTATION_URL_FORMAT = + "%s%s/data-prepper/pipelines/configuration/%s/%s/"; + static final Map, String> PLUGIN_TYPE_TO_URI_PARAMETER_MAP = Map.of( + Source.class, "sources", + Processor.class, "processors", + ConditionalRoute.class, "processors", + Buffer.class, "buffers", + Sink.class, "sinks" + ); + static final String CONDITIONAL_ROUTE_PROCESSOR_NAME = "routes"; + static final Map> PLUGIN_TYPE_NAME_TO_CLASS_MAP = Map.of( + SOURCE_PLUGIN_TYPE, Source.class, + PROCESSOR_PLUGIN_TYPE, Processor.class, + ROUTE_PLUGIN_TYPE, ConditionalRoute.class, + BUFFER_PLUGIN_TYPE, Buffer.class, + SINK_PLUGIN_TYPE, Sink.class); + + private final String siteUrl; + private final String siteBaseUrl; + private final Reflections reflections; + private final JsonSchemaConverter jsonSchemaConverter; + + public PluginConfigsJsonSchemaConverter( + final Reflections reflections, + final JsonSchemaConverter jsonSchemaConverter, + final String siteUrl, + final String siteBaseUrl) { + this.reflections = reflections; + this.jsonSchemaConverter = jsonSchemaConverter; + this.siteUrl = siteUrl == null ? SITE_URL_PLACEHOLDER : siteUrl; + this.siteBaseUrl = siteBaseUrl == null ? SITE_BASE_URL_PLACEHOLDER : siteBaseUrl; + } + + public Set validPluginTypeNames() { + return PLUGIN_TYPE_NAME_TO_CLASS_MAP.keySet(); + } + + public Class pluginTypeNameToPluginType(final String pluginTypeName) { + final Class pluginType = PLUGIN_TYPE_NAME_TO_CLASS_MAP.get(pluginTypeName); + if (pluginType == null) { + throw new IllegalArgumentException(String.format("Invalid plugin type name: %s.", pluginTypeName)); + } + return pluginType; + } + + public Map convertPluginConfigsIntoJsonSchemas( + final SchemaVersion schemaVersion, final OptionPreset optionPreset, final Class pluginType) { + final Map> nameToConfigClass = scanForPluginConfigs(pluginType); + return nameToConfigClass.entrySet().stream() + .flatMap(entry -> { + final String pluginName = entry.getKey(); + String value; + try { + final ObjectNode jsonSchemaNode = jsonSchemaConverter.convertIntoJsonSchema( + schemaVersion, optionPreset, entry.getValue()); + addPluginName(jsonSchemaNode, pluginName); + addDocumentationLink(jsonSchemaNode, pluginName, pluginType); + value = jsonSchemaNode.toPrettyString(); + } catch (JsonProcessingException e) { + LOG.error("Encountered error retrieving JSON schema for {}", pluginName); + return Stream.empty(); + } + return Stream.of(Map.entry(entry.getKey(), value)); + }) + .filter(entry -> Objects.nonNull(entry.getValue())) + .collect(Collectors.toMap( + Map.Entry::getKey, + Map.Entry::getValue + )); + } + + private Map> scanForPluginConfigs(final Class pluginType) { + if (ConditionalRoute.class.equals(pluginType)) { + return Map.of(CONDITIONAL_ROUTE_PROCESSOR_NAME, ConditionalRoute.class); + } + return reflections.getTypesAnnotatedWith(DataPrepperPlugin.class).stream() + .map(clazz -> clazz.getAnnotation(DataPrepperPlugin.class)) + .filter(dataPrepperPlugin -> pluginType.equals(dataPrepperPlugin.pluginType())) + .collect(Collectors.toMap( + DataPrepperPlugin::name, + DataPrepperPlugin::pluginConfigurationType + )); + } + + private void addDocumentationLink(final ObjectNode jsonSchemaNode, + final String pluginName, + final Class pluginType) { + jsonSchemaNode.put(DOCUMENTATION_LINK_KEY, + String.format( + PLUGIN_DOCUMENTATION_URL_FORMAT, + siteUrl, + siteBaseUrl, + PLUGIN_TYPE_TO_URI_PARAMETER_MAP.get(pluginType), + pluginName)); + } + + private void addPluginName(final ObjectNode jsonSchemaNode, + final String pluginName) { + jsonSchemaNode.put(PLUGIN_NAME_KEY, pluginName); + } +} diff --git a/data-prepper-plugin-schema-cli/src/main/java/org/opensearch/dataprepper/schemas/module/CustomJacksonModule.java b/data-prepper-plugin-schema-cli/src/main/java/org/opensearch/dataprepper/schemas/module/CustomJacksonModule.java new file mode 100644 index 0000000000..09c649cc4c --- /dev/null +++ b/data-prepper-plugin-schema-cli/src/main/java/org/opensearch/dataprepper/schemas/module/CustomJacksonModule.java @@ -0,0 +1,31 @@ +package org.opensearch.dataprepper.schemas.module; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.PropertyNamingStrategies; +import com.github.victools.jsonschema.generator.MemberScope; +import com.github.victools.jsonschema.module.jackson.JacksonModule; +import com.github.victools.jsonschema.module.jackson.JacksonOption; + +public class CustomJacksonModule extends JacksonModule { + + public CustomJacksonModule() { + super(); + } + + public CustomJacksonModule(JacksonOption... options) { + super(options); + } + + @Override + protected String getPropertyNameOverrideBasedOnJsonPropertyAnnotation(MemberScope member) { + JsonProperty annotation = member.getAnnotationConsideringFieldAndGetter(JsonProperty.class); + if (annotation != null) { + String nameOverride = annotation.value(); + // check for invalid overrides + if (nameOverride != null && !nameOverride.isEmpty() && !nameOverride.equals(member.getDeclaredName())) { + return nameOverride; + } + } + return PropertyNamingStrategies.SNAKE_CASE.nameForField(null, null, member.getName()); + } +} diff --git a/data-prepper-plugin-schema-cli/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterTest.java b/data-prepper-plugin-schema-cli/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterTest.java new file mode 100644 index 0000000000..d5d172f8c0 --- /dev/null +++ b/data-prepper-plugin-schema-cli/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterTest.java @@ -0,0 +1,60 @@ +package org.opensearch.dataprepper.schemas; + +import com.fasterxml.jackson.annotation.JsonClassDescription; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.github.victools.jsonschema.generator.Module; +import com.github.victools.jsonschema.generator.OptionPreset; +import com.github.victools.jsonschema.generator.SchemaVersion; +import org.junit.jupiter.api.Test; +import org.opensearch.dataprepper.schemas.module.CustomJacksonModule; + +import java.util.Collections; +import java.util.List; + +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; + +class JsonSchemaConverterTest { + + public JsonSchemaConverter createObjectUnderTest(final List modules) { + return new JsonSchemaConverter(modules); + } + + @Test + void testConvertIntoJsonSchemaWithDefaultModules() throws JsonProcessingException { + final JsonSchemaConverter jsonSchemaConverter = createObjectUnderTest(Collections.emptyList()); + final ObjectNode jsonSchemaNode = jsonSchemaConverter.convertIntoJsonSchema( + SchemaVersion.DRAFT_2020_12, OptionPreset.PLAIN_JSON, TestConfig.class); + assertThat(jsonSchemaNode, instanceOf(ObjectNode.class)); + } + + @Test + void testConvertIntoJsonSchemaWithCustomJacksonModule() throws JsonProcessingException { + final JsonSchemaConverter jsonSchemaConverter = createObjectUnderTest( + Collections.singletonList(new CustomJacksonModule())); + final ObjectNode jsonSchemaNode = jsonSchemaConverter.convertIntoJsonSchema( + SchemaVersion.DRAFT_2020_12, OptionPreset.PLAIN_JSON, TestConfig.class); + assertThat(jsonSchemaNode, instanceOf(ObjectNode.class)); + assertThat(jsonSchemaNode.has("description"), is(true)); + final JsonNode propertiesNode = jsonSchemaNode.at("/properties"); + assertThat(propertiesNode, instanceOf(ObjectNode.class)); + assertThat(propertiesNode.has("test_attribute_with_getter"), is(true)); + assertThat(propertiesNode.has("custom_test_attribute"), is(true)); + } + + @JsonClassDescription("test config") + static class TestConfig { + private String testAttributeWithGetter; + + @JsonProperty("custom_test_attribute") + private String testAttributeWithJsonPropertyAnnotation; + + public String getTestAttributeWithGetter() { + return testAttributeWithGetter; + } + } +} \ No newline at end of file diff --git a/data-prepper-plugin-schema-cli/src/test/java/org/opensearch/dataprepper/schemas/PluginConfigsJsonSchemaConverterIT.java b/data-prepper-plugin-schema-cli/src/test/java/org/opensearch/dataprepper/schemas/PluginConfigsJsonSchemaConverterIT.java new file mode 100644 index 0000000000..71e9bf5faa --- /dev/null +++ b/data-prepper-plugin-schema-cli/src/test/java/org/opensearch/dataprepper/schemas/PluginConfigsJsonSchemaConverterIT.java @@ -0,0 +1,80 @@ +package org.opensearch.dataprepper.schemas; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.victools.jsonschema.generator.Module; +import com.github.victools.jsonschema.generator.OptionPreset; +import com.github.victools.jsonschema.generator.SchemaVersion; +import com.github.victools.jsonschema.module.jackson.JacksonModule; +import com.github.victools.jsonschema.module.jakarta.validation.JakartaValidationModule; +import com.github.victools.jsonschema.module.jakarta.validation.JakartaValidationOption; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.reflections.Reflections; +import org.reflections.scanners.Scanners; +import org.reflections.util.ClasspathHelper; +import org.reflections.util.ConfigurationBuilder; + +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.stream.Stream; + +import static com.github.victools.jsonschema.module.jackson.JacksonOption.RESPECT_JSONPROPERTY_REQUIRED; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.opensearch.dataprepper.schemas.PluginConfigsJsonSchemaConverter.DOCUMENTATION_LINK_KEY; +import static org.opensearch.dataprepper.schemas.PluginConfigsJsonSchemaConverter.PLUGIN_NAME_KEY; + +class PluginConfigsJsonSchemaConverterIT { + static final String DEFAULT_PLUGINS_CLASSPATH = "org.opensearch.dataprepper.plugins"; + private static final String TEST_URL = String.format("https://%s/", UUID.randomUUID()); + private static final String TEST_BASE_URL = String.format("/%s", UUID.randomUUID()); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + static final TypeReference> MAP_TYPE_REFERENCE = new TypeReference<>() {}; + + private PluginConfigsJsonSchemaConverter objectUnderTest; + + @BeforeEach + void setUp() { + final List modules = List.of( + new JacksonModule(RESPECT_JSONPROPERTY_REQUIRED), + new JakartaValidationModule(JakartaValidationOption.NOT_NULLABLE_FIELD_IS_REQUIRED, + JakartaValidationOption.INCLUDE_PATTERN_EXPRESSIONS) + ); + final Reflections reflections = new Reflections(new ConfigurationBuilder() + .setUrls(ClasspathHelper.forPackage(DEFAULT_PLUGINS_CLASSPATH)) + .setScanners(Scanners.TypesAnnotated, Scanners.SubTypes)); + objectUnderTest = new PluginConfigsJsonSchemaConverter( + reflections, new JsonSchemaConverter(modules), TEST_URL, TEST_BASE_URL); + } + + @ParameterizedTest + @MethodSource("getValidPluginTypes") + void testConvertPluginConfigsIntoJsonSchemas(final Class pluginType) { + final Map result = objectUnderTest.convertPluginConfigsIntoJsonSchemas( + SchemaVersion.DRAFT_2020_12, OptionPreset.PLAIN_JSON, pluginType); + assertThat(result.isEmpty(), is(false)); + result.values().forEach(schema -> { + final Map schemaMap; + try { + schemaMap = OBJECT_MAPPER.readValue(schema, MAP_TYPE_REFERENCE); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + assertThat(schemaMap, notNullValue()); + assertThat(schemaMap.containsKey(PLUGIN_NAME_KEY), is(true)); + assertThat(((String) schemaMap.get(DOCUMENTATION_LINK_KEY)).startsWith(TEST_URL + TEST_BASE_URL), + is(true)); + }); + } + + private static Stream getValidPluginTypes() { + return PluginConfigsJsonSchemaConverter.PLUGIN_TYPE_TO_URI_PARAMETER_MAP.keySet() + .stream().flatMap(clazz -> Stream.of(Arguments.of(clazz))); + } +} \ No newline at end of file diff --git a/data-prepper-plugin-schema-cli/src/test/java/org/opensearch/dataprepper/schemas/PluginConfigsJsonSchemaConverterTest.java b/data-prepper-plugin-schema-cli/src/test/java/org/opensearch/dataprepper/schemas/PluginConfigsJsonSchemaConverterTest.java new file mode 100644 index 0000000000..3d1c1b585a --- /dev/null +++ b/data-prepper-plugin-schema-cli/src/test/java/org/opensearch/dataprepper/schemas/PluginConfigsJsonSchemaConverterTest.java @@ -0,0 +1,110 @@ +package org.opensearch.dataprepper.schemas; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.github.victools.jsonschema.generator.OptionPreset; +import com.github.victools.jsonschema.generator.SchemaVersion; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; +import org.reflections.Reflections; + +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.opensearch.dataprepper.schemas.PluginConfigsJsonSchemaConverter.DOCUMENTATION_LINK_KEY; +import static org.opensearch.dataprepper.schemas.PluginConfigsJsonSchemaConverter.PLUGIN_NAME_KEY; +import static org.opensearch.dataprepper.schemas.PluginConfigsJsonSchemaConverter.PLUGIN_TYPE_NAME_TO_CLASS_MAP; + +@ExtendWith(MockitoExtension.class) +class PluginConfigsJsonSchemaConverterTest { + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + static final TypeReference> MAP_TYPE_REFERENCE = new TypeReference<>() {}; + + @Mock + private JsonSchemaConverter jsonSchemaConverter; + + @Mock + private Reflections reflections; + + @InjectMocks + private PluginConfigsJsonSchemaConverter objectUnderTest; + + @Test + void testValidPluginTypeNames() { + assertThat(PLUGIN_TYPE_NAME_TO_CLASS_MAP.keySet().containsAll(objectUnderTest.validPluginTypeNames()), + is(true)); + } + + @Test + void testPluginTypeNameToPluginTypeWithValidInput() { + objectUnderTest.validPluginTypeNames().forEach( + pluginType -> assertThat(objectUnderTest.pluginTypeNameToPluginType(pluginType), + equalTo(PLUGIN_TYPE_NAME_TO_CLASS_MAP.get(pluginType)))); + } + + @Test + void testPluginTypeNameToPluginTypeWithInValidInput() { + final String inValidPluginType = "invalid-" + UUID.randomUUID(); + assertThrows( + IllegalArgumentException.class, () -> objectUnderTest.pluginTypeNameToPluginType(inValidPluginType)); + } + + @Test + void testConvertPluginConfigsIntoJsonSchemasHappyPath() throws JsonProcessingException { + when(reflections.getTypesAnnotatedWith(eq(DataPrepperPlugin.class))).thenReturn(Set.of(TestPlugin.class)); + final ObjectNode objectNode = OBJECT_MAPPER.createObjectNode(); + when(jsonSchemaConverter.convertIntoJsonSchema( + any(SchemaVersion.class), any(OptionPreset.class), eq(TestPluginConfig.class))).thenReturn(objectNode); + final Map result = objectUnderTest.convertPluginConfigsIntoJsonSchemas( + SchemaVersion.DRAFT_2020_12, OptionPreset.PLAIN_JSON, TestPluginType.class); + assertThat(result.size(), equalTo(1)); + final Map schemaMap = OBJECT_MAPPER.readValue(result.get("test_plugin"), MAP_TYPE_REFERENCE); + assertThat(schemaMap, notNullValue()); + assertThat(schemaMap.get(DOCUMENTATION_LINK_KEY), equalTo( + "{{site.url}}{{site.baseurl}}/data-prepper/pipelines/configuration/null/test_plugin/" + )); + assertThat(schemaMap.containsKey(PLUGIN_NAME_KEY), is(true)); + } + + @Test + void testConvertPluginConfigsIntoJsonSchemasWithError() throws JsonProcessingException { + when(reflections.getTypesAnnotatedWith(eq(DataPrepperPlugin.class))).thenReturn(Set.of(TestPlugin.class)); + final JsonProcessingException jsonProcessingException = mock(JsonProcessingException.class); + when(jsonSchemaConverter.convertIntoJsonSchema( + any(SchemaVersion.class), any(OptionPreset.class), eq(TestPluginConfig.class))).thenThrow( + jsonProcessingException); + final Map result = objectUnderTest.convertPluginConfigsIntoJsonSchemas( + SchemaVersion.DRAFT_2020_12, OptionPreset.PLAIN_JSON, TestPluginType.class); + assertThat(result.isEmpty(), is(true)); + } + + @DataPrepperPlugin( + name = "test_plugin", pluginType = TestPluginType.class, pluginConfigurationType = TestPluginConfig.class) + static class TestPlugin { + + } + + static class TestPluginConfig { + + } + + static class TestPluginType { + + } +} \ No newline at end of file diff --git a/settings.gradle b/settings.gradle index cb7e888c53..18ccd4dc7b 100644 --- a/settings.gradle +++ b/settings.gradle @@ -104,6 +104,7 @@ include 'data-prepper-core' include 'data-prepper-main' include 'data-prepper-pipeline-parser' include 'data-prepper-plugin-framework' +include 'data-prepper-plugin-schema-cli' include 'data-prepper-plugins:common' include 'data-prepper-plugins:armeria-common' include 'data-prepper-plugins:anomaly-detector-processor'