diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/DefaultTransformExtension.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/DefaultTransformExtension.java new file mode 100644 index 0000000000000..9cccbade339dc --- /dev/null +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/DefaultTransformExtension.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.transform; + +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Settings; + +public class DefaultTransformExtension implements TransformExtension { + + @Override + public boolean includeNodeInfo() { + return true; + } + + @Override + public Settings getTransformInternalIndexAdditionalSettings() { + return Settings.EMPTY; + } + + /** + * Provides destination index settings, hardcoded at the moment. In future this might be customizable or generation could be based on + * source settings. + */ + @Override + public Settings getTransformDestinationIndexSettings() { + return Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-1") + .build(); + } +} diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java index 86cdce7a92b50..6eebc97541123 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java @@ -137,6 +137,7 @@ public class Transform extends Plugin implements SystemIndexPlugin, PersistentTa private final Settings settings; private final SetOnce transformServices = new SetOnce<>(); + private final TransformExtension transformExtension = new DefaultTransformExtension(); public static final Integer DEFAULT_INITIAL_MAX_PAGE_SEARCH_SIZE = Integer.valueOf(500); public static final TimeValue DEFAULT_TRANSFORM_FREQUENCY = TimeValue.timeValueSeconds(60); @@ -250,7 +251,12 @@ public Collection createComponents( client, xContentRegistry ); - TransformAuditor auditor = new TransformAuditor(client, clusterService.getNodeName(), clusterService, includeNodeInfo()); + TransformAuditor auditor = new TransformAuditor( + client, + clusterService.getNodeName(), + clusterService, + getTransformExtension().includeNodeInfo() + ); Clock clock = Clock.systemUTC(); TransformCheckpointService checkpointService = new TransformCheckpointService( clock, @@ -264,7 +270,11 @@ public Collection createComponents( transformServices.set(new TransformServices(configManager, checkpointService, auditor, scheduler)); - return Arrays.asList(transformServices.get(), new TransformClusterStateListener(clusterService, client)); + return Arrays.asList( + transformServices.get(), + new TransformClusterStateListener(clusterService, client), + new TransformExtensionHolder(getTransformExtension()) + ); } @Override @@ -285,7 +295,7 @@ public List> getPersistentTasksExecutor( threadPool, clusterService, settingsModule.getSettings(), - getTransformInternalIndexAdditionalSettings(), + getTransformExtension().getTransformInternalIndexAdditionalSettings(), expressionResolver ) ); @@ -354,7 +364,9 @@ public UnaryOperator> getIndexTemplateMetadat @Override public Collection getSystemIndexDescriptors(Settings settings) { try { - return List.of(TransformInternalIndex.getSystemIndexDescriptor(getTransformInternalIndexAdditionalSettings())); + return List.of( + TransformInternalIndex.getSystemIndexDescriptor(getTransformExtension().getTransformInternalIndexAdditionalSettings()) + ); } catch (IOException e) { throw new UncheckedIOException(e); } @@ -467,11 +479,17 @@ public String getFeatureDescription() { return "Manages configuration and state for transforms"; } + public TransformExtension getTransformExtension() { + return transformExtension; + } + + @Deprecated public boolean includeNodeInfo() { - return true; + return getTransformExtension().includeNodeInfo(); } + @Deprecated public Settings getTransformInternalIndexAdditionalSettings() { - return Settings.EMPTY; + return getTransformExtension().getTransformInternalIndexAdditionalSettings(); } } diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformExtension.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformExtension.java new file mode 100644 index 0000000000000..c919f4dd4c550 --- /dev/null +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformExtension.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.transform; + +import org.elasticsearch.common.settings.Settings; + +public interface TransformExtension { + + boolean includeNodeInfo(); + + Settings getTransformInternalIndexAdditionalSettings(); + + /** + * Provides destination index settings, hardcoded at the moment. In future this might be customizable or generation could be based on + * source settings. + */ + Settings getTransformDestinationIndexSettings(); +} diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformExtensionHolder.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformExtensionHolder.java new file mode 100644 index 0000000000000..93f285f14cf69 --- /dev/null +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformExtensionHolder.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.transform; + +import org.elasticsearch.node.Node; + +import java.util.Objects; + +/** + * Wrapper for the {@link TransformExtension} interface that allows it to be used + * given the way {@link Node} does Guice bindings for plugin components. + * TODO: remove this class entirely once Guice is removed entirely. + */ +public class TransformExtensionHolder { + + private final TransformExtension transformExtension; + + /** + * Used by Guice. + */ + public TransformExtensionHolder() { + this.transformExtension = null; + } + + public TransformExtensionHolder(TransformExtension transformExtension) { + this.transformExtension = Objects.requireNonNull(transformExtension); + } + + public boolean isEmpty() { + return transformExtension == null; + } + + public TransformExtension getTransformExtension() { + return transformExtension; + } +} diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransformUpdater.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransformUpdater.java index 791dc75d5f0b8..98777b47543cb 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransformUpdater.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransformUpdater.java @@ -125,6 +125,7 @@ public static void updateTransform( final boolean dryRun, final boolean checkAccess, final TimeValue timeout, + final Settings destIndexSettings, ActionListener listener ) { // rewrite config into a new format if necessary @@ -185,6 +186,7 @@ public static void updateTransform( destIndexMappings, seqNoPrimaryTermAndIndex, clusterState, + destIndexSettings, ActionListener.wrap(r -> updateTransformListener.onResponse(null), listener::onFailure) ); }, listener::onFailure); @@ -300,6 +302,7 @@ private static void updateTransformConfiguration( Map mappings, SeqNoPrimaryTermAndIndex seqNoPrimaryTermAndIndex, ClusterState clusterState, + Settings destIndexSettings, ActionListener listener ) { // <3> Return to the listener @@ -351,6 +354,7 @@ private static void updateTransformConfiguration( indexNameExpressionResolver, clusterState, config, + destIndexSettings, mappings, createDestinationListener ); diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportPreviewTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportPreviewTransformAction.java index cdaba16182348..a7d7851704054 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportPreviewTransformAction.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportPreviewTransformAction.java @@ -50,6 +50,7 @@ import org.elasticsearch.xpack.core.transform.transforms.SyncConfig; import org.elasticsearch.xpack.core.transform.transforms.TransformConfig; import org.elasticsearch.xpack.core.transform.transforms.TransformDestIndexSettings; +import org.elasticsearch.xpack.transform.TransformExtensionHolder; import org.elasticsearch.xpack.transform.persistence.TransformIndex; import org.elasticsearch.xpack.transform.transforms.Function; import org.elasticsearch.xpack.transform.transforms.FunctionFactory; @@ -78,6 +79,7 @@ public class TransportPreviewTransformAction extends HandledTransportAction>> previewListener = ActionListener.wrap(docs -> { if (pipeline == null) { TransformDestIndexSettings generatedDestIndexSettings = TransformIndex.createTransformDestIndexSettings( + destIndexSettings, mappings.get(), transformId, Clock.systemUTC() diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportResetTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportResetTransformAction.java index d60fb4af651f9..bff969cf0b856 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportResetTransformAction.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportResetTransformAction.java @@ -37,6 +37,7 @@ import org.elasticsearch.xpack.core.transform.action.StopTransformAction; import org.elasticsearch.xpack.core.transform.transforms.TransformConfig; import org.elasticsearch.xpack.core.transform.transforms.TransformConfigUpdate; +import org.elasticsearch.xpack.transform.TransformExtensionHolder; import org.elasticsearch.xpack.transform.TransformServices; import org.elasticsearch.xpack.transform.notifications.TransformAuditor; import org.elasticsearch.xpack.transform.persistence.SeqNoPrimaryTermAndIndex; @@ -58,6 +59,7 @@ public class TransportResetTransformAction extends AcknowledgedTransportMasterNo private final Client client; private final SecurityContext securityContext; private final Settings settings; + private final Settings destIndexSettings; @Inject public TransportResetTransformAction( @@ -68,7 +70,8 @@ public TransportResetTransformAction( IndexNameExpressionResolver indexNameExpressionResolver, TransformServices transformServices, Client client, - Settings settings + Settings settings, + TransformExtensionHolder transformExtensionHolder ) { super( ResetTransformAction.NAME, @@ -87,6 +90,7 @@ public TransportResetTransformAction( ? new SecurityContext(settings, threadPool.getThreadContext()) : null; this.settings = settings; + this.destIndexSettings = transformExtensionHolder.getTransformExtension().getTransformDestinationIndexSettings(); } @Override @@ -131,6 +135,7 @@ protected void masterOperation(Task task, Request request, ClusterState state, A false, // dry run false, // check access request.timeout(), + destIndexSettings, updateTransformListener ); }, diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStartTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStartTransformAction.java index fa6272a30ae89..8776f112e6178 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStartTransformAction.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStartTransformAction.java @@ -23,6 +23,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.health.HealthStatus; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; @@ -40,6 +41,7 @@ import org.elasticsearch.xpack.core.transform.transforms.TransformState; import org.elasticsearch.xpack.core.transform.transforms.TransformTaskParams; import org.elasticsearch.xpack.core.transform.transforms.TransformTaskState; +import org.elasticsearch.xpack.transform.TransformExtensionHolder; import org.elasticsearch.xpack.transform.TransformServices; import org.elasticsearch.xpack.transform.notifications.TransformAuditor; import org.elasticsearch.xpack.transform.persistence.AuthorizationStatePersistenceUtils; @@ -62,6 +64,7 @@ public class TransportStartTransformAction extends TransportMasterNodeAction li false, // dryRun true, // checkAccess request.getTimeout(), + destIndexSettings, ActionListener.wrap(updateResult -> { TransformConfig originalConfig = configAndVersion.v1(); TransformConfig updatedConfig = updateResult.getConfig(); diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpgradeTransformsAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpgradeTransformsAction.java index 2e28106b77606..6b01f6d7966a0 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpgradeTransformsAction.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpgradeTransformsAction.java @@ -35,6 +35,7 @@ import org.elasticsearch.xpack.core.transform.action.UpgradeTransformsAction.Response; import org.elasticsearch.xpack.core.transform.transforms.TransformConfig; import org.elasticsearch.xpack.core.transform.transforms.TransformConfigUpdate; +import org.elasticsearch.xpack.transform.TransformExtensionHolder; import org.elasticsearch.xpack.transform.TransformServices; import org.elasticsearch.xpack.transform.action.TransformUpdater.UpdateResult; import org.elasticsearch.xpack.transform.notifications.TransformAuditor; @@ -56,6 +57,7 @@ public class TransportUpgradeTransformsAction extends TransportMasterNodeAction< private final Settings settings; private final Client client; private final TransformAuditor auditor; + private final Settings destIndexSettings; @Inject public TransportUpgradeTransformsAction( @@ -66,7 +68,8 @@ public TransportUpgradeTransformsAction( IndexNameExpressionResolver indexNameExpressionResolver, TransformServices transformServices, Client client, - Settings settings + Settings settings, + TransformExtensionHolder transformExtensionHolder ) { super( UpgradeTransformsAction.NAME, @@ -88,6 +91,7 @@ public TransportUpgradeTransformsAction( this.securityContext = XPackSettings.SECURITY_ENABLED.get(settings) ? new SecurityContext(settings, threadPool.getThreadContext()) : null; + this.destIndexSettings = transformExtensionHolder.getTransformExtension().getTransformDestinationIndexSettings(); } @Override @@ -163,6 +167,7 @@ private void updateOneTransform(String id, boolean dryRun, TimeValue timeout, Ac dryRun, false, // check access, timeout, + destIndexSettings, listener ); }, failure -> { diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformIndex.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformIndex.java index e8ef290bd9db1..39183a1ca8502 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformIndex.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformIndex.java @@ -23,7 +23,6 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.common.settings.Settings; @@ -106,6 +105,7 @@ public static void createDestinationIndex( IndexNameExpressionResolver indexNameExpressionResolver, ClusterState clusterState, TransformConfig config, + Settings destIndexSettings, Map destIndexMappings, ActionListener listener ) { @@ -137,6 +137,7 @@ public static void createDestinationIndex( if (dest.length == 0) { TransformDestIndexSettings generatedDestIndexSettings = createTransformDestIndexSettings( + destIndexSettings, destIndexMappings, config.getId(), Clock.systemUTC() @@ -248,13 +249,16 @@ static void setUpDestinationAliases(Client client, TransformConfig config, Actio ); } - public static TransformDestIndexSettings createTransformDestIndexSettings(Map mappings, String id, Clock clock) { + public static TransformDestIndexSettings createTransformDestIndexSettings( + Settings settings, + Map mappings, + String id, + Clock clock + ) { Map indexMappings = new HashMap<>(); indexMappings.put(PROPERTIES, createMappingsFromStringMap(mappings)); indexMappings.put(META, createMetadata(id, clock)); - Settings settings = createSettings(); - // transform does not create aliases, however the user might customize this in future Set aliases = null; return new TransformDestIndexSettings(indexMappings, settings, aliases); @@ -288,17 +292,6 @@ private static Map createMetadata(String id, Clock clock) { return metadata; } - /** - * creates generated index settings, hardcoded at the moment, in future this might be customizable or generation could - * be based on source settings. - */ - private static Settings createSettings() { - return Settings.builder() // <1> - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-1") - .build(); - } - /** * This takes the a {@code Map} of the type "fieldname: fieldtype" and transforms it into the * typical mapping format. diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransformUpdaterTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransformUpdaterTests.java index 6c59f2b34f0c0..27ca18d0e1d2f 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransformUpdaterTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransformUpdaterTests.java @@ -43,6 +43,7 @@ import org.elasticsearch.xpack.core.transform.transforms.TransformStoredDoc; import org.elasticsearch.xpack.core.transform.transforms.TransformTaskState; import org.elasticsearch.xpack.core.transform.utils.TransformConfigVersionUtils; +import org.elasticsearch.xpack.transform.DefaultTransformExtension; import org.elasticsearch.xpack.transform.action.TransformUpdater.UpdateResult; import org.elasticsearch.xpack.transform.notifications.MockTransformAuditor; import org.elasticsearch.xpack.transform.notifications.TransformAuditor; @@ -76,6 +77,7 @@ public class TransformUpdaterTests extends ESTestCase { private ClusterService clusterService = mock(ClusterService.class); private TransformAuditor auditor = new MockTransformAuditor(clusterService); private final Settings settings = Settings.builder().put(XPackSettings.SECURITY_ENABLED.getKey(), true).build(); + private final Settings destIndexSettings = new DefaultTransformExtension().getTransformDestinationIndexSettings(); private static class MyMockClient extends NoOpClient { @@ -157,6 +159,7 @@ public void testTransformUpdateNoAction() throws InterruptedException { false, false, AcknowledgedRequest.DEFAULT_ACK_TIMEOUT, + destIndexSettings, listener ), updateResult -> { @@ -192,6 +195,7 @@ public void testTransformUpdateNoAction() throws InterruptedException { false, false, AcknowledgedRequest.DEFAULT_ACK_TIMEOUT, + destIndexSettings, listener ), updateResult -> { @@ -264,6 +268,7 @@ public void testTransformUpdateRewrite() throws InterruptedException { false, false, AcknowledgedRequest.DEFAULT_ACK_TIMEOUT, + destIndexSettings, listener ), updateResult -> { @@ -331,6 +336,7 @@ public void testTransformUpdateDryRun() throws InterruptedException { true, false, AcknowledgedRequest.DEFAULT_ACK_TIMEOUT, + destIndexSettings, listener ), updateResult -> { @@ -378,6 +384,7 @@ public void testTransformUpdateCheckAccessSuccess() throws InterruptedException false, true, AcknowledgedRequest.DEFAULT_ACK_TIMEOUT, + destIndexSettings, listener ), updateResult -> { @@ -420,6 +427,7 @@ public void testTransformUpdateCheckAccessFailureDeferValidation() throws Interr false, true, AcknowledgedRequest.DEFAULT_ACK_TIMEOUT, + destIndexSettings, listener ), updateResult -> { @@ -454,6 +462,7 @@ public void testTransformUpdateCheckAccessFailureNoDeferValidation() { false, true, AcknowledgedRequest.DEFAULT_ACK_TIMEOUT, + destIndexSettings, ActionListener.wrap( r -> fail("Should fail due to missing privileges"), e -> assertThat(e.getMessage(), is(equalTo("missing privileges"))) diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/persistence/TransformIndexTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/persistence/TransformIndexTests.java index 9f704faf970c8..5afb6db1856fe 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/persistence/TransformIndexTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/persistence/TransformIndexTests.java @@ -146,7 +146,7 @@ public void testCreateDestinationIndex() throws IOException { TransformIndex.createDestinationIndex( client, TransformConfigTests.randomTransformConfig(TRANSFORM_ID), - TransformIndex.createTransformDestIndexSettings(new HashMap<>(), TRANSFORM_ID, clock), + TransformIndex.createTransformDestIndexSettings(Settings.EMPTY, new HashMap<>(), TRANSFORM_ID, clock), ActionTestUtils.assertNoFailureListener(Assert::assertTrue) );