diff --git a/src/Nest/XPack/Transform/PreviewTransform/PreviewTransformRequest.cs b/src/Nest/XPack/Transform/PreviewTransform/PreviewTransformRequest.cs index 5accd1d978e..2aa92fd8ee6 100644 --- a/src/Nest/XPack/Transform/PreviewTransform/PreviewTransformRequest.cs +++ b/src/Nest/XPack/Transform/PreviewTransform/PreviewTransformRequest.cs @@ -29,6 +29,10 @@ public partial interface IPreviewTransformRequest /// [DataMember(Name = "sync")] public ITransformSyncContainer Sync { get; set; } + + /// + [DataMember(Name = "settings")] + public ITransformSettings Settings { get; set; } } public partial class PreviewTransformRequest @@ -50,6 +54,10 @@ public partial class PreviewTransformRequest /// public ITransformSyncContainer Sync { get; set; } + + /// + [DataMember(Name = "settings")] + public ITransformSettings Settings { get; set; } } public partial class PreviewTransformDescriptor : IPreviewTransformRequest where TDocument : class @@ -60,6 +68,7 @@ public partial class PreviewTransformDescriptor : IPreviewTransformRe Time IPreviewTransformRequest.Frequency { get; set; } ITransformPivot IPreviewTransformRequest.Pivot { get; set; } ITransformSyncContainer IPreviewTransformRequest.Sync { get; set; } + ITransformSettings IPreviewTransformRequest.Settings { get; set; } /// public PreviewTransformDescriptor Description(string description) => @@ -83,5 +92,9 @@ public PreviewTransformDescriptor Pivot(Func public PreviewTransformDescriptor Sync(Func, ITransformSyncContainer> selector) => Assign(selector.InvokeOrDefault(new TransformSyncContainerDescriptor()), (a, v) => a.Sync = v); + + /// + public PreviewTransformDescriptor Settings(Func selector) => + Assign(selector?.Invoke(new TransformSettingsDescriptor()), (a, v) => a.Settings = v); } } diff --git a/src/Nest/XPack/Transform/PutTransform/PutTransformRequest.cs b/src/Nest/XPack/Transform/PutTransform/PutTransformRequest.cs index 529e93d1e43..13d904f5a06 100644 --- a/src/Nest/XPack/Transform/PutTransform/PutTransformRequest.cs +++ b/src/Nest/XPack/Transform/PutTransform/PutTransformRequest.cs @@ -29,6 +29,10 @@ public partial interface IPutTransformRequest /// [DataMember(Name = "sync")] public ITransformSyncContainer Sync { get; set; } + + /// + [DataMember(Name = "settings")] + public ITransformSettings Settings { get; set; } } /// @@ -51,6 +55,9 @@ public partial class PutTransformRequest /// public ITransformSyncContainer Sync { get; set; } + + /// + public ITransformSettings Settings { get; set; } } public partial class PutTransformDescriptor : IPutTransformRequest where TDocument : class @@ -61,6 +68,7 @@ public partial class PutTransformDescriptor : IPutTransformRequest wh Time IPutTransformRequest.Frequency { get; set; } ITransformPivot IPutTransformRequest.Pivot { get; set; } ITransformSyncContainer IPutTransformRequest.Sync { get; set; } + ITransformSettings IPutTransformRequest.Settings { get; set; } /// public PutTransformDescriptor Description(string description) => @@ -84,5 +92,9 @@ public PutTransformDescriptor Pivot(Func public PutTransformDescriptor Sync(Func, ITransformSyncContainer> selector) => Assign(selector?.Invoke(new TransformSyncContainerDescriptor()), (a, v) => a.Sync = v); + + /// + public PutTransformDescriptor Settings(Func selector) => + Assign(selector?.Invoke(new TransformSettingsDescriptor()), (a, v) => a.Settings = v); } } diff --git a/src/Nest/XPack/Transform/Transform.cs b/src/Nest/XPack/Transform/Transform.cs index 057db4b15b5..fec14741c69 100644 --- a/src/Nest/XPack/Transform/Transform.cs +++ b/src/Nest/XPack/Transform/Transform.cs @@ -42,5 +42,9 @@ public class Transform /// [DataMember(Name = "sync")] public ITransformSyncContainer Sync { get; set; } + + /// + [DataMember(Name = "settings")] + public ITransformSettings Settings { get; set; } } } diff --git a/src/Nest/XPack/Transform/TransformSettings.cs b/src/Nest/XPack/Transform/TransformSettings.cs new file mode 100644 index 00000000000..53c8ae4c586 --- /dev/null +++ b/src/Nest/XPack/Transform/TransformSettings.cs @@ -0,0 +1,49 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Runtime.Serialization; +using Elasticsearch.Net.Utf8Json; + +namespace Nest +{ + /// + /// Transform settings + /// + /// Valid in Elasticsearch 7.8.0+ + /// + [InterfaceDataContract] + [ReadAs(typeof(TransformSettings))] + public interface ITransformSettings + { + [DataMember(Name = "docs_per_second")] + public float? DocsPerSecond { get; set; } + + [DataMember(Name = "max_page_search_size")] + public int? MaxPageSearchSize { get; set; } + } + + /// + public class TransformSettings : ITransformSettings + { + /// + public float? DocsPerSecond { get; set; } + + /// + public int? MaxPageSearchSize { get; set; } + } + + public class TransformSettingsDescriptor : DescriptorBase, ITransformSettings + { + float? ITransformSettings.DocsPerSecond { get; set; } + int? ITransformSettings.MaxPageSearchSize { get; set; } + + /// + public TransformSettingsDescriptor DocsPerSecond(float? docsPerSecond) => + Assign(docsPerSecond, (a, v) => a.DocsPerSecond = v); + + /// + public TransformSettingsDescriptor MaxPageSearchSize(int? maxPageSearchSize) => + Assign(maxPageSearchSize, (a, v) => a.MaxPageSearchSize = v); + } +} diff --git a/src/Nest/XPack/Transform/UpdateTransform/UpdateTransformRequest.cs b/src/Nest/XPack/Transform/UpdateTransform/UpdateTransformRequest.cs index ecec53785de..e8d8a2f6ea7 100644 --- a/src/Nest/XPack/Transform/UpdateTransform/UpdateTransformRequest.cs +++ b/src/Nest/XPack/Transform/UpdateTransform/UpdateTransformRequest.cs @@ -25,24 +25,31 @@ public partial interface IUpdateTransformRequest /// [DataMember(Name = "sync")] public ITransformSyncContainer Sync { get; set; } + + /// + [DataMember(Name = "settings")] + public ITransformSettings Settings { get; set; } } public partial class UpdateTransformRequest { - /// + /// public string Description { get; set; } - /// + /// public ITransformSource Source { get; set; } - /// + /// public ITransformDestination Destination { get; set; } - /// + /// public Time Frequency { get; set; } - /// + /// public ITransformSyncContainer Sync { get; set; } + + /// + public ITransformSettings Settings { get; set; } } public partial class UpdateTransformDescriptor : IUpdateTransformRequest where TDocument : class @@ -52,6 +59,7 @@ public partial class UpdateTransformDescriptor : IUpdateTransformRequ ITransformDestination IUpdateTransformRequest.Destination { get; set; } Time IUpdateTransformRequest.Frequency { get; set; } ITransformSyncContainer IUpdateTransformRequest.Sync { get; set; } + ITransformSettings IUpdateTransformRequest.Settings { get; set; } /// public UpdateTransformDescriptor Description(string description) => @@ -71,5 +79,9 @@ public UpdateTransformDescriptor Destination(Func public UpdateTransformDescriptor Sync(Func, ITransformSyncContainer> selector) => Assign(selector?.Invoke(new TransformSyncContainerDescriptor()), (a, v) => a.Sync = v); + + /// + public UpdateTransformDescriptor Settings(Func selector) => + Assign(selector?.Invoke(new TransformSettingsDescriptor()), (a, v) => a.Settings = v); } } diff --git a/tests/Tests/XPack/Transform/TransformApiWithSettingsTests.cs b/tests/Tests/XPack/Transform/TransformApiWithSettingsTests.cs new file mode 100644 index 00000000000..9dc9f3a2c8a --- /dev/null +++ b/tests/Tests/XPack/Transform/TransformApiWithSettingsTests.cs @@ -0,0 +1,169 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System; +using System.Collections.Generic; +using Elastic.Elasticsearch.Xunit.XunitPlumbing; +using Elasticsearch.Net; +using Nest; +using Tests.Core.ManagedElasticsearch.Clusters; +using Tests.Domain; +using Tests.Framework.EndpointTests; +using Tests.Framework.EndpointTests.TestState; +using static Nest.Infer; + +namespace Tests.XPack.Transform +{ + [SkipVersion("<7.8.0", "Settings introduced in 7.8.0")] + public class TransformApiWithSettingsTests : ApiIntegrationTestBase, IPreviewTransformRequest, PreviewTransformDescriptor, PreviewTransformRequest> + { + public TransformApiWithSettingsTests(WritableCluster cluster, EndpointUsage usage) : base(cluster, usage) { } + + protected override LazyResponses ClientUsage() => Calls( + (client, f) => client.Transform.Preview(f), + (client, f) => client.Transform.PreviewAsync(f), + (client, r) => client.Transform.Preview(r), + (client, r) => client.Transform.PreviewAsync(r) + ); + + protected override HttpMethod HttpMethod => HttpMethod.POST; + protected override string UrlPath => $"_transform/_preview"; + protected override bool ExpectIsValid => true; + protected override int ExpectStatusCode => 200; + protected override bool SupportsDeserialization => false; + protected override object ExpectJson => new + { + description = CallIsolatedValue, + frequency = "1s", + source = new { index = new[] { "project" }, query = new { match_all = new { } } }, + dest = new { index = $"transform-{CallIsolatedValue}" }, + pivot = new + { + aggregations = new + { + averageCommits = new + { + avg = new + { + field = "numberOfCommits" + } + }, + sumIntoMaster = new + { + scripted_metric = new + { + combine_script = new + { + source = "long sum = 0; for (s in state.masterCommits) { sum += s } return sum" + }, + init_script = new + { + source = "state.masterCommits = []" + }, + map_script = new + { + source = "state.masterCommits.add(doc['branches.keyword'].contains('master')? 1 : 0)" + }, + reduce_script = new + { + source = "long sum = 0; for (s in states) { sum += s } return sum" + } + } + } + }, + group_by = new + { + weekStartedOn = new + { + date_histogram = new + { + calendar_interval = "week", + field = "startedOn" + } + } + } + }, + sync = new + { + time = new + { + field = "lastActivity" + } + }, + settings = new + { + docs_per_second = 200.0, + max_page_search_size = 200 + } + }; + + protected override PreviewTransformRequest Initializer => new PreviewTransformRequest + { + Description = CallIsolatedValue, + Frequency = "1s", + Source = new TransformSource { Index = Index(), Query = new MatchAllQuery() }, + Destination = new TransformDestination { Index = $"transform-{CallIsolatedValue}" }, + Pivot = new TransformPivot + { + Aggregations = + new AverageAggregation("averageCommits", Field(f => f.NumberOfCommits)) && + new ScriptedMetricAggregation("sumIntoMaster") + { + InitScript = new InlineScript("state.masterCommits = []"), + MapScript = new InlineScript("state.masterCommits.add(doc['branches.keyword'].contains('master')? 1 : 0)"), + CombineScript = new InlineScript("long sum = 0; for (s in state.masterCommits) { sum += s } return sum"), + ReduceScript = new InlineScript("long sum = 0; for (s in states) { sum += s } return sum") + }, + GroupBy = new Dictionary + { + { + "weekStartedOn", + new DateHistogramGroupSource() { Field = Field(f => f.StartedOn), CalendarInterval = DateInterval.Week } + } + } + }, + Sync = new TransformSyncContainer(new TransformTimeSync { Field = Field(f => f.LastActivity) }), + Settings = new TransformSettings { MaxPageSearchSize = 200, DocsPerSecond = 200 } + }; + + protected override Func, IPreviewTransformRequest> Fluent => f => f + .Description(CallIsolatedValue) + .Frequency(new Time(1, TimeUnit.Second)) + .Source(s => s + .Index() + .Query(q => q.MatchAll()) + ) + .Destination(de => de + .Index($"transform-{CallIsolatedValue}") + ) + .Pivot(p => p + .Aggregations(a => a + .Average("averageCommits", avg => avg + .Field(f => f.NumberOfCommits) + ) + .ScriptedMetric("sumIntoMaster", sm => sm + .InitScript("state.masterCommits = []") + .MapScript("state.masterCommits.add(doc['branches.keyword'].contains('master')? 1 : 0)") + .CombineScript("long sum = 0; for (s in state.masterCommits) { sum += s } return sum") + .ReduceScript("long sum = 0; for (s in states) { sum += s } return sum") + ) + ) + .GroupBy(g => g + .DateHistogram("weekStartedOn", dh => dh + .Field(f => f.StartedOn) + .CalendarInterval(DateInterval.Week) + ) + ) + ) + .Sync(sy => sy + .Time(t => t + .Field(f => f.LastActivity) + ) + ) + .Settings(s => s + .MaxPageSearchSize(200) + .DocsPerSecond(200) + ); + } +}