From a830b205350b0cfb29814a195c838c87208bd228 Mon Sep 17 00:00:00 2001 From: Mpdreamz Date: Tue, 18 Dec 2018 15:26:48 +0100 Subject: [PATCH] Add support for the auto_date_histogram aggregation elastic/elasticsearch#28993 --- src/Nest/Aggregations/AggregateDictionary.cs | 14 +++ .../Aggregations/AggregateJsonConverter.cs | 7 ++ src/Nest/Aggregations/AggregationContainer.cs | 14 +++ .../AutoDateHistogramAggregation.cs | 111 ++++++++++++++++ .../AutoDateHistogramBucket.cs | 7 ++ .../Aggregations/Bucket/BucketAggregate.cs | 1 + .../DateHistogram/DateHistogramAggregation.cs | 1 + .../AutoDateHistogramAggregationUsageTests.cs | 118 ++++++++++++++++++ 8 files changed, 273 insertions(+) create mode 100644 src/Nest/Aggregations/Bucket/AutoDateHistogram/AutoDateHistogramAggregation.cs create mode 100644 src/Nest/Aggregations/Bucket/AutoDateHistogram/AutoDateHistogramBucket.cs create mode 100644 src/Tests/Tests/Aggregations/Bucket/AutoDateHistogram/AutoDateHistogramAggregationUsageTests.cs diff --git a/src/Nest/Aggregations/AggregateDictionary.cs b/src/Nest/Aggregations/AggregateDictionary.cs index fa46421ce6f..26c0f3e7bff 100644 --- a/src/Nest/Aggregations/AggregateDictionary.cs +++ b/src/Nest/Aggregations/AggregateDictionary.cs @@ -177,6 +177,20 @@ public TermsAggregate Terms(string key) public MultiBucketAggregate DateHistogram(string key) => GetMultiBucketAggregate(key); + public AutoDateHistogramAggregate AutoDateHistogram(string key) + { + var bucket = TryGet(key); + if (bucket == null) return null; + + return new AutoDateHistogramAggregate + { + Buckets = bucket.Items.OfType().ToList(), + Meta = bucket.Meta, + Interval = bucket.Interval + }; + } + + public CompositeBucketAggregate Composite(string key) { var bucket = TryGet(key); diff --git a/src/Nest/Aggregations/AggregateJsonConverter.cs b/src/Nest/Aggregations/AggregateJsonConverter.cs index 3585924fe4e..501fcdc546f 100644 --- a/src/Nest/Aggregations/AggregateJsonConverter.cs +++ b/src/Nest/Aggregations/AggregateJsonConverter.cs @@ -490,6 +490,12 @@ private IAggregate GetMultiBucketAggregate(JsonReader reader, JsonSerializer ser } while (reader.TokenType != JsonToken.EndArray); bucket.Items = items; reader.Read(); + if (reader.TokenType == JsonToken.PropertyName && (string)reader.Value == Parser.Interval) + { + var interval = reader.ReadAsString(); + bucket.Interval = new Time(interval); + } + return bucket; } @@ -755,6 +761,7 @@ private static class Parser public const string DocCountErrorUpperBound = "doc_count_error_upper_bound"; public const string Fields = "fields"; public const string From = "from"; + public const string Interval = "interval"; public const string FromAsString = "from_as_string"; public const string Hits = "hits"; diff --git a/src/Nest/Aggregations/AggregationContainer.cs b/src/Nest/Aggregations/AggregationContainer.cs index fab58e88c56..b591f53270f 100644 --- a/src/Nest/Aggregations/AggregationContainer.cs +++ b/src/Nest/Aggregations/AggregationContainer.cs @@ -104,6 +104,9 @@ public interface IAggregationContainer [JsonProperty("date_histogram")] IDateHistogramAggregation DateHistogram { get; set; } + [JsonProperty("auto_date_histogram")] + IAutoDateHistogramAggregation AutoDateHistogram { get; set; } + [JsonProperty("date_range")] IDateRangeAggregation DateRange { get; set; } @@ -253,11 +256,15 @@ public class AggregationContainer : IAggregationContainer public ICompositeAggregation Composite { get; set; } public ICumulativeSumAggregation CumulativeSum { get; set; } + public IDateHistogramAggregation DateHistogram { get; set; } + public IAutoDateHistogramAggregation AutoDateHistogram { get; set; } + public IDateRangeAggregation DateRange { get; set; } public IDerivativeAggregation Derivative { get; set; } + public IExtendedStatsAggregation ExtendedStats { get; set; } public IExtendedStatsBucketAggregation ExtendedStatsBucket { get; set; } @@ -386,6 +393,8 @@ public class AggregationContainerDescriptor : DescriptorBase, IDateHistogramAggregation> selector ) => _SetInnerAggregation(name, selector, (a, d) => a.DateHistogram = d); + public AggregationContainerDescriptor AutoDateHistogram(string name, + Func, IAutoDateHistogramAggregation> selector + ) => + _SetInnerAggregation(name, selector, (a, d) => a.AutoDateHistogram = d); + public AggregationContainerDescriptor Percentiles(string name, Func, IPercentilesAggregation> selector ) => diff --git a/src/Nest/Aggregations/Bucket/AutoDateHistogram/AutoDateHistogramAggregation.cs b/src/Nest/Aggregations/Bucket/AutoDateHistogram/AutoDateHistogramAggregation.cs new file mode 100644 index 00000000000..6a3c6d6d620 --- /dev/null +++ b/src/Nest/Aggregations/Bucket/AutoDateHistogram/AutoDateHistogramAggregation.cs @@ -0,0 +1,111 @@ +using System; +using System.Collections.Generic; +using System.Linq.Expressions; +using Newtonsoft.Json; + +namespace Nest +{ + [JsonObject(MemberSerialization = MemberSerialization.OptIn)] + [ContractJsonConverter(typeof(AggregationJsonConverter))] + public interface IAutoDateHistogramAggregation : IBucketAggregation + { + [JsonProperty("field")] + Field Field { get; set; } + + [JsonProperty("format")] + string Format { get; set; } + + [JsonProperty("missing")] + DateTime? Missing { get; set; } + + [JsonProperty("offset")] + string Offset { get; set; } + + [JsonProperty("params")] + IDictionary Params { get; set; } + + [JsonProperty("script")] + IScript Script { get; set; } + + [JsonProperty("time_zone")] + string TimeZone { get; set; } + } + + public class AutoDateHistogramAggregation : BucketAggregationBase, IAutoDateHistogramAggregation + { + private string _format; + + internal AutoDateHistogramAggregation() { } + + public AutoDateHistogramAggregation(string name) : base(name) { } + + public Field Field { get; set; } + + //see: https://github.com/elastic/elasticsearch/issues/9725 + public string Format + { + get => !string.IsNullOrEmpty(_format) && + !_format.Contains("date_optional_time") && + (Missing.HasValue) + ? _format + "||date_optional_time" + : _format; + set => _format = value; + } + + public DateTime? Missing { get; set; } + public string Offset { get; set; } + public IDictionary Params { get; set; } + public IScript Script { get; set; } + public string TimeZone { get; set; } + + internal override void WrapInContainer(AggregationContainer c) => c.AutoDateHistogram = this; + } + + public class AutoDateHistogramAggregationDescriptor + : BucketAggregationDescriptorBase, IAutoDateHistogramAggregation, T> + , IAutoDateHistogramAggregation + where T : class + { + private string _format; + + Field IAutoDateHistogramAggregation.Field { get; set; } + + //see: https://github.com/elastic/elasticsearch/issues/9725 + string IAutoDateHistogramAggregation.Format + { + get => !string.IsNullOrEmpty(_format) && + !_format.Contains("date_optional_time") && + (Self.Missing.HasValue) + ? _format + "||date_optional_time" + : _format; + set => _format = value; + } + + DateTime? IAutoDateHistogramAggregation.Missing { get; set; } + + string IAutoDateHistogramAggregation.Offset { get; set; } + + IDictionary IAutoDateHistogramAggregation.Params { get; set; } + + IScript IAutoDateHistogramAggregation.Script { get; set; } + + string IAutoDateHistogramAggregation.TimeZone { get; set; } + + public AutoDateHistogramAggregationDescriptor Field(Field field) => Assign(a => a.Field = field); + + public AutoDateHistogramAggregationDescriptor Field(Expression> field) => Assign(a => a.Field = field); + + public AutoDateHistogramAggregationDescriptor Script(string script) => Assign(a => a.Script = (InlineScript)script); + + public AutoDateHistogramAggregationDescriptor Script(Func scriptSelector) => + Assign(a => a.Script = scriptSelector?.Invoke(new ScriptDescriptor())); + + public AutoDateHistogramAggregationDescriptor Format(string format) => Assign(a => a.Format = format); + + public AutoDateHistogramAggregationDescriptor TimeZone(string timeZone) => Assign(a => a.TimeZone = timeZone); + + public AutoDateHistogramAggregationDescriptor Offset(string offset) => Assign(a => a.Offset = offset); + + public AutoDateHistogramAggregationDescriptor Missing(DateTime? missing) => Assign(a => a.Missing = missing); + } +} diff --git a/src/Nest/Aggregations/Bucket/AutoDateHistogram/AutoDateHistogramBucket.cs b/src/Nest/Aggregations/Bucket/AutoDateHistogram/AutoDateHistogramBucket.cs new file mode 100644 index 00000000000..14ffecea0f6 --- /dev/null +++ b/src/Nest/Aggregations/Bucket/AutoDateHistogram/AutoDateHistogramBucket.cs @@ -0,0 +1,7 @@ +namespace Nest +{ + public class AutoDateHistogramAggregate : MultiBucketAggregate + { + public Time Interval { get; internal set; } + } +} diff --git a/src/Nest/Aggregations/Bucket/BucketAggregate.cs b/src/Nest/Aggregations/Bucket/BucketAggregate.cs index a65dfb5af16..88114ab1338 100644 --- a/src/Nest/Aggregations/Bucket/BucketAggregate.cs +++ b/src/Nest/Aggregations/Bucket/BucketAggregate.cs @@ -70,5 +70,6 @@ public class BucketAggregate : IAggregate public IReadOnlyCollection Items { get; set; } = EmptyReadOnly.Collection; public IReadOnlyDictionary Meta { get; set; } = EmptyReadOnly.Dictionary; public long? SumOtherDocCount { get; set; } + public Time Interval { get; set; } } } diff --git a/src/Nest/Aggregations/Bucket/DateHistogram/DateHistogramAggregation.cs b/src/Nest/Aggregations/Bucket/DateHistogram/DateHistogramAggregation.cs index 469577c23b1..2f92916d584 100644 --- a/src/Nest/Aggregations/Bucket/DateHistogram/DateHistogramAggregation.cs +++ b/src/Nest/Aggregations/Bucket/DateHistogram/DateHistogramAggregation.cs @@ -87,6 +87,7 @@ public class DateHistogramAggregationDescriptor ExtendedBounds IDateHistogramAggregation.ExtendedBounds { get; set; } Field IDateHistogramAggregation.Field { get; set; } + //see: https://github.com/elastic/elasticsearch/issues/9725 string IDateHistogramAggregation.Format { get => !string.IsNullOrEmpty(_format) && diff --git a/src/Tests/Tests/Aggregations/Bucket/AutoDateHistogram/AutoDateHistogramAggregationUsageTests.cs b/src/Tests/Tests/Aggregations/Bucket/AutoDateHistogram/AutoDateHistogramAggregationUsageTests.cs new file mode 100644 index 00000000000..7f53579dbb6 --- /dev/null +++ b/src/Tests/Tests/Aggregations/Bucket/AutoDateHistogram/AutoDateHistogramAggregationUsageTests.cs @@ -0,0 +1,118 @@ +using System; +using System.Linq; +using FluentAssertions; +using Nest; +using Tests.Core.Extensions; +using Tests.Core.ManagedElasticsearch.Clusters; +using Tests.Domain; +using Tests.Framework.Integration; +using static Nest.Infer; +using static Tests.Domain.Helpers.TestValueHelper; + +namespace Tests.Aggregations.Bucket.AutoDateHistogram +{ + /** + * A multi-bucket aggregation similar to the Date Histogram Aggregation except instead of providing an interval to + * use as the width of each bucket, a target number of buckets is provided indicating the number of buckets needed + * and the interval of the buckets is automatically chosen to best achieve that target. The number of buckets + * returned will always be less than or equal to this target number. + * + * NOTE: When specifying a `format` **and** `extended_bounds` or `missing`, in order for Elasticsearch to be able to parse + * the serialized `DateTime` of `extended_bounds` or `missing` correctly, the `date_optional_time` format is included + * as part of the `format` value. + * + * Be sure to read the Elasticsearch documentation on {ref_current}/search-aggregations-bucket-autodatehistogram-aggregation.html[Auto Date Histogram Aggregation]. + */ + public class AutoDateHistogramAggregationUsageTests : ProjectsOnlyAggregationUsageTestBase + { + public AutoDateHistogramAggregationUsageTests(ReadOnlyCluster i, EndpointUsage usage) : base(i, usage) { } + + protected override object AggregationJson => new + { + projects_started_per_month = new + { + auto_date_histogram = new + { + field = "startedOn", + format = "yyyy-MM-dd'T'HH:mm:ss||date_optional_time", //<1> Note the inclusion of `date_optional_time` to `format` + missing = FixedDate + }, + aggs = new + { + project_tags = new + { + nested = new + { + path = "tags" + }, + aggs = new + { + tags = new + { + terms = new { field = "tags.name" } + } + } + } + } + } + }; + + protected override Func, IAggregationContainer> FluentAggs => a => a + .AutoDateHistogram("projects_started_per_month", date => date + .Field(p => p.StartedOn) + .Format("yyyy-MM-dd'T'HH:mm:ss") + .Missing(FixedDate) + .Aggregations(childAggs => childAggs + .Nested("project_tags", n => n + .Path(p => p.Tags) + .Aggregations(nestedAggs => nestedAggs + .Terms("tags", avg => avg.Field(p => p.Tags.First().Name)) + ) + ) + ) + ); + + protected override AggregationDictionary InitializerAggs => + new AutoDateHistogramAggregation("projects_started_per_month") + { + Field = Field(p => p.StartedOn), + Format = "yyyy-MM-dd'T'HH:mm:ss", + Missing = FixedDate, + Aggregations = new NestedAggregation("project_tags") + { + Path = Field(p => p.Tags), + Aggregations = new TermsAggregation("tags") + { + Field = Field(p => p.Tags.First().Name) + } + } + }; + + protected override void ExpectResponse(ISearchResponse response) + { + /** ==== Handling responses + * The `AggregateDictionary found on `.Aggregations` on `ISearchResponse` has several helper methods + * so we can fetch our aggregation results easily in the correct type. + * <> + */ + response.ShouldBeValid(); + + var dateHistogram = response.Aggregations.AutoDateHistogram("projects_started_per_month"); + dateHistogram.Should().NotBeNull(); + dateHistogram.Interval.Should().NotBeNull(); + dateHistogram.Buckets.Should().NotBeNull(); + dateHistogram.Buckets.Count.Should().BeGreaterThan(1); + foreach (var item in dateHistogram.Buckets) + { + item.Date.Should().NotBe(default); + item.DocCount.Should().BeGreaterThan(0); + + var nested = item.Nested("project_tags"); + nested.Should().NotBeNull(); + + var nestedTerms = nested.Terms("tags"); + nestedTerms.Buckets.Count.Should().BeGreaterThan(0); + } + } + } +}