From d58162c106f61bbbb293d0bd9c9b7b902a2e4c06 Mon Sep 17 00:00:00 2001 From: Ke Li Date: Thu, 15 Feb 2018 16:23:20 +0800 Subject: [PATCH] Upgrade t-digest to 3.2 (#28295) (#28305) --- .../metrics/percentile-aggregation.asciidoc | 24 ++-- .../180_percentiles_tdigest_metric.yml | 122 +++++++++--------- server/build.gradle | 2 +- server/licenses/t-digest-3.0.jar.sha1 | 1 - server/licenses/t-digest-3.2.jar.sha1 | 1 + .../metrics/TDigestPercentileRanksIT.java | 32 ++--- .../TDigestPercentilesAggregatorTests.java | 18 +-- 7 files changed, 98 insertions(+), 102 deletions(-) delete mode 100644 server/licenses/t-digest-3.0.jar.sha1 create mode 100644 server/licenses/t-digest-3.2.jar.sha1 diff --git a/docs/reference/aggregations/metrics/percentile-aggregation.asciidoc b/docs/reference/aggregations/metrics/percentile-aggregation.asciidoc index f7a594dec881d..4ca9c849b9b61 100644 --- a/docs/reference/aggregations/metrics/percentile-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/percentile-aggregation.asciidoc @@ -53,13 +53,13 @@ percentiles: `[ 1, 5, 25, 50, 75, 95, 99 ]`. The response will look like this: "aggregations": { "load_time_outlier": { "values" : { - "1.0": 9.9, - "5.0": 29.500000000000004, - "25.0": 167.5, + "1.0": 5.0, + "5.0": 25.0, + "25.0": 165.0, "50.0": 445.0, - "75.0": 722.5, - "95.0": 940.5, - "99.0": 980.1000000000001 + "75.0": 725.0, + "95.0": 945.0, + "99.0": 985.0 } } } @@ -129,15 +129,15 @@ Response: "values": [ { "key": 1.0, - "value": 9.9 + "value": 5.0 }, { "key": 5.0, - "value": 29.500000000000004 + "value": 25.0 }, { "key": 25.0, - "value": 167.5 + "value": 165.0 }, { "key": 50.0, @@ -145,15 +145,15 @@ Response: }, { "key": 75.0, - "value": 722.5 + "value": 725.0 }, { "key": 95.0, - "value": 940.5 + "value": 945.0 }, { "key": 99.0, - "value": 980.1000000000001 + "value": 985.0 } ] } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/180_percentiles_tdigest_metric.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/180_percentiles_tdigest_metric.yml index 808e22909318a..bf22b9169863b 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/180_percentiles_tdigest_metric.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/180_percentiles_tdigest_metric.yml @@ -65,21 +65,21 @@ setup: - match: { hits.total: 4 } - length: { hits.hits: 4 } - - match: { aggregations.percentiles_int.values.1\.0: 2.5 } - - match: { aggregations.percentiles_int.values.5\.0: 8.500000000000002 } - - match: { aggregations.percentiles_int.values.25\.0: 38.5 } + - match: { aggregations.percentiles_int.values.1\.0: 1.0 } + - match: { aggregations.percentiles_int.values.5\.0: 1.0 } + - match: { aggregations.percentiles_int.values.25\.0: 26.0 } - match: { aggregations.percentiles_int.values.50\.0: 76.0 } - - match: { aggregations.percentiles_int.values.75\.0: 113.5 } - - match: { aggregations.percentiles_int.values.95\.0: 143.49999999999997 } - - match: { aggregations.percentiles_int.values.99\.0: 149.5 } + - match: { aggregations.percentiles_int.values.75\.0: 126.0 } + - match: { aggregations.percentiles_int.values.95\.0: 151.0 } + - match: { aggregations.percentiles_int.values.99\.0: 151.0 } - - match: { aggregations.percentiles_double.values.1\.0: 2.5 } - - match: { aggregations.percentiles_double.values.5\.0: 8.500000000000002 } - - match: { aggregations.percentiles_double.values.25\.0: 38.5 } + - match: { aggregations.percentiles_double.values.1\.0: 1.0 } + - match: { aggregations.percentiles_double.values.5\.0: 1.0 } + - match: { aggregations.percentiles_double.values.25\.0: 26.0 } - match: { aggregations.percentiles_double.values.50\.0: 76.0 } - - match: { aggregations.percentiles_double.values.75\.0: 113.5 } - - match: { aggregations.percentiles_double.values.95\.0: 143.49999999999997 } - - match: { aggregations.percentiles_double.values.99\.0: 149.5 } + - match: { aggregations.percentiles_double.values.75\.0: 126.0 } + - match: { aggregations.percentiles_double.values.95\.0: 151.0 } + - match: { aggregations.percentiles_double.values.99\.0: 151.0 } - do: search: @@ -100,21 +100,21 @@ setup: - match: { hits.total: 4 } - length: { hits.hits: 4 } - - match: { aggregations.percentiles_int.values.1\.0: 2.5 } - - match: { aggregations.percentiles_int.values.5\.0: 8.500000000000002 } - - match: { aggregations.percentiles_int.values.25\.0: 38.5 } + - match: { aggregations.percentiles_int.values.1\.0: 1.0 } + - match: { aggregations.percentiles_int.values.5\.0: 1.0 } + - match: { aggregations.percentiles_int.values.25\.0: 26.0 } - match: { aggregations.percentiles_int.values.50\.0: 76.0 } - - match: { aggregations.percentiles_int.values.75\.0: 113.5 } - - match: { aggregations.percentiles_int.values.95\.0: 143.49999999999997 } - - match: { aggregations.percentiles_int.values.99\.0: 149.5 } + - match: { aggregations.percentiles_int.values.75\.0: 126.0 } + - match: { aggregations.percentiles_int.values.95\.0: 151.0 } + - match: { aggregations.percentiles_int.values.99\.0: 151.0 } - - match: { aggregations.percentiles_double.values.1\.0: 2.5 } - - match: { aggregations.percentiles_double.values.5\.0: 8.500000000000002 } - - match: { aggregations.percentiles_double.values.25\.0: 38.5 } + - match: { aggregations.percentiles_double.values.1\.0: 1.0 } + - match: { aggregations.percentiles_double.values.5\.0: 1.0 } + - match: { aggregations.percentiles_double.values.25\.0: 26.0 } - match: { aggregations.percentiles_double.values.50\.0: 76.0 } - - match: { aggregations.percentiles_double.values.75\.0: 113.5 } - - match: { aggregations.percentiles_double.values.95\.0: 143.49999999999997 } - - match: { aggregations.percentiles_double.values.99\.0: 149.5 } + - match: { aggregations.percentiles_double.values.75\.0: 126.0 } + - match: { aggregations.percentiles_double.values.95\.0: 151.0 } + - match: { aggregations.percentiles_double.values.99\.0: 151.0 } --- @@ -135,21 +135,21 @@ setup: - match: { hits.total: 4 } - length: { hits.hits: 0 } - - match: { aggregations.percentiles_int.values.1\.0: 2.5 } - - match: { aggregations.percentiles_int.values.5\.0: 8.500000000000002 } - - match: { aggregations.percentiles_int.values.25\.0: 38.5 } + - match: { aggregations.percentiles_int.values.1\.0: 1.0 } + - match: { aggregations.percentiles_int.values.5\.0: 1.0 } + - match: { aggregations.percentiles_int.values.25\.0: 26.0 } - match: { aggregations.percentiles_int.values.50\.0: 76.0 } - - match: { aggregations.percentiles_int.values.75\.0: 113.5 } - - match: { aggregations.percentiles_int.values.95\.0: 143.49999999999997 } - - match: { aggregations.percentiles_int.values.99\.0: 149.5 } + - match: { aggregations.percentiles_int.values.75\.0: 126.0 } + - match: { aggregations.percentiles_int.values.95\.0: 151.0 } + - match: { aggregations.percentiles_int.values.99\.0: 151.0 } - - match: { aggregations.percentiles_double.values.1\.0: 2.5 } - - match: { aggregations.percentiles_double.values.5\.0: 8.500000000000002 } - - match: { aggregations.percentiles_double.values.25\.0: 38.5 } + - match: { aggregations.percentiles_double.values.1\.0: 1.0 } + - match: { aggregations.percentiles_double.values.5\.0: 1.0 } + - match: { aggregations.percentiles_double.values.25\.0: 26.0 } - match: { aggregations.percentiles_double.values.50\.0: 76.0 } - - match: { aggregations.percentiles_double.values.75\.0: 113.5 } - - match: { aggregations.percentiles_double.values.95\.0: 143.49999999999997 } - - match: { aggregations.percentiles_double.values.99\.0: 149.5 } + - match: { aggregations.percentiles_double.values.75\.0: 126.0 } + - match: { aggregations.percentiles_double.values.95\.0: 151.0 } + - match: { aggregations.percentiles_double.values.99\.0: 151.0 } @@ -176,21 +176,21 @@ setup: - match: { hits.total: 3 } - length: { hits.hits: 3 } - - match: { aggregations.percentiles_int.values.1\.0: 52.0 } - - match: { aggregations.percentiles_int.values.5\.0: 56.0 } - - match: { aggregations.percentiles_int.values.25\.0: 76.0 } + - match: { aggregations.percentiles_int.values.1\.0: 51.0 } + - match: { aggregations.percentiles_int.values.5\.0: 51.0 } + - match: { aggregations.percentiles_int.values.25\.0: 63.5 } - match: { aggregations.percentiles_int.values.50\.0: 101.0 } - - match: { aggregations.percentiles_int.values.75\.0: 126.0 } - - match: { aggregations.percentiles_int.values.95\.0: 146.0 } - - match: { aggregations.percentiles_int.values.99\.0: 150.0 } + - match: { aggregations.percentiles_int.values.75\.0: 138.5 } + - match: { aggregations.percentiles_int.values.95\.0: 151.0 } + - match: { aggregations.percentiles_int.values.99\.0: 151.0 } - - match: { aggregations.percentiles_double.values.1\.0: 52.0 } - - match: { aggregations.percentiles_double.values.5\.0: 56.0 } - - match: { aggregations.percentiles_double.values.25\.0: 76.0 } + - match: { aggregations.percentiles_double.values.1\.0: 51.0 } + - match: { aggregations.percentiles_double.values.5\.0: 51.0 } + - match: { aggregations.percentiles_double.values.25\.0: 63.5 } - match: { aggregations.percentiles_double.values.50\.0: 101.0 } - - match: { aggregations.percentiles_double.values.75\.0: 126.0 } - - match: { aggregations.percentiles_double.values.95\.0: 146.0 } - - match: { aggregations.percentiles_double.values.99\.0: 150.0 } + - match: { aggregations.percentiles_double.values.75\.0: 138.5 } + - match: { aggregations.percentiles_double.values.95\.0: 151.0 } + - match: { aggregations.percentiles_double.values.99\.0: 151.0 } --- "Missing field with missing param": @@ -248,13 +248,13 @@ setup: - match: { aggregations.percentiles_int.meta.foo: "bar" } - - match: { aggregations.percentiles_int.values.1\.0: 2.5 } - - match: { aggregations.percentiles_int.values.5\.0: 8.500000000000002 } - - match: { aggregations.percentiles_int.values.25\.0: 38.5 } + - match: { aggregations.percentiles_int.values.1\.0: 1.0 } + - match: { aggregations.percentiles_int.values.5\.0: 1.0 } + - match: { aggregations.percentiles_int.values.25\.0: 26.0 } - match: { aggregations.percentiles_int.values.50\.0: 76.0 } - - match: { aggregations.percentiles_int.values.75\.0: 113.5 } - - match: { aggregations.percentiles_int.values.95\.0: 143.49999999999997 } - - match: { aggregations.percentiles_int.values.99\.0: 149.5 } + - match: { aggregations.percentiles_int.values.75\.0: 126.0 } + - match: { aggregations.percentiles_int.values.95\.0: 151.0 } + - match: { aggregations.percentiles_int.values.99\.0: 151.0 } --- "Invalid params test": @@ -337,12 +337,12 @@ setup: - match: { hits.total: 4 } - length: { hits.hits: 4 } - - match: { aggregations.percentiles_int.values.5\.0: 8.500000000000002 } - - match: { aggregations.percentiles_int.values.25\.0: 38.5 } - - match: { aggregations.percentiles_int.values.50\.0: 76.0 } + - match: { aggregations.percentiles_int.values.5\.0: 1.0 } + - match: { aggregations.percentiles_int.values.25\.0: 26.0 } + - match: { aggregations.percentiles_int.values.50\.0: 76.0 } - - match: { aggregations.percentiles_double.values.5\.0: 8.500000000000002 } - - match: { aggregations.percentiles_double.values.25\.0: 38.5 } + - match: { aggregations.percentiles_double.values.5\.0: 1.0 } + - match: { aggregations.percentiles_double.values.25\.0: 26.0 } - match: { aggregations.percentiles_double.values.50\.0: 76.0 } --- @@ -363,9 +363,9 @@ setup: - length: { hits.hits: 4 } - match: { aggregations.percentiles_int.values.0.key: 5.0 } - - match: { aggregations.percentiles_int.values.0.value: 8.500000000000002 } + - match: { aggregations.percentiles_int.values.0.value: 1.0 } - match: { aggregations.percentiles_int.values.1.key: 25.0 } - - match: { aggregations.percentiles_int.values.1.value: 38.5 } + - match: { aggregations.percentiles_int.values.1.value: 26.0 } - match: { aggregations.percentiles_int.values.2.key: 50.0 } - match: { aggregations.percentiles_int.values.2.value: 76.0 } diff --git a/server/build.gradle b/server/build.gradle index 9ec3d73e3cc67..7b30f57d885e8 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -99,7 +99,7 @@ dependencies { compile "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}" // percentiles aggregation - compile 'com.tdunning:t-digest:3.0' + compile 'com.tdunning:t-digest:3.2' // precentil ranks aggregation compile 'org.hdrhistogram:HdrHistogram:2.1.9' diff --git a/server/licenses/t-digest-3.0.jar.sha1 b/server/licenses/t-digest-3.0.jar.sha1 deleted file mode 100644 index ce2f2e2f04098..0000000000000 --- a/server/licenses/t-digest-3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -84ccf145ac2215e6bfa63baa3101c0af41017cfc \ No newline at end of file diff --git a/server/licenses/t-digest-3.2.jar.sha1 b/server/licenses/t-digest-3.2.jar.sha1 new file mode 100644 index 0000000000000..de6e848545f38 --- /dev/null +++ b/server/licenses/t-digest-3.2.jar.sha1 @@ -0,0 +1 @@ +2ab94758b0276a8a26102adf8d528cf6d0567b9a \ No newline at end of file diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksIT.java index debd51a2633b4..3846168009dc6 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksIT.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksIT.java @@ -69,7 +69,6 @@ protected Collection> nodePlugins() { } private static double[] randomPercents(long minValue, long maxValue) { - final int length = randomIntBetween(1, 20); final double[] percents = new double[length]; for (int i = 0; i < percents.length; ++i) { @@ -97,7 +96,7 @@ private static PercentileRanksAggregationBuilder randomCompression(PercentileRan return builder; } - private void assertConsistent(double[] pcts, PercentileRanks values, long minValue, long maxValue) { + private void assertConsistent(double[] pcts, PercentileRanks values, long minValue) { final List percentileList = CollectionUtils.iterableAsArrayList(values); assertEquals(pcts.length, percentileList.size()); for (int i = 0; i < pcts.length; ++i) { @@ -109,9 +108,6 @@ private void assertConsistent(double[] pcts, PercentileRanks values, long minVal if (percentile.getPercent() == 0) { assertThat(percentile.getValue(), lessThanOrEqualTo((double) minValue)); } - if (percentile.getPercent() == 100) { - assertThat(percentile.getValue(), greaterThanOrEqualTo((double) maxValue)); - } } for (int i = 1; i < percentileList.size(); ++i) { @@ -193,7 +189,7 @@ public void testSingleValuedField() throws Exception { assertHitCount(searchResponse, 10); final PercentileRanks values = searchResponse.getAggregations().get("percentile_ranks"); - assertConsistent(pcts, values, minValue, maxValue); + assertConsistent(pcts, values, minValue); } @Override @@ -233,7 +229,7 @@ public void testSingleValuedFieldOutsideRange() throws Exception { assertHitCount(searchResponse, 10); final PercentileRanks values = searchResponse.getAggregations().get("percentile_ranks"); - assertConsistent(pcts, values, minValue, maxValue); + assertConsistent(pcts, values, minValue); } @Override @@ -248,7 +244,7 @@ public void testSingleValuedFieldPartiallyUnmapped() throws Exception { assertHitCount(searchResponse, 10); final PercentileRanks values = searchResponse.getAggregations().get("percentile_ranks"); - assertConsistent(pcts, values, minValue, maxValue); + assertConsistent(pcts, values, minValue); } @Override @@ -266,7 +262,7 @@ public void testSingleValuedFieldWithValueScript() throws Exception { assertHitCount(searchResponse, 10); final PercentileRanks values = searchResponse.getAggregations().get("percentile_ranks"); - assertConsistent(pcts, values, minValue - 1, maxValue - 1); + assertConsistent(pcts, values, minValue - 1); } @Override @@ -286,7 +282,7 @@ public void testSingleValuedFieldWithValueScriptWithParams() throws Exception { assertHitCount(searchResponse, 10); final PercentileRanks values = searchResponse.getAggregations().get("percentile_ranks"); - assertConsistent(pcts, values, minValue - 1, maxValue - 1); + assertConsistent(pcts, values, minValue - 1); } @Override @@ -301,7 +297,7 @@ public void testMultiValuedField() throws Exception { assertHitCount(searchResponse, 10); final PercentileRanks values = searchResponse.getAggregations().get("percentile_ranks"); - assertConsistent(pcts, values, minValues, maxValues); + assertConsistent(pcts, values, minValues); } @Override @@ -319,7 +315,7 @@ public void testMultiValuedFieldWithValueScript() throws Exception { assertHitCount(searchResponse, 10); final PercentileRanks values = searchResponse.getAggregations().get("percentile_ranks"); - assertConsistent(pcts, values, minValues - 1, maxValues - 1); + assertConsistent(pcts, values, minValues - 1); } public void testMultiValuedFieldWithValueScriptReverse() throws Exception { @@ -336,7 +332,7 @@ public void testMultiValuedFieldWithValueScriptReverse() throws Exception { assertHitCount(searchResponse, 10); final PercentileRanks values = searchResponse.getAggregations().get("percentile_ranks"); - assertConsistent(pcts, values, -maxValues, -minValues); + assertConsistent(pcts, values, -maxValues); } @Override @@ -356,7 +352,7 @@ public void testMultiValuedFieldWithValueScriptWithParams() throws Exception { assertHitCount(searchResponse, 10); final PercentileRanks values = searchResponse.getAggregations().get("percentile_ranks"); - assertConsistent(pcts, values, minValues - 1, maxValues - 1); + assertConsistent(pcts, values, minValues - 1); } @Override @@ -373,7 +369,7 @@ public void testScriptSingleValued() throws Exception { assertHitCount(searchResponse, 10); final PercentileRanks values = searchResponse.getAggregations().get("percentile_ranks"); - assertConsistent(pcts, values, minValue, maxValue); + assertConsistent(pcts, values, minValue); } @Override @@ -394,7 +390,7 @@ public void testScriptSingleValuedWithParams() throws Exception { assertHitCount(searchResponse, 10); final PercentileRanks values = searchResponse.getAggregations().get("percentile_ranks"); - assertConsistent(pcts, values, minValue - 1, maxValue - 1); + assertConsistent(pcts, values, minValue - 1); } @Override @@ -412,7 +408,7 @@ public void testScriptMultiValued() throws Exception { assertHitCount(searchResponse, 10); final PercentileRanks values = searchResponse.getAggregations().get("percentile_ranks"); - assertConsistent(pcts, values, minValues, maxValues); + assertConsistent(pcts, values, minValues); } @Override @@ -431,7 +427,7 @@ public void testScriptMultiValuedWithParams() throws Exception { assertHitCount(searchResponse, 10); final PercentileRanks values = searchResponse.getAggregations().get("percentile_ranks"); - assertConsistent(pcts, values, minValues - 1, maxValues - 1); + assertConsistent(pcts, values, minValues - 1); } public void testOrderBySubAggregation() { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/TDigestPercentilesAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/TDigestPercentilesAggregatorTests.java index 1605e3710e25b..85ab361a8b337 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/TDigestPercentilesAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/TDigestPercentilesAggregatorTests.java @@ -74,12 +74,12 @@ public void testSomeMatchesSortedNumericDocValues() throws IOException { }, tdigest -> { assertEquals(7L, tdigest.state.size()); assertEquals(7L, tdigest.state.centroidCount()); - assertEquals(4.0d, tdigest.percentile(75), 0.0d); - assertEquals("4.0", tdigest.percentileAsString(75)); + assertEquals(4.5d, tdigest.percentile(75), 0.0d); + assertEquals("4.5", tdigest.percentileAsString(75)); assertEquals(2.0d, tdigest.percentile(50), 0.0d); assertEquals("2.0", tdigest.percentileAsString(50)); - assertEquals(1.0d, tdigest.percentile(20), 0.0d); - assertEquals("1.0", tdigest.percentileAsString(20)); + assertEquals(1.0d, tdigest.percentile(22), 0.0d); + assertEquals("1.0", tdigest.percentileAsString(22)); }); } @@ -97,14 +97,14 @@ public void testSomeMatchesNumericDocValues() throws IOException { assertEquals(tdigest.state.centroidCount(), 7L); assertEquals(8.0d, tdigest.percentile(100), 0.0d); assertEquals("8.0", tdigest.percentileAsString(100)); - assertEquals(5.48d, tdigest.percentile(86), 0.0d); - assertEquals("5.48", tdigest.percentileAsString(86)); + assertEquals(6.98d, tdigest.percentile(88), 0.0d); + assertEquals("6.98", tdigest.percentileAsString(88)); assertEquals(1.0d, tdigest.percentile(33), 0.0d); assertEquals("1.0", tdigest.percentileAsString(33)); assertEquals(1.0d, tdigest.percentile(25), 0.0d); assertEquals("1.0", tdigest.percentileAsString(25)); - assertEquals(0.06d, tdigest.percentile(1), 0.0d); - assertEquals("0.06", tdigest.percentileAsString(1)); + assertEquals(0.0d, tdigest.percentile(1), 0.0d); + assertEquals("0.0", tdigest.percentileAsString(1)); }); } @@ -124,7 +124,7 @@ public void testQueryFiltering() throws IOException { assertEquals(4L, tdigest.state.centroidCount()); assertEquals(2.0d, tdigest.percentile(100), 0.0d); assertEquals(1.0d, tdigest.percentile(50), 0.0d); - assertEquals(0.75d, tdigest.percentile(25), 0.0d); + assertEquals(0.5d, tdigest.percentile(25), 0.0d); }); testCase(LongPoint.newRangeQuery("row", 100, 110), docs, tdigest -> {