From 801adf4ae54ef8ff451d738c60fac990946b68f8 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Tue, 25 Sep 2018 14:11:25 +0200 Subject: [PATCH 1/2] add support for missing_bucket on composite aggregation as per https://github.com/elastic/elasticsearch/pull/29465 --- .../Composite/CompositeAggregationSource.cs | 14 +++ src/Tests/Tests.Domain/Project.cs | 2 +- .../CompositeAggregationUsageTests.cs | 118 +++++++++++++++++- 3 files changed, 131 insertions(+), 3 deletions(-) diff --git a/src/Nest/Aggregations/Bucket/Composite/CompositeAggregationSource.cs b/src/Nest/Aggregations/Bucket/Composite/CompositeAggregationSource.cs index 23754272e33..31444f5c72b 100644 --- a/src/Nest/Aggregations/Bucket/Composite/CompositeAggregationSource.cs +++ b/src/Nest/Aggregations/Bucket/Composite/CompositeAggregationSource.cs @@ -37,6 +37,13 @@ public interface ICompositeAggregationSource /// [JsonProperty("order")] SortOrder? Order { get; set; } + + /// + /// By default documents without a value for a given source are ignored. It is possible to include + /// them in the response as null by setting this to true + /// + [JsonProperty("missing_bucket")] + bool? MissingBucket { get; set; } } /// @@ -59,6 +66,9 @@ protected CompositeAggregationSourceBase(string name) => /// public SortOrder? Order { get; set; } + + /// + public bool? MissingBucket { get; set; } } /// @@ -93,6 +103,7 @@ public abstract class CompositeAggregationSourceDescriptorBase _sourceType; Field ICompositeAggregationSource.Field { get; set; } SortOrder? ICompositeAggregationSource.Order { get; set; } + bool? ICompositeAggregationSource.MissingBucket { get; set; } protected CompositeAggregationSourceDescriptorBase(string name, string sourceType) { @@ -108,6 +119,9 @@ protected CompositeAggregationSourceDescriptorBase(string name, string sourceTyp /// public TDescriptor Order(SortOrder? order) => Assign(a => a.Order = order); + + /// + public TDescriptor MissingBucket(bool? includeMissing = true) => Assign(a => a.MissingBucket = includeMissing); } internal class CompositeAggregationSourceConverter : ReserializeJsonConverter diff --git a/src/Tests/Tests.Domain/Project.cs b/src/Tests/Tests.Domain/Project.cs index 233a9248712..62e20b06cc9 100644 --- a/src/Tests/Tests.Domain/Project.cs +++ b/src/Tests/Tests.Domain/Project.cs @@ -57,7 +57,7 @@ public class Project .RuleFor(p => p.NumberOfCommits, f => Gimme.Random.Number(1, 1000)) .RuleFor(p => p.NumberOfContributors, f => Gimme.Random.Number(1, 200)) .RuleFor(p => p.Ranges, f => Ranges.Generator.Generate()) - .RuleFor(p => p.Branches, f => Gimme.Random.ListItems(new List { "master", "dev", "release", "qa", "test" }, 2)) + .RuleFor(p => p.Branches, f => Gimme.Random.ListItems(new List { "master", "dev", "release", "qa", "test" })) .RuleFor(p => p.SourceOnly, f => TestConfiguration.Instance.Random.SourceSerializer ? new SourceOnlyObject() : null ) diff --git a/src/Tests/Tests/Aggregations/Bucket/Composite/CompositeAggregationUsageTests.cs b/src/Tests/Tests/Aggregations/Bucket/Composite/CompositeAggregationUsageTests.cs index 0f22d0ebcf7..dfd53a9cc09 100644 --- a/src/Tests/Tests/Aggregations/Bucket/Composite/CompositeAggregationUsageTests.cs +++ b/src/Tests/Tests/Aggregations/Bucket/Composite/CompositeAggregationUsageTests.cs @@ -32,7 +32,7 @@ namespace Tests.Aggregations.Bucket.Composite * * Be sure to read the Elasticsearch documentation on {ref_current}/search-aggregations-bucket-composite-aggregation.html[Composite Aggregation]. */ - [SkipVersion("<6.1.0", "Composite Aggregation is only available in Elasticsearch 6.1.0+")] + //[SkipVersion("<6.1.0", "Composite Aggregation is only available in Elasticsearch 6.1.0+")] public class CompositeAggregationUsageTests : ProjectsOnlyAggregationUsageTestBase { public CompositeAggregationUsageTests(ReadOnlyCluster i, EndpointUsage usage) : base(i, usage) { } @@ -196,9 +196,123 @@ protected override void ExpectResponse(ISearchResponse response) } } + public class CompositeAggregationMissingBucketUsageTests : ProjectsOnlyAggregationUsageTestBase + { + public CompositeAggregationMissingBucketUsageTests(ReadOnlyCluster i, EndpointUsage usage) : base(i, usage) { } + + protected override object AggregationJson => new + { + my_buckets = new + { + composite = new + { + sources = new object[] + { + new + { + branches = new + { + terms = new + { + field = "branches.keyword", + order = "asc", + missing_bucket = true + } + } + }, + } + }, + aggs = new + { + project_tags = new + { + nested = new { path = "tags" }, + aggs = new + { + tags = new { terms = new {field = "tags.name"} } + } + } + } + } + }; + + protected override Func, IAggregationContainer> FluentAggs => a => a + .Composite("my_buckets", date => date + .Sources(s => s + .Terms("branches", t => t + .Field(f => f.Branches.Suffix("keyword")) + .MissingBucket() + .Order(SortOrder.Ascending) + ) + ) + .Aggregations(childAggs => childAggs + .Nested("project_tags", n => n + .Path(p => p.Tags) + .Aggregations(nestedAggs => nestedAggs + .Terms("tags", avg => avg.Field(p => p.Tags.First().Name)) + ) + ) + ) + ); + + protected override AggregationDictionary InitializerAggs => + new CompositeAggregation("my_buckets") + { + Sources = new List + { + new TermsCompositeAggregationSource("branches") + { + Field = Infer.Field(f => f.Branches.Suffix("keyword")), + MissingBucket = true, + Order = SortOrder.Ascending + } + }, + Aggregations = new NestedAggregation("project_tags") + { + Path = Field(p => p.Tags), + Aggregations = new TermsAggregation("tags") + { + Field = Field(p => p.Tags.First().Name) + } + } + }; + + /**==== Handling Responses + * Each Composite aggregation bucket key is an `CompositeKey`, a specialized + * `IReadOnlyDictionary` type with methods to convert values to supported types + */ + protected override void ExpectResponse(ISearchResponse response) + { + response.ShouldBeValid(); + + var composite = response.Aggregations.Composite("my_buckets"); + composite.Should().NotBeNull(); + composite.Buckets.Should().NotBeNullOrEmpty(); + composite.AfterKey.Should().NotBeNull(); + if (TestConfiguration.Instance.InRange(">=6.3.0")) + composite.AfterKey.Should().HaveCount(1).And.ContainKeys("branches"); + var i = 0; + foreach (var item in composite.Buckets) + { + var key = item.Key; + key.Should().NotBeNull(); + + key.TryGetValue("branches", out string branches).Should().BeTrue("expected to find 'branches' in composite bucket"); + if (i == 0) branches.Should().BeNull("First key should be null as we expect to have some projects with no branches"); + else branches.Should().NotBeNullOrEmpty(); + + var nested = item.Nested("project_tags"); + nested.Should().NotBeNull(); + + var nestedTerms = nested.Terms("tags"); + nestedTerms.Buckets.Count.Should().BeGreaterThan(0); + i++; + } + } + } //hide - [SkipVersion("<6.3.0", "Date histogram source only supports format starting from Elasticsearch 6.3.0+")] + //[SkipVersion("<6.3.0", "Date histogram source only supports format starting from Elasticsearch 6.3.0+")] public class DateFormatCompositeAggregationUsageTests : ProjectsOnlyAggregationUsageTestBase { public DateFormatCompositeAggregationUsageTests(ReadOnlyCluster i, EndpointUsage usage) : base(i, usage) { } From 757e826f6f5391d84518fbb26725cd2d9129849a Mon Sep 17 00:00:00 2001 From: Russ Cam Date: Thu, 18 Oct 2018 10:18:06 +1000 Subject: [PATCH 2/2] Address PR comments --- .../Composite/CompositeAggregationSource.cs | 4 ++-- .../CompositeAggregationUsageTests.cs | 20 ++++++++++++------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/Nest/Aggregations/Bucket/Composite/CompositeAggregationSource.cs b/src/Nest/Aggregations/Bucket/Composite/CompositeAggregationSource.cs index 31444f5c72b..cb736b1f704 100644 --- a/src/Nest/Aggregations/Bucket/Composite/CompositeAggregationSource.cs +++ b/src/Nest/Aggregations/Bucket/Composite/CompositeAggregationSource.cs @@ -67,7 +67,7 @@ protected CompositeAggregationSourceBase(string name) => /// public SortOrder? Order { get; set; } - /// + /// public bool? MissingBucket { get; set; } } @@ -120,7 +120,7 @@ protected CompositeAggregationSourceDescriptorBase(string name, string sourceTyp /// public TDescriptor Order(SortOrder? order) => Assign(a => a.Order = order); - /// + /// public TDescriptor MissingBucket(bool? includeMissing = true) => Assign(a => a.MissingBucket = includeMissing); } diff --git a/src/Tests/Tests/Aggregations/Bucket/Composite/CompositeAggregationUsageTests.cs b/src/Tests/Tests/Aggregations/Bucket/Composite/CompositeAggregationUsageTests.cs index dfd53a9cc09..de33f936b45 100644 --- a/src/Tests/Tests/Aggregations/Bucket/Composite/CompositeAggregationUsageTests.cs +++ b/src/Tests/Tests/Aggregations/Bucket/Composite/CompositeAggregationUsageTests.cs @@ -32,7 +32,7 @@ namespace Tests.Aggregations.Bucket.Composite * * Be sure to read the Elasticsearch documentation on {ref_current}/search-aggregations-bucket-composite-aggregation.html[Composite Aggregation]. */ - //[SkipVersion("<6.1.0", "Composite Aggregation is only available in Elasticsearch 6.1.0+")] + [SkipVersion("<6.1.0", "Composite Aggregation is only available in Elasticsearch 6.1.0+")] public class CompositeAggregationUsageTests : ProjectsOnlyAggregationUsageTestBase { public CompositeAggregationUsageTests(ReadOnlyCluster i, EndpointUsage usage) : base(i, usage) { } @@ -155,7 +155,7 @@ public CompositeAggregationUsageTests(ReadOnlyCluster i, EndpointUsage usage) : }; /**==== Handling Responses - * Each Composite aggregation bucket key is an `CompositeKey`, a specialized + * Each Composite aggregation bucket key is a `CompositeKey` type, a specialized * `IReadOnlyDictionary` type with methods to convert values to supported types */ protected override void ExpectResponse(ISearchResponse response) @@ -196,6 +196,14 @@ protected override void ExpectResponse(ISearchResponse response) } } + /**[float] + * == Missing buckets + * By default documents without a value for a given source are ignored. + * It is possible to include them in the response by setting missing_bucket to `true` (defaults to `false`): + * + * NOTE: Only available in Elasticsearch 6.4.0+ + */ + [SkipVersion("<6.4.0", "Missing buckets added to Composite Aggregation Elasticsearch 6.4.0+")] public class CompositeAggregationMissingBucketUsageTests : ProjectsOnlyAggregationUsageTestBase { public CompositeAggregationMissingBucketUsageTests(ReadOnlyCluster i, EndpointUsage usage) : base(i, usage) { } @@ -289,8 +297,10 @@ protected override void ExpectResponse(ISearchResponse response) composite.Should().NotBeNull(); composite.Buckets.Should().NotBeNullOrEmpty(); composite.AfterKey.Should().NotBeNull(); + if (TestConfiguration.Instance.InRange(">=6.3.0")) composite.AfterKey.Should().HaveCount(1).And.ContainKeys("branches"); + var i = 0; foreach (var item in composite.Buckets) { @@ -312,7 +322,7 @@ protected override void ExpectResponse(ISearchResponse response) } //hide - //[SkipVersion("<6.3.0", "Date histogram source only supports format starting from Elasticsearch 6.3.0+")] + [SkipVersion("<6.3.0", "Date histogram source only supports format starting from Elasticsearch 6.3.0+")] public class DateFormatCompositeAggregationUsageTests : ProjectsOnlyAggregationUsageTestBase { public DateFormatCompositeAggregationUsageTests(ReadOnlyCluster i, EndpointUsage usage) : base(i, usage) { } @@ -400,10 +410,6 @@ public DateFormatCompositeAggregationUsageTests(ReadOnlyCluster i, EndpointUsage } }; - /**==== Handling Responses - * Each Composite aggregation bucket key is an `CompositeKey`, a specialized - * `IReadOnlyDictionary` type with methods to convert values to supported types - */ protected override void ExpectResponse(ISearchResponse response) { response.ShouldBeValid();