Feature/7.x/cumulative cardinality agg (#4147)

* Add Cumulative Cardinality agg (and Data Science plugin)

Addresses elastic/elasticsearch#43661

* skipversion on cumulative cardinality api tests

Author: Mpdreamz

src/Nest/Aggregations/AggregateDictionary.cs

@@ -55,6 +55,8 @@ internal static string[] TypedKeyTokens(string key)
 
 		public ValueAggregate CumulativeSum(string key) => TryGet<ValueAggregate>(key);
 
+		public ValueAggregate CumulativeCardinality(string key) => TryGet<ValueAggregate>(key);
+
 		public ValueAggregate BucketScript(string key) => TryGet<ValueAggregate>(key);
 
 		public ValueAggregate SerialDifferencing(string key) => TryGet<ValueAggregate>(key);

src/Nest/Aggregations/AggregationContainer.cs

@@ -113,6 +113,9 @@ public interface IAggregationContainer
 		[DataMember(Name = "cumulative_sum")]
 		ICumulativeSumAggregation CumulativeSum { get; set; }
 
+		[DataMember(Name = "cumulative_cardinality")]
+		ICumulativeCardinalityAggregation CumulativeCardinality { get; set; }
+
 		[DataMember(Name = "date_histogram")]
 		IDateHistogramAggregation DateHistogram { get; set; }
 
@@ -282,6 +285,8 @@ public class AggregationContainer : IAggregationContainer
 
 		public ICumulativeSumAggregation CumulativeSum { get; set; }
 
+		public ICumulativeCardinalityAggregation CumulativeCardinality { get; set; }
+
 		public IDateHistogramAggregation DateHistogram { get; set; }
 
 		public IAutoDateHistogramAggregation AutoDateHistogram { get; set; }
@@ -358,6 +363,7 @@ public class AggregationContainer : IAggregationContainer
 		public IStatsAggregation Stats { get; set; }
 
 		public IStatsBucketAggregation StatsBucket { get; set; }
+
 		public ISumAggregation Sum { get; set; }
 
 		public ISumBucketAggregation SumBucket { get; set; }
@@ -425,6 +431,8 @@ public class AggregationContainerDescriptor<T> : DescriptorBase<AggregationConta
 
 		ICumulativeSumAggregation IAggregationContainer.CumulativeSum { get; set; }
 
+		ICumulativeCardinalityAggregation IAggregationContainer.CumulativeCardinality { get; set; }
+
 		IDateHistogramAggregation IAggregationContainer.DateHistogram { get; set; }
 
 		IAutoDateHistogramAggregation IAggregationContainer.AutoDateHistogram { get; set; }
@@ -750,6 +758,11 @@ Func<CumulativeSumAggregationDescriptor, ICumulativeSumAggregation> selector
 		) =>
 			_SetInnerAggregation(name, selector, (a, d) => a.CumulativeSum = d);
 
+		public AggregationContainerDescriptor<T> CumulativeCardinality(string name,
+			Func<CumulativeCardinalityAggregationDescriptor, ICumulativeCardinalityAggregation> selector
+		) =>
+			_SetInnerAggregation(name, selector, (a, d) => a.CumulativeCardinality = d);
+
 		public AggregationContainerDescriptor<T> SerialDifferencing(string name,
 			Func<SerialDifferencingAggregationDescriptor, ISerialDifferencingAggregation> selector
 		) =>

src/Nest/Aggregations/Pipeline/CumulativeCardinality/CumulativeCardinalityAggregation.cs

@@ -0,0 +1,23 @@
+using Elasticsearch.Net.Utf8Json;
+
+namespace Nest
+{
+	[InterfaceDataContract]
+	[ReadAs(typeof(CumulativeCardinalityAggregation))]
+	public interface ICumulativeCardinalityAggregation : IPipelineAggregation { }
+
+	public class CumulativeCardinalityAggregation
+		: PipelineAggregationBase, ICumulativeCardinalityAggregation
+	{
+		internal CumulativeCardinalityAggregation() { }
+
+		public CumulativeCardinalityAggregation(string name, SingleBucketsPath bucketsPath)
+			: base(name, bucketsPath) { }
+
+		internal override void WrapInContainer(AggregationContainer c) => c.CumulativeCardinality = this;
+	}
+
+	public class CumulativeCardinalityAggregationDescriptor
+		: PipelineAggregationDescriptorBase<CumulativeCardinalityAggregationDescriptor, ICumulativeCardinalityAggregation, SingleBucketsPath>
+			, ICumulativeCardinalityAggregation { }
+}

src/Nest/Aggregations/Visitor/AggregationVisitor.cs

@@ -110,6 +110,8 @@ public interface IAggregationVisitor
 
 		void Visit(ICumulativeSumAggregation aggregation);
 
+		void Visit(ICumulativeCardinalityAggregation aggregation);
+
 		void Visit(ISerialDifferencingAggregation aggregation);
 
 		void Visit(IBucketScriptAggregation aggregation);
@@ -179,6 +181,8 @@ public virtual void Visit(IPercentilesBucketAggregation aggregation) { }
 
 		public virtual void Visit(ICumulativeSumAggregation aggregation) { }
 
+		public virtual void Visit(ICumulativeCardinalityAggregation aggregation) { }
+
 		public virtual void Visit(IBucketScriptAggregation aggregation) { }
 
 		public virtual void Visit(ISamplerAggregation aggregation) { }

src/Nest/Aggregations/Visitor/AggregationWalker.cs

@@ -48,6 +48,7 @@ public void Walk(IAggregationContainer aggregation, IAggregationVisitor visitor)
 				Accept(v, d.Aggregations);
 			});
 			AcceptAggregation(aggregation.CumulativeSum, visitor, (v, d) => v.Visit(d));
+			AcceptAggregation(aggregation.CumulativeCardinality, visitor, (v, d) => v.Visit(d));
 			AcceptAggregation(aggregation.DateHistogram, visitor, (v, d) =>
 			{
 				v.Visit(d);

src/Nest/XPack/Info/XPackInfo/XPackInfoResponse.cs

@@ -53,6 +53,9 @@ public class XPackFeatures
 		[DataMember(Name = "flattened")]
 		public XPackFeature Flattened { get; internal set; }
 
+		[DataMember(Name = "data_science")]
+		public XPackFeature DataScience { get; internal set; }
+
 		[DataMember(Name = "graph")]
 		public XPackFeature Graph { get; internal set; }
 

src/Nest/XPack/Info/XPackUsage/XPackUsageResponse.cs

@@ -51,6 +51,9 @@ public class XPackUsageResponse : ResponseBase
 		[DataMember(Name = "flattened")]
 		public XPackUsage Flattened { get; internal set; }
 
+		[DataMember(Name = "data_science")]
+		public XPackUsage DataScience { get; internal set; }
+
 		[DataMember(Name = "ilm")]
 		public IlmUsage IndexLifecycleManagement { get; internal set; }
 

src/Tests/Tests/Aggregations/Pipeline/CumulativeCardinality/CumulativeCardinalityAggregationUsageTests.cs

@@ -0,0 +1,87 @@
+using System;
+using Elastic.Xunit.XunitPlumbing;
+using FluentAssertions;
+using Nest;
+using Tests.Core.Extensions;
+using Tests.Core.ManagedElasticsearch.Clusters;
+using Tests.Domain;
+using Tests.Framework.EndpointTests.TestState;
+
+namespace Tests.Aggregations.Pipeline.CumulativeCardinality
+{
+	[SkipVersion("<7.4.0", "Introduced in 7.4")]
+	public class CumulativeCardinalityAggregationUsageTests : AggregationUsageTestBase
+	{
+		public CumulativeCardinalityAggregationUsageTests(ReadOnlyCluster cluster, EndpointUsage usage) : base(cluster, usage) { }
+
+		protected override object AggregationJson => new
+		{
+			projects_started_per_month = new
+			{
+				date_histogram = new
+				{
+					field = "startedOn",
+					calendar_interval = "month",
+				},
+				aggs = new
+				{
+					commits = new
+					{
+						cardinality = new
+						{
+							field = "numberOfCommits"
+						}
+					},
+					cumulative_commits = new
+					{
+						cumulative_cardinality = new
+						{
+							buckets_path = "commits"
+						}
+					}
+				}
+			}
+		};
+
+		protected override Func<AggregationContainerDescriptor<Project>, IAggregationContainer> FluentAggs => a => a
+			.DateHistogram("projects_started_per_month", dh => dh
+				.Field(p => p.StartedOn)
+				.CalendarInterval(DateInterval.Month)
+				.Aggregations(aa => aa
+					.Cardinality("commits", sm => sm
+						.Field(p => p.NumberOfCommits)
+					)
+					.CumulativeCardinality("cumulative_commits", d => d
+						.BucketsPath("commits")
+					)
+				)
+			);
+
+		protected override AggregationDictionary InitializerAggs =>
+			new DateHistogramAggregation("projects_started_per_month")
+			{
+				Field = "startedOn",
+				CalendarInterval = DateInterval.Month,
+				Aggregations =
+					new CardinalityAggregation("commits", "numberOfCommits") &&
+					new CumulativeCardinalityAggregation("cumulative_commits", "commits")
+			};
+
+		protected override void ExpectResponse(ISearchResponse<Project> response)
+		{
+			response.ShouldBeValid();
+
+			var projectsPerMonth = response.Aggregations.DateHistogram("projects_started_per_month");
+			projectsPerMonth.Should().NotBeNull();
+			projectsPerMonth.Buckets.Should().NotBeNull();
+			projectsPerMonth.Buckets.Count.Should().BeGreaterThan(0);
+
+			foreach (var item in projectsPerMonth.Buckets)
+			{
+				var commitsDerivative = item.CumulativeCardinality("cumulative_commits");
+				commitsDerivative.Should().NotBeNull();
+				commitsDerivative.Value.Should().NotBe(null);
+			}
+		}
+	}
+}

src/Tests/Tests/Aggregations/Pipeline/CumulativeSum/CumulativeSumAggregationUsageTests.cs

@@ -78,7 +78,7 @@ protected override void ExpectResponse(ISearchResponse<Project> response)
 
 			foreach (var item in projectsPerMonth.Buckets)
 			{
-				var commitsDerivative = item.Derivative("cumulative_commits");
+				var commitsDerivative = item.CumulativeSum("cumulative_commits");
 				commitsDerivative.Should().NotBeNull();
 				commitsDerivative.Value.Should().NotBe(null);
 			}