Add analytics plugin usage stats to _xpack/usage (elastic#54911)

Adds analytics plugin usage stats to _xpack/usage.

Closes elastic#54847

Author: imotov

docs/reference/rest-api/usage.asciidoc

@@ -16,7 +16,7 @@ Provides usage information about the installed {xpack} features.
 === {api-description-title}
 
 This API provides information about which features are currently enabled and
-available under the current license and some usage statistics. 
+available under the current license and some usage statistics.
 
 [discrete]
 [[usage-api-query-parms]]
@@ -291,9 +291,9 @@ GET /_xpack/usage
 // 1. Handling eql, which is disabled by default on release builds and enabled
 //    everywhere else during the initial implementation phase until its release
 // 2. Ignore the contents of the `ilm` and `slm` objects because they don't know
-//    all of the policies that will be in them. 
+//    all of the policies that will be in them.
 // 3. Ignore the contents of the `analytics` object because it might contain
 //    additional stats
 // 4. All of the numbers and strings on the right hand side of *every* field in
 //    the response are ignored. So we're really only asserting things about the
-//    the shape of this response, not the values in it.
+//    the shape of this response, not the values in it.

x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/AnalyticsPlugin.java

@@ -74,7 +74,7 @@ public List<PipelineAggregationSpec> getPipelineAggregations() {
                 CumulativeCardinalityPipelineAggregationBuilder.NAME,
                 CumulativeCardinalityPipelineAggregationBuilder::new,
                 CumulativeCardinalityPipelineAggregator::new,
-                usage.track(AnalyticsUsage.Item.CUMULATIVE_CARDINALITY,
+                usage.track(AnalyticsStatsAction.Item.CUMULATIVE_CARDINALITY,
                         checkLicense(CumulativeCardinalityPipelineAggregationBuilder.PARSER)))
         );
     }
@@ -85,22 +85,22 @@ public List<AggregationSpec> getAggregations() {
             new AggregationSpec(
                 StringStatsAggregationBuilder.NAME,
                 StringStatsAggregationBuilder::new,
-                usage.track(AnalyticsUsage.Item.STRING_STATS, checkLicense(StringStatsAggregationBuilder.PARSER)))
+                usage.track(AnalyticsStatsAction.Item.STRING_STATS, checkLicense(StringStatsAggregationBuilder.PARSER)))
                 .addResultReader(InternalStringStats::new),
             new AggregationSpec(
                 BoxplotAggregationBuilder.NAME,
                 BoxplotAggregationBuilder::new,
-                usage.track(AnalyticsUsage.Item.BOXPLOT, checkLicense(BoxplotAggregationBuilder.PARSER)))
+                usage.track(AnalyticsStatsAction.Item.BOXPLOT, checkLicense(BoxplotAggregationBuilder.PARSER)))
                 .addResultReader(InternalBoxplot::new),
             new AggregationSpec(
                 TopMetricsAggregationBuilder.NAME,
                 TopMetricsAggregationBuilder::new,
-                usage.track(AnalyticsUsage.Item.TOP_METRICS, checkLicense(TopMetricsAggregationBuilder.PARSER)))
+                usage.track(AnalyticsStatsAction.Item.TOP_METRICS, checkLicense(TopMetricsAggregationBuilder.PARSER)))
                 .addResultReader(InternalTopMetrics::new),
             new AggregationSpec(
                 TTestAggregationBuilder.NAME,
                 TTestAggregationBuilder::new,
-                usage.track(AnalyticsUsage.Item.T_TEST, checkLicense(TTestAggregationBuilder.PARSER)))
+                usage.track(AnalyticsStatsAction.Item.T_TEST, checkLicense(TTestAggregationBuilder.PARSER)))
                 .addResultReader(InternalTTest::new)
         );
     }
@@ -138,7 +138,7 @@ public Collection<Object> createComponents(Client client, ClusterService cluster
             ResourceWatcherService resourceWatcherService, ScriptService scriptService, NamedXContentRegistry xContentRegistry,
             Environment environment, NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry,
             IndexNameExpressionResolver indexNameExpressionResolver) {
-        return singletonList(new AnalyticsUsage());
+        return singletonList(usage);
     }
 
     @Override

x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/AnalyticsUsage.java

@@ -8,54 +8,32 @@
 
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.xcontent.ContextParser;
+import org.elasticsearch.xpack.core.analytics.EnumCounters;
 import org.elasticsearch.xpack.core.analytics.action.AnalyticsStatsAction;
 
-import java.util.EnumMap;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicLong;
-
 /**
  * Tracks usage of the Analytics aggregations.
  */
 public class AnalyticsUsage {
-    /**
-     * Items to track.
-     */
-    public enum Item {
-        BOXPLOT,
-        CUMULATIVE_CARDINALITY,
-        STRING_STATS,
-        TOP_METRICS,
-        T_TEST;
-    }
 
-    private final Map<Item, AtomicLong> trackers = new EnumMap<>(Item.class);
+    private final EnumCounters<AnalyticsStatsAction.Item> counters = new EnumCounters<>(AnalyticsStatsAction.Item.class);
 
     public AnalyticsUsage() {
-        for (Item item: Item.values()) {
-            trackers.put(item, new AtomicLong(0));
-        }
     }
 
     /**
      * Track successful parsing.
      */
-    public <C, T> ContextParser<C, T> track(Item item, ContextParser<C, T> realParser) {
-        AtomicLong usage = trackers.get(item);
+    public <C, T> ContextParser<C, T> track(AnalyticsStatsAction.Item item, ContextParser<C, T> realParser) {
         return (parser, context) -> {
             T value = realParser.parse(parser, context);
             // Intentionally doesn't count unless the parser returns cleanly.
-            usage.incrementAndGet();
+            counters.inc(item);
             return value;
         };
     }
 
     public AnalyticsStatsAction.NodeResponse stats(DiscoveryNode node) {
-        return new AnalyticsStatsAction.NodeResponse(node,
-                trackers.get(Item.BOXPLOT).get(),
-                trackers.get(Item.CUMULATIVE_CARDINALITY).get(),
-                trackers.get(Item.STRING_STATS).get(),
-                trackers.get(Item.TOP_METRICS).get(),
-                trackers.get(Item.T_TEST).get());
+        return new AnalyticsStatsAction.NodeResponse(node, counters);
     }
 }

x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/AnalyticsStatsActionNodeResponseTests.java

@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.analytics.action;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.xpack.core.analytics.EnumCounters;
+import org.elasticsearch.xpack.core.analytics.action.AnalyticsStatsAction;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class AnalyticsStatsActionNodeResponseTests extends AbstractWireSerializingTestCase<AnalyticsStatsAction.NodeResponse> {
+
+    @Override
+    protected Writeable.Reader<AnalyticsStatsAction.NodeResponse> instanceReader() {
+        return AnalyticsStatsAction.NodeResponse::new;
+    }
+
+    @Override
+    protected AnalyticsStatsAction.NodeResponse createTestInstance() {
+        String nodeName = randomAlphaOfLength(10);
+        DiscoveryNode node = new DiscoveryNode(nodeName, buildNewFakeTransportAddress(), Version.CURRENT);
+        EnumCounters<AnalyticsStatsAction.Item> counters = new EnumCounters<>(AnalyticsStatsAction.Item.class);
+        for (AnalyticsStatsAction.Item item : AnalyticsStatsAction.Item.values()) {
+            if (randomBoolean()) {
+                counters.inc(item, randomLongBetween(0, 1000));
+            }
+        }
+        return new AnalyticsStatsAction.NodeResponse(node, counters);
+    }
+
+    public void testItemEnum() {
+        int i = 0;
+        // We rely on the ordinals for serialization, so they shouldn't change between version
+        assertThat(AnalyticsStatsAction.Item.BOXPLOT.ordinal(), equalTo(i++));
+        assertThat(AnalyticsStatsAction.Item.CUMULATIVE_CARDINALITY.ordinal(), equalTo(i++));
+        assertThat(AnalyticsStatsAction.Item.STRING_STATS.ordinal(), equalTo(i++));
+        assertThat(AnalyticsStatsAction.Item.TOP_METRICS.ordinal(), equalTo(i++));
+        assertThat(AnalyticsStatsAction.Item.T_TEST.ordinal(), equalTo(i++));
+        // Please add tests for newly added items here
+        assertThat(AnalyticsStatsAction.Item.values().length, equalTo(i));
+    }
+}

x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/TransportAnalyticsStatsActionTests.java

@@ -22,6 +22,7 @@
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.analytics.AnalyticsUsage;
+import org.elasticsearch.xpack.core.analytics.AnalyticsFeatureSetUsage;
 import org.elasticsearch.xpack.core.analytics.action.AnalyticsStatsAction;
 
 import java.io.IOException;
@@ -33,6 +34,7 @@
 import static java.util.Collections.emptyList;
 import static java.util.stream.Collectors.toList;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.xpack.analytics.action.AnalyticsUsageTransportAction.usageFeatureResponse;
 import static org.hamcrest.Matchers.equalTo;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
@@ -58,20 +60,18 @@ public TransportAnalyticsStatsAction action(AnalyticsUsage usage) {
     }
 
     public void test() throws IOException {
-        for (AnalyticsUsage.Item item : AnalyticsUsage.Item.values()) {
+        for (AnalyticsStatsAction.Item item : AnalyticsStatsAction.Item.values()) {
             AnalyticsUsage realUsage = new AnalyticsUsage();
             AnalyticsUsage emptyUsage = new AnalyticsUsage();
             ContextParser<Void, Void> parser = realUsage.track(item, (p, c) -> c);
             ObjectPath unused = run(realUsage, emptyUsage);
-            assertThat(unused.evaluate("stats.0." + item.name().toLowerCase(Locale.ROOT) + "_usage"), equalTo(0));
-            assertThat(unused.evaluate("stats.1." + item.name().toLowerCase(Locale.ROOT) + "_usage"), equalTo(0));
+            assertThat(unused.evaluate("stats." + item.name().toLowerCase(Locale.ROOT) + "_usage"), equalTo(0));
             int count = between(1, 10000);
             for (int i = 0; i < count; i++) {
                 assertNull(parser.parse(null, null));
             }
             ObjectPath used = run(realUsage, emptyUsage);
-            assertThat(item.name(), used.evaluate("stats.0." + item.name().toLowerCase(Locale.ROOT) + "_usage"), equalTo(count));
-            assertThat(item.name(), used.evaluate("stats.1." + item.name().toLowerCase(Locale.ROOT) + "_usage"), equalTo(0));
+            assertThat(item.name(), used.evaluate("stats." + item.name().toLowerCase(Locale.ROOT) + "_usage"), equalTo(count));
         }
     }
 
@@ -83,11 +83,9 @@ private ObjectPath run(AnalyticsUsage... nodeUsages) throws IOException {
         AnalyticsStatsAction.Response response = new AnalyticsStatsAction.Response(
                 new ClusterName("cluster_name"), nodeResponses, emptyList());
 
+        AnalyticsFeatureSetUsage usage = usageFeatureResponse(true, true, response);
         try (XContentBuilder builder = jsonBuilder()) {
-            builder.startObject();
-            response.toXContent(builder, ToXContent.EMPTY_PARAMS);
-            builder.endObject();
-
+            usage.toXContent(builder, ToXContent.EMPTY_PARAMS);
             return ObjectPath.createFromXContent(JsonXContent.jsonXContent, BytesReference.bytes(builder));
         }
     }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageResponse.java

@@ -43,4 +43,4 @@ public void writeTo(StreamOutput out) throws IOException {
         }
     }
 
-    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/analytics/EnumCounters.java

@@ -0,0 +1,138 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.analytics;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLongArray;
+
+/**
+ * Utility class similar to org.elasticsearch.xpack.core.watcher.common.stats.Counters, but it is using Enum instead
+ * of string to identify the counter. The serialization happens using enum ordinals similar to
+ * {@link StreamOutput#writeEnum(Enum)}, which means that ordinal for existing enums should remain the same for backward
+ * and forward compatibility of the serialization protocol.
+ */
+public class EnumCounters<E extends Enum<E>> implements Writeable {
+    private final AtomicLongArray counters;
+    private final E[] enums;
+
+    public EnumCounters(Class<E> enumClass) {
+        counters = new AtomicLongArray(enumClass.getEnumConstants().length);
+        enums = enumClass.getEnumConstants();
+    }
+
+    public EnumCounters(StreamInput in, Class<E> enumClass) throws IOException {
+        int size = in.readVInt();
+        enums = enumClass.getEnumConstants();
+        long[] vals = new long[enums.length];
+        for (int i = 0; i < size; i++) {
+            long val = in.readVLong();
+            if (i < vals.length) {
+                vals[i] = val;
+            }
+        }
+        counters = new AtomicLongArray(vals);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeVInt(counters.length());
+        for (int i = 0; i < counters.length(); i++) {
+            out.writeVLong(counters.get(i));
+        }
+    }
+
+    public void set(E name) {
+        counters.set(name.ordinal(), 0);
+    }
+
+    public void inc(E name) {
+        counters.incrementAndGet(name.ordinal());
+    }
+
+    public void inc(E name, long count) {
+        counters.addAndGet(name.ordinal(), count);
+    }
+
+    public long get(E name) {
+        return counters.get(name.ordinal());
+    }
+
+    public long size() {
+        return counters.length();
+    }
+
+    public boolean hasCounters() {
+        return size() > 0;
+    }
+
+    public Map<String, Object> toMap() {
+        Map<String, Object> map = new HashMap<>();
+        for (E e : enums) {
+            map.put(e.name().toLowerCase(Locale.ROOT), counters.get(e.ordinal()));
+        }
+        return map;
+    }
+
+    public static <E extends Enum<E>> EnumCounters<E> merge(Class<E> enumClass, List<EnumCounters<E>> counters) {
+        EnumCounters<E> result = new EnumCounters<>(enumClass);
+        E[] enums = enumClass.getEnumConstants();
+        for (EnumCounters<E> c : counters) {
+            for (E e : enums) {
+                result.inc(e, c.get(e));
+            }
+        }
+        return result;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        EnumCounters<?> that = (EnumCounters<?>) o;
+        return Arrays.equals(toArray(), that.toArray()) &&
+            Arrays.equals(enums, that.enums);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = Arrays.hashCode(toArray());
+        result = 31 * result + Arrays.hashCode(enums);
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder buf = new StringBuilder("[");
+        boolean first = true;
+        for (E e : enums) {
+            buf.append(e.name().toLowerCase(Locale.ROOT)).append(": ").append(get(e));
+            if (first) {
+                buf.append(", ");
+                first = false;
+            }
+        }
+        buf.append("]");
+        return buf.toString();
+    }
+
+    private long[] toArray() {
+        long[] res = new long[enums.length];
+        for (int i = 0; i < res.length; i++) {
+            res[i] = counters.get(i);
+        }
+        return res;
+    }
+}