Skip to content

Commit 749120e

Browse files
przemekwitekprzemekwitek
authored
[Transform] Add telemetry support for transform features (#71607)
1 parent c831d88 commit 749120e

File tree

10 files changed

+173
-50
lines changed

10 files changed

+173
-50
lines changed

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformFeatureSetUsage.java

+22-3
Original file line numberDiff line numberDiff line change
@@ -17,25 +17,36 @@
1717
import org.elasticsearch.xpack.core.XPackField;
1818

1919
import java.io.IOException;
20+
import java.util.Collections;
2021
import java.util.Map;
2122
import java.util.Map.Entry;
2223
import java.util.Objects;
2324

2425
public class TransformFeatureSetUsage extends Usage {
2526

27+
private static final String FEATURE_COUNTS = "feature_counts";
28+
2629
private final Map<String, Long> transformCountByState;
30+
private final Map<String, Long> transformCountByFeature;
2731
private final TransformIndexerStats accumulatedStats;
2832

2933
public TransformFeatureSetUsage(StreamInput in) throws IOException {
3034
super(in);
3135
this.transformCountByState = in.readMap(StreamInput::readString, StreamInput::readLong);
36+
if (in.getVersion().onOrAfter(Version.V_8_0_0)) { // TODO: V_7_13_0
37+
this.transformCountByFeature = in.readMap(StreamInput::readString, StreamInput::readLong);
38+
} else {
39+
this.transformCountByFeature = Collections.emptyMap();
40+
}
3241
this.accumulatedStats = new TransformIndexerStats(in);
3342
}
3443

3544
public TransformFeatureSetUsage(Map<String, Long> transformCountByState,
36-
TransformIndexerStats accumulatedStats) {
45+
Map<String, Long> transformCountByFeature,
46+
TransformIndexerStats accumulatedStats) {
3747
super(XPackField.TRANSFORM, true, true);
3848
this.transformCountByState = Objects.requireNonNull(transformCountByState);
49+
this.transformCountByFeature = Objects.requireNonNull(transformCountByFeature);
3950
this.accumulatedStats = Objects.requireNonNull(accumulatedStats);
4051
}
4152

@@ -48,30 +59,37 @@ public Version getMinimalSupportedVersion() {
4859
public void writeTo(StreamOutput out) throws IOException {
4960
super.writeTo(out);
5061
out.writeMap(transformCountByState, StreamOutput::writeString, StreamOutput::writeLong);
62+
if (out.getVersion().onOrAfter(Version.V_8_0_0)) { // TODO: V_7_13_0
63+
out.writeMap(transformCountByFeature, StreamOutput::writeString, StreamOutput::writeLong);
64+
}
5165
accumulatedStats.writeTo(out);
5266
}
5367

5468
@Override
5569
protected void innerXContent(XContentBuilder builder, Params params) throws IOException {
5670
super.innerXContent(builder, params);
5771
if (transformCountByState.isEmpty() == false) {
72+
// Transforms by state
5873
builder.startObject(TransformField.TRANSFORMS.getPreferredName());
5974
long all = 0L;
6075
for (Entry<String, Long> entry : transformCountByState.entrySet()) {
6176
builder.field(entry.getKey(), entry.getValue());
62-
all+=entry.getValue();
77+
all += entry.getValue();
6378
}
6479
builder.field(Metadata.ALL, all);
6580
builder.endObject();
6681

82+
// Transform count for each feature
83+
builder.field(FEATURE_COUNTS, transformCountByFeature);
84+
6785
// if there are no transforms, do not show any stats
6886
builder.field(TransformField.STATS_FIELD.getPreferredName(), accumulatedStats);
6987
}
7088
}
7189

7290
@Override
7391
public int hashCode() {
74-
return Objects.hash(enabled, available, transformCountByState, accumulatedStats);
92+
return Objects.hash(enabled, available, transformCountByState, transformCountByFeature, accumulatedStats);
7593
}
7694

7795
@Override
@@ -85,6 +103,7 @@ public boolean equals(Object obj) {
85103
TransformFeatureSetUsage other = (TransformFeatureSetUsage) obj;
86104
return Objects.equals(name, other.name) && available == other.available && enabled == other.enabled
87105
&& Objects.equals(transformCountByState, other.transformCountByState)
106+
&& Objects.equals(transformCountByFeature, other.transformCountByFeature)
88107
&& Objects.equals(accumulatedStats, other.accumulatedStats);
89108
}
90109
}

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
import org.elasticsearch.common.ParseField;
1111

1212
/*
13-
* Utility class to hold common fields and strings for data frame.
13+
* Utility class to hold common fields and strings for transform.
1414
*/
1515
public final class TransformField {
1616

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java

+18-7
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import java.time.Instant;
3636
import java.util.Collections;
3737
import java.util.List;
38+
import java.util.Locale;
3839
import java.util.Map;
3940
import java.util.Objects;
4041

@@ -51,10 +52,20 @@ public class TransformConfig extends AbstractDiffable<TransformConfig> implement
5152
/** Version in which {@code FieldCapabilitiesRequest.runtime_fields} field was introduced. */
5253
private static final Version FIELD_CAPS_RUNTIME_MAPPINGS_INTRODUCED_VERSION = Version.V_7_12_0;
5354

54-
// types of transforms
55-
public static final ParseField PIVOT_TRANSFORM = new ParseField("pivot");
56-
public static final ParseField LATEST_TRANSFORM = new ParseField("latest");
55+
/** Specifies all the possible transform functions. */
56+
public enum Function {
57+
PIVOT, LATEST;
5758

59+
private final ParseField parseField;
60+
61+
Function() {
62+
this.parseField = new ParseField(name().toLowerCase(Locale.ROOT));
63+
}
64+
65+
public ParseField getParseField() {
66+
return parseField;
67+
}
68+
}
5869
private static final ConstructingObjectParser<TransformConfig, String> STRICT_PARSER = createParser(false);
5970
private static final ConstructingObjectParser<TransformConfig, String> LENIENT_PARSER = createParser(true);
6071
static final int MAX_DESCRIPTION_LENGTH = 1_000;
@@ -149,8 +160,8 @@ private static ConstructingObjectParser<TransformConfig, String> createParser(bo
149160
parser.declareNamedObject(optionalConstructorArg(), (p, c, n) -> p.namedObject(SyncConfig.class, n, c), TransformField.SYNC);
150161
parser.declareString(optionalConstructorArg(), TransformField.INDEX_DOC_TYPE);
151162
parser.declareObject(optionalConstructorArg(), (p, c) -> p.mapStrings(), HEADERS);
152-
parser.declareObject(optionalConstructorArg(), (p, c) -> PivotConfig.fromXContent(p, lenient), PIVOT_TRANSFORM);
153-
parser.declareObject(optionalConstructorArg(), (p, c) -> LatestConfig.fromXContent(p, lenient), LATEST_TRANSFORM);
163+
parser.declareObject(optionalConstructorArg(), (p, c) -> PivotConfig.fromXContent(p, lenient), Function.PIVOT.getParseField());
164+
parser.declareObject(optionalConstructorArg(), (p, c) -> LatestConfig.fromXContent(p, lenient), Function.LATEST.getParseField());
154165
parser.declareString(optionalConstructorArg(), TransformField.DESCRIPTION);
155166
parser.declareObject(optionalConstructorArg(), (p, c) -> SettingsConfig.fromXContent(p, lenient), TransformField.SETTINGS);
156167
parser.declareNamedObject(
@@ -429,10 +440,10 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa
429440
builder.endObject();
430441
}
431442
if (pivotConfig != null) {
432-
builder.field(PIVOT_TRANSFORM.getPreferredName(), pivotConfig);
443+
builder.field(Function.PIVOT.getParseField().getPreferredName(), pivotConfig);
433444
}
434445
if (latestConfig != null) {
435-
builder.field(LATEST_TRANSFORM.getPreferredName(), latestConfig);
446+
builder.field(Function.LATEST.getParseField().getPreferredName(), latestConfig);
436447
}
437448
if (description != null) {
438449
builder.field(TransformField.DESCRIPTION.getPreferredName(), description);

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/persistence/TransformInternalIndexConstants.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,8 @@ public final class TransformInternalIndexConstants {
3131
public static final String TRANSFORM_PREFIX_DEPRECATED = ".data-frame-";
3232

3333
// version is not a rollover pattern, however padded because sort is string based
34-
public static final Version INDEX_VERSION_LAST_CHANGED = Version.V_7_12_0;
35-
public static final String INDEX_VERSION = "006";
34+
public static final Version INDEX_VERSION_LAST_CHANGED = Version.V_7_13_0;
35+
public static final String INDEX_VERSION = "007";
3636
public static final String INDEX_PATTERN = TRANSFORM_PREFIX + "internal-";
3737
public static final String LATEST_INDEX_VERSIONED_NAME = INDEX_PATTERN + INDEX_VERSION;
3838
public static final String LATEST_INDEX_NAME = LATEST_INDEX_VERSIONED_NAME;

‎x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/TransformFeatureSetUsageTests.java

+12-9
Original file line numberDiff line numberDiff line change
@@ -10,27 +10,30 @@
1010
import org.elasticsearch.common.io.stream.Writeable.Reader;
1111
import org.elasticsearch.test.AbstractWireSerializingTestCase;
1212
import org.elasticsearch.xpack.core.indexing.IndexerState;
13+
import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats;
1314
import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStatsTests;
1415

15-
import java.util.HashMap;
16+
import java.util.Arrays;
1617
import java.util.Map;
1718

19+
import static java.util.stream.Collectors.toMap;
20+
1821
public class TransformFeatureSetUsageTests extends AbstractWireSerializingTestCase<TransformFeatureSetUsage> {
1922

2023
@Override
2124
protected TransformFeatureSetUsage createTestInstance() {
22-
Map<String, Long> transformCountByState = new HashMap<>();
23-
24-
if (randomBoolean()) {
25-
transformCountByState.put(randomFrom(IndexerState.values()).toString(), randomLong());
26-
}
27-
28-
return new TransformFeatureSetUsage(transformCountByState, TransformIndexerStatsTests.randomStats());
25+
Map<String, Long> transformCountByState =
26+
randomSubsetOf(Arrays.asList(IndexerState.values())).stream()
27+
.collect(toMap(state -> state.value(), state -> randomLong()));
28+
Map<String, Long> transformCountByFeature =
29+
randomList(10, () -> randomAlphaOfLength(10)).stream()
30+
.collect(toMap(f -> f, f -> randomLong()));
31+
TransformIndexerStats accumulatedStats = TransformIndexerStatsTests.randomStats();
32+
return new TransformFeatureSetUsage(transformCountByState, transformCountByFeature, accumulatedStats);
2933
}
3034

3135
@Override
3236
protected Reader<TransformFeatureSetUsage> instanceReader() {
3337
return TransformFeatureSetUsage::new;
3438
}
35-
3639
}

‎x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java

+20-8
Original file line numberDiff line numberDiff line change
@@ -247,8 +247,6 @@ protected void createReviewsIndexNano() throws IOException {
247247

248248
protected void createContinuousPivotReviewsTransform(String transformId, String transformIndex, String authHeader) throws IOException {
249249

250-
final Request createTransformRequest = createRequestWithAuth("PUT", getTransformEndpoint() + transformId, authHeader);
251-
252250
String config = "{ \"dest\": {\"index\":\"" + transformIndex + "\"}," + " \"source\": {\"index\":\"" + REVIEWS_INDEX_NAME + "\"},"
253251
// Set frequency high for testing
254252
+ " \"sync\": {\"time\":{\"field\": \"timestamp\", \"delay\": \"15m\"}},"
@@ -266,10 +264,7 @@ protected void createContinuousPivotReviewsTransform(String transformId, String
266264
+ " } } } }"
267265
+ "}";
268266

269-
createTransformRequest.setJsonEntity(config);
270-
271-
Map<String, Object> createTransformResponse = entityAsMap(client().performRequest(createTransformRequest));
272-
assertThat(createTransformResponse.get("acknowledged"), equalTo(Boolean.TRUE));
267+
createReviewsTransform(transformId, authHeader, config);
273268
}
274269

275270
protected void createPivotReviewsTransform(
@@ -280,8 +275,6 @@ protected void createPivotReviewsTransform(
280275
String authHeader,
281276
String sourceIndex
282277
) throws IOException {
283-
final Request createTransformRequest = createRequestWithAuth("PUT", getTransformEndpoint() + transformId, authHeader);
284-
285278
String config = "{";
286279

287280
if (pipeline != null) {
@@ -315,6 +308,25 @@ protected void createPivotReviewsTransform(
315308
+ "\"frequency\":\"1s\""
316309
+ "}";
317310

311+
createReviewsTransform(transformId, authHeader, config);
312+
}
313+
314+
protected void createLatestReviewsTransform(String transformId, String transformIndex) throws IOException {
315+
String config = "{"
316+
+ " \"dest\": {\"index\":\"" + transformIndex + "\"},"
317+
+ " \"source\": {\"index\":\"" + REVIEWS_INDEX_NAME + "\"},"
318+
+ " \"latest\": {"
319+
+ " \"unique_key\": [ \"user_id\" ],"
320+
+ " \"sort\": \"@timestamp\""
321+
+ " },"
322+
+ "\"frequency\":\"1s\""
323+
+ "}";
324+
325+
createReviewsTransform(transformId, null, config);
326+
}
327+
328+
private void createReviewsTransform(String transformId, String authHeader, String config) throws IOException {
329+
final Request createTransformRequest = createRequestWithAuth("PUT", getTransformEndpoint() + transformId, authHeader);
318330
createTransformRequest.setJsonEntity(config);
319331

320332
Map<String, Object> createTransformResponse = entityAsMap(client().performRequest(createTransformRequest));

‎x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformUsageIT.java

+23-9
Original file line numberDiff line numberDiff line change
@@ -40,17 +40,23 @@ public void testUsage() throws Exception {
4040
assertTrue((boolean) XContentMapValues.extractValue("transform.available", usageAsMap));
4141
assertTrue((boolean) XContentMapValues.extractValue("transform.enabled", usageAsMap));
4242
// no transforms, no stats
43-
assertEquals(null, XContentMapValues.extractValue("transform.transforms", usageAsMap));
44-
assertEquals(null, XContentMapValues.extractValue("transform.stats", usageAsMap));
43+
assertNull(XContentMapValues.extractValue("transform.transforms", usageAsMap));
44+
assertNull(XContentMapValues.extractValue("transform.feature_counts", usageAsMap));
45+
assertNull(XContentMapValues.extractValue("transform.stats", usageAsMap));
4546

4647
// create transforms
4748
createPivotReviewsTransform("test_usage", "pivot_reviews", null);
4849
createPivotReviewsTransform("test_usage_no_stats", "pivot_reviews_no_stats", null);
4950
createContinuousPivotReviewsTransform("test_usage_continuous", "pivot_reviews_continuous", null);
51+
createLatestReviewsTransform("test_usage_latest", "latest_reviews");
5052
usageResponse = client().performRequest(new Request("GET", "_xpack/usage"));
5153
usageAsMap = entityAsMap(usageResponse);
52-
assertEquals(3, XContentMapValues.extractValue("transform.transforms._all", usageAsMap));
53-
assertEquals(3, XContentMapValues.extractValue("transform.transforms.stopped", usageAsMap));
54+
assertEquals(4, XContentMapValues.extractValue("transform.transforms._all", usageAsMap));
55+
assertEquals(4, XContentMapValues.extractValue("transform.transforms.stopped", usageAsMap));
56+
assertEquals(3, XContentMapValues.extractValue("transform.feature_counts.pivot", usageAsMap));
57+
assertEquals(1, XContentMapValues.extractValue("transform.feature_counts.latest", usageAsMap));
58+
assertEquals(0, XContentMapValues.extractValue("transform.feature_counts.retention_policy", usageAsMap));
59+
assertEquals(1, XContentMapValues.extractValue("transform.feature_counts.sync", usageAsMap));
5460

5561
startAndWaitForTransform("test_usage", "pivot_reviews");
5662
stopTransform("test_usage", false);
@@ -100,9 +106,13 @@ public void testUsage() throws Exception {
100106
Response response = client().performRequest(new Request("GET", "_xpack/usage"));
101107
Map<String, Object> statsMap = entityAsMap(response);
102108
// we should see some stats
103-
assertEquals(3, XContentMapValues.extractValue("transform.transforms._all", statsMap));
104-
assertEquals(2, XContentMapValues.extractValue("transform.transforms.stopped", statsMap));
109+
assertEquals(4, XContentMapValues.extractValue("transform.transforms._all", statsMap));
110+
assertEquals(3, XContentMapValues.extractValue("transform.transforms.stopped", statsMap));
105111
assertEquals(1, XContentMapValues.extractValue("transform.transforms.started", statsMap));
112+
assertEquals(3, XContentMapValues.extractValue("transform.feature_counts.pivot", statsMap));
113+
assertEquals(1, XContentMapValues.extractValue("transform.feature_counts.latest", statsMap));
114+
assertEquals(0, XContentMapValues.extractValue("transform.feature_counts.retention_policy", statsMap));
115+
assertEquals(1, XContentMapValues.extractValue("transform.feature_counts.sync", statsMap));
106116
for (String statName : PROVIDED_STATS) {
107117
// the trigger count can be off: e.g. if the scheduler kicked in before usage has been called,
108118
// or if the scheduler triggered later, but state hasn't been persisted (by design)
@@ -130,11 +140,15 @@ public void testUsage() throws Exception {
130140
usageResponse = client().performRequest(new Request("GET", "_xpack/usage"));
131141
usageAsMap = entityAsMap(usageResponse);
132142

133-
assertEquals(3, XContentMapValues.extractValue("transform.transforms._all", usageAsMap));
134-
assertEquals(3, XContentMapValues.extractValue("transform.transforms.stopped", usageAsMap));
143+
assertEquals(4, XContentMapValues.extractValue("transform.transforms._all", usageAsMap));
144+
assertEquals(4, XContentMapValues.extractValue("transform.transforms.stopped", usageAsMap));
145+
assertEquals(3, XContentMapValues.extractValue("transform.feature_counts.pivot", usageAsMap));
146+
assertEquals(1, XContentMapValues.extractValue("transform.feature_counts.latest", usageAsMap));
147+
assertEquals(0, XContentMapValues.extractValue("transform.feature_counts.retention_policy", usageAsMap));
148+
assertEquals(1, XContentMapValues.extractValue("transform.feature_counts.sync", usageAsMap));
135149
}
136150

137-
private double extractStatsAsDouble(Object statsObject) {
151+
private static double extractStatsAsDouble(Object statsObject) {
138152
if (statsObject instanceof Integer) {
139153
return ((Integer) statsObject).doubleValue();
140154
} else if (statsObject instanceof Double) {

0 commit comments

Comments
 (0)