Skip to content

Commit 34734ae

Browse files
author
Hendrik Muhs
authored
[Transform] provide exponential_avg* stats for batch transforms (#52041)
provide exponential_avg* stats for batch transforms, avoids confusion why those values are all 0 otherwise
1 parent 6ace2ef commit 34734ae

File tree

8 files changed

+297
-165
lines changed

8 files changed

+297
-165
lines changed

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStats.java

Lines changed: 87 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88

99
import org.elasticsearch.Version;
1010
import org.elasticsearch.common.ParseField;
11+
import org.elasticsearch.common.Strings;
1112
import org.elasticsearch.common.io.stream.StreamInput;
1213
import org.elasticsearch.common.io.stream.StreamOutput;
1314
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
@@ -36,23 +37,34 @@ public class TransformIndexerStats extends IndexerJobStats {
3637
public static ParseField SEARCH_TOTAL = new ParseField("search_total");
3738
public static ParseField SEARCH_FAILURES = new ParseField("search_failures");
3839
public static ParseField INDEX_FAILURES = new ParseField("index_failures");
39-
public static ParseField EXPONENTIAL_AVG_CHECKPOINT_DURATION_MS =
40-
new ParseField("exponential_avg_checkpoint_duration_ms");
41-
public static ParseField EXPONENTIAL_AVG_DOCUMENTS_INDEXED =
42-
new ParseField("exponential_avg_documents_indexed");
43-
public static ParseField EXPONENTIAL_AVG_DOCUMENTS_PROCESSED =
44-
new ParseField("exponential_avg_documents_processed");
40+
public static ParseField EXPONENTIAL_AVG_CHECKPOINT_DURATION_MS = new ParseField("exponential_avg_checkpoint_duration_ms");
41+
public static ParseField EXPONENTIAL_AVG_DOCUMENTS_INDEXED = new ParseField("exponential_avg_documents_indexed");
42+
public static ParseField EXPONENTIAL_AVG_DOCUMENTS_PROCESSED = new ParseField("exponential_avg_documents_processed");
4543

4644
// This changes how much "weight" past calculations have.
4745
// The shorter the window, the less "smoothing" will occur.
4846
private static final int EXP_AVG_WINDOW = 10;
49-
private static final double ALPHA = 2.0/(EXP_AVG_WINDOW + 1);
47+
private static final double ALPHA = 2.0 / (EXP_AVG_WINDOW + 1);
5048

5149
private static final ConstructingObjectParser<TransformIndexerStats, Void> LENIENT_PARSER = new ConstructingObjectParser<>(
52-
NAME, true,
53-
args -> new TransformIndexerStats(
54-
(long) args[0], (long) args[1], (long) args[2], (long) args[3], (long) args[4], (long) args[5], (long) args[6],
55-
(long) args[7], (long) args[8], (long) args[9], (Double) args[10], (Double) args[11], (Double) args[12]));
50+
NAME,
51+
true,
52+
args -> new TransformIndexerStats(
53+
(long) args[0],
54+
(long) args[1],
55+
(long) args[2],
56+
(long) args[3],
57+
(long) args[4],
58+
(long) args[5],
59+
(long) args[6],
60+
(long) args[7],
61+
(long) args[8],
62+
(long) args[9],
63+
(Double) args[10],
64+
(Double) args[11],
65+
(Double) args[12]
66+
)
67+
);
5668

5769
static {
5870
LENIENT_PARSER.declareLong(constructorArg(), NUM_PAGES);
@@ -73,37 +85,62 @@ public class TransformIndexerStats extends IndexerJobStats {
7385
private double expAvgCheckpointDurationMs;
7486
private double expAvgDocumentsIndexed;
7587
private double expAvgDocumentsProcessed;
88+
7689
/**
7790
* Create with all stats set to zero
7891
*/
7992
public TransformIndexerStats() {
8093
super();
8194
}
8295

83-
public TransformIndexerStats(long numPages, long numInputDocuments, long numOutputDocuments,
84-
long numInvocations, long indexTime, long searchTime, long indexTotal, long searchTotal,
85-
long indexFailures, long searchFailures, Double expAvgCheckpointDurationMs,
86-
Double expAvgDocumentsIndexed, Double expAvgDocumentsProcessed ) {
87-
super(numPages, numInputDocuments, numOutputDocuments, numInvocations, indexTime, searchTime, indexTotal, searchTotal,
88-
indexFailures, searchFailures);
96+
public TransformIndexerStats(
97+
long numPages,
98+
long numInputDocuments,
99+
long numOutputDocuments,
100+
long numInvocations,
101+
long indexTime,
102+
long searchTime,
103+
long indexTotal,
104+
long searchTotal,
105+
long indexFailures,
106+
long searchFailures,
107+
Double expAvgCheckpointDurationMs,
108+
Double expAvgDocumentsIndexed,
109+
Double expAvgDocumentsProcessed
110+
) {
111+
super(
112+
numPages,
113+
numInputDocuments,
114+
numOutputDocuments,
115+
numInvocations,
116+
indexTime,
117+
searchTime,
118+
indexTotal,
119+
searchTotal,
120+
indexFailures,
121+
searchFailures
122+
);
89123
this.expAvgCheckpointDurationMs = expAvgCheckpointDurationMs == null ? 0.0 : expAvgCheckpointDurationMs;
90124
this.expAvgDocumentsIndexed = expAvgDocumentsIndexed == null ? 0.0 : expAvgDocumentsIndexed;
91125
this.expAvgDocumentsProcessed = expAvgDocumentsProcessed == null ? 0.0 : expAvgDocumentsProcessed;
92126
}
93127

94-
public TransformIndexerStats(long numPages, long numInputDocuments, long numOutputDocuments,
95-
long numInvocations, long indexTime, long searchTime, long indexTotal, long searchTotal,
96-
long indexFailures, long searchFailures) {
97-
this(numPages, numInputDocuments, numOutputDocuments, numInvocations, indexTime, searchTime, indexTotal, searchTotal,
98-
indexFailures, searchFailures, 0.0, 0.0, 0.0);
99-
}
100-
101128
public TransformIndexerStats(TransformIndexerStats other) {
102-
this(other.numPages, other.numInputDocuments, other.numOuputDocuments, other.numInvocations,
103-
other.indexTime, other.searchTime, other.indexTotal, other.searchTotal, other.indexFailures, other.searchFailures);
104-
this.expAvgCheckpointDurationMs = other.expAvgCheckpointDurationMs;
105-
this.expAvgDocumentsIndexed = other.expAvgDocumentsIndexed;
106-
this.expAvgDocumentsProcessed = other.expAvgDocumentsProcessed;
129+
this(
130+
other.numPages,
131+
other.numInputDocuments,
132+
other.numOuputDocuments,
133+
other.numInvocations,
134+
other.indexTime,
135+
other.searchTime,
136+
other.indexTotal,
137+
other.searchTotal,
138+
other.indexFailures,
139+
other.searchFailures,
140+
other.expAvgCheckpointDurationMs,
141+
other.expAvgDocumentsIndexed,
142+
other.expAvgDocumentsProcessed
143+
);
107144
}
108145

109146
public TransformIndexerStats(StreamInput in) throws IOException {
@@ -180,7 +217,7 @@ public void incrementCheckpointExponentialAverages(long checkpointDurationMs, lo
180217
}
181218

182219
private double calculateExpAvg(double previousExpValue, double alpha, long observedValue) {
183-
return alpha * observedValue + (1-alpha) * previousExpValue;
220+
return alpha * observedValue + (1 - alpha) * previousExpValue;
184221
}
185222

186223
@Override
@@ -212,9 +249,26 @@ public boolean equals(Object other) {
212249

213250
@Override
214251
public int hashCode() {
215-
return Objects.hash(numPages, numInputDocuments, numOuputDocuments, numInvocations,
216-
indexTime, searchTime, indexFailures, searchFailures, indexTotal, searchTotal,
217-
expAvgCheckpointDurationMs, expAvgDocumentsIndexed, expAvgDocumentsProcessed);
252+
return Objects.hash(
253+
numPages,
254+
numInputDocuments,
255+
numOuputDocuments,
256+
numInvocations,
257+
indexTime,
258+
searchTime,
259+
indexFailures,
260+
searchFailures,
261+
indexTotal,
262+
searchTotal,
263+
expAvgCheckpointDurationMs,
264+
expAvgDocumentsIndexed,
265+
expAvgDocumentsProcessed
266+
);
267+
}
268+
269+
@Override
270+
public String toString() {
271+
return Strings.toString(this);
218272
}
219273

220274
public static TransformIndexerStats fromXContent(XContentParser parser) {

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStatsTests.java

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,13 +31,21 @@ protected TransformIndexerStats doParseInstance(XContentParser parser) {
3131
}
3232

3333
public static TransformIndexerStats randomStats() {
34-
return new TransformIndexerStats(randomLongBetween(10L, 10000L),
35-
randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L),
36-
randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L),
34+
return new TransformIndexerStats(
35+
randomLongBetween(10L, 10000L),
36+
randomLongBetween(0L, 10000L),
37+
randomLongBetween(0L, 10000L),
38+
randomLongBetween(0L, 10000L),
39+
randomLongBetween(0L, 10000L),
40+
randomLongBetween(0L, 10000L),
41+
randomLongBetween(0L, 10000L),
42+
randomLongBetween(0L, 10000L),
43+
randomLongBetween(0L, 10000L),
3744
randomLongBetween(0L, 10000L),
3845
randomBoolean() ? randomDouble() : null,
3946
randomBoolean() ? randomDouble() : null,
40-
randomBoolean() ? randomDouble() : null);
47+
randomBoolean() ? randomDouble() : null
48+
);
4149
}
4250

4351
public void testExpAvgIncrement() {

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformStatsTests.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ public void testBwcWith73() throws IOException {
6969
STARTED,
7070
randomBoolean() ? null : randomAlphaOfLength(100),
7171
randomBoolean() ? null : NodeAttributeTests.randomNodeAttributes(),
72-
new TransformIndexerStats(1, 2, 3, 4, 5, 6, 7, 8, 9, 10),
72+
new TransformIndexerStats(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 0.0, 0.0, 0.0),
7373
new TransformCheckpointingInfo(
7474
new TransformCheckpointStats(0, null, null, 10, 100),
7575
new TransformCheckpointStats(0, null, null, 100, 1000),

0 commit comments

Comments
 (0)