Skip to content

Commit 5fb9081

Browse files
benwtrentbenwtrent
authored
[ML][Transforms] adjusting stats.progress for cont. transforms (#45361)
* [ML][Transforms] adjusting stats.progress for cont. transforms * addressing PR comments * rename fix
1 parent c980043 commit 5fb9081

27 files changed

+706
-215
lines changed

‎client/rest-high-level/src/main/java/org/elasticsearch/client/core/IndexerJobStats.java

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -36,16 +36,16 @@ public abstract class IndexerJobStats {
3636
public static ParseField SEARCH_FAILURES = new ParseField("search_failures");
3737
public static ParseField INDEX_FAILURES = new ParseField("index_failures");
3838

39-
private final long numPages;
40-
private final long numInputDocuments;
41-
private final long numOuputDocuments;
42-
private final long numInvocations;
43-
private final long indexTime;
44-
private final long indexTotal;
45-
private final long searchTime;
46-
private final long searchTotal;
47-
private final long indexFailures;
48-
private final long searchFailures;
39+
protected final long numPages;
40+
protected final long numInputDocuments;
41+
protected final long numOuputDocuments;
42+
protected final long numInvocations;
43+
protected final long indexTime;
44+
protected final long indexTotal;
45+
protected final long searchTime;
46+
protected final long searchTotal;
47+
protected final long indexFailures;
48+
protected final long searchFailures;
4949

5050
public IndexerJobStats(long numPages, long numInputDocuments, long numOutputDocuments, long numInvocations,
5151
long indexTime, long searchTime, long indexTotal, long searchTotal, long indexFailures, long searchFailures) {

‎client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameIndexerTransformStats.java

Lines changed: 71 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,18 +20,28 @@
2020
package org.elasticsearch.client.dataframe.transforms;
2121

2222
import org.elasticsearch.client.core.IndexerJobStats;
23+
import org.elasticsearch.common.ParseField;
2324
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
2425
import org.elasticsearch.common.xcontent.XContentParser;
2526

2627
import java.io.IOException;
28+
import java.util.Objects;
2729

2830
import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
31+
import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
2932

3033
public class DataFrameIndexerTransformStats extends IndexerJobStats {
3134

35+
static ParseField EXPONENTIAL_AVG_CHECKPOINT_DURATION_MS = new ParseField("exponential_avg_checkpoint_duration_ms");
36+
static ParseField EXPONENTIAL_AVG_DOCUMENTS_INDEXED = new ParseField("exponential_avg_documents_indexed");
37+
static ParseField EXPONENTIAL_AVG_DOCUMENTS_PROCESSED = new ParseField("exponential_avg_documents_processed");
38+
3239
public static final ConstructingObjectParser<DataFrameIndexerTransformStats, Void> LENIENT_PARSER = new ConstructingObjectParser<>(
33-
NAME, true, args -> new DataFrameIndexerTransformStats((long) args[0], (long) args[1], (long) args[2],
34-
(long) args[3], (long) args[4], (long) args[5], (long) args[6], (long) args[7], (long) args[8], (long) args[9]));
40+
NAME,
41+
true,
42+
args -> new DataFrameIndexerTransformStats((long) args[0], (long) args[1], (long) args[2],
43+
(long) args[3], (long) args[4], (long) args[5], (long) args[6], (long) args[7], (long) args[8], (long) args[9],
44+
(Double) args[10], (Double) args[11], (Double) args[12]));
3545

3646
static {
3747
LENIENT_PARSER.declareLong(constructorArg(), NUM_PAGES);
@@ -44,16 +54,74 @@ public class DataFrameIndexerTransformStats extends IndexerJobStats {
4454
LENIENT_PARSER.declareLong(constructorArg(), SEARCH_TOTAL);
4555
LENIENT_PARSER.declareLong(constructorArg(), INDEX_FAILURES);
4656
LENIENT_PARSER.declareLong(constructorArg(), SEARCH_FAILURES);
57+
LENIENT_PARSER.declareDouble(optionalConstructorArg(), EXPONENTIAL_AVG_CHECKPOINT_DURATION_MS);
58+
LENIENT_PARSER.declareDouble(optionalConstructorArg(), EXPONENTIAL_AVG_DOCUMENTS_INDEXED);
59+
LENIENT_PARSER.declareDouble(optionalConstructorArg(), EXPONENTIAL_AVG_DOCUMENTS_PROCESSED);
4760
}
4861

4962
public static DataFrameIndexerTransformStats fromXContent(XContentParser parser) throws IOException {
5063
return LENIENT_PARSER.parse(parser, null);
5164
}
5265

66+
private final double expAvgCheckpointDurationMs;
67+
private final double expAvgDocumentsIndexed;
68+
private final double expAvgDocumentsProcessed;
69+
5370
public DataFrameIndexerTransformStats(long numPages, long numInputDocuments, long numOuputDocuments,
5471
long numInvocations, long indexTime, long searchTime,
55-
long indexTotal, long searchTotal, long indexFailures, long searchFailures) {
72+
long indexTotal, long searchTotal, long indexFailures, long searchFailures,
73+
Double expAvgCheckpointDurationMs, Double expAvgDocumentsIndexed,
74+
Double expAvgDocumentsProcessed) {
5675
super(numPages, numInputDocuments, numOuputDocuments, numInvocations, indexTime, searchTime,
5776
indexTotal, searchTotal, indexFailures, searchFailures);
77+
this.expAvgCheckpointDurationMs = expAvgCheckpointDurationMs == null ? 0.0 : expAvgCheckpointDurationMs;
78+
this.expAvgDocumentsIndexed = expAvgDocumentsIndexed == null ? 0.0 : expAvgDocumentsIndexed;
79+
this.expAvgDocumentsProcessed = expAvgDocumentsProcessed == null ? 0.0 : expAvgDocumentsProcessed;
80+
}
81+
82+
public double getExpAvgCheckpointDurationMs() {
83+
return expAvgCheckpointDurationMs;
84+
}
85+
86+
public double getExpAvgDocumentsIndexed() {
87+
return expAvgDocumentsIndexed;
88+
}
89+
90+
public double getExpAvgDocumentsProcessed() {
91+
return expAvgDocumentsProcessed;
92+
}
93+
94+
@Override
95+
public boolean equals(Object other) {
96+
if (this == other) {
97+
return true;
98+
}
99+
100+
if (other == null || getClass() != other.getClass()) {
101+
return false;
102+
}
103+
104+
DataFrameIndexerTransformStats that = (DataFrameIndexerTransformStats) other;
105+
106+
return Objects.equals(this.numPages, that.numPages)
107+
&& Objects.equals(this.numInputDocuments, that.numInputDocuments)
108+
&& Objects.equals(this.numOuputDocuments, that.numOuputDocuments)
109+
&& Objects.equals(this.numInvocations, that.numInvocations)
110+
&& Objects.equals(this.indexTime, that.indexTime)
111+
&& Objects.equals(this.searchTime, that.searchTime)
112+
&& Objects.equals(this.indexFailures, that.indexFailures)
113+
&& Objects.equals(this.searchFailures, that.searchFailures)
114+
&& Objects.equals(this.indexTotal, that.indexTotal)
115+
&& Objects.equals(this.searchTotal, that.searchTotal)
116+
&& Objects.equals(this.expAvgCheckpointDurationMs, that.expAvgCheckpointDurationMs)
117+
&& Objects.equals(this.expAvgDocumentsIndexed, that.expAvgDocumentsIndexed)
118+
&& Objects.equals(this.expAvgDocumentsProcessed, that.expAvgDocumentsProcessed);
119+
}
120+
121+
@Override
122+
public int hashCode() {
123+
return Objects.hash(numPages, numInputDocuments, numOuputDocuments, numInvocations,
124+
indexTime, searchTime, indexFailures, searchFailures, indexTotal, searchTotal,
125+
expAvgCheckpointDurationMs, expAvgDocumentsIndexed, expAvgDocumentsProcessed);
58126
}
59127
}

‎client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformCheckpointingInfo.java

Lines changed: 32 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -19,30 +19,40 @@
1919

2020
package org.elasticsearch.client.dataframe.transforms;
2121

22+
import org.elasticsearch.client.dataframe.transforms.util.TimeUtil;
23+
import org.elasticsearch.common.Nullable;
2224
import org.elasticsearch.common.ParseField;
2325
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
26+
import org.elasticsearch.common.xcontent.ObjectParser;
2427
import org.elasticsearch.common.xcontent.XContentParser;
2528

29+
import java.time.Instant;
2630
import java.util.Objects;
2731

2832
public class DataFrameTransformCheckpointingInfo {
2933

3034
public static final ParseField LAST_CHECKPOINT = new ParseField("last", "current");
3135
public static final ParseField NEXT_CHECKPOINT = new ParseField("next", "in_progress");
3236
public static final ParseField OPERATIONS_BEHIND = new ParseField("operations_behind");
37+
public static final ParseField CHANGES_LAST_DETECTED_AT = new ParseField("changes_last_detected_at");
3338

3439
private final DataFrameTransformCheckpointStats last;
3540
private final DataFrameTransformCheckpointStats next;
3641
private final long operationsBehind;
42+
private final Instant changesLastDetectedAt;
3743

3844
private static final ConstructingObjectParser<DataFrameTransformCheckpointingInfo, Void> LENIENT_PARSER =
3945
new ConstructingObjectParser<>(
40-
"data_frame_transform_checkpointing_info", true, a -> {
46+
"data_frame_transform_checkpointing_info",
47+
true,
48+
a -> {
4149
long behind = a[2] == null ? 0L : (Long) a[2];
42-
50+
Instant changesLastDetectedAt = (Instant)a[3];
4351
return new DataFrameTransformCheckpointingInfo(
44-
a[0] == null ? DataFrameTransformCheckpointStats.EMPTY : (DataFrameTransformCheckpointStats) a[0],
45-
a[1] == null ? DataFrameTransformCheckpointStats.EMPTY : (DataFrameTransformCheckpointStats) a[1], behind);
52+
a[0] == null ? DataFrameTransformCheckpointStats.EMPTY : (DataFrameTransformCheckpointStats) a[0],
53+
a[1] == null ? DataFrameTransformCheckpointStats.EMPTY : (DataFrameTransformCheckpointStats) a[1],
54+
behind,
55+
changesLastDetectedAt);
4656
});
4757

4858
static {
@@ -51,13 +61,20 @@ public class DataFrameTransformCheckpointingInfo {
5161
LENIENT_PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(),
5262
(p, c) -> DataFrameTransformCheckpointStats.fromXContent(p), NEXT_CHECKPOINT);
5363
LENIENT_PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), OPERATIONS_BEHIND);
64+
LENIENT_PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(),
65+
p -> TimeUtil.parseTimeFieldToInstant(p, CHANGES_LAST_DETECTED_AT.getPreferredName()),
66+
CHANGES_LAST_DETECTED_AT,
67+
ObjectParser.ValueType.VALUE);
5468
}
5569

56-
public DataFrameTransformCheckpointingInfo(DataFrameTransformCheckpointStats last, DataFrameTransformCheckpointStats next,
57-
long operationsBehind) {
70+
public DataFrameTransformCheckpointingInfo(DataFrameTransformCheckpointStats last,
71+
DataFrameTransformCheckpointStats next,
72+
long operationsBehind,
73+
Instant changesLastDetectedAt) {
5874
this.last = Objects.requireNonNull(last);
5975
this.next = Objects.requireNonNull(next);
6076
this.operationsBehind = operationsBehind;
77+
this.changesLastDetectedAt = changesLastDetectedAt;
6178
}
6279

6380
public DataFrameTransformCheckpointStats getLast() {
@@ -72,13 +89,18 @@ public long getOperationsBehind() {
7289
return operationsBehind;
7390
}
7491

92+
@Nullable
93+
public Instant getChangesLastDetectedAt() {
94+
return changesLastDetectedAt;
95+
}
96+
7597
public static DataFrameTransformCheckpointingInfo fromXContent(XContentParser p) {
7698
return LENIENT_PARSER.apply(p, null);
7799
}
78100

79101
@Override
80102
public int hashCode() {
81-
return Objects.hash(last, next, operationsBehind);
103+
return Objects.hash(last, next, operationsBehind, changesLastDetectedAt);
82104
}
83105

84106
@Override
@@ -94,8 +116,9 @@ public boolean equals(Object other) {
94116
DataFrameTransformCheckpointingInfo that = (DataFrameTransformCheckpointingInfo) other;
95117

96118
return Objects.equals(this.last, that.last) &&
97-
Objects.equals(this.next, that.next) &&
98-
this.operationsBehind == that.operationsBehind;
119+
Objects.equals(this.next, that.next) &&
120+
this.operationsBehind == that.operationsBehind &&
121+
Objects.equals(this.changesLastDetectedAt, that.changesLastDetectedAt);
99122
}
100123

101124
}

‎client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformProgress.java

Lines changed: 38 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -19,58 +19,81 @@
1919

2020
package org.elasticsearch.client.dataframe.transforms;
2121

22+
import org.elasticsearch.common.Nullable;
2223
import org.elasticsearch.common.ParseField;
2324
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
2425
import org.elasticsearch.common.xcontent.XContentParser;
2526

2627
import java.util.Objects;
2728

28-
import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
2929
import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
3030

3131
public class DataFrameTransformProgress {
3232

3333
public static final ParseField TOTAL_DOCS = new ParseField("total_docs");
3434
public static final ParseField DOCS_REMAINING = new ParseField("docs_remaining");
3535
public static final ParseField PERCENT_COMPLETE = new ParseField("percent_complete");
36+
public static final ParseField DOCS_PROCESSED = new ParseField("docs_processed");
37+
public static final ParseField DOCS_INDEXED = new ParseField("docs_indexed");
3638

3739
public static final ConstructingObjectParser<DataFrameTransformProgress, Void> PARSER = new ConstructingObjectParser<>(
3840
"data_frame_transform_progress",
3941
true,
40-
a -> new DataFrameTransformProgress((Long) a[0], (Long)a[1], (Double)a[2]));
42+
a -> new DataFrameTransformProgress((Long) a[0], (Long)a[1], (Double)a[2], (Long)a[3], (Long)a[4]));
4143

4244
static {
43-
PARSER.declareLong(constructorArg(), TOTAL_DOCS);
45+
PARSER.declareLong(optionalConstructorArg(), TOTAL_DOCS);
4446
PARSER.declareLong(optionalConstructorArg(), DOCS_REMAINING);
4547
PARSER.declareDouble(optionalConstructorArg(), PERCENT_COMPLETE);
48+
PARSER.declareLong(optionalConstructorArg(), DOCS_PROCESSED);
49+
PARSER.declareLong(optionalConstructorArg(), DOCS_INDEXED);
4650
}
4751

4852
public static DataFrameTransformProgress fromXContent(XContentParser parser) {
4953
return PARSER.apply(parser, null);
5054
}
5155

52-
private final long totalDocs;
53-
private final long remainingDocs;
54-
private final double percentComplete;
55-
56-
public DataFrameTransformProgress(long totalDocs, Long remainingDocs, double percentComplete) {
56+
private final Long totalDocs;
57+
private final Long remainingDocs;
58+
private final Double percentComplete;
59+
private final long documentsProcessed;
60+
private final long documentsIndexed;
61+
62+
public DataFrameTransformProgress(Long totalDocs,
63+
Long remainingDocs,
64+
Double percentComplete,
65+
Long documentsProcessed,
66+
Long documentsIndexed) {
5767
this.totalDocs = totalDocs;
5868
this.remainingDocs = remainingDocs == null ? totalDocs : remainingDocs;
5969
this.percentComplete = percentComplete;
70+
this.documentsProcessed = documentsProcessed == null ? 0 : documentsProcessed;
71+
this.documentsIndexed = documentsIndexed == null ? 0 : documentsIndexed;
6072
}
6173

62-
public double getPercentComplete() {
74+
@Nullable
75+
public Double getPercentComplete() {
6376
return percentComplete;
6477
}
6578

66-
public long getTotalDocs() {
79+
@Nullable
80+
public Long getTotalDocs() {
6781
return totalDocs;
6882
}
6983

70-
public long getRemainingDocs() {
84+
@Nullable
85+
public Long getRemainingDocs() {
7186
return remainingDocs;
7287
}
7388

89+
public long getDocumentsProcessed() {
90+
return documentsProcessed;
91+
}
92+
93+
public long getDocumentsIndexed() {
94+
return documentsIndexed;
95+
}
96+
7497
@Override
7598
public boolean equals(Object other) {
7699
if (other == this) {
@@ -84,11 +107,13 @@ public boolean equals(Object other) {
84107
DataFrameTransformProgress that = (DataFrameTransformProgress) other;
85108
return Objects.equals(this.remainingDocs, that.remainingDocs)
86109
&& Objects.equals(this.totalDocs, that.totalDocs)
87-
&& Objects.equals(this.percentComplete, that.percentComplete);
110+
&& Objects.equals(this.percentComplete, that.percentComplete)
111+
&& Objects.equals(this.documentsIndexed, that.documentsIndexed)
112+
&& Objects.equals(this.documentsProcessed, that.documentsProcessed);
88113
}
89114

90115
@Override
91116
public int hashCode(){
92-
return Objects.hash(remainingDocs, totalDocs, percentComplete);
117+
return Objects.hash(remainingDocs, totalDocs, percentComplete, documentsIndexed, documentsProcessed);
93118
}
94119
}

‎client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameTransformIT.java

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -406,7 +406,20 @@ public void testGetStats() throws Exception {
406406
DataFrameTransformStats stats = statsResponse.getTransformsStats().get(0);
407407
assertEquals(DataFrameTransformStats.State.STOPPED, stats.getState());
408408

409-
DataFrameIndexerTransformStats zeroIndexerStats = new DataFrameIndexerTransformStats(0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L);
409+
DataFrameIndexerTransformStats zeroIndexerStats = new DataFrameIndexerTransformStats(
410+
0L,
411+
0L,
412+
0L,
413+
0L,
414+
0L,
415+
0L,
416+
0L,
417+
0L,
418+
0L,
419+
0L,
420+
0.0,
421+
0.0,
422+
0.0);
410423
assertEquals(zeroIndexerStats, stats.getIndexerStats());
411424

412425
// start the transform

‎client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameIndexerTransformStatsTests.java

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,10 @@ public void testFromXContent() throws IOException {
4242
public static DataFrameIndexerTransformStats randomStats() {
4343
return new DataFrameIndexerTransformStats(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
4444
randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
45-
randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong());
45+
randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
46+
randomBoolean() ? null : randomDouble(),
47+
randomBoolean() ? null : randomDouble(),
48+
randomBoolean() ? null : randomDouble());
4649
}
4750

4851
public static void toXContent(DataFrameIndexerTransformStats stats, XContentBuilder builder) throws IOException {
@@ -57,6 +60,12 @@ public static void toXContent(DataFrameIndexerTransformStats stats, XContentBuil
5760
builder.field(IndexerJobStats.SEARCH_TIME_IN_MS.getPreferredName(), stats.getSearchTime());
5861
builder.field(IndexerJobStats.SEARCH_TOTAL.getPreferredName(), stats.getSearchTotal());
5962
builder.field(IndexerJobStats.SEARCH_FAILURES.getPreferredName(), stats.getSearchFailures());
63+
builder.field(DataFrameIndexerTransformStats.EXPONENTIAL_AVG_CHECKPOINT_DURATION_MS.getPreferredName(),
64+
stats.getExpAvgCheckpointDurationMs());
65+
builder.field(DataFrameIndexerTransformStats.EXPONENTIAL_AVG_DOCUMENTS_INDEXED.getPreferredName(),
66+
stats.getExpAvgDocumentsIndexed());
67+
builder.field(DataFrameIndexerTransformStats.EXPONENTIAL_AVG_DOCUMENTS_PROCESSED.getPreferredName(),
68+
stats.getExpAvgDocumentsProcessed());
6069
builder.endObject();
6170
}
6271
}

0 commit comments

Comments
 (0)