Skip to content

Commit c5ded5c

Browse files
dimitris-athanasioudimitris-athanasiou
authored
[ML] Remove parsing of old progress format in DF Analytics (#55711)
Since #55580 we've introduced a new format for parsing progress from the data frame analytics process. As the process is now writing out progress in this new way, we can remove the parsing of the old format.
1 parent 9ac7a7a commit c5ded5c

File tree

5 files changed

+18
-47
lines changed

5 files changed

+18
-47
lines changed

‎x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessor.java

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@
1818
import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
1919
import org.elasticsearch.xpack.core.ml.dataframe.analyses.Classification;
2020
import org.elasticsearch.xpack.core.ml.dataframe.analyses.Regression;
21-
import org.elasticsearch.xpack.core.ml.dataframe.stats.common.MemoryUsage;
2221
import org.elasticsearch.xpack.core.ml.dataframe.stats.classification.ClassificationStats;
22+
import org.elasticsearch.xpack.core.ml.dataframe.stats.common.MemoryUsage;
2323
import org.elasticsearch.xpack.core.ml.dataframe.stats.outlierdetection.OutlierDetectionStats;
2424
import org.elasticsearch.xpack.core.ml.dataframe.stats.regression.RegressionStats;
2525
import org.elasticsearch.xpack.core.ml.inference.TrainedModelConfig;
@@ -171,14 +171,6 @@ private void processResult(AnalyticsResult result, DataFrameRowsJoiner resultsJo
171171
phaseProgress.getProgressPercent());
172172
statsHolder.getProgressTracker().analyzingPercent.set(phaseProgress.getProgressPercent());
173173
}
174-
175-
// TODO remove after process is writing out phase_progress
176-
Integer progressPercent = result.getProgressPercent();
177-
if (progressPercent != null) {
178-
LOGGER.debug("[{}] Analyzing progress updated to [{}]", analytics.getId(), progressPercent);
179-
statsHolder.getProgressTracker().analyzingPercent.set(progressPercent);
180-
}
181-
182174
TrainedModelDefinition.Builder inferenceModelBuilder = result.getInferenceModelBuilder();
183175
if (inferenceModelBuilder != null) {
184176
createAndIndexInferenceModel(inferenceModelBuilder);

‎x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/results/AnalyticsResult.java

Lines changed: 7 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,6 @@ public class AnalyticsResult implements ToXContentObject {
3030
public static final ParseField TYPE = new ParseField("analytics_result");
3131

3232
private static final ParseField PHASE_PROGRESS = new ParseField("phase_progress");
33-
private static final ParseField PROGRESS_PERCENT = new ParseField("progress_percent");
3433
private static final ParseField INFERENCE_MODEL = new ParseField("inference_model");
3534
private static final ParseField ANALYTICS_MEMORY_USAGE = new ParseField("analytics_memory_usage");
3635
private static final ParseField OUTLIER_DETECTION_STATS = new ParseField("outlier_detection_stats");
@@ -41,18 +40,16 @@ public class AnalyticsResult implements ToXContentObject {
4140
a -> new AnalyticsResult(
4241
(RowResults) a[0],
4342
(PhaseProgress) a[1],
44-
(Integer) a[2],
45-
(TrainedModelDefinition.Builder) a[3],
46-
(MemoryUsage) a[4],
47-
(OutlierDetectionStats) a[5],
48-
(ClassificationStats) a[6],
49-
(RegressionStats) a[7]
43+
(TrainedModelDefinition.Builder) a[2],
44+
(MemoryUsage) a[3],
45+
(OutlierDetectionStats) a[4],
46+
(ClassificationStats) a[5],
47+
(RegressionStats) a[6]
5048
));
5149

5250
static {
5351
PARSER.declareObject(optionalConstructorArg(), RowResults.PARSER, RowResults.TYPE);
5452
PARSER.declareObject(optionalConstructorArg(), PhaseProgress.PARSER, PHASE_PROGRESS);
55-
PARSER.declareInt(optionalConstructorArg(), PROGRESS_PERCENT);
5653
// TODO change back to STRICT_PARSER once native side is aligned
5754
PARSER.declareObject(optionalConstructorArg(), TrainedModelDefinition.LENIENT_PARSER, INFERENCE_MODEL);
5855
PARSER.declareObject(optionalConstructorArg(), MemoryUsage.STRICT_PARSER, ANALYTICS_MEMORY_USAGE);
@@ -63,10 +60,6 @@ public class AnalyticsResult implements ToXContentObject {
6360

6461
private final RowResults rowResults;
6562
private final PhaseProgress phaseProgress;
66-
67-
// TODO remove after process is writing out phase_progress
68-
private final Integer progressPercent;
69-
7063
private final TrainedModelDefinition.Builder inferenceModelBuilder;
7164
private final TrainedModelDefinition inferenceModel;
7265
private final MemoryUsage memoryUsage;
@@ -76,15 +69,13 @@ public class AnalyticsResult implements ToXContentObject {
7669

7770
public AnalyticsResult(@Nullable RowResults rowResults,
7871
@Nullable PhaseProgress phaseProgress,
79-
@Nullable Integer progressPercent,
8072
@Nullable TrainedModelDefinition.Builder inferenceModelBuilder,
8173
@Nullable MemoryUsage memoryUsage,
8274
@Nullable OutlierDetectionStats outlierDetectionStats,
8375
@Nullable ClassificationStats classificationStats,
8476
@Nullable RegressionStats regressionStats) {
8577
this.rowResults = rowResults;
8678
this.phaseProgress = phaseProgress;
87-
this.progressPercent = progressPercent;
8879
this.inferenceModelBuilder = inferenceModelBuilder;
8980
this.inferenceModel = inferenceModelBuilder == null ? null : inferenceModelBuilder.build();
9081
this.memoryUsage = memoryUsage;
@@ -101,10 +92,6 @@ public PhaseProgress getPhaseProgress() {
10192
return phaseProgress;
10293
}
10394

104-
public Integer getProgressPercent() {
105-
return progressPercent;
106-
}
107-
10895
public TrainedModelDefinition.Builder getInferenceModelBuilder() {
10996
return inferenceModelBuilder;
11097
}
@@ -134,9 +121,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
134121
if (phaseProgress != null) {
135122
builder.field(PHASE_PROGRESS.getPreferredName(), phaseProgress);
136123
}
137-
if (progressPercent != null) {
138-
builder.field(PROGRESS_PERCENT.getPreferredName(), progressPercent);
139-
}
140124
if (inferenceModel != null) {
141125
builder.field(INFERENCE_MODEL.getPreferredName(),
142126
inferenceModel,
@@ -170,7 +154,6 @@ public boolean equals(Object other) {
170154
AnalyticsResult that = (AnalyticsResult) other;
171155
return Objects.equals(rowResults, that.rowResults)
172156
&& Objects.equals(phaseProgress, that.phaseProgress)
173-
&& Objects.equals(progressPercent, that.progressPercent)
174157
&& Objects.equals(inferenceModel, that.inferenceModel)
175158
&& Objects.equals(memoryUsage, that.memoryUsage)
176159
&& Objects.equals(outlierDetectionStats, that.outlierDetectionStats)
@@ -180,7 +163,7 @@ public boolean equals(Object other) {
180163

181164
@Override
182165
public int hashCode() {
183-
return Objects.hash(rowResults, phaseProgress, progressPercent, inferenceModel, memoryUsage, outlierDetectionStats,
184-
classificationStats, regressionStats);
166+
return Objects.hash(rowResults, phaseProgress, inferenceModel, memoryUsage, outlierDetectionStats, classificationStats,
167+
regressionStats);
185168
}
186169
}

‎x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManagerTests.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ public class AnalyticsProcessManagerTests extends ESTestCase {
5858
private static final String CONFIG_ID = "config-id";
5959
private static final int NUM_ROWS = 100;
6060
private static final int NUM_COLS = 4;
61-
private static final AnalyticsResult PROCESS_RESULT = new AnalyticsResult(null, null, null, null, null, null, null, null);
61+
private static final AnalyticsResult PROCESS_RESULT = new AnalyticsResult(null, null, null, null, null, null, null);
6262

6363
private Client client;
6464
private DataFrameAnalyticsAuditor auditor;

‎x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessorTests.java

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -105,8 +105,8 @@ public void testProcess_GivenNoResults() {
105105
public void testProcess_GivenEmptyResults() {
106106
givenDataFrameRows(2);
107107
givenProcessResults(Arrays.asList(
108-
new AnalyticsResult(null, null,50, null, null, null, null, null),
109-
new AnalyticsResult(null, null, 100, null, null, null, null, null)));
108+
new AnalyticsResult(null, null,null, null, null, null, null),
109+
new AnalyticsResult(null, null, null, null, null, null, null)));
110110
AnalyticsResultProcessor resultProcessor = createResultProcessor();
111111

112112
resultProcessor.process(process);
@@ -121,8 +121,8 @@ public void testProcess_GivenRowResults() {
121121
givenDataFrameRows(2);
122122
RowResults rowResults1 = mock(RowResults.class);
123123
RowResults rowResults2 = mock(RowResults.class);
124-
givenProcessResults(Arrays.asList(new AnalyticsResult(rowResults1, null,50, null, null, null, null, null),
125-
new AnalyticsResult(rowResults2, null, 100, null, null, null, null, null)));
124+
givenProcessResults(Arrays.asList(new AnalyticsResult(rowResults1, null,null, null, null, null, null),
125+
new AnalyticsResult(rowResults2, null, null, null, null, null, null)));
126126
AnalyticsResultProcessor resultProcessor = createResultProcessor();
127127

128128
resultProcessor.process(process);
@@ -139,8 +139,8 @@ public void testProcess_GivenDataFrameRowsJoinerFails() {
139139
givenDataFrameRows(2);
140140
RowResults rowResults1 = mock(RowResults.class);
141141
RowResults rowResults2 = mock(RowResults.class);
142-
givenProcessResults(Arrays.asList(new AnalyticsResult(rowResults1, null,50, null, null, null, null, null),
143-
new AnalyticsResult(rowResults2, null, 100, null, null, null, null, null)));
142+
givenProcessResults(Arrays.asList(new AnalyticsResult(rowResults1, null,null, null, null, null, null),
143+
new AnalyticsResult(rowResults2, null, null, null, null, null, null)));
144144

145145
doThrow(new RuntimeException("some failure")).when(dataFrameRowsJoiner).processRowResults(any(RowResults.class));
146146

@@ -174,7 +174,7 @@ public void testProcess_GivenInferenceModelIsStoredSuccessfully() {
174174
extractedFieldList.add(new DocValueField("baz", Collections.emptySet()));
175175
TargetType targetType = analyticsConfig.getAnalysis() instanceof Regression ? TargetType.REGRESSION : TargetType.CLASSIFICATION;
176176
TrainedModelDefinition.Builder inferenceModel = TrainedModelDefinitionTests.createRandomBuilder(targetType);
177-
givenProcessResults(Arrays.asList(new AnalyticsResult(null, null, null, inferenceModel, null, null, null, null)));
177+
givenProcessResults(Arrays.asList(new AnalyticsResult(null, null, inferenceModel, null, null, null, null)));
178178
AnalyticsResultProcessor resultProcessor = createResultProcessor(extractedFieldList);
179179

180180
resultProcessor.process(process);
@@ -238,7 +238,7 @@ public void testProcess_GivenInferenceModelFailedToStore() {
238238

239239
TargetType targetType = analyticsConfig.getAnalysis() instanceof Regression ? TargetType.REGRESSION : TargetType.CLASSIFICATION;
240240
TrainedModelDefinition.Builder inferenceModel = TrainedModelDefinitionTests.createRandomBuilder(targetType);
241-
givenProcessResults(Arrays.asList(new AnalyticsResult(null, null, null, inferenceModel, null, null, null, null)));
241+
givenProcessResults(Arrays.asList(new AnalyticsResult(null, null, inferenceModel, null, null, null, null)));
242242
AnalyticsResultProcessor resultProcessor = createResultProcessor();
243243

244244
resultProcessor.process(process);

‎x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/results/AnalyticsResultTests.java

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,6 @@ protected NamedXContentRegistry xContentRegistry() {
4343
protected AnalyticsResult createTestInstance() {
4444
RowResults rowResults = null;
4545
PhaseProgress phaseProgress = null;
46-
Integer progressPercent = null;
4746
TrainedModelDefinition.Builder inferenceModel = null;
4847
MemoryUsage memoryUsage = null;
4948
OutlierDetectionStats outlierDetectionStats = null;
@@ -55,9 +54,6 @@ protected AnalyticsResult createTestInstance() {
5554
if (randomBoolean()) {
5655
phaseProgress = new PhaseProgress(randomAlphaOfLength(10), randomIntBetween(0, 100));
5756
}
58-
if (randomBoolean()) {
59-
progressPercent = randomIntBetween(0, 100);
60-
}
6157
if (randomBoolean()) {
6258
inferenceModel = TrainedModelDefinitionTests.createRandomBuilder();
6359
}
@@ -73,7 +69,7 @@ protected AnalyticsResult createTestInstance() {
7369
if (randomBoolean()) {
7470
regressionStats = RegressionStatsTests.createRandom();
7571
}
76-
return new AnalyticsResult(rowResults, phaseProgress, progressPercent, inferenceModel, memoryUsage, outlierDetectionStats,
72+
return new AnalyticsResult(rowResults, phaseProgress, inferenceModel, memoryUsage, outlierDetectionStats,
7773
classificationStats, regressionStats);
7874
}
7975

0 commit comments

Comments
 (0)