Skip to content

Commit c16acef

Browse files
davidkyledavidkyle
committed
Rewrite Inference yml tests for better clean up (elastic#61180)
Inference processors asynchronously usage write stats to the .ml-stats index after they used. In tests the write can leak into the next test causing failures depending on which test follows. This change waits for the usage stats docs to be written at the end of the test
1 parent 97fd4dd commit c16acef

File tree

3 files changed

+174
-50
lines changed

3 files changed

+174
-50
lines changed

‎x-pack/plugin/ml/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/InferenceProcessorIT.java

+170
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,170 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
package org.elasticsearch.xpack.ml.integration;
8+
9+
import org.elasticsearch.client.Request;
10+
import org.elasticsearch.client.RequestOptions;
11+
import org.elasticsearch.client.Response;
12+
import org.elasticsearch.client.ResponseException;
13+
import org.elasticsearch.test.rest.ESRestTestCase;
14+
import org.junit.After;
15+
import org.junit.Before;
16+
17+
import java.io.IOException;
18+
import java.util.Map;
19+
20+
import static org.hamcrest.Matchers.equalTo;
21+
22+
23+
public class InferenceProcessorIT extends ESRestTestCase {
24+
25+
private static final String MODEL_ID = "a-perfect-regression-model";
26+
27+
@Before
28+
public void enableLogging() throws IOException {
29+
Request setTrace = new Request("PUT", "_cluster/settings");
30+
setTrace.setJsonEntity(
31+
"{\"transient\": {\"logger.org.elasticsearch.xpack.ml.inference\": \"TRACE\"}}"
32+
);
33+
assertThat(client().performRequest(setTrace).getStatusLine().getStatusCode(), equalTo(200));
34+
}
35+
36+
private void putRegressionModel() throws IOException {
37+
38+
Request model = new Request("PUT", "_ml/inference/" + MODEL_ID);
39+
model.setJsonEntity(
40+
" {\n" +
41+
" \"description\": \"empty model for tests\",\n" +
42+
" \"tags\": [\"regression\", \"tag1\"],\n" +
43+
" \"input\": {\"field_names\": [\"field1\", \"field2\"]},\n" +
44+
" \"inference_config\": { \"regression\": {\"results_field\": \"my_regression\"}},\n" +
45+
" \"definition\": {\n" +
46+
" \"preprocessors\": [],\n" +
47+
" \"trained_model\": {\n" +
48+
" \"tree\": {\n" +
49+
" \"feature_names\": [\"field1\", \"field2\"],\n" +
50+
" \"tree_structure\": [\n" +
51+
" {\"node_index\": 0, \"leaf_value\": 42}\n" +
52+
" ],\n" +
53+
" \"target_type\": \"regression\"\n" +
54+
" }\n" +
55+
" }\n" +
56+
" }\n" +
57+
" }"
58+
);
59+
60+
assertThat(client().performRequest(model).getStatusLine().getStatusCode(), equalTo(200));
61+
}
62+
63+
public void testCreateAndDeletePipelineWithInferenceProcessor() throws IOException {
64+
putRegressionModel();
65+
66+
Request putPipeline = new Request("PUT", "_ingest/pipeline/regression-model-pipeline");
67+
putPipeline.setJsonEntity(
68+
" {\n" +
69+
" \"processors\": [\n" +
70+
" {\n" +
71+
" \"inference\" : {\n" +
72+
" \"model_id\" : \"a-perfect-regression-model\",\n" +
73+
" \"inference_config\": {\"regression\": {}},\n" +
74+
" \"target_field\": \"regression_field\",\n" +
75+
" \"field_map\": {}\n" +
76+
" }\n" +
77+
" }\n" +
78+
" ]\n" +
79+
" }"
80+
);
81+
82+
assertThat(client().performRequest(putPipeline).getStatusLine().getStatusCode(), equalTo(200));
83+
84+
// using the model will ensure it is loaded and stats will be written before it is deleted
85+
infer("regression-model-pipeline");
86+
87+
Request deletePipeline = new Request("DELETE", "_ingest/pipeline/regression-model-pipeline");
88+
assertThat(client().performRequest(deletePipeline).getStatusLine().getStatusCode(), equalTo(200));
89+
}
90+
91+
public void testCreateProcessorWithDeprecatedFields() throws IOException {
92+
putRegressionModel();
93+
94+
Request putPipeline = new Request("PUT", "_ingest/pipeline/regression-model-deprecated-pipeline");
95+
putPipeline.setJsonEntity(
96+
"{\n" +
97+
" \"processors\": [\n" +
98+
" {\n" +
99+
" \"inference\" : {\n" +
100+
" \"model_id\" : \"a-perfect-regression-model\",\n" +
101+
" \"inference_config\": {\"regression\": {}},\n" +
102+
" \"field_mappings\": {}\n" +
103+
" }\n" +
104+
" }\n" +
105+
" ]\n" +
106+
"}"
107+
);
108+
109+
RequestOptions ro = expectWarnings("Deprecated field [field_mappings] used, expected [field_map] instead");
110+
putPipeline.setOptions(ro);
111+
Response putResponse = client().performRequest(putPipeline);
112+
assertThat(putResponse.getStatusLine().getStatusCode(), equalTo(200));
113+
114+
// using the model will ensure it is loaded and stats will be written before it is deleted
115+
infer("regression-model-deprecated-pipeline");
116+
117+
Request deletePipeline = new Request("DELETE", "_ingest/pipeline/regression-model-deprecated-pipeline");
118+
Response deleteResponse = client().performRequest(deletePipeline);
119+
assertThat(deleteResponse.getStatusLine().getStatusCode(), equalTo(200));
120+
}
121+
122+
public void infer(String pipelineId) throws IOException {
123+
Request putDoc = new Request("POST", "any_index/_doc?pipeline=" + pipelineId);
124+
putDoc.setJsonEntity("{\"field1\": 1, \"field2\": 2}");
125+
126+
Response response = client().performRequest(putDoc);
127+
assertThat(response.getStatusLine().getStatusCode(), equalTo(201));
128+
}
129+
130+
@After
131+
@SuppressWarnings("unchecked")
132+
public void waitForStatsDoc() throws Exception {
133+
assertBusy( () -> {
134+
Request searchForStats = new Request("GET", ".ml-stats-*/_search?rest_total_hits_as_int");
135+
searchForStats.setJsonEntity(
136+
"{\n" +
137+
" \"query\": {\n" +
138+
" \"bool\": {\n" +
139+
" \"filter\": [\n" +
140+
" {\n" +
141+
" \"term\": {\n" +
142+
" \"type\": \"inference_stats\"\n" +
143+
" }\n" +
144+
" },\n" +
145+
" {\n" +
146+
" \"term\": {\n" +
147+
" \"model_id\": \"" + MODEL_ID + "\"\n" +
148+
" }\n" +
149+
" }\n" +
150+
" ]\n" +
151+
" }\n" +
152+
" }\n" +
153+
"}"
154+
);
155+
156+
try {
157+
Response searchResponse = client().performRequest(searchForStats);
158+
159+
Map<String, Object> responseAsMap = entityAsMap(searchResponse);
160+
Map<String, Object> hits = (Map<String, Object>)responseAsMap.get("hits");
161+
assertThat(responseAsMap.toString(), hits.get("total"), equalTo(1));
162+
} catch (ResponseException e) {
163+
// the search may fail because the index is not ready yet in which case retry
164+
if (e.getMessage().contains("search_phase_execution_exception") == false) {
165+
throw e;
166+
}
167+
}
168+
});
169+
}
170+
}

‎x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/loadingservice/ModelLoadingService.java

+4
Original file line numberDiff line numberDiff line change
@@ -430,6 +430,10 @@ private void cacheEvictionListener(RemovalNotification<String, ModelAndConsumer>
430430
INFERENCE_MODEL_CACHE_TTL.getKey());
431431
auditIfNecessary(notification.getKey(), msg);
432432
}
433+
434+
logger.trace(() -> new ParameterizedMessage("Persisting stats for evicted model [{}]",
435+
notification.getValue().model.getModelId()));
436+
433437
// If the model is no longer referenced, flush the stats to persist as soon as possible
434438
notification.getValue().model.persistStats(referencedModels.contains(notification.getKey()) == false);
435439
} finally {

‎x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_processor.yml

-50
Original file line numberDiff line numberDiff line change
@@ -29,28 +29,6 @@ setup:
2929
}
3030
3131
---
32-
"Test create and delete pipeline with inference processor":
33-
- do:
34-
ingest.put_pipeline:
35-
id: "regression-model-pipeline"
36-
body: >
37-
{
38-
"processors": [
39-
{
40-
"inference" : {
41-
"model_id" : "a-perfect-regression-model",
42-
"inference_config": {"regression": {}},
43-
"target_field": "regression_field",
44-
"field_map": {}
45-
}
46-
}
47-
]
48-
}
49-
- match: { acknowledged: true }
50-
- do:
51-
ingest.delete_pipeline:
52-
id: "regression-model-pipeline"
53-
---
5432
"Test create processor with missing mandatory fields":
5533
- do:
5634
catch: /\[model_id\] required property is missing/
@@ -68,35 +46,7 @@ setup:
6846
}
6947
]
7048
}
71-
---
72-
"Test create processor with deprecated fields":
73-
- skip:
74-
features:
75-
- "warnings"
76-
- "allowed_warnings"
77-
- do:
78-
warnings:
79-
- 'Deprecated field [field_mappings] used, expected [field_map] instead'
80-
ingest.put_pipeline:
81-
id: "regression-model-pipeline"
82-
body: >
83-
{
84-
"processors": [
85-
{
86-
"inference" : {
87-
"model_id" : "a-perfect-regression-model",
88-
"inference_config": {"regression": {}},
89-
"field_mappings": {}
90-
}
91-
}
92-
]
93-
}
9449
95-
- do:
96-
allowed_warnings:
97-
- 'Deprecated field [field_mappings] used, expected [field_map] instead'
98-
ingest.delete_pipeline:
99-
id: "regression-model-pipeline"
10050
---
10151
"Test simulate":
10252
- do:

0 commit comments

Comments
 (0)