Skip to content

Commit eea47bb

Browse files
benwtrentbenwtrent
authored
[ML][Data Frame] Add deduced mappings to _preview response payload (#43742)
* [ML][Data Frame] Add deduced mappings to _preview response payload * updating preview docs
1 parent d38a48b commit eea47bb

File tree

9 files changed

+126
-29
lines changed

9 files changed

+126
-29
lines changed

‎client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/PreviewDataFrameTransformResponse.java

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -29,23 +29,32 @@
2929
public class PreviewDataFrameTransformResponse {
3030

3131
private static final String PREVIEW = "preview";
32+
private static final String MAPPINGS = "mappings";
3233

3334
@SuppressWarnings("unchecked")
3435
public static PreviewDataFrameTransformResponse fromXContent(final XContentParser parser) throws IOException {
35-
Object previewDocs = parser.map().get(PREVIEW);
36-
return new PreviewDataFrameTransformResponse((List<Map<String, Object>>) previewDocs);
36+
Map<String, Object> previewMap = parser.mapOrdered();
37+
Object previewDocs = previewMap.get(PREVIEW);
38+
Object mappings = previewMap.get(MAPPINGS);
39+
return new PreviewDataFrameTransformResponse((List<Map<String, Object>>) previewDocs, (Map<String, Object>) mappings);
3740
}
3841

3942
private List<Map<String, Object>> docs;
43+
private Map<String, Object> mappings;
4044

41-
public PreviewDataFrameTransformResponse(List<Map<String, Object>> docs) {
45+
public PreviewDataFrameTransformResponse(List<Map<String, Object>> docs, Map<String, Object> mappings) {
4246
this.docs = docs;
47+
this.mappings = mappings;
4348
}
4449

4550
public List<Map<String, Object>> getDocs() {
4651
return docs;
4752
}
4853

54+
public Map<String, Object> getMappings() {
55+
return mappings;
56+
}
57+
4958
@Override
5059
public boolean equals(Object obj) {
5160
if (obj == this) {
@@ -57,12 +66,12 @@ public boolean equals(Object obj) {
5766
}
5867

5968
PreviewDataFrameTransformResponse other = (PreviewDataFrameTransformResponse) obj;
60-
return Objects.equals(other.docs, docs);
69+
return Objects.equals(other.docs, docs) && Objects.equals(other.mappings, mappings);
6170
}
6271

6372
@Override
6473
public int hashCode() {
65-
return Objects.hashCode(docs);
74+
return Objects.hash(docs, mappings);
6675
}
6776

6877
}

‎client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameTransformIT.java

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@
7171
import static org.hamcrest.Matchers.empty;
7272
import static org.hamcrest.Matchers.equalTo;
7373
import static org.hamcrest.Matchers.greaterThan;
74+
import static org.hamcrest.Matchers.hasKey;
7475
import static org.hamcrest.Matchers.hasSize;
7576
import static org.hamcrest.Matchers.is;
7677
import static org.hamcrest.Matchers.oneOf;
@@ -277,6 +278,7 @@ public void testStartStop() throws IOException {
277278
assertThat(taskState, is(DataFrameTransformTaskState.STOPPED));
278279
}
279280

281+
@SuppressWarnings("unchecked")
280282
public void testPreview() throws IOException {
281283
String sourceIndex = "transform-source";
282284
createIndex(sourceIndex);
@@ -298,6 +300,12 @@ public void testPreview() throws IOException {
298300
Optional<Map<String, Object>> michel = docs.stream().filter(doc -> "michel".equals(doc.get("reviewer"))).findFirst();
299301
assertTrue(michel.isPresent());
300302
assertEquals(3.6d, (double) michel.get().get("avg_rating"), 0.1d);
303+
304+
Map<String, Object> mappings = preview.getMappings();
305+
assertThat(mappings, hasKey("properties"));
306+
Map<String, Object> fields = (Map<String, Object>)mappings.get("properties");
307+
assertThat(fields.get("reviewer"), equalTo(Map.of("type", "keyword")));
308+
assertThat(fields.get("avg_rating"), equalTo(Map.of("type", "double")));
301309
}
302310

303311
private DataFrameTransformConfig validDataFrameTransformConfig(String id, String source, String destination) {

‎client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/PreviewDataFrameTransformResponseTests.java

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,8 +53,13 @@ private PreviewDataFrameTransformResponse createTestInstance() {
5353
}
5454
docs.add(doc);
5555
}
56+
int numMappingEntries = randomIntBetween(5, 10);
57+
Map<String, Object> mappings = new HashMap<>(numMappingEntries);
58+
for (int i = 0; i < numMappingEntries; i++) {
59+
mappings.put(randomAlphaOfLength(10), Map.of("type", randomAlphaOfLength(10)));
60+
}
5661

57-
return new PreviewDataFrameTransformResponse(docs);
62+
return new PreviewDataFrameTransformResponse(docs, mappings);
5863
}
5964

6065
private void toXContent(PreviewDataFrameTransformResponse response, XContentBuilder builder) throws IOException {
@@ -64,6 +69,7 @@ private void toXContent(PreviewDataFrameTransformResponse response, XContentBuil
6469
builder.map(doc);
6570
}
6671
builder.endArray();
72+
builder.field("mappings", response.getMappings());
6773
builder.endObject();
6874
}
6975
}

‎client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/DataFrameTransformDocumentationIT.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -447,6 +447,7 @@ public void testPreview() throws IOException, InterruptedException {
447447
// end::preview-data-frame-transform-execute
448448

449449
assertNotNull(response.getDocs());
450+
assertNotNull(response.getMappings());
450451
}
451452
{
452453
// tag::preview-data-frame-transform-execute-listener

‎docs/reference/data-frames/apis/preview-transform.asciidoc

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,17 @@ The data that is returned for this example is as follows:
9090
"customer_id" : "12"
9191
}
9292
...
93-
]
93+
],
94+
"mappings": {
95+
"properties": {
96+
"max_price": {
97+
"type": "double"
98+
},
99+
"customer_id": {
100+
"type": "keyword"
101+
}
102+
}
103+
}
94104
}
95105
----
96106
// NOTCONSOLE

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/PreviewDataFrameTransformAction.java

Lines changed: 48 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
package org.elasticsearch.xpack.core.dataframe.action;
88

9+
import org.elasticsearch.Version;
910
import org.elasticsearch.action.ActionType;
1011
import org.elasticsearch.action.ActionRequestValidationException;
1112
import org.elasticsearch.action.ActionResponse;
@@ -137,11 +138,14 @@ public boolean equals(Object obj) {
137138
public static class Response extends ActionResponse implements ToXContentObject {
138139

139140
private List<Map<String, Object>> docs;
141+
private Map<String, Object> mappings;
140142
public static ParseField PREVIEW = new ParseField("preview");
143+
public static ParseField MAPPINGS = new ParseField("mappings");
141144

142145
static ObjectParser<Response, Void> PARSER = new ObjectParser<>("data_frame_transform_preview", Response::new);
143146
static {
144147
PARSER.declareObjectArray(Response::setDocs, (p, c) -> p.mapOrdered(), PREVIEW);
148+
PARSER.declareObject(Response::setMappings, (p, c) -> p.mapOrdered(), MAPPINGS);
145149
}
146150
public Response() {}
147151

@@ -151,6 +155,10 @@ public Response(StreamInput in) throws IOException {
151155
for (int i = 0; i < size; i++) {
152156
this.docs.add(in.readMap());
153157
}
158+
if (in.getVersion().onOrAfter(Version.V_7_3_0)) {
159+
Map<String, Object> objectMap = in.readMap();
160+
this.mappings = objectMap == null ? null : Map.copyOf(objectMap);
161+
}
154162
}
155163

156164
public Response(List<Map<String, Object>> docs) {
@@ -161,18 +169,56 @@ public void setDocs(List<Map<String, Object>> docs) {
161169
this.docs = new ArrayList<>(docs);
162170
}
163171

172+
public void setMappings(Map<String, Object> mappings) {
173+
this.mappings = Map.copyOf(mappings);
174+
}
175+
176+
/**
177+
* This takes the a {@code Map<String, String>} of the type "fieldname: fieldtype" and transforms it into the
178+
* typical mapping format.
179+
*
180+
* Example:
181+
*
182+
* input:
183+
* {"field1.subField1": "long", "field2": "keyword"}
184+
*
185+
* output:
186+
* {
187+
* "properties": {
188+
* "field1.subField1": {
189+
* "type": "long"
190+
* },
191+
* "field2": {
192+
* "type": "keyword"
193+
* }
194+
* }
195+
* }
196+
* @param mappings A Map of the form {"fieldName": "fieldType"}
197+
*/
198+
public void setMappingsFromStringMap(Map<String, String> mappings) {
199+
Map<String, Object> fieldMappings = new HashMap<>();
200+
mappings.forEach((k, v) -> fieldMappings.put(k, Map.of("type", v)));
201+
this.mappings = Map.of("properties", fieldMappings);
202+
}
203+
164204
@Override
165205
public void writeTo(StreamOutput out) throws IOException {
166206
out.writeInt(docs.size());
167207
for (Map<String, Object> doc : docs) {
168208
out.writeMapWithConsistentOrder(doc);
169209
}
210+
if (out.getVersion().onOrAfter(Version.V_7_3_0)) {
211+
out.writeMap(mappings);
212+
}
170213
}
171214

172215
@Override
173216
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
174217
builder.startObject();
175218
builder.field(PREVIEW.getPreferredName(), docs);
219+
if (mappings != null) {
220+
builder.field(MAPPINGS.getPreferredName(), mappings);
221+
}
176222
builder.endObject();
177223
return builder;
178224
}
@@ -188,12 +234,12 @@ public boolean equals(Object obj) {
188234
}
189235

190236
Response other = (Response) obj;
191-
return Objects.equals(other.docs, docs);
237+
return Objects.equals(other.docs, docs) && Objects.equals(other.mappings, mappings);
192238
}
193239

194240
@Override
195241
public int hashCode() {
196-
return Objects.hashCode(docs);
242+
return Objects.hash(docs, mappings);
197243
}
198244

199245
public static Response fromXContent(final XContentParser parser) throws IOException {

‎x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/PreviewDataFrameTransformsActionResponseTests.java

Lines changed: 21 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -35,13 +35,28 @@ protected Response createTestInstance() {
3535
int size = randomIntBetween(0, 10);
3636
List<Map<String, Object>> data = new ArrayList<>(size);
3737
for (int i = 0; i < size; i++) {
38-
Map<String, Object> datum = new HashMap<>();
39-
Map<String, Object> entry = new HashMap<>();
40-
entry.put("value1", randomIntBetween(1, 100));
41-
datum.put(randomAlphaOfLength(10), entry);
42-
data.add(datum);
38+
data.add(Map.of(randomAlphaOfLength(10), Map.of("value1", randomIntBetween(1, 100))));
4339
}
44-
return new Response(data);
40+
41+
Response response = new Response(data);
42+
if (randomBoolean()) {
43+
size = randomIntBetween(0, 10);
44+
if (randomBoolean()) {
45+
Map<String, Object> mappings = new HashMap<>(size);
46+
for (int i = 0; i < size; i++) {
47+
mappings.put(randomAlphaOfLength(10), Map.of("type", randomAlphaOfLength(10)));
48+
}
49+
response.setMappings(mappings);
50+
} else {
51+
Map<String, String> mappings = new HashMap<>(size);
52+
for (int i = 0; i < size; i++) {
53+
mappings.put(randomAlphaOfLength(10), randomAlphaOfLength(10));
54+
}
55+
response.setMappingsFromStringMap(mappings);
56+
}
57+
}
58+
59+
return response;
4560
}
4661

4762
@Override

‎x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportPreviewDataFrameTransformAction.java

Lines changed: 8 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -104,25 +104,16 @@ protected void doExecute(Task task,
104104

105105
Pivot pivot = new Pivot(config.getPivotConfig());
106106

107-
getPreview(pivot,
108-
config.getSource(),
109-
config.getDestination().getPipeline(),
110-
config.getDestination().getIndex(),
111-
ActionListener.wrap(
112-
previewResponse -> listener.onResponse(new PreviewDataFrameTransformAction.Response(previewResponse)),
113-
error -> {
114-
logger.error("Failure gathering preview", error);
115-
listener.onFailure(error);
116-
}
117-
));
107+
getPreview(pivot, config.getSource(), config.getDestination().getPipeline(), config.getDestination().getIndex(), listener);
118108
}
119109

120110
@SuppressWarnings("unchecked")
121111
private void getPreview(Pivot pivot,
122112
SourceConfig source,
123113
String pipeline,
124114
String dest,
125-
ActionListener<List<Map<String, Object>>> listener) {
115+
ActionListener<PreviewDataFrameTransformAction.Response> listener) {
116+
final PreviewDataFrameTransformAction.Response previewResponse = new PreviewDataFrameTransformAction.Response();
126117
ActionListener<SimulatePipelineResponse> pipelineResponseActionListener = ActionListener.wrap(
127118
simulatePipelineResponse -> {
128119
List<Map<String, Object>> response = new ArrayList<>(simulatePipelineResponse.getResults().size());
@@ -135,12 +126,14 @@ private void getPreview(Pivot pivot,
135126
response.add((Map<String, Object>)XContentMapValues.extractValue("doc._source", tempMap));
136127
}
137128
}
138-
listener.onResponse(response);
129+
previewResponse.setDocs(response);
130+
listener.onResponse(previewResponse);
139131
},
140132
listener::onFailure
141133
);
142134
pivot.deduceMappings(client, source, ActionListener.wrap(
143135
deducedMappings -> {
136+
previewResponse.setMappingsFromStringMap(deducedMappings);
144137
ClientHelper.executeWithHeadersAsync(threadPool.getThreadContext().getHeaders(),
145138
ClientHelper.DATA_FRAME_ORIGIN,
146139
client,
@@ -157,7 +150,8 @@ private void getPreview(Pivot pivot,
157150
List<Map<String, Object>> results = pivot.extractResults(agg, deducedMappings, stats)
158151
.peek(doc -> doc.keySet().removeIf(k -> k.startsWith("_")))
159152
.collect(Collectors.toList());
160-
listener.onResponse(results);
153+
previewResponse.setDocs(results);
154+
listener.onResponse(previewResponse);
161155
} else {
162156
List<Map<String, Object>> results = pivot.extractResults(agg, deducedMappings, stats)
163157
.map(doc -> {

‎x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/preview_transforms.yml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,11 @@ setup:
9898
- match: { preview.2.avg_response: 42.0 }
9999
- match: { preview.2.time.max: "2017-02-18T01:01:00.000Z" }
100100
- match: { preview.2.time.min: "2017-02-18T01:01:00.000Z" }
101+
- match: { mappings.properties.airline.type: "keyword" }
102+
- match: { mappings.properties.by-hour.type: "date" }
103+
- match: { mappings.properties.avg_response.type: "double" }
104+
- match: { mappings.properties.time\.max.type: "date" }
105+
- match: { mappings.properties.time\.min.type: "date" }
101106

102107
- do:
103108
ingest.put_pipeline:
@@ -141,6 +146,9 @@ setup:
141146
- match: { preview.2.by-hour: 1487379600000 }
142147
- match: { preview.2.avg_response: 42.0 }
143148
- match: { preview.2.my_field: 42 }
149+
- match: { mappings.properties.airline.type: "keyword" }
150+
- match: { mappings.properties.by-hour.type: "date" }
151+
- match: { mappings.properties.avg_response.type: "double" }
144152

145153
---
146154
"Test preview transform with invalid config":

0 commit comments

Comments
 (0)