Skip to content

Commit eac38e9

Browse files
benwtrentbenwtrent
authored
[ML] Add indices_options to datafeed config and update (#52793) (#52905)
This adds a new configurable field called `indices_options`. This allows users to create or update the indices_options used when a datafeed reads from an index. This is necessary for the following use cases: - Reading from frozen indices - Allowing certain indices in multiple index patterns to not exist yet These index options are available on datafeed creation and update. Users may specify them as URL parameters or within the configuration object. closes #48056
1 parent f46b370 commit eac38e9

File tree

46 files changed

+732
-111
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

46 files changed

+732
-111
lines changed

‎client/rest-high-level/src/main/java/org/elasticsearch/client/ml/datafeed/DatafeedConfig.java

Lines changed: 28 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
*/
1919
package org.elasticsearch.client.ml.datafeed;
2020

21+
import org.elasticsearch.action.support.IndicesOptions;
2122
import org.elasticsearch.client.ml.job.config.Job;
2223
import org.elasticsearch.common.ParseField;
2324
import org.elasticsearch.common.bytes.BytesArray;
@@ -63,6 +64,7 @@ public class DatafeedConfig implements ToXContentObject {
6364
public static final ParseField CHUNKING_CONFIG = new ParseField("chunking_config");
6465
public static final ParseField DELAYED_DATA_CHECK_CONFIG = new ParseField("delayed_data_check_config");
6566
public static final ParseField MAX_EMPTY_SEARCHES = new ParseField("max_empty_searches");
67+
public static final ParseField INDICES_OPTIONS = new ParseField("indices_options");
6668

6769
public static final ConstructingObjectParser<Builder, Void> PARSER = new ConstructingObjectParser<>(
6870
"datafeed_config", true, a -> new Builder((String)a[0], (String)a[1]));
@@ -90,6 +92,9 @@ public class DatafeedConfig implements ToXContentObject {
9092
PARSER.declareObject(Builder::setChunkingConfig, ChunkingConfig.PARSER, CHUNKING_CONFIG);
9193
PARSER.declareObject(Builder::setDelayedDataCheckConfig, DelayedDataCheckConfig.PARSER, DELAYED_DATA_CHECK_CONFIG);
9294
PARSER.declareInt(Builder::setMaxEmptySearches, MAX_EMPTY_SEARCHES);
95+
PARSER.declareObject(Builder::setIndicesOptions,
96+
(p, c) -> IndicesOptions.fromMap(p.map(), new IndicesOptions(IndicesOptions.Option.NONE, IndicesOptions.WildcardStates.NONE)),
97+
INDICES_OPTIONS);
9398
}
9499

95100
private static BytesReference parseBytes(XContentParser parser) throws IOException {
@@ -110,11 +115,12 @@ private static BytesReference parseBytes(XContentParser parser) throws IOExcepti
110115
private final ChunkingConfig chunkingConfig;
111116
private final DelayedDataCheckConfig delayedDataCheckConfig;
112117
private final Integer maxEmptySearches;
118+
private final IndicesOptions indicesOptions;
113119

114120
private DatafeedConfig(String id, String jobId, TimeValue queryDelay, TimeValue frequency, List<String> indices, BytesReference query,
115121
BytesReference aggregations, List<SearchSourceBuilder.ScriptField> scriptFields, Integer scrollSize,
116122
ChunkingConfig chunkingConfig, DelayedDataCheckConfig delayedDataCheckConfig,
117-
Integer maxEmptySearches) {
123+
Integer maxEmptySearches, IndicesOptions indicesOptions) {
118124
this.id = id;
119125
this.jobId = jobId;
120126
this.queryDelay = queryDelay;
@@ -127,6 +133,7 @@ private DatafeedConfig(String id, String jobId, TimeValue queryDelay, TimeValue
127133
this.chunkingConfig = chunkingConfig;
128134
this.delayedDataCheckConfig = delayedDataCheckConfig;
129135
this.maxEmptySearches = maxEmptySearches;
136+
this.indicesOptions = indicesOptions;
130137
}
131138

132139
public String getId() {
@@ -177,6 +184,10 @@ public Integer getMaxEmptySearches() {
177184
return maxEmptySearches;
178185
}
179186

187+
public IndicesOptions getIndicesOptions() {
188+
return indicesOptions;
189+
}
190+
180191
@Override
181192
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
182193
builder.startObject();
@@ -216,6 +227,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
216227
if (maxEmptySearches != null) {
217228
builder.field(MAX_EMPTY_SEARCHES.getPreferredName(), maxEmptySearches);
218229
}
230+
if (indicesOptions != null) {
231+
builder.startObject(INDICES_OPTIONS.getPreferredName());
232+
indicesOptions.toXContent(builder, params);
233+
builder.endObject();
234+
}
219235

220236
builder.endObject();
221237
return builder;
@@ -257,7 +273,8 @@ public boolean equals(Object other) {
257273
&& Objects.equals(this.scriptFields, that.scriptFields)
258274
&& Objects.equals(this.chunkingConfig, that.chunkingConfig)
259275
&& Objects.equals(this.delayedDataCheckConfig, that.delayedDataCheckConfig)
260-
&& Objects.equals(this.maxEmptySearches, that.maxEmptySearches);
276+
&& Objects.equals(this.maxEmptySearches, that.maxEmptySearches)
277+
&& Objects.equals(this.indicesOptions, that.indicesOptions);
261278
}
262279

263280
/**
@@ -268,7 +285,7 @@ public boolean equals(Object other) {
268285
@Override
269286
public int hashCode() {
270287
return Objects.hash(id, jobId, frequency, queryDelay, indices, asMap(query), scrollSize, asMap(aggregations), scriptFields,
271-
chunkingConfig, delayedDataCheckConfig, maxEmptySearches);
288+
chunkingConfig, delayedDataCheckConfig, maxEmptySearches, indicesOptions);
272289
}
273290

274291
public static Builder builder(String id, String jobId) {
@@ -289,6 +306,7 @@ public static class Builder {
289306
private ChunkingConfig chunkingConfig;
290307
private DelayedDataCheckConfig delayedDataCheckConfig;
291308
private Integer maxEmptySearches;
309+
private IndicesOptions indicesOptions;
292310

293311
public Builder(String id, String jobId) {
294312
this.id = Objects.requireNonNull(id, ID.getPreferredName());
@@ -308,6 +326,7 @@ public Builder(DatafeedConfig config) {
308326
this.chunkingConfig = config.chunkingConfig;
309327
this.delayedDataCheckConfig = config.getDelayedDataCheckConfig();
310328
this.maxEmptySearches = config.getMaxEmptySearches();
329+
this.indicesOptions = config.indicesOptions;
311330
}
312331

313332
public Builder setIndices(List<String> indices) {
@@ -395,9 +414,14 @@ public Builder setMaxEmptySearches(int maxEmptySearches) {
395414
return this;
396415
}
397416

417+
public Builder setIndicesOptions(IndicesOptions indicesOptions) {
418+
this.indicesOptions = indicesOptions;
419+
return this;
420+
}
421+
398422
public DatafeedConfig build() {
399423
return new DatafeedConfig(id, jobId, queryDelay, frequency, indices, query, aggregations, scriptFields, scrollSize,
400-
chunkingConfig, delayedDataCheckConfig, maxEmptySearches);
424+
chunkingConfig, delayedDataCheckConfig, maxEmptySearches, indicesOptions);
401425
}
402426

403427
private static BytesReference xContentToBytes(ToXContentObject object) throws IOException {

‎client/rest-high-level/src/main/java/org/elasticsearch/client/ml/datafeed/DatafeedUpdate.java

Lines changed: 27 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
package org.elasticsearch.client.ml.datafeed;
2020

2121
import org.elasticsearch.client.ml.job.config.Job;
22+
import org.elasticsearch.action.support.IndicesOptions;
2223
import org.elasticsearch.common.ParseField;
2324
import org.elasticsearch.common.bytes.BytesArray;
2425
import org.elasticsearch.common.bytes.BytesReference;
@@ -80,6 +81,9 @@ public class DatafeedUpdate implements ToXContentObject {
8081
DelayedDataCheckConfig.PARSER,
8182
DatafeedConfig.DELAYED_DATA_CHECK_CONFIG);
8283
PARSER.declareInt(Builder::setMaxEmptySearches, DatafeedConfig.MAX_EMPTY_SEARCHES);
84+
PARSER.declareObject(Builder::setIndicesOptions,
85+
(p, c) -> IndicesOptions.fromMap(p.map(), new IndicesOptions(IndicesOptions.Option.NONE, IndicesOptions.WildcardStates.NONE)),
86+
DatafeedConfig.INDICES_OPTIONS);
8387
}
8488

8589
private static BytesReference parseBytes(XContentParser parser) throws IOException {
@@ -100,11 +104,12 @@ private static BytesReference parseBytes(XContentParser parser) throws IOExcepti
100104
private final ChunkingConfig chunkingConfig;
101105
private final DelayedDataCheckConfig delayedDataCheckConfig;
102106
private final Integer maxEmptySearches;
107+
private final IndicesOptions indicesOptions;
103108

104109
private DatafeedUpdate(String id, String jobId, TimeValue queryDelay, TimeValue frequency, List<String> indices, BytesReference query,
105110
BytesReference aggregations, List<SearchSourceBuilder.ScriptField> scriptFields, Integer scrollSize,
106111
ChunkingConfig chunkingConfig, DelayedDataCheckConfig delayedDataCheckConfig,
107-
Integer maxEmptySearches) {
112+
Integer maxEmptySearches, IndicesOptions indicesOptions) {
108113
this.id = id;
109114
this.jobId = jobId;
110115
this.queryDelay = queryDelay;
@@ -117,6 +122,7 @@ private DatafeedUpdate(String id, String jobId, TimeValue queryDelay, TimeValue
117122
this.chunkingConfig = chunkingConfig;
118123
this.delayedDataCheckConfig = delayedDataCheckConfig;
119124
this.maxEmptySearches = maxEmptySearches;
125+
this.indicesOptions = indicesOptions;
120126
}
121127

122128
/**
@@ -157,6 +163,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
157163
addOptionalField(builder, DatafeedConfig.SCROLL_SIZE, scrollSize);
158164
addOptionalField(builder, DatafeedConfig.CHUNKING_CONFIG, chunkingConfig);
159165
addOptionalField(builder, DatafeedConfig.MAX_EMPTY_SEARCHES, maxEmptySearches);
166+
if (indicesOptions != null) {
167+
builder.startObject(DatafeedConfig.INDICES_OPTIONS.getPreferredName());
168+
indicesOptions.toXContent(builder, params);
169+
builder.endObject();
170+
}
160171
builder.endObject();
161172
return builder;
162173
}
@@ -211,6 +222,10 @@ public Integer getMaxEmptySearches() {
211222
return maxEmptySearches;
212223
}
213224

225+
public IndicesOptions getIndicesOptions() {
226+
return indicesOptions;
227+
}
228+
214229
private static Map<String, Object> asMap(BytesReference bytesReference) {
215230
return bytesReference == null ? null : XContentHelper.convertToMap(bytesReference, true, XContentType.JSON).v2();
216231
}
@@ -247,7 +262,8 @@ public boolean equals(Object other) {
247262
&& Objects.equals(this.delayedDataCheckConfig, that.delayedDataCheckConfig)
248263
&& Objects.equals(this.scriptFields, that.scriptFields)
249264
&& Objects.equals(this.chunkingConfig, that.chunkingConfig)
250-
&& Objects.equals(this.maxEmptySearches, that.maxEmptySearches);
265+
&& Objects.equals(this.maxEmptySearches, that.maxEmptySearches)
266+
&& Objects.equals(this.indicesOptions, that.indicesOptions);
251267
}
252268

253269
/**
@@ -258,7 +274,7 @@ public boolean equals(Object other) {
258274
@Override
259275
public int hashCode() {
260276
return Objects.hash(id, jobId, frequency, queryDelay, indices, asMap(query), scrollSize, asMap(aggregations), scriptFields,
261-
chunkingConfig, delayedDataCheckConfig, maxEmptySearches);
277+
chunkingConfig, delayedDataCheckConfig, maxEmptySearches, indicesOptions);
262278
}
263279

264280
public static Builder builder(String id) {
@@ -279,6 +295,7 @@ public static class Builder {
279295
private ChunkingConfig chunkingConfig;
280296
private DelayedDataCheckConfig delayedDataCheckConfig;
281297
private Integer maxEmptySearches;
298+
private IndicesOptions indicesOptions;
282299

283300
public Builder(String id) {
284301
this.id = Objects.requireNonNull(id, DatafeedConfig.ID.getPreferredName());
@@ -297,6 +314,7 @@ public Builder(DatafeedUpdate config) {
297314
this.chunkingConfig = config.chunkingConfig;
298315
this.delayedDataCheckConfig = config.delayedDataCheckConfig;
299316
this.maxEmptySearches = config.maxEmptySearches;
317+
this.indicesOptions = config.indicesOptions;
300318
}
301319

302320
@Deprecated
@@ -381,9 +399,14 @@ public Builder setMaxEmptySearches(int maxEmptySearches) {
381399
return this;
382400
}
383401

402+
public Builder setIndicesOptions(IndicesOptions indicesOptions) {
403+
this.indicesOptions = indicesOptions;
404+
return this;
405+
}
406+
384407
public DatafeedUpdate build() {
385408
return new DatafeedUpdate(id, jobId, queryDelay, frequency, indices, query, aggregations, scriptFields, scrollSize,
386-
chunkingConfig, delayedDataCheckConfig, maxEmptySearches);
409+
chunkingConfig, delayedDataCheckConfig, maxEmptySearches, indicesOptions);
387410
}
388411

389412
private static BytesReference xContentToBytes(ToXContentObject object) throws IOException {

‎client/rest-high-level/src/test/java/org/elasticsearch/client/ml/datafeed/DatafeedConfigTests.java

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
package org.elasticsearch.client.ml.datafeed;
2020

2121
import com.carrotsearch.randomizedtesting.generators.CodepointSetGenerator;
22+
import org.elasticsearch.action.support.IndicesOptions;
2223
import org.elasticsearch.common.unit.TimeValue;
2324
import org.elasticsearch.common.xcontent.DeprecationHandler;
2425
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
@@ -109,6 +110,13 @@ public static DatafeedConfig.Builder createRandomBuilder() {
109110
if (randomBoolean()) {
110111
builder.setMaxEmptySearches(randomIntBetween(10, 100));
111112
}
113+
if (randomBoolean()) {
114+
builder.setIndicesOptions(IndicesOptions.fromOptions(randomBoolean(),
115+
randomBoolean(),
116+
randomBoolean(),
117+
randomBoolean(),
118+
randomBoolean()));
119+
}
112120
return builder;
113121
}
114122

‎client/rest-high-level/src/test/java/org/elasticsearch/client/ml/datafeed/DatafeedUpdateTests.java

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
*/
1919
package org.elasticsearch.client.ml.datafeed;
2020

21+
import org.elasticsearch.action.support.IndicesOptions;
2122
import org.elasticsearch.common.unit.TimeValue;
2223
import org.elasticsearch.common.xcontent.XContentParser;
2324
import org.elasticsearch.index.query.QueryBuilders;
@@ -86,6 +87,13 @@ public static DatafeedUpdate createRandom() {
8687
if (randomBoolean()) {
8788
builder.setMaxEmptySearches(randomIntBetween(10, 100));
8889
}
90+
if (randomBoolean()) {
91+
builder.setIndicesOptions(IndicesOptions.fromOptions(randomBoolean(),
92+
randomBoolean(),
93+
randomBoolean(),
94+
randomBoolean(),
95+
randomBoolean()));
96+
}
8997
return builder.build();
9098
}
9199

‎docs/reference/ml/anomaly-detection/apis/put-datafeed.asciidoc

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,11 @@ include::{docdir}/ml/ml-shared.asciidoc[tag=script-fields]
9898
(Optional, unsigned integer)
9999
include::{docdir}/ml/ml-shared.asciidoc[tag=scroll-size]
100100

101+
`indices_options`::
102+
(Optional, object)
103+
include::{docdir}/ml/ml-shared.asciidoc[tag=indices-options]
104+
105+
101106
[[ml-put-datafeed-example]]
102107
==== {api-examples-title}
103108

‎docs/reference/ml/anomaly-detection/apis/update-datafeed.asciidoc

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,9 @@ include::{docdir}/ml/ml-shared.asciidoc[tag=script-fields]
101101
(Optional, unsigned integer)
102102
include::{docdir}/ml/ml-shared.asciidoc[tag=scroll-size]
103103

104+
`indices_options`::
105+
(Optional, object)
106+
include::{docdir}/ml/ml-shared.asciidoc[tag=indices-options]
104107

105108
[[ml-update-datafeed-example]]
106109
==== {api-examples-title}

‎docs/reference/ml/ml-shared.asciidoc

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -665,6 +665,19 @@ not be set to `false` on any {ml} nodes.
665665
--
666666
end::indices[]
667667

668+
tag::indices-options[]
669+
Object specifying index expansion options used during search.
670+
For example:
671+
```
672+
{
673+
"expand_wildcards": ["all"],
674+
"ignore_unavailable": true,
675+
"allow_no_indices": "false",
676+
"ignore_throttled": true
677+
}
678+
```
679+
end::indices-options[]
680+
668681
tag::influencers[]
669682
A comma separated list of influencer field names. Typically these can be the by,
670683
over, or partition fields that are used in the detector configuration. You might

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutDatafeedAction.java

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
import org.elasticsearch.action.ActionRequestValidationException;
1010
import org.elasticsearch.action.ActionResponse;
1111
import org.elasticsearch.action.ActionType;
12+
import org.elasticsearch.action.support.IndicesOptions;
1213
import org.elasticsearch.action.support.master.AcknowledgedRequest;
1314
import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder;
1415
import org.elasticsearch.client.ElasticsearchClient;
@@ -33,8 +34,11 @@ private PutDatafeedAction() {
3334

3435
public static class Request extends AcknowledgedRequest<Request> implements ToXContentObject {
3536

36-
public static Request parseRequest(String datafeedId, XContentParser parser) {
37+
public static Request parseRequest(String datafeedId, IndicesOptions indicesOptions, XContentParser parser) {
3738
DatafeedConfig.Builder datafeed = DatafeedConfig.STRICT_PARSER.apply(parser, null);
39+
if (datafeed.getIndicesOptions() == null) {
40+
datafeed.setIndicesOptions(indicesOptions);
41+
}
3842
datafeed.setId(datafeedId);
3943
return new Request(datafeed.build());
4044
}

0 commit comments

Comments
 (0)