Skip to content

Commit 1204608

Browse files
authored
Initial data stream commit (#53666)
* Initial data stream commit This commits adds a data stream feature flag, initial definition of a data stream and the stubs for the data stream create, delete and get APIs. Also simple serialization tests are added and a rest test to thest the data stream API stubs. This is a large amount of code and mainly mechanical, but this commit should be straightforward to review, because there isn't any real logic. The data stream transport and rest action are behind the data stream feature flag and are only intialized if the feature flag is enabled. The feature flag is enabled if elasticsearch is build as snapshot or a release build and the 'es.datastreams_feature_flag_registered' is enabled. The integ-test-zip sets the feature flag if building a release build, otherwise rest tests would fail. Relates to #53100 * fixed hlrc test * ignore bwc until this change has been backported to 7.x branch * changed data stream apis to be a cluster based action. before this commit the data steams api were indices based actions, but data streams aren't indices, data streams encapsulates indices, but are indices themselves. It is a cluster level attribute, and therefor cluster based action fits best for now. Perhaps in the future we will have data stream based actions and then this would be a right fit for the data stream crud apis. * this should have been part of the previous commit * fixed yaml test * Also add feature flag in other modules that run the yaml test if a release build is executed * Reverted the commits that make data stream a cluster based api This reverts commit e362eeb. * Make data stream crud apis work like a indices based api. * renamed timestamp field * fixed compile error after merging in master * fixed merge mistake * moved setting system property * applied review comments
1 parent 8aee647 commit 1204608

File tree

24 files changed

+1143
-3
lines changed

24 files changed

+1143
-3
lines changed

client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java

+4-1
Original file line numberDiff line numberDiff line change
@@ -798,7 +798,10 @@ public void testApiNamingConventions() throws Exception {
798798
"scripts_painless_execute",
799799
"cluster.put_component_template",
800800
"cluster.get_component_template",
801-
"cluster.delete_component_template"
801+
"cluster.delete_component_template",
802+
"indices.create_data_stream",
803+
"indices.get_data_streams",
804+
"indices.delete_data_stream"
802805
};
803806
//These API are not required for high-level client feature completeness
804807
String[] notRequiredApi = new String[] {

distribution/archives/integ-test-zip/build.gradle

+1
Original file line numberDiff line numberDiff line change
@@ -37,5 +37,6 @@ integTest.runner {
3737
testClusters.integTest {
3838
if (BuildParams.isSnapshotBuild() == false) {
3939
systemProperty 'es.itv2_feature_flag_registered', 'true'
40+
systemProperty 'es.datastreams_feature_flag_registered', 'true'
4041
}
4142
}

qa/smoke-test-multinode/build.gradle

+2
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
* specific language governing permissions and limitations
1717
* under the License.
1818
*/
19+
import org.elasticsearch.gradle.info.BuildParams
1920

2021
import org.elasticsearch.gradle.info.BuildParams
2122

@@ -52,5 +53,6 @@ integTest.runner {
5253
testClusters.integTest {
5354
if (BuildParams.isSnapshotBuild() == false) {
5455
systemProperty 'es.itv2_feature_flag_registered', 'true'
56+
systemProperty 'es.datastreams_feature_flag_registered', 'true'
5557
}
5658
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
{
2+
"indices.create_data_stream":{
3+
"documentation":{
4+
"url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/data-streams.html",
5+
"description":"Creates or updates a data stream"
6+
},
7+
"stability":"experimental",
8+
"url":{
9+
"paths":[
10+
{
11+
"path":"/_data_stream/{name}",
12+
"methods":[
13+
"PUT"
14+
],
15+
"parts":{
16+
"name":{
17+
"type":"string",
18+
"description":"The name of the data stream"
19+
}
20+
}
21+
}
22+
]
23+
},
24+
"params":{
25+
},
26+
"body":{
27+
"description":"The data stream definition",
28+
"required":true
29+
}
30+
}
31+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
{
2+
"indices.delete_data_stream":{
3+
"documentation":{
4+
"url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/data-streams.html",
5+
"description":"Deletes a data stream."
6+
},
7+
"stability":"experimental",
8+
"url":{
9+
"paths":[
10+
{
11+
"path":"/_data_stream/{name}",
12+
"methods":[
13+
"DELETE"
14+
],
15+
"parts":{
16+
"name":{
17+
"type":"string",
18+
"description":"The name of the data stream"
19+
}
20+
}
21+
}
22+
]
23+
},
24+
"params":{}
25+
}
26+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
{
2+
"indices.get_data_streams":{
3+
"documentation":{
4+
"url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/data-streams.html",
5+
"description":"Returns data streams."
6+
},
7+
"stability":"experimental",
8+
"url":{
9+
"paths":[
10+
{
11+
"path":"/_data_streams",
12+
"methods":[
13+
"GET"
14+
]
15+
},
16+
{
17+
"path":"/_data_streams/{name}",
18+
"methods":[
19+
"GET"
20+
],
21+
"parts":{
22+
"name":{
23+
"type":"list",
24+
"description":"The comma separated names of data streams"
25+
}
26+
}
27+
}
28+
]
29+
},
30+
"params":{
31+
}
32+
}
33+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
---
2+
"Test stubs":
3+
- skip:
4+
version: " - 7.99.99"
5+
reason: not backported yet
6+
7+
- do:
8+
indices.create_data_stream:
9+
name: data-stream2
10+
body:
11+
timestamp_field: "@timestamp"
12+
- is_true: acknowledged
13+
14+
- do:
15+
indices.get_data_streams: {}
16+
- match: { 0.name: my_data_stream1 }
17+
- match: { 0.timestamp_field: '@timestamp' }
18+
- match: { 0.indices: ['my_data_stream1-000000'] }
19+
- match: { 1.name: my_data_stream2 }
20+
- match: { 1.timestamp_field: '@timestamp' }
21+
- match: { 1.indices: [] }
22+
23+
- do:
24+
indices.delete_data_stream:
25+
name: data-stream2
26+
- is_true: acknowledged

server/src/main/java/org/elasticsearch/action/ActionModule.java

+38
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,9 @@
2828
import org.elasticsearch.action.admin.cluster.configuration.ClearVotingConfigExclusionsAction;
2929
import org.elasticsearch.action.admin.cluster.configuration.TransportAddVotingConfigExclusionsAction;
3030
import org.elasticsearch.action.admin.cluster.configuration.TransportClearVotingConfigExclusionsAction;
31+
import org.elasticsearch.action.admin.indices.datastream.DeleteDataStreamAction;
32+
import org.elasticsearch.action.admin.indices.datastream.GetDataStreamsAction;
33+
import org.elasticsearch.action.admin.indices.datastream.CreateDataStreamAction;
3134
import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
3235
import org.elasticsearch.action.admin.cluster.health.TransportClusterHealthAction;
3336
import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsAction;
@@ -250,9 +253,11 @@
250253
import org.elasticsearch.rest.action.admin.cluster.RestClusterStatsAction;
251254
import org.elasticsearch.rest.action.admin.cluster.RestClusterUpdateSettingsAction;
252255
import org.elasticsearch.rest.action.admin.cluster.RestCreateSnapshotAction;
256+
import org.elasticsearch.rest.action.admin.indices.RestDeleteDataStreamAction;
253257
import org.elasticsearch.rest.action.admin.cluster.RestDeleteRepositoryAction;
254258
import org.elasticsearch.rest.action.admin.cluster.RestDeleteSnapshotAction;
255259
import org.elasticsearch.rest.action.admin.cluster.RestDeleteStoredScriptAction;
260+
import org.elasticsearch.rest.action.admin.indices.RestGetDataStreamsAction;
256261
import org.elasticsearch.rest.action.admin.cluster.RestGetRepositoriesAction;
257262
import org.elasticsearch.rest.action.admin.cluster.RestGetScriptContextAction;
258263
import org.elasticsearch.rest.action.admin.cluster.RestGetScriptLanguageAction;
@@ -265,6 +270,7 @@
265270
import org.elasticsearch.rest.action.admin.cluster.RestNodesStatsAction;
266271
import org.elasticsearch.rest.action.admin.cluster.RestNodesUsageAction;
267272
import org.elasticsearch.rest.action.admin.cluster.RestPendingClusterTasksAction;
273+
import org.elasticsearch.rest.action.admin.indices.RestCreateDataStreamAction;
268274
import org.elasticsearch.rest.action.admin.cluster.RestPutRepositoryAction;
269275
import org.elasticsearch.rest.action.admin.cluster.RestPutStoredScriptAction;
270276
import org.elasticsearch.rest.action.admin.cluster.RestReloadSecureSettingsAction;
@@ -388,6 +394,24 @@ public class ActionModule extends AbstractModule {
388394
}
389395
}
390396

397+
private static final boolean DATASTREAMS_FEATURE_FLAG_REGISTERED;
398+
399+
static {
400+
final String property = System.getProperty("es.datastreams_feature_flag_registered");
401+
if (Build.CURRENT.isSnapshot() && property != null) {
402+
throw new IllegalArgumentException("es.datastreams_feature_flag_registered is only supported in non-snapshot builds");
403+
}
404+
if (Build.CURRENT.isSnapshot() || "true".equals(property)) {
405+
DATASTREAMS_FEATURE_FLAG_REGISTERED = true;
406+
} else if ("false".equals(property) || property == null) {
407+
DATASTREAMS_FEATURE_FLAG_REGISTERED = false;
408+
} else {
409+
throw new IllegalArgumentException(
410+
"expected es.datastreams_feature_flag_registered to be unset or [true|false] but was [" + property + "]"
411+
);
412+
}
413+
}
414+
391415
private final Settings settings;
392416
private final IndexNameExpressionResolver indexNameExpressionResolver;
393417
private final IndexScopedSettings indexScopedSettings;
@@ -565,6 +589,13 @@ public <Request extends ActionRequest, Response extends ActionResponse> void reg
565589

566590
actionPlugins.stream().flatMap(p -> p.getActions().stream()).forEach(actions::register);
567591

592+
// Data streams:
593+
if (DATASTREAMS_FEATURE_FLAG_REGISTERED) {
594+
actions.register(CreateDataStreamAction.INSTANCE, CreateDataStreamAction.TransportAction.class);
595+
actions.register(DeleteDataStreamAction.INSTANCE, DeleteDataStreamAction.TransportAction.class);
596+
actions.register(GetDataStreamsAction.INSTANCE, GetDataStreamsAction.TransportAction.class);
597+
}
598+
568599
// Persistent tasks:
569600
actions.register(StartPersistentTaskAction.INSTANCE, StartPersistentTaskAction.TransportAction.class);
570601
actions.register(UpdatePersistentTaskStatusAction.INSTANCE, UpdatePersistentTaskStatusAction.TransportAction.class);
@@ -717,6 +748,13 @@ public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster) {
717748
registerHandler.accept(new RestDeletePipelineAction());
718749
registerHandler.accept(new RestSimulatePipelineAction());
719750

751+
// Data Stream API
752+
if (DATASTREAMS_FEATURE_FLAG_REGISTERED) {
753+
registerHandler.accept(new RestCreateDataStreamAction());
754+
registerHandler.accept(new RestDeleteDataStreamAction());
755+
registerHandler.accept(new RestGetDataStreamsAction());
756+
}
757+
720758
// CAT API
721759
registerHandler.accept(new RestAllocationAction());
722760
registerHandler.accept(new RestShardsAction());
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.elasticsearch.action.admin.indices.datastream;
20+
21+
import org.elasticsearch.action.ActionListener;
22+
import org.elasticsearch.action.ActionRequestValidationException;
23+
import org.elasticsearch.action.ActionType;
24+
import org.elasticsearch.action.support.ActionFilters;
25+
import org.elasticsearch.action.support.master.AcknowledgedResponse;
26+
import org.elasticsearch.action.support.master.MasterNodeRequest;
27+
import org.elasticsearch.action.support.master.TransportMasterNodeAction;
28+
import org.elasticsearch.cluster.ClusterState;
29+
import org.elasticsearch.cluster.block.ClusterBlockException;
30+
import org.elasticsearch.cluster.block.ClusterBlockLevel;
31+
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
32+
import org.elasticsearch.cluster.service.ClusterService;
33+
import org.elasticsearch.common.inject.Inject;
34+
import org.elasticsearch.common.io.stream.StreamInput;
35+
import org.elasticsearch.common.io.stream.StreamOutput;
36+
import org.elasticsearch.tasks.Task;
37+
import org.elasticsearch.threadpool.ThreadPool;
38+
import org.elasticsearch.transport.TransportService;
39+
40+
import java.io.IOException;
41+
import java.util.Objects;
42+
43+
public class CreateDataStreamAction extends ActionType<AcknowledgedResponse> {
44+
45+
public static final CreateDataStreamAction INSTANCE = new CreateDataStreamAction();
46+
public static final String NAME = "indices:admin/data_stream/create";
47+
48+
private CreateDataStreamAction() {
49+
super(NAME, AcknowledgedResponse::new);
50+
}
51+
52+
public static class Request extends MasterNodeRequest<Request> {
53+
54+
private final String name;
55+
private String timestampFieldName;
56+
57+
public Request(String name) {
58+
this.name = name;
59+
}
60+
61+
public void setTimestampFieldName(String timestampFieldName) {
62+
this.timestampFieldName = timestampFieldName;
63+
}
64+
65+
@Override
66+
public ActionRequestValidationException validate() {
67+
return null;
68+
}
69+
70+
public Request(StreamInput in) throws IOException {
71+
super(in);
72+
this.name = in.readString();
73+
this.timestampFieldName = in.readString();
74+
}
75+
76+
@Override
77+
public void writeTo(StreamOutput out) throws IOException {
78+
super.writeTo(out);
79+
out.writeString(name);
80+
out.writeString(timestampFieldName);
81+
}
82+
83+
@Override
84+
public boolean equals(Object o) {
85+
if (this == o) return true;
86+
if (o == null || getClass() != o.getClass()) return false;
87+
Request request = (Request) o;
88+
return name.equals(request.name) &&
89+
timestampFieldName.equals(request.timestampFieldName);
90+
}
91+
92+
@Override
93+
public int hashCode() {
94+
return Objects.hash(name, timestampFieldName);
95+
}
96+
}
97+
98+
public static class TransportAction extends TransportMasterNodeAction<Request, AcknowledgedResponse> {
99+
100+
@Inject
101+
public TransportAction(TransportService transportService, ClusterService clusterService, ThreadPool threadPool,
102+
ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
103+
super(NAME, transportService, clusterService, threadPool, actionFilters, Request::new, indexNameExpressionResolver);
104+
}
105+
106+
@Override
107+
protected String executor() {
108+
return ThreadPool.Names.SAME;
109+
}
110+
111+
@Override
112+
protected AcknowledgedResponse read(StreamInput in) throws IOException {
113+
return new AcknowledgedResponse(in);
114+
}
115+
116+
@Override
117+
protected void masterOperation(Task task, Request request, ClusterState state,
118+
ActionListener<AcknowledgedResponse> listener) throws Exception {
119+
listener.onResponse(new AcknowledgedResponse(true));
120+
}
121+
122+
@Override
123+
protected ClusterBlockException checkBlock(Request request, ClusterState state) {
124+
return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
125+
}
126+
}
127+
128+
}

0 commit comments

Comments
 (0)