Skip to content

Commit b537e59

Browse files
martijnvgmartijnvg
committed
Initial data stream commit
This commits adds a data stream feature flag, initial definition of a data stream and the stubs for the data stream create, delete and get APIs. Also simple serialization tests are added and a rest test to thest the data stream API stubs. This is a large amount of code and mainly mechanical, but this commit should be straightforward to review, because there isn't any real logic. The data stream transport and rest action are behind the data stream feature flag and are only intialized if the feature flag is enabled. The feature flag is enabled if elasticsearch is build as snapshot or a release build and the 'es.datastreams_feature_flag_registered' is enabled. The integ-test-zip sets the feature flag if building a release build, otherwise rest tests would fail. Relates to elastic#53100
1 parent 1f439a2 commit b537e59

File tree

20 files changed

+1134
-0
lines changed

20 files changed

+1134
-0
lines changed

‎distribution/archives/integ-test-zip/build.gradle

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import org.elasticsearch.gradle.info.BuildParams
12
/*
23
* Licensed to Elasticsearch under one or more contributor
34
* license agreements. See the NOTICE file distributed with
@@ -32,3 +33,9 @@ integTest.runner {
3233
systemProperty 'tests.logfile', '--external--'
3334
}
3435
}
36+
37+
testClusters.integTest {
38+
if (BuildParams.isSnapshotBuild() == false) {
39+
systemProperty 'es.datastreams_feature_flag_registered', 'true'
40+
}
41+
}

‎rest-api-spec/src/main/resources/rest-api-spec/api/indices.create_data_stream.json

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
{
2+
"indices.create_data_stream":{
3+
"documentation":{
4+
"url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/data-streams.html",
5+
"description":"Creates or updates a data stream"
6+
},
7+
"stability":"experimental",
8+
"url":{
9+
"paths":[
10+
{
11+
"path":"/_data_stream/{name}",
12+
"methods":[
13+
"PUT"
14+
],
15+
"parts":{
16+
"name":{
17+
"type":"string",
18+
"description":"The name of the data stream"
19+
}
20+
}
21+
}
22+
]
23+
},
24+
"params":{
25+
},
26+
"body":{
27+
"description":"The data stream definition",
28+
"required":true
29+
}
30+
}
31+
}

‎rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_data_stream.json

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
{
2+
"indices.delete_data_stream":{
3+
"documentation":{
4+
"url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/data-streams.html",
5+
"description":"Deletes a data stream."
6+
},
7+
"stability":"experimental",
8+
"url":{
9+
"paths":[
10+
{
11+
"path":"/_data_stream/{name}",
12+
"methods":[
13+
"DELETE"
14+
],
15+
"parts":{
16+
"name":{
17+
"type":"string",
18+
"description":"The name of the data stream"
19+
}
20+
}
21+
}
22+
]
23+
},
24+
"params":{}
25+
}
26+
}

‎rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_streams.json

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
{
2+
"indices.get_data_streams":{
3+
"documentation":{
4+
"url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/data-streams.html",
5+
"description":"Returns data streams."
6+
},
7+
"stability":"experimental",
8+
"url":{
9+
"paths":[
10+
{
11+
"path":"/_data_streams",
12+
"methods":[
13+
"GET"
14+
]
15+
},
16+
{
17+
"path":"/_data_streams/{name}",
18+
"methods":[
19+
"GET"
20+
],
21+
"parts":{
22+
"name":{
23+
"type":"list",
24+
"description":"The comma separated names of data streams"
25+
}
26+
}
27+
}
28+
]
29+
},
30+
"params":{
31+
}
32+
}
33+
}

‎rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
---
2+
"Test stubs":
3+
- do:
4+
indices.create_data_stream:
5+
name: data-stream2
6+
body:
7+
timestamp_field_name: "@timestamp"
8+
- is_true: acknowledged
9+
10+
- do:
11+
indices.get_data_streams: {}
12+
- match: { 0.name: my_data_stream1 }
13+
- match: { 0.timestamp_field_name: '@timestamp' }
14+
- match: { 0.indices: ['my_data_stream1-000000'] }
15+
- match: { 1.name: my_data_stream2 }
16+
- match: { 1.timestamp_field_name: '@timestamp' }
17+
- match: { 1.indices: [] }
18+
19+
- do:
20+
indices.delete_data_stream:
21+
name: data-stream2
22+
- is_true: acknowledged

‎server/src/main/java/org/elasticsearch/action/ActionModule.java

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,16 @@
2121

2222
import org.apache.logging.log4j.LogManager;
2323
import org.apache.logging.log4j.Logger;
24+
import org.elasticsearch.Build;
2425
import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplainAction;
2526
import org.elasticsearch.action.admin.cluster.allocation.TransportClusterAllocationExplainAction;
2627
import org.elasticsearch.action.admin.cluster.configuration.AddVotingConfigExclusionsAction;
2728
import org.elasticsearch.action.admin.cluster.configuration.ClearVotingConfigExclusionsAction;
2829
import org.elasticsearch.action.admin.cluster.configuration.TransportAddVotingConfigExclusionsAction;
2930
import org.elasticsearch.action.admin.cluster.configuration.TransportClearVotingConfigExclusionsAction;
31+
import org.elasticsearch.action.admin.indices.datastream.DeleteDataStreamAction;
32+
import org.elasticsearch.action.admin.indices.datastream.GetDataStreamsAction;
33+
import org.elasticsearch.action.admin.indices.datastream.CreateDataStreamAction;
3034
import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
3135
import org.elasticsearch.action.admin.cluster.health.TransportClusterHealthAction;
3236
import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsAction;
@@ -243,9 +247,11 @@
243247
import org.elasticsearch.rest.action.admin.cluster.RestClusterStatsAction;
244248
import org.elasticsearch.rest.action.admin.cluster.RestClusterUpdateSettingsAction;
245249
import org.elasticsearch.rest.action.admin.cluster.RestCreateSnapshotAction;
250+
import org.elasticsearch.rest.action.admin.cluster.RestDeleteDataStreamAction;
246251
import org.elasticsearch.rest.action.admin.cluster.RestDeleteRepositoryAction;
247252
import org.elasticsearch.rest.action.admin.cluster.RestDeleteSnapshotAction;
248253
import org.elasticsearch.rest.action.admin.cluster.RestDeleteStoredScriptAction;
254+
import org.elasticsearch.rest.action.admin.cluster.RestGetDataStreamsAction;
249255
import org.elasticsearch.rest.action.admin.cluster.RestGetRepositoriesAction;
250256
import org.elasticsearch.rest.action.admin.cluster.RestGetScriptContextAction;
251257
import org.elasticsearch.rest.action.admin.cluster.RestGetScriptLanguageAction;
@@ -258,6 +264,7 @@
258264
import org.elasticsearch.rest.action.admin.cluster.RestNodesStatsAction;
259265
import org.elasticsearch.rest.action.admin.cluster.RestNodesUsageAction;
260266
import org.elasticsearch.rest.action.admin.cluster.RestPendingClusterTasksAction;
267+
import org.elasticsearch.rest.action.admin.cluster.RestCreateDataStreamAction;
261268
import org.elasticsearch.rest.action.admin.cluster.RestPutRepositoryAction;
262269
import org.elasticsearch.rest.action.admin.cluster.RestPutStoredScriptAction;
263270
import org.elasticsearch.rest.action.admin.cluster.RestReloadSecureSettingsAction;
@@ -361,6 +368,24 @@ public class ActionModule extends AbstractModule {
361368

362369
private static final Logger logger = LogManager.getLogger(ActionModule.class);
363370

371+
private static final boolean DATASTREAMS_FEATURE_FLAG_REGISTERED;
372+
373+
static {
374+
final String property = System.getProperty("es.datastreams_feature_flag_registered");
375+
if (Build.CURRENT.isSnapshot() && property != null) {
376+
throw new IllegalArgumentException("es.datastreams_feature_flag_registered is only supported in non-snapshot builds");
377+
}
378+
if (Build.CURRENT.isSnapshot() || "true".equals(property)) {
379+
DATASTREAMS_FEATURE_FLAG_REGISTERED = true;
380+
} else if ("false".equals(property) || property == null) {
381+
DATASTREAMS_FEATURE_FLAG_REGISTERED = false;
382+
} else {
383+
throw new IllegalArgumentException(
384+
"expected es.datastreams_feature_flag_registered to be unset or [true|false] but was [" + property + "]"
385+
);
386+
}
387+
}
388+
364389
private final Settings settings;
365390
private final IndexNameExpressionResolver indexNameExpressionResolver;
366391
private final IndexScopedSettings indexScopedSettings;
@@ -533,6 +558,13 @@ public <Request extends ActionRequest, Response extends ActionResponse> void reg
533558

534559
actionPlugins.stream().flatMap(p -> p.getActions().stream()).forEach(actions::register);
535560

561+
// Data streams:
562+
if (DATASTREAMS_FEATURE_FLAG_REGISTERED) {
563+
actions.register(CreateDataStreamAction.INSTANCE, CreateDataStreamAction.TransportAction.class);
564+
actions.register(DeleteDataStreamAction.INSTANCE, DeleteDataStreamAction.TransportAction.class);
565+
actions.register(GetDataStreamsAction.INSTANCE, GetDataStreamsAction.TransportAction.class);
566+
}
567+
536568
// Persistent tasks:
537569
actions.register(StartPersistentTaskAction.INSTANCE, StartPersistentTaskAction.TransportAction.class);
538570
actions.register(UpdatePersistentTaskStatusAction.INSTANCE, UpdatePersistentTaskStatusAction.TransportAction.class);
@@ -680,6 +712,13 @@ public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster) {
680712
registerHandler.accept(new RestDeletePipelineAction());
681713
registerHandler.accept(new RestSimulatePipelineAction());
682714

715+
// Data Stream API
716+
if (DATASTREAMS_FEATURE_FLAG_REGISTERED) {
717+
registerHandler.accept(new RestCreateDataStreamAction());
718+
registerHandler.accept(new RestDeleteDataStreamAction());
719+
registerHandler.accept(new RestGetDataStreamsAction());
720+
}
721+
683722
// CAT API
684723
registerHandler.accept(new RestAllocationAction());
685724
registerHandler.accept(new RestShardsAction());

‎server/src/main/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamAction.java

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.elasticsearch.action.admin.indices.datastream;
20+
21+
import org.elasticsearch.action.ActionListener;
22+
import org.elasticsearch.action.ActionRequestValidationException;
23+
import org.elasticsearch.action.ActionType;
24+
import org.elasticsearch.action.support.ActionFilters;
25+
import org.elasticsearch.action.support.master.AcknowledgedResponse;
26+
import org.elasticsearch.action.support.master.MasterNodeRequest;
27+
import org.elasticsearch.action.support.master.TransportMasterNodeAction;
28+
import org.elasticsearch.cluster.ClusterState;
29+
import org.elasticsearch.cluster.block.ClusterBlockException;
30+
import org.elasticsearch.cluster.block.ClusterBlockLevel;
31+
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
32+
import org.elasticsearch.cluster.service.ClusterService;
33+
import org.elasticsearch.common.inject.Inject;
34+
import org.elasticsearch.common.io.stream.StreamInput;
35+
import org.elasticsearch.common.io.stream.StreamOutput;
36+
import org.elasticsearch.tasks.Task;
37+
import org.elasticsearch.threadpool.ThreadPool;
38+
import org.elasticsearch.transport.TransportService;
39+
40+
import java.io.IOException;
41+
import java.util.Objects;
42+
43+
public class CreateDataStreamAction extends ActionType<AcknowledgedResponse> {
44+
45+
public static final CreateDataStreamAction INSTANCE = new CreateDataStreamAction();
46+
public static final String NAME = "indices:admin/data_stream/create";
47+
48+
private CreateDataStreamAction() {
49+
super(NAME, AcknowledgedResponse::new);
50+
}
51+
52+
public static class Request extends MasterNodeRequest<Request> {
53+
54+
private final String name;
55+
private String timestampFieldName;
56+
57+
public Request(String name) {
58+
this.name = name;
59+
}
60+
61+
public void setTimestampFieldName(String timestampFieldName) {
62+
this.timestampFieldName = timestampFieldName;
63+
}
64+
65+
@Override
66+
public ActionRequestValidationException validate() {
67+
return null;
68+
}
69+
70+
public Request(StreamInput in) throws IOException {
71+
super(in);
72+
this.name = in.readString();
73+
this.timestampFieldName = in.readString();
74+
}
75+
76+
@Override
77+
public void writeTo(StreamOutput out) throws IOException {
78+
super.writeTo(out);
79+
out.writeString(name);
80+
out.writeString(timestampFieldName);
81+
}
82+
83+
@Override
84+
public boolean equals(Object o) {
85+
if (this == o) return true;
86+
if (o == null || getClass() != o.getClass()) return false;
87+
Request request = (Request) o;
88+
return name.equals(request.name) &&
89+
timestampFieldName.equals(request.timestampFieldName);
90+
}
91+
92+
@Override
93+
public int hashCode() {
94+
return Objects.hash(name, timestampFieldName);
95+
}
96+
}
97+
98+
public static class TransportAction extends TransportMasterNodeAction<Request, AcknowledgedResponse> {
99+
100+
@Inject
101+
public TransportAction(TransportService transportService, ClusterService clusterService, ThreadPool threadPool,
102+
ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
103+
super(NAME, transportService, clusterService, threadPool, actionFilters, Request::new, indexNameExpressionResolver);
104+
}
105+
106+
@Override
107+
protected String executor() {
108+
return ThreadPool.Names.SAME;
109+
}
110+
111+
@Override
112+
protected AcknowledgedResponse read(StreamInput in) throws IOException {
113+
return new AcknowledgedResponse(in);
114+
}
115+
116+
@Override
117+
protected void masterOperation(Task task, Request request, ClusterState state,
118+
ActionListener<AcknowledgedResponse> listener) throws Exception {
119+
listener.onResponse(new AcknowledgedResponse(true));
120+
}
121+
122+
@Override
123+
protected ClusterBlockException checkBlock(Request request, ClusterState state) {
124+
return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
125+
}
126+
}
127+
128+
}

0 commit comments

Comments
 (0)