Skip to content

Commit 0c0f455

Browse files
dnhatndnhatn
authored
Deprecate indices without soft-deletes (#50502)
Soft-deletes will be enabled for all indices in 8.0. Hence, we should deprecate new indices without soft-deletes in 7.x.
1 parent 5ea750f commit 0c0f455

File tree

7 files changed

+115
-20
lines changed

7 files changed

+115
-20
lines changed

‎docs/reference/index-modules/history-retention.asciidoc

+3
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,9 @@ there>>. {ccr-cap} will not function if soft deletes are disabled.
6565
configured at index creation and only on indices created on or after 6.5.0.
6666
The default value is `true`.
6767

68+
deprecated::[7.6, Creating indices with soft-deletes disabled is
69+
deprecated and will be removed in future Elasticsearch versions.]
70+
6871
`index.soft_deletes.retention_lease.period`::
6972

7073
The maximum length of time to retain a shard history retention lease before

‎qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java

+25-4
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import org.elasticsearch.cluster.metadata.MetaDataIndexStateService;
2929
import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
3030
import org.elasticsearch.common.Booleans;
31+
import org.elasticsearch.common.Strings;
3132
import org.elasticsearch.common.settings.Settings;
3233
import org.elasticsearch.common.util.concurrent.AbstractRunnable;
3334
import org.elasticsearch.common.xcontent.support.XContentMapValues;
@@ -281,7 +282,7 @@ public void testRelocationWithConcurrentIndexing() throws Exception {
281282
}
282283
}
283284

284-
public void testRecoveryWithSoftDeletes() throws Exception {
285+
public void testRecovery() throws Exception {
285286
final String index = "recover_with_soft_deletes";
286287
if (CLUSTER_TYPE == ClusterType.OLD) {
287288
Settings.Builder settings = Settings.builder()
@@ -294,7 +295,7 @@ public void testRecoveryWithSoftDeletes() throws Exception {
294295
.put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms")
295296
.put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0"); // fail faster
296297
if (randomBoolean()) {
297-
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true);
298+
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
298299
}
299300
createIndex(index, settings.build());
300301
int numDocs = randomInt(10);
@@ -325,7 +326,7 @@ public void testRetentionLeasesEstablishedWhenPromotingPrimary() throws Exceptio
325326
.put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), between(1, 2)) // triggers nontrivial promotion
326327
.put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms")
327328
.put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0") // fail faster
328-
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true);
329+
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
329330
createIndex(index, settings.build());
330331
int numDocs = randomInt(10);
331332
indexDocs(index, 0, numDocs);
@@ -348,7 +349,7 @@ public void testRetentionLeasesEstablishedWhenRelocatingPrimary() throws Excepti
348349
.put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), between(0, 1))
349350
.put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms")
350351
.put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0") // fail faster
351-
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true);
352+
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
352353
createIndex(index, settings.build());
353354
int numDocs = randomInt(10);
354355
indexDocs(index, 0, numDocs);
@@ -740,6 +741,26 @@ public void testAutoExpandIndicesDuringRollingUpgrade() throws Exception {
740741
}
741742
}
742743

744+
public void testSoftDeletesDisabledWarning() throws Exception {
745+
final String indexName = "test_soft_deletes_disabled_warning";
746+
if (CLUSTER_TYPE == ClusterType.OLD) {
747+
boolean softDeletesEnabled = true;
748+
Settings.Builder settings = Settings.builder();
749+
if (randomBoolean()) {
750+
softDeletesEnabled = randomBoolean();
751+
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), softDeletesEnabled);
752+
}
753+
Request request = new Request("PUT", "/" + indexName);
754+
request.setJsonEntity("{\"settings\": " + Strings.toString(settings.build()) + "}");
755+
if (softDeletesEnabled == false) {
756+
expectSoftDeletesWarning(request, indexName);
757+
}
758+
client().performRequest(request);
759+
}
760+
ensureGreen(indexName);
761+
indexDocs(indexName, randomInt(100), randomInt(100));
762+
}
763+
743764
@SuppressWarnings("unchecked")
744765
private Map<String, Object> getIndexSettingsAsMap(String index) throws IOException {
745766
Map<String, Object> indexSettings = getIndexSettings(index);

‎rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yml

+17
Original file line numberDiff line numberDiff line change
@@ -119,3 +119,20 @@
119119
properties:
120120
"":
121121
type: keyword
122+
123+
---
124+
"Create index without soft deletes":
125+
- skip:
126+
version: " - 7.9.99"
127+
reason: "indices without soft deletes are deprecated in 8.0"
128+
features: "warnings"
129+
130+
- do:
131+
warnings:
132+
- Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions.
133+
Please do not specify value for setting [index.soft_deletes.enabled] of index [test_index].
134+
indices.create:
135+
index: test_index
136+
body:
137+
settings:
138+
soft_deletes.enabled: false

‎rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/20_translog.yml

+19-6
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,19 @@
11
---
22
"Translog retention without soft_deletes":
3+
- skip:
4+
version: " - 7.9.99"
5+
reason: "indices without soft deletes are deprecated in 8.0"
6+
features: "warnings"
7+
38
- do:
49
indices.create:
510
index: test
611
body:
712
settings:
813
soft_deletes.enabled: false
14+
warnings:
15+
- Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions.
16+
Please do not specify value for setting [index.soft_deletes.enabled] of index [test].
917
- do:
1018
cluster.health:
1119
wait_for_no_initializing_shards: true
@@ -68,8 +76,8 @@
6876
---
6977
"Translog retention with soft_deletes":
7078
- skip:
71-
version: " - 7.9.99"
72-
reason: "start ignoring translog retention policy with soft-deletes enabled in 8.0"
79+
version: " - 7.3.99"
80+
reason: "start ignoring translog retention policy with soft-deletes enabled in 7.4"
7381
- do:
7482
indices.create:
7583
index: test
@@ -130,15 +138,20 @@
130138
---
131139
"Translog stats on closed indices without soft-deletes":
132140
- skip:
133-
version: " - 7.2.99"
134-
reason: "closed indices have translog stats starting version 7.3.0"
141+
version: " - 7.9.99"
142+
reason: "indices without soft deletes are deprecated in 8.0"
143+
features: "warnings"
135144

136145
- do:
137146
indices.create:
138147
index: test
139148
body:
140149
settings:
141150
soft_deletes.enabled: false
151+
warnings:
152+
- Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions.
153+
Please do not specify value for setting [index.soft_deletes.enabled] of index [test].
154+
142155
- do:
143156
cluster.health:
144157
wait_for_no_initializing_shards: true
@@ -184,8 +197,8 @@
184197
---
185198
"Translog stats on closed indices with soft-deletes":
186199
- skip:
187-
version: " - 7.9.99"
188-
reason: "start ignoring translog retention policy with soft-deletes enabled in 8.0"
200+
version: " - 7.3.99"
201+
reason: "start ignoring translog retention policy with soft-deletes enabled in 7.4"
189202
- do:
190203
indices.create:
191204
index: test

‎server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java

+8
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@
5353
import org.elasticsearch.common.ValidationException;
5454
import org.elasticsearch.common.compress.CompressedXContent;
5555
import org.elasticsearch.common.io.PathUtils;
56+
import org.elasticsearch.common.logging.DeprecationLogger;
5657
import org.elasticsearch.common.settings.IndexScopedSettings;
5758
import org.elasticsearch.common.settings.Setting;
5859
import org.elasticsearch.common.settings.Settings;
@@ -62,6 +63,7 @@
6263
import org.elasticsearch.index.Index;
6364
import org.elasticsearch.index.IndexNotFoundException;
6465
import org.elasticsearch.index.IndexService;
66+
import org.elasticsearch.index.IndexSettings;
6567
import org.elasticsearch.index.mapper.DocumentMapper;
6668
import org.elasticsearch.index.mapper.MapperService;
6769
import org.elasticsearch.index.mapper.MapperService.MergeReason;
@@ -102,6 +104,7 @@
102104
*/
103105
public class MetaDataCreateIndexService {
104106
private static final Logger logger = LogManager.getLogger(MetaDataCreateIndexService.class);
107+
private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(logger);
105108

106109
public static final int MAX_INDEX_NAME_BYTES = 255;
107110

@@ -434,6 +437,11 @@ static Settings aggregateIndexSettings(ClusterState currentState, CreateIndexClu
434437
* that will be used to create this index.
435438
*/
436439
MetaDataCreateIndexService.checkShardLimit(indexSettings, currentState);
440+
if (indexSettings.getAsBoolean(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true) == false) {
441+
DEPRECATION_LOGGER.deprecatedAndMaybeLog("soft_deletes_disabled",
442+
"Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions. " +
443+
"Please do not specify value for setting [index.soft_deletes.enabled] of index [" + request.index() + "].");
444+
}
437445
return indexSettings;
438446
}
439447

‎server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java

+15
Original file line numberDiff line numberDiff line change
@@ -828,6 +828,21 @@ public void testGetIndexNumberOfRoutingShardsYieldsSourceNumberOfShards() {
828828
assertThat(targetRoutingNumberOfShards, is(6));
829829
}
830830

831+
public void testSoftDeletesDisabledDeprecation() {
832+
request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test");
833+
request.settings(Settings.builder().put(INDEX_SOFT_DELETES_SETTING.getKey(), false).build());
834+
aggregateIndexSettings(ClusterState.EMPTY_STATE, request, List.of(), Map.of(),
835+
null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS);
836+
assertWarnings("Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions. "
837+
+ "Please do not specify value for setting [index.soft_deletes.enabled] of index [test].");
838+
request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test");
839+
if (randomBoolean()) {
840+
request.settings(Settings.builder().put(INDEX_SOFT_DELETES_SETTING.getKey(), true).build());
841+
}
842+
aggregateIndexSettings(ClusterState.EMPTY_STATE, request, List.of(), Map.of(),
843+
null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS);
844+
}
845+
831846
private IndexTemplateMetaData addMatchingTemplate(Consumer<IndexTemplateMetaData.Builder> configurator) {
832847
IndexTemplateMetaData.Builder builder = templateMetaDataBuilder("template1", "te*");
833848
configurator.accept(builder);

‎test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

+28-10
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@
5252
import org.elasticsearch.common.xcontent.json.JsonXContent;
5353
import org.elasticsearch.common.xcontent.support.XContentMapValues;
5454
import org.elasticsearch.core.internal.io.IOUtils;
55+
import org.elasticsearch.index.IndexSettings;
5556
import org.elasticsearch.index.seqno.ReplicationTracker;
5657
import org.elasticsearch.rest.RestStatus;
5758
import org.elasticsearch.snapshots.SnapshotState;
@@ -964,23 +965,27 @@ protected static void ensureNoInitializingShards() throws IOException {
964965
}
965966

966967
protected static void createIndex(String name, Settings settings) throws IOException {
967-
Request request = new Request("PUT", "/" + name);
968-
request.setJsonEntity("{\n \"settings\": " + Strings.toString(settings) + "}");
969-
client().performRequest(request);
968+
createIndex(name, settings, null);
970969
}
971970

972971
protected static void createIndex(String name, Settings settings, String mapping) throws IOException {
973-
Request request = new Request("PUT", "/" + name);
974-
request.setJsonEntity("{\n \"settings\": " + Strings.toString(settings)
975-
+ ", \"mappings\" : {" + mapping + "} }");
976-
client().performRequest(request);
972+
createIndex(name, settings, mapping, null);
977973
}
978974

979975
protected static void createIndex(String name, Settings settings, String mapping, String aliases) throws IOException {
980976
Request request = new Request("PUT", "/" + name);
981-
request.setJsonEntity("{\n \"settings\": " + Strings.toString(settings)
982-
+ ", \"mappings\" : {" + mapping + "}"
983-
+ ", \"aliases\": {" + aliases + "} }");
977+
String entity = "{\"settings\": " + Strings.toString(settings);
978+
if (mapping != null) {
979+
entity += ",\"mappings\" : {" + mapping + "}";
980+
}
981+
if (aliases != null) {
982+
entity += ",\"aliases\": {" + aliases + "}";
983+
}
984+
entity += "}";
985+
if (settings.getAsBoolean(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true) == false) {
986+
expectSoftDeletesWarning(request, name);
987+
}
988+
request.setJsonEntity(entity);
984989
client().performRequest(request);
985990
}
986991

@@ -999,6 +1004,19 @@ private static void updateIndexSettings(String index, Settings settings) throws
9991004
client().performRequest(request);
10001005
}
10011006

1007+
protected static void expectSoftDeletesWarning(Request request, String indexName) {
1008+
final List<String> expectedWarnings = List.of(
1009+
"Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions. " +
1010+
"Please do not specify value for setting [index.soft_deletes.enabled] of index [" + indexName + "].");
1011+
if (nodeVersions.stream().allMatch(version -> version.onOrAfter(Version.V_8_0_0))) {
1012+
request.setOptions(RequestOptions.DEFAULT.toBuilder()
1013+
.setWarningsHandler(warnings -> warnings.equals(expectedWarnings) == false));
1014+
} else if (nodeVersions.stream().anyMatch(version -> version.onOrAfter(Version.V_8_0_0))) {
1015+
request.setOptions(RequestOptions.DEFAULT.toBuilder()
1016+
.setWarningsHandler(warnings -> warnings.isEmpty() == false && warnings.equals(expectedWarnings) == false));
1017+
}
1018+
}
1019+
10021020
protected static Map<String, Object> getIndexSettings(String index) throws IOException {
10031021
Request request = new Request("GET", "/" + index + "/_settings");
10041022
request.addParameter("flat_settings", "true");

0 commit comments

Comments
 (0)