Skip to content

Commit 0d45752

Browse files
authored
Fix IndexMetaData loads after rollover (#33394)
When we rollover and index we write the conditions of the rollover that the old index met into the old index. Loading this index metadata requires a working `NamedXContentRegistry` that has been populated with parsers from the rollover infrastructure. We had a few loads that didn't use a working `NamedXContentRegistry` and so would fail if they ever encountered an index that had been rolled over. Here are the locations of the loads and how I fixed them: * IndexFolderUpgrader - removed entirely. It existed to support opening indices made in Elasticsearch 2.x. Since we only need this change as far back as 6.4.1 which will supports reading from indices created as far back as 5.0.0 we should be good here. * TransportNodesListGatewayStartedShards - wired the `NamedXContentRegistry` into place. * TransportNodesListShardStoreMetaData - wired the `NamedXContentRegistry` into place. * OldIndexUtils - removed entirely. It existed to support the zip based index backwards compatibility tests which we've since replaced with code that actually runs old versions of Elasticsearch. In addition to fixing the actual problem I added full cluster restart integration tests for rollover which would have caught this problem and I added an extra assertion to IndexMetaData's deserialization code which will trip if we try to deserialize and index's metadata without a fully formed `NamedXContentRegistry`. It won't catch if use the *wrong* `NamedXContentRegistry` but it is better than nothing. Closes #33316
1 parent 6a3adbd commit 0d45752

File tree

8 files changed

+71
-544
lines changed

8 files changed

+71
-544
lines changed

qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java

Lines changed: 57 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -477,6 +477,62 @@ public void testShrinkAfterUpgrade() throws IOException {
477477
}
478478
}
479479

480+
/**
481+
* Test upgrading after a rollover. Specifically:
482+
* <ol>
483+
* <li>Create an index with a write alias
484+
* <li>Write some documents to the write alias
485+
* <li>Roll over the index
486+
* <li>Make sure the document count is correct
487+
* <li>Upgrade
488+
* <li>Write some more documents to the write alias
489+
* <li>Make sure the document count is correct
490+
* </ol>
491+
*/
492+
public void testRollover() throws IOException {
493+
if (runningAgainstOldCluster) {
494+
Request createIndex = new Request("PUT", "/" + index + "-000001");
495+
createIndex.setJsonEntity("{"
496+
+ " \"aliases\": {"
497+
+ " \"" + index + "_write\": {}"
498+
+ " }"
499+
+ "}");
500+
client().performRequest(createIndex);
501+
}
502+
503+
int bulkCount = 10;
504+
StringBuilder bulk = new StringBuilder();
505+
for (int i = 0; i < bulkCount; i++) {
506+
bulk.append("{\"index\":{}}\n");
507+
bulk.append("{\"test\":\"test\"}\n");
508+
}
509+
Request bulkRequest = new Request("POST", "/" + index + "_write/doc/_bulk");
510+
bulkRequest.setJsonEntity(bulk.toString());
511+
bulkRequest.addParameter("refresh", "");
512+
assertThat(EntityUtils.toString(client().performRequest(bulkRequest).getEntity()), containsString("\"errors\":false"));
513+
514+
if (runningAgainstOldCluster) {
515+
Request rolloverRequest = new Request("POST", "/" + index + "_write/_rollover");
516+
rolloverRequest.setJsonEntity("{"
517+
+ " \"conditions\": {"
518+
+ " \"max_docs\": 5"
519+
+ " }"
520+
+ "}");
521+
client().performRequest(rolloverRequest);
522+
523+
assertThat(EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices?v")).getEntity()),
524+
containsString("testrollover-000002"));
525+
}
526+
527+
Request countRequest = new Request("POST", "/" + index + "-*/_search");
528+
countRequest.addParameter("size", "0");
529+
Map<String, Object> count = entityAsMap(client().performRequest(countRequest));
530+
assertNoFailures(count);
531+
532+
int expectedCount = bulkCount + (runningAgainstOldCluster ? 0 : bulkCount);
533+
assertEquals(expectedCount, (int) XContentMapValues.extractValue("hits.total", count));
534+
}
535+
480536
void assertBasicSearchWorks(int count) throws IOException {
481537
logger.info("--> testing basic search");
482538
{
@@ -947,7 +1003,7 @@ private void checkSnapshot(String snapshotName, int count, Version tookOnVersion
9471003
Request writeToRestoredRequest = new Request("POST", "/restored_" + index + "/doc/_bulk");
9481004
writeToRestoredRequest.addParameter("refresh", "true");
9491005
writeToRestoredRequest.setJsonEntity(bulk.toString());
950-
client().performRequest(writeToRestoredRequest);
1006+
assertThat(EntityUtils.toString(client().performRequest(writeToRestoredRequest).getEntity()), containsString("\"errors\":false"));
9511007

9521008
// And count to make sure the add worked
9531009
// Make sure search finds all documents

server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
import org.elasticsearch.common.settings.Setting;
4646
import org.elasticsearch.common.settings.Setting.Property;
4747
import org.elasticsearch.common.settings.Settings;
48+
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
4849
import org.elasticsearch.common.xcontent.ToXContent;
4950
import org.elasticsearch.common.xcontent.ToXContentFragment;
5051
import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -1346,6 +1347,8 @@ public void toXContent(XContentBuilder builder, IndexMetaData state) throws IOEx
13461347

13471348
@Override
13481349
public IndexMetaData fromXContent(XContentParser parser) throws IOException {
1350+
assert parser.getXContentRegistry() != NamedXContentRegistry.EMPTY
1351+
: "loading index metadata requires a working named xcontent registry";
13491352
return Builder.fromXContent(parser);
13501353
}
13511354
};

server/src/main/java/org/elasticsearch/common/util/IndexFolderUpgrader.java

Lines changed: 0 additions & 134 deletions
This file was deleted.

server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,6 @@
3535
import org.elasticsearch.common.component.AbstractComponent;
3636
import org.elasticsearch.common.settings.Settings;
3737
import org.elasticsearch.common.unit.TimeValue;
38-
import org.elasticsearch.common.util.IndexFolderUpgrader;
3938
import org.elasticsearch.env.NodeEnvironment;
4039
import org.elasticsearch.index.Index;
4140
import org.elasticsearch.plugins.MetaDataUpgrader;
@@ -84,7 +83,6 @@ public GatewayMetaState(Settings settings, NodeEnvironment nodeEnv, MetaStateSer
8483
if (DiscoveryNode.isMasterNode(settings) || DiscoveryNode.isDataNode(settings)) {
8584
try {
8685
ensureNoPre019State();
87-
IndexFolderUpgrader.upgradeIndicesIfNeeded(settings, nodeEnv);
8886
final MetaData metaData = metaStateService.loadFullState();
8987
final MetaData upgradedMetaData = upgradeMetaData(metaData, metaDataIndexUpgradeService, metaDataUpgrader);
9088
// We finished global state validation and successfully checked all indices for backward compatibility

server/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -69,15 +69,18 @@ public class TransportNodesListGatewayStartedShards extends
6969
public static final String ACTION_NAME = "internal:gateway/local/started_shards";
7070
private final NodeEnvironment nodeEnv;
7171
private final IndicesService indicesService;
72+
private final NamedXContentRegistry namedXContentRegistry;
7273

7374
@Inject
7475
public TransportNodesListGatewayStartedShards(Settings settings, ThreadPool threadPool, ClusterService clusterService,
7576
TransportService transportService, ActionFilters actionFilters,
76-
NodeEnvironment env, IndicesService indicesService) {
77+
NodeEnvironment env, IndicesService indicesService,
78+
NamedXContentRegistry namedXContentRegistry) {
7779
super(settings, ACTION_NAME, threadPool, clusterService, transportService, actionFilters,
7880
Request::new, NodeRequest::new, ThreadPool.Names.FETCH_SHARD_STARTED, NodeGatewayStartedShards.class);
7981
this.nodeEnv = env;
8082
this.indicesService = indicesService;
83+
this.namedXContentRegistry = namedXContentRegistry;
8184
}
8285

8386
@Override
@@ -112,15 +115,15 @@ protected NodeGatewayStartedShards nodeOperation(NodeRequest request) {
112115
try {
113116
final ShardId shardId = request.getShardId();
114117
logger.trace("{} loading local shard state info", shardId);
115-
ShardStateMetaData shardStateMetaData = ShardStateMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY,
118+
ShardStateMetaData shardStateMetaData = ShardStateMetaData.FORMAT.loadLatestState(logger, namedXContentRegistry,
116119
nodeEnv.availableShardPaths(request.shardId));
117120
if (shardStateMetaData != null) {
118121
IndexMetaData metaData = clusterService.state().metaData().index(shardId.getIndex());
119122
if (metaData == null) {
120123
// we may send this requests while processing the cluster state that recovered the index
121124
// sometimes the request comes in before the local node processed that cluster state
122125
// in such cases we can load it from disk
123-
metaData = IndexMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY,
126+
metaData = IndexMetaData.FORMAT.loadLatestState(logger, namedXContentRegistry,
124127
nodeEnv.indexPaths(shardId.getIndex()));
125128
}
126129
if (metaData == null) {

server/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -67,17 +67,19 @@ public class TransportNodesListShardStoreMetaData extends TransportNodesAction<T
6767
public static final String ACTION_NAME = "internal:cluster/nodes/indices/shard/store";
6868

6969
private final IndicesService indicesService;
70-
7170
private final NodeEnvironment nodeEnv;
71+
private final NamedXContentRegistry namedXContentRegistry;
7272

7373
@Inject
7474
public TransportNodesListShardStoreMetaData(Settings settings, ThreadPool threadPool,
7575
ClusterService clusterService, TransportService transportService,
76-
IndicesService indicesService, NodeEnvironment nodeEnv, ActionFilters actionFilters) {
76+
IndicesService indicesService, NodeEnvironment nodeEnv,
77+
ActionFilters actionFilters, NamedXContentRegistry namedXContentRegistry) {
7778
super(settings, ACTION_NAME, threadPool, clusterService, transportService, actionFilters,
7879
Request::new, NodeRequest::new, ThreadPool.Names.FETCH_SHARD_STORE, NodeStoreFilesMetaData.class);
7980
this.indicesService = indicesService;
8081
this.nodeEnv = nodeEnv;
82+
this.namedXContentRegistry = namedXContentRegistry;
8183
}
8284

8385
@Override
@@ -129,7 +131,7 @@ private StoreFilesMetaData listStoreMetaData(ShardId shardId) throws IOException
129131
// we may send this requests while processing the cluster state that recovered the index
130132
// sometimes the request comes in before the local node processed that cluster state
131133
// in such cases we can load it from disk
132-
metaData = IndexMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY,
134+
metaData = IndexMetaData.FORMAT.loadLatestState(logger, namedXContentRegistry,
133135
nodeEnv.indexPaths(shardId.getIndex()));
134136
}
135137
if (metaData == null) {

0 commit comments

Comments
 (0)