Skip to content

Commit 6620ae1

Browse files
johtanijohtani
committed
Index Template: parse and validate mappings in template when template puts
Share applying template with MetaDataCreateIndexService and MetaDataIndexTemplateService Add some unit test Collapse addMappingsToMapperService and move it to MapperService Extract validateTemplate method use expectThrows in testcase Add TODO comment Related elastic#2415 Backport elastic#8802 (cherry picked from commit fd76291)
1 parent 5167b9f commit 6620ae1

File tree

8 files changed

+380
-99
lines changed

8 files changed

+380
-99
lines changed

‎core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java

Lines changed: 8 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -53,9 +53,7 @@
5353
import org.elasticsearch.common.io.Streams;
5454
import org.elasticsearch.common.regex.Regex;
5555
import org.elasticsearch.common.settings.Settings;
56-
import org.elasticsearch.common.xcontent.XContentFactory;
5756
import org.elasticsearch.common.xcontent.XContentHelper;
58-
import org.elasticsearch.common.xcontent.XContentParser;
5957
import org.elasticsearch.env.Environment;
6058
import org.elasticsearch.env.NodeEnvironment;
6159
import org.elasticsearch.index.Index;
@@ -209,7 +207,7 @@ public ClusterState execute(ClusterState currentState) throws Exception {
209207
List<String> templateNames = new ArrayList<>();
210208

211209
for (Map.Entry<String, String> entry : request.mappings().entrySet()) {
212-
mappings.put(entry.getKey(), parseMapping(entry.getValue()));
210+
mappings.put(entry.getKey(), MapperService.parseMapping(entry.getValue()));
213211
}
214212

215213
for (Map.Entry<String, Custom> entry : request.customs().entrySet()) {
@@ -221,9 +219,9 @@ public ClusterState execute(ClusterState currentState) throws Exception {
221219
templateNames.add(template.getName());
222220
for (ObjectObjectCursor<String, CompressedXContent> cursor : template.mappings()) {
223221
if (mappings.containsKey(cursor.key)) {
224-
XContentHelper.mergeDefaults(mappings.get(cursor.key), parseMapping(cursor.value.string()));
222+
XContentHelper.mergeDefaults(mappings.get(cursor.key), MapperService.parseMapping(cursor.value.string()));
225223
} else {
226-
mappings.put(cursor.key, parseMapping(cursor.value.string()));
224+
mappings.put(cursor.key, MapperService.parseMapping(cursor.value.string()));
227225
}
228226
}
229227
// handle custom
@@ -311,26 +309,11 @@ public ClusterState execute(ClusterState currentState) throws Exception {
311309
// now add the mappings
312310
IndexService indexService = indicesService.indexServiceSafe(request.index());
313311
MapperService mapperService = indexService.mapperService();
314-
// first, add the default mapping
315-
if (mappings.containsKey(MapperService.DEFAULT_MAPPING)) {
316-
try {
317-
mapperService.merge(MapperService.DEFAULT_MAPPING, new CompressedXContent(XContentFactory.jsonBuilder().map(mappings.get(MapperService.DEFAULT_MAPPING)).string()), MapperService.MergeReason.MAPPING_UPDATE, request.updateAllTypes());
318-
} catch (Exception e) {
319-
removalReason = "failed on parsing default mapping on index creation";
320-
throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, MapperService.DEFAULT_MAPPING, e.getMessage());
321-
}
322-
}
323-
for (Map.Entry<String, Map<String, Object>> entry : mappings.entrySet()) {
324-
if (entry.getKey().equals(MapperService.DEFAULT_MAPPING)) {
325-
continue;
326-
}
327-
try {
328-
// apply the default here, its the first time we parse it
329-
mapperService.merge(entry.getKey(), new CompressedXContent(XContentFactory.jsonBuilder().map(entry.getValue()).string()), MapperService.MergeReason.MAPPING_UPDATE, request.updateAllTypes());
330-
} catch (Exception e) {
331-
removalReason = "failed on parsing mappings on index creation";
332-
throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, entry.getKey(), e.getMessage());
333-
}
312+
try {
313+
mapperService.merge(mappings, request.updateAllTypes());
314+
} catch (MapperParsingException mpe) {
315+
removalReason = "failed on parsing default mapping/mappings on index creation";
316+
throw mpe;
334317
}
335318

336319
IndexQueryParserService indexQueryParserService = indexService.queryParserService();
@@ -422,35 +405,6 @@ public ClusterState execute(ClusterState currentState) throws Exception {
422405
});
423406
}
424407

425-
private Map<String, Object> parseMapping(String mappingSource) throws Exception {
426-
try (XContentParser parser = XContentFactory.xContent(mappingSource).createParser(mappingSource)) {
427-
return parser.map();
428-
}
429-
}
430-
431-
private void addMappings(Map<String, Map<String, Object>> mappings, Path mappingsDir) throws IOException {
432-
try (DirectoryStream<Path> stream = Files.newDirectoryStream(mappingsDir)) {
433-
for (Path mappingFile : stream) {
434-
final String fileName = mappingFile.getFileName().toString();
435-
if (FileSystemUtils.isHidden(mappingFile)) {
436-
continue;
437-
}
438-
int lastDotIndex = fileName.lastIndexOf('.');
439-
String mappingType = lastDotIndex != -1 ? mappingFile.getFileName().toString().substring(0, lastDotIndex) : mappingFile.getFileName().toString();
440-
try (BufferedReader reader = Files.newBufferedReader(mappingFile, Charsets.UTF_8)) {
441-
String mappingSource = Streams.copyToString(reader);
442-
if (mappings.containsKey(mappingType)) {
443-
XContentHelper.mergeDefaults(mappings.get(mappingType), parseMapping(mappingSource));
444-
} else {
445-
mappings.put(mappingType, parseMapping(mappingSource));
446-
}
447-
} catch (Exception e) {
448-
logger.warn("failed to read / parse mapping [" + mappingType + "] from location [" + mappingFile + "], ignoring...", e);
449-
}
450-
}
451-
}
452-
}
453-
454408
private List<IndexTemplateMetaData> findTemplates(CreateIndexClusterStateUpdateRequest request, ClusterState state, IndexTemplateFilter indexTemplateFilter) throws IOException {
455409
List<IndexTemplateMetaData> templates = new ArrayList<>();
456410
for (ObjectCursor<IndexTemplateMetaData> cursor : state.metaData().templates().values()) {

‎core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java

Lines changed: 67 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import com.carrotsearch.hppc.cursors.ObjectCursor;
2222
import com.google.common.collect.Maps;
2323
import com.google.common.collect.Sets;
24+
import org.elasticsearch.Version;
2425
import org.elasticsearch.action.admin.indices.alias.Alias;
2526
import org.elasticsearch.action.support.master.MasterNodeRequest;
2627
import org.elasticsearch.cluster.ClusterService;
@@ -34,11 +35,17 @@
3435
import org.elasticsearch.common.regex.Regex;
3536
import org.elasticsearch.common.settings.Settings;
3637
import org.elasticsearch.common.unit.TimeValue;
38+
import org.elasticsearch.index.Index;
39+
import org.elasticsearch.index.IndexService;
40+
import org.elasticsearch.index.mapper.MapperParsingException;
41+
import org.elasticsearch.index.mapper.MapperService;
3742
import org.elasticsearch.indices.IndexTemplateAlreadyExistsException;
3843
import org.elasticsearch.indices.IndexTemplateMissingException;
44+
import org.elasticsearch.indices.IndicesService;
3945
import org.elasticsearch.indices.InvalidIndexTemplateException;
4046

4147
import java.util.ArrayList;
48+
import java.util.HashMap;
4249
import java.util.List;
4350
import java.util.Locale;
4451
import java.util.Map;
@@ -51,13 +58,15 @@ public class MetaDataIndexTemplateService extends AbstractComponent {
5158

5259
private final ClusterService clusterService;
5360
private final AliasValidator aliasValidator;
61+
private final IndicesService indicesService;
5462
private final MetaDataCreateIndexService metaDataCreateIndexService;
5563

5664
@Inject
57-
public MetaDataIndexTemplateService(Settings settings, ClusterService clusterService, MetaDataCreateIndexService metaDataCreateIndexService, AliasValidator aliasValidator) {
65+
public MetaDataIndexTemplateService(Settings settings, ClusterService clusterService, MetaDataCreateIndexService metaDataCreateIndexService, AliasValidator aliasValidator, IndicesService indicesService) {
5866
super(settings);
5967
this.clusterService = clusterService;
6068
this.aliasValidator = aliasValidator;
69+
this.indicesService = indicesService;
6170
this.metaDataCreateIndexService = metaDataCreateIndexService;
6271
}
6372

@@ -125,28 +134,7 @@ public void putTemplate(final PutRequest request, final PutListener listener) {
125134
return;
126135
}
127136

128-
IndexTemplateMetaData.Builder templateBuilder;
129-
try {
130-
templateBuilder = IndexTemplateMetaData.builder(request.name);
131-
templateBuilder.order(request.order);
132-
templateBuilder.template(request.template);
133-
templateBuilder.settings(request.settings);
134-
for (Map.Entry<String, String> entry : request.mappings.entrySet()) {
135-
templateBuilder.putMapping(entry.getKey(), entry.getValue());
136-
}
137-
for (Alias alias : request.aliases) {
138-
AliasMetaData aliasMetaData = AliasMetaData.builder(alias.name()).filter(alias.filter())
139-
.indexRouting(alias.indexRouting()).searchRouting(alias.searchRouting()).build();
140-
templateBuilder.putAlias(aliasMetaData);
141-
}
142-
for (Map.Entry<String, IndexMetaData.Custom> entry : request.customs.entrySet()) {
143-
templateBuilder.putCustom(entry.getKey(), entry.getValue());
144-
}
145-
} catch (Throwable e) {
146-
listener.onFailure(e);
147-
return;
148-
}
149-
final IndexTemplateMetaData template = templateBuilder.build();
137+
final IndexTemplateMetaData.Builder templateBuilder = IndexTemplateMetaData.builder(request.name);
150138

151139
clusterService.submitStateUpdateTask("create-index-template [" + request.name + "], cause [" + request.cause + "]",
152140
new ClusterStateUpdateTask(Priority.URGENT) {
@@ -161,22 +149,76 @@ public void onFailure(String source, Throwable t) {
161149
}
162150

163151
@Override
164-
public ClusterState execute(ClusterState currentState) {
152+
public ClusterState execute(ClusterState currentState) throws Exception {
165153
if (request.create && currentState.metaData().templates().containsKey(request.name)) {
166154
throw new IndexTemplateAlreadyExistsException(request.name);
167155
}
156+
157+
validateAndAddTemplate(request, templateBuilder, indicesService, metaDataCreateIndexService);
158+
159+
for (Alias alias : request.aliases) {
160+
AliasMetaData aliasMetaData = AliasMetaData.builder(alias.name()).filter(alias.filter())
161+
.indexRouting(alias.indexRouting()).searchRouting(alias.searchRouting()).build();
162+
templateBuilder.putAlias(aliasMetaData);
163+
}
164+
for (Map.Entry<String, IndexMetaData.Custom> entry : request.customs.entrySet()) {
165+
templateBuilder.putCustom(entry.getKey(), entry.getValue());
166+
}
167+
IndexTemplateMetaData template = templateBuilder.build();
168+
168169
MetaData.Builder builder = MetaData.builder(currentState.metaData()).put(template);
169170

170171
return ClusterState.builder(currentState).metaData(builder).build();
171172
}
172173

173174
@Override
174175
public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
175-
listener.onResponse(new PutResponse(true, template));
176+
listener.onResponse(new PutResponse(true, templateBuilder.build()));
176177
}
177178
});
178179
}
179180

181+
private void validateAndAddTemplate(final PutRequest request, IndexTemplateMetaData.Builder templateBuilder, IndicesService indicesService,
182+
MetaDataCreateIndexService metaDataCreateIndexService) throws Exception {
183+
Index createdIndex = null;
184+
final String temporaryIndexName = Strings.randomBase64UUID();
185+
try {
186+
187+
//create index service for parsing and validating "mappings"
188+
Settings dummySettings = Settings.builder()
189+
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
190+
.put(request.settings)
191+
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
192+
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
193+
.put(IndexMetaData.SETTING_INDEX_UUID, Strings.randomBase64UUID())
194+
.build();
195+
196+
IndexService dummyIndexService = indicesService.createIndex(temporaryIndexName, dummySettings, clusterService.localNode().id());
197+
createdIndex = dummyIndexService.index();
198+
199+
templateBuilder.order(request.order);
200+
templateBuilder.template(request.template);
201+
templateBuilder.settings(request.settings);
202+
203+
Map<String, Map<String, Object>> mappingsForValidation = new HashMap<>();
204+
for (Map.Entry<String, String> entry : request.mappings.entrySet()) {
205+
try {
206+
templateBuilder.putMapping(entry.getKey(), entry.getValue());
207+
} catch (Exception e) {
208+
throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, entry.getKey(), e.getMessage());
209+
}
210+
mappingsForValidation.put(entry.getKey(), MapperService.parseMapping(entry.getValue()));
211+
}
212+
213+
dummyIndexService.mapperService().merge(mappingsForValidation, false);
214+
215+
} finally {
216+
if (createdIndex != null) {
217+
indicesService.removeIndex(temporaryIndexName, " created for parsing template mapping");
218+
}
219+
}
220+
}
221+
180222
private void validate(PutRequest request) {
181223
List<String> validationErrors = new ArrayList<>();
182224
if (request.name.contains(" ")) {

‎core/src/main/java/org/elasticsearch/index/mapper/MapperService.java

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,8 @@
4646
import org.elasticsearch.common.lucene.search.Queries;
4747
import org.elasticsearch.common.regex.Regex;
4848
import org.elasticsearch.common.settings.Settings;
49+
import org.elasticsearch.common.xcontent.XContentFactory;
50+
import org.elasticsearch.common.xcontent.XContentParser;
4951
import org.elasticsearch.index.AbstractIndexComponent;
5052
import org.elasticsearch.index.Index;
5153
import org.elasticsearch.index.analysis.AnalysisService;
@@ -262,6 +264,35 @@ public void removeTypeListener(DocumentTypeListener listener) {
262264
typeListeners.remove(listener);
263265
}
264266

267+
public static Map<String, Object> parseMapping(String mappingSource) throws Exception {
268+
try (XContentParser parser = XContentFactory.xContent(mappingSource).createParser(mappingSource)) {
269+
return parser.map();
270+
}
271+
}
272+
273+
//TODO: make this atomic
274+
public void merge(Map<String, Map<String, Object>> mappings, boolean updateAllTypes) throws MapperParsingException {
275+
// first, add the default mapping
276+
if (mappings.containsKey(DEFAULT_MAPPING)) {
277+
try {
278+
this.merge(DEFAULT_MAPPING, new CompressedXContent(XContentFactory.jsonBuilder().map(mappings.get(DEFAULT_MAPPING)).string()), MergeReason.MAPPING_UPDATE, updateAllTypes);
279+
} catch (Exception e) {
280+
throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, DEFAULT_MAPPING, e.getMessage());
281+
}
282+
}
283+
for (Map.Entry<String, Map<String, Object>> entry : mappings.entrySet()) {
284+
if (entry.getKey().equals(DEFAULT_MAPPING)) {
285+
continue;
286+
}
287+
try {
288+
// apply the default here, its the first time we parse it
289+
this.merge(entry.getKey(), new CompressedXContent(XContentFactory.jsonBuilder().map(entry.getValue()).string()), MergeReason.MAPPING_UPDATE, updateAllTypes);
290+
} catch (Exception e) {
291+
throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, entry.getKey(), e.getMessage());
292+
}
293+
}
294+
}
295+
265296
public DocumentMapper merge(String type, CompressedXContent mappingSource, MergeReason reason, boolean updateAllTypes) {
266297
if (DEFAULT_MAPPING.equals(type)) {
267298
// verify we can parse it

0 commit comments

Comments
 (0)