|
| 1 | +/* |
| 2 | + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one |
| 3 | + * or more contributor license agreements. Licensed under the Elastic License; |
| 4 | + * you may not use this file except in compliance with the Elastic License. |
| 5 | + */ |
| 6 | + |
| 7 | +package org.elasticsearch.xpack.runtimefields.rest; |
| 8 | + |
| 9 | +import com.carrotsearch.randomizedtesting.annotations.Name; |
| 10 | +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; |
| 11 | + |
| 12 | +import org.elasticsearch.index.mapper.IpFieldMapper; |
| 13 | +import org.elasticsearch.index.mapper.KeywordFieldMapper; |
| 14 | +import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; |
| 15 | +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; |
| 16 | +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; |
| 17 | +import org.elasticsearch.test.rest.yaml.section.DoSection; |
| 18 | +import org.elasticsearch.test.rest.yaml.section.ExecutableSection; |
| 19 | + |
| 20 | +import java.util.ArrayList; |
| 21 | +import java.util.HashMap; |
| 22 | +import java.util.List; |
| 23 | +import java.util.Map; |
| 24 | +import java.util.Objects; |
| 25 | + |
| 26 | +public class CoreTestsWithRuntimeFieldsIT extends ESClientYamlSuiteTestCase { |
| 27 | + public CoreTestsWithRuntimeFieldsIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { |
| 28 | + super(testCandidate); |
| 29 | + } |
| 30 | + |
| 31 | + /** |
| 32 | + * Builds test parameters similarly to {@link ESClientYamlSuiteTestCase#createParameters()}, |
| 33 | + * replacing the body of index creation commands so that fields are {@code runtime_script}s |
| 34 | + * that load from {@code source} instead of their original type. Test configurations that |
| 35 | + * do are not modified to contain runtime fields are not returned as they are tested |
| 36 | + * elsewhere. |
| 37 | + */ |
| 38 | + @ParametersFactory |
| 39 | + public static Iterable<Object[]> parameters() throws Exception { |
| 40 | + /* |
| 41 | + * Map of "setup"s that we've seen - from path to whether or |
| 42 | + * not we the setup was modified to include a runtime_script |
| 43 | + */ |
| 44 | + Map<String, Boolean> seenSetups = new HashMap<>(); |
| 45 | + List<Object[]> result = new ArrayList<>(); |
| 46 | + for (Object[] orig : ESClientYamlSuiteTestCase.createParameters()) { |
| 47 | + assert orig.length == 1; |
| 48 | + ClientYamlTestCandidate candidate = (ClientYamlTestCandidate) orig[0]; |
| 49 | + boolean modifiedSetup = seenSetups.computeIfAbsent( |
| 50 | + candidate.getName(), |
| 51 | + k -> modifySection(candidate.getSuitePath() + "/setup", candidate.getSetupSection().getExecutableSections()) |
| 52 | + ); |
| 53 | + boolean modifiedTest = modifySection(candidate.getTestPath(), candidate.getTestSection().getExecutableSections()); |
| 54 | + if (modifiedSetup || modifiedTest) { |
| 55 | + result.add(new Object[] { candidate }); |
| 56 | + } |
| 57 | + } |
| 58 | + return result; |
| 59 | + } |
| 60 | + |
| 61 | + /** |
| 62 | + * Replace property configuration in {@code indices.create} with scripts |
| 63 | + * that load from the source. |
| 64 | + * @return {@code true} if any fields were rewritten into runtime_scripts, {@code false} otherwise. |
| 65 | + */ |
| 66 | + private static boolean modifySection(String sectionName, List<ExecutableSection> executables) { |
| 67 | + boolean include = false; |
| 68 | + for (ExecutableSection section : executables) { |
| 69 | + if (false == (section instanceof DoSection)) { |
| 70 | + continue; |
| 71 | + } |
| 72 | + DoSection doSection = (DoSection) section; |
| 73 | + if (false == doSection.getApiCallSection().getApi().equals("indices.create")) { |
| 74 | + continue; |
| 75 | + } |
| 76 | + for (Map<?, ?> body : doSection.getApiCallSection().getBodies()) { |
| 77 | + Object settings = body.get("settings"); |
| 78 | + if (settings instanceof Map && ((Map<?, ?>) settings).containsKey("sort.field")) { |
| 79 | + /* |
| 80 | + * You can't sort the index on a runtime_keyword and it is |
| 81 | + * hard to figure out if the sort was a runtime_keyword so |
| 82 | + * let's just skip this test. |
| 83 | + */ |
| 84 | + continue; |
| 85 | + } |
| 86 | + Object mappings = body.get("mappings"); |
| 87 | + if (false == (mappings instanceof Map)) { |
| 88 | + continue; |
| 89 | + } |
| 90 | + Object properties = ((Map<?, ?>) mappings).get("properties"); |
| 91 | + if (false == (properties instanceof Map)) { |
| 92 | + continue; |
| 93 | + } |
| 94 | + for (Map.Entry<?, ?> property : ((Map<?, ?>) properties).entrySet()) { |
| 95 | + if (false == property.getValue() instanceof Map) { |
| 96 | + continue; |
| 97 | + } |
| 98 | + @SuppressWarnings("unchecked") |
| 99 | + Map<String, Object> propertyMap = (Map<String, Object>) property.getValue(); |
| 100 | + String name = property.getKey().toString(); |
| 101 | + String type = Objects.toString(propertyMap.get("type")); |
| 102 | + if ("false".equals(Objects.toString(propertyMap.get("doc_values")))) { |
| 103 | + // If doc_values is false we can't emulate with scripts. `null` and `true` are fine. |
| 104 | + continue; |
| 105 | + } |
| 106 | + if ("false".equals(Objects.toString(propertyMap.get("index")))) { |
| 107 | + // If index is false we can't emulate with scripts |
| 108 | + continue; |
| 109 | + } |
| 110 | + if ("true".equals(Objects.toString(propertyMap.get("store")))) { |
| 111 | + // If store is true we can't emulate with scripts |
| 112 | + continue; |
| 113 | + } |
| 114 | + if (propertyMap.containsKey("ignore_above")) { |
| 115 | + // Scripts don't support ignore_above so we skip those fields |
| 116 | + continue; |
| 117 | + } |
| 118 | + if (propertyMap.containsKey("ignore_malformed")) { |
| 119 | + // Our source reading script doesn't emulate ignore_malformed |
| 120 | + continue; |
| 121 | + } |
| 122 | + String toLoad = painlessToLoadFromSource(name, type); |
| 123 | + if (toLoad == null) { |
| 124 | + continue; |
| 125 | + } |
| 126 | + propertyMap.put("type", "runtime_script"); |
| 127 | + propertyMap.put("runtime_type", type); |
| 128 | + propertyMap.put("script", toLoad); |
| 129 | + propertyMap.remove("store"); |
| 130 | + propertyMap.remove("index"); |
| 131 | + propertyMap.remove("doc_values"); |
| 132 | + include = true; |
| 133 | + } |
| 134 | + } |
| 135 | + } |
| 136 | + return include; |
| 137 | + } |
| 138 | + |
| 139 | + private static String painlessToLoadFromSource(String name, String type) { |
| 140 | + String emit = PAINLESS_TO_EMIT.get(type); |
| 141 | + if (emit == null) { |
| 142 | + return null; |
| 143 | + } |
| 144 | + StringBuilder b = new StringBuilder(); |
| 145 | + b.append("def v = source['").append(name).append("'];\n"); |
| 146 | + b.append("if (v instanceof Iterable) {\n"); |
| 147 | + b.append(" for (def vv : ((Iterable) v)) {\n"); |
| 148 | + b.append(" if (vv != null) {\n"); |
| 149 | + b.append(" def value = vv;\n"); |
| 150 | + b.append(" ").append(emit).append("\n"); |
| 151 | + b.append(" }\n"); |
| 152 | + b.append(" }\n"); |
| 153 | + b.append("} else {\n"); |
| 154 | + b.append(" if (v != null) {\n"); |
| 155 | + b.append(" def value = v;\n"); |
| 156 | + b.append(" ").append(emit).append("\n"); |
| 157 | + b.append(" }\n"); |
| 158 | + b.append("}\n"); |
| 159 | + return b.toString(); |
| 160 | + } |
| 161 | + |
| 162 | + private static final Map<String, String> PAINLESS_TO_EMIT = Map.ofEntries( |
| 163 | + // TODO implement dates against the parser |
| 164 | + Map.entry( |
| 165 | + NumberType.DOUBLE.typeName(), |
| 166 | + "value(value instanceof Number ? ((Number) value).doubleValue() : Double.parseDouble(value.toString()));" |
| 167 | + ), |
| 168 | + Map.entry(KeywordFieldMapper.CONTENT_TYPE, "value(value.toString());"), |
| 169 | + Map.entry(IpFieldMapper.CONTENT_TYPE, "stringValue(value.toString());"), |
| 170 | + Map.entry( |
| 171 | + NumberType.LONG.typeName(), |
| 172 | + "value(value instanceof Number ? ((Number) value).longValue() : Long.parseLong(value.toString()));" |
| 173 | + ) |
| 174 | + ); |
| 175 | + |
| 176 | +} |
0 commit comments