Skip to content

Commit 37a23d7

Browse files
nik9000nik9000
authored
TSDB: Basic rolling upgrade test (#78028)
Adds an index in time_series mode to the rolling upgrade tests.
1 parent 9dda447 commit 37a23d7

File tree

1 file changed

+144
-2
lines changed

1 file changed

+144
-2
lines changed

‎qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java

Lines changed: 144 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,23 +7,33 @@
77
*/
88
package org.elasticsearch.upgrades;
99

10+
import io.github.nik9000.mapmatcher.ListMatcher;
11+
1012
import org.apache.http.util.EntityUtils;
1113
import org.elasticsearch.Version;
1214
import org.elasticsearch.client.Request;
1315
import org.elasticsearch.client.Response;
1416
import org.elasticsearch.client.ResponseException;
15-
import org.elasticsearch.core.Booleans;
1617
import org.elasticsearch.common.Strings;
1718
import org.elasticsearch.common.xcontent.XContentBuilder;
1819
import org.elasticsearch.common.xcontent.XContentType;
20+
import org.elasticsearch.common.xcontent.json.JsonXContent;
1921
import org.elasticsearch.common.xcontent.support.XContentMapValues;
22+
import org.elasticsearch.core.Booleans;
23+
import org.elasticsearch.index.mapper.DateFieldMapper;
24+
import org.hamcrest.Matcher;
2025

2126
import java.io.IOException;
2227
import java.nio.charset.StandardCharsets;
2328
import java.util.List;
2429
import java.util.Map;
30+
import java.util.concurrent.TimeUnit;
2531

32+
import static io.github.nik9000.mapmatcher.ListMatcher.matchesList;
33+
import static io.github.nik9000.mapmatcher.MapMatcher.assertMap;
34+
import static io.github.nik9000.mapmatcher.MapMatcher.matchesMap;
2635
import static org.elasticsearch.rest.action.search.RestSearchAction.TOTAL_HITS_AS_INT_PARAM;
36+
import static org.hamcrest.Matchers.closeTo;
2737
import static org.hamcrest.Matchers.containsString;
2838
import static org.hamcrest.Matchers.either;
2939
import static org.hamcrest.Matchers.equalTo;
@@ -228,12 +238,144 @@ private void bulk(String index, String valueSuffix, int count) throws IOExceptio
228238
b.append("{\"index\": {\"_index\": \"").append(index).append("\"}}\n");
229239
b.append("{\"f1\": \"v").append(i).append(valueSuffix).append("\", \"f2\": ").append(i).append("}\n");
230240
}
241+
bulk(index, b.toString());
242+
}
243+
244+
private static final List<String> TSDB_DIMS = List.of("6a841a21", "947e4ced", "a4c385a1", "b47a2f4e", "df3145b3");
245+
private static final long[] TSDB_TIMES;
246+
static {
247+
String[] times = new String[] {
248+
"2021-01-01T00:00:00Z",
249+
"2021-01-02T00:00:00Z",
250+
"2021-01-02T00:10:00Z",
251+
"2021-01-02T00:20:00Z",
252+
"2021-01-02T00:30:00Z" };
253+
TSDB_TIMES = new long[times.length];
254+
for (int i = 0; i < times.length; i++) {
255+
TSDB_TIMES[i] = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis(times[i]);
256+
}
257+
}
258+
259+
public void testTsdb() throws IOException {
260+
assumeTrue("tsdb added in 8.0.0", UPGRADE_FROM_VERSION.onOrAfter(Version.V_8_0_0));
261+
262+
StringBuilder bulk = new StringBuilder();
263+
switch (CLUSTER_TYPE) {
264+
case OLD:
265+
createTsdbIndex();
266+
tsdbBulk(bulk, TSDB_DIMS.get(0), TSDB_TIMES[0], TSDB_TIMES[1], 0.1);
267+
tsdbBulk(bulk, TSDB_DIMS.get(1), TSDB_TIMES[0], TSDB_TIMES[1], -0.1);
268+
bulk("tsdb", bulk.toString());
269+
assertTsdbAgg(closeTo(215.95, 0.005), closeTo(-215.95, 0.005));
270+
return;
271+
case MIXED:
272+
if (FIRST_MIXED_ROUND) {
273+
tsdbBulk(bulk, TSDB_DIMS.get(0), TSDB_TIMES[1], TSDB_TIMES[2], 0.1);
274+
tsdbBulk(bulk, TSDB_DIMS.get(1), TSDB_TIMES[1], TSDB_TIMES[2], -0.1);
275+
tsdbBulk(bulk, TSDB_DIMS.get(2), TSDB_TIMES[0], TSDB_TIMES[2], 1.1);
276+
bulk("tsdb", bulk.toString());
277+
assertTsdbAgg(closeTo(217.45, 0.005), closeTo(-217.45, 0.005), closeTo(2391.95, 0.005));
278+
return;
279+
}
280+
tsdbBulk(bulk, TSDB_DIMS.get(0), TSDB_TIMES[2], TSDB_TIMES[3], 0.1);
281+
tsdbBulk(bulk, TSDB_DIMS.get(1), TSDB_TIMES[2], TSDB_TIMES[3], -0.1);
282+
tsdbBulk(bulk, TSDB_DIMS.get(2), TSDB_TIMES[2], TSDB_TIMES[3], 1.1);
283+
tsdbBulk(bulk, TSDB_DIMS.get(3), TSDB_TIMES[0], TSDB_TIMES[3], 10);
284+
bulk("tsdb", bulk.toString());
285+
assertTsdbAgg(closeTo(218.95, 0.005), closeTo(-218.95, 0.005), closeTo(2408.45, 0.005), closeTo(21895, 0.5));
286+
return;
287+
case UPGRADED:
288+
tsdbBulk(bulk, TSDB_DIMS.get(0), TSDB_TIMES[3], TSDB_TIMES[4], 0.1);
289+
tsdbBulk(bulk, TSDB_DIMS.get(1), TSDB_TIMES[3], TSDB_TIMES[4], -0.1);
290+
tsdbBulk(bulk, TSDB_DIMS.get(2), TSDB_TIMES[3], TSDB_TIMES[4], 1.1);
291+
tsdbBulk(bulk, TSDB_DIMS.get(3), TSDB_TIMES[3], TSDB_TIMES[4], 10);
292+
tsdbBulk(bulk, TSDB_DIMS.get(4), TSDB_TIMES[0], TSDB_TIMES[4], -5);
293+
bulk("tsdb", bulk.toString());
294+
assertTsdbAgg(
295+
closeTo(220.45, 0.005),
296+
closeTo(-220.45, 0.005),
297+
closeTo(2424.95, 0.005),
298+
closeTo(22045, 0.5),
299+
closeTo(-11022.5, 0.5)
300+
);
301+
return;
302+
}
303+
}
304+
305+
private void bulk(String index, String entity) throws IOException {
231306
Request bulk = new Request("POST", "/_bulk");
232307
bulk.addParameter("refresh", "true");
233-
bulk.setJsonEntity(b.toString());
308+
bulk.setJsonEntity(entity.toString());
234309
client().performRequest(bulk);
235310
}
236311

312+
private void createTsdbIndex() throws IOException {
313+
Request createIndex = new Request("PUT", "/tsdb");
314+
XContentBuilder indexSpec = XContentBuilder.builder(XContentType.JSON.xContent()).startObject();
315+
indexSpec.startObject("mappings").startObject("properties");
316+
{
317+
indexSpec.startObject("@timestamp").field("type", "date").endObject();
318+
indexSpec.startObject("dim").field("type", "keyword").field("time_series_dimension", true).endObject();
319+
}
320+
indexSpec.endObject().endObject();
321+
indexSpec.startObject("settings").field("mode", "time_series").endObject();
322+
createIndex.setJsonEntity(Strings.toString(indexSpec.endObject()));
323+
client().performRequest(createIndex);
324+
}
325+
326+
private void tsdbBulk(StringBuilder bulk, String dim, long timeStart, long timeEnd, double rate) throws IOException {
327+
long delta = TimeUnit.SECONDS.toMillis(20);
328+
double value = (timeStart - TSDB_TIMES[0]) / TimeUnit.SECONDS.toMillis(20) * rate;
329+
for (long t = timeStart; t < timeEnd; t += delta) {
330+
bulk.append("{\"index\": {\"_index\": \"tsdb\"}}\n");
331+
bulk.append("{\"@timestamp\": ").append(t);
332+
bulk.append(", \"dim\": \"").append(dim).append("\"");
333+
bulk.append(", \"value\": ").append(value).append("}\n");
334+
value += rate;
335+
}
336+
}
337+
338+
private void assertTsdbAgg(Matcher<?>... expected) throws IOException {
339+
Request request = new Request("POST", "/tsdb/_search");
340+
request.addParameter("size", "0");
341+
XContentBuilder body = JsonXContent.contentBuilder().startObject();
342+
// TODO replace tsid runtime field with real tsid
343+
body.startObject("runtime_mappings");
344+
{
345+
body.startObject("tsid");
346+
{
347+
body.field("type", "keyword");
348+
body.field("script", "emit('dim:' + doc['dim'].value)");
349+
}
350+
body.endObject();
351+
}
352+
body.endObject();
353+
body.startObject("aggs").startObject("tsids");
354+
{
355+
body.startObject("terms").field("field", "tsid").endObject();
356+
body.startObject("aggs").startObject("avg");
357+
{
358+
body.startObject("avg").field("field", "value").endObject();
359+
}
360+
body.endObject().endObject();
361+
}
362+
body.endObject().endObject();
363+
request.setJsonEntity(Strings.toString(body.endObject()));
364+
ListMatcher tsidsExpected = matchesList();
365+
for (int d = 0; d < expected.length; d++) {
366+
// Object key = Map.of("dim", TSDB_DIMS.get(d)); TODO use this once tsid is real
367+
Object key = "dim:" + TSDB_DIMS.get(d);
368+
tsidsExpected = tsidsExpected.item(
369+
matchesMap().extraOk().entry("key", key).entry("avg", Map.of("value", expected[d]))
370+
);
371+
}
372+
assertMap(
373+
entityAsMap(client().performRequest(request)),
374+
matchesMap().extraOk()
375+
.entry("aggregations", matchesMap().entry("tsids", matchesMap().extraOk().entry("buckets", tsidsExpected)))
376+
);
377+
}
378+
237379
private void assertCount(String index, int count) throws IOException {
238380
Request searchTestIndexRequest = new Request("POST", "/" + index + "/_search");
239381
searchTestIndexRequest.addParameter(TOTAL_HITS_AS_INT_PARAM, "true");

0 commit comments

Comments
 (0)