Skip to content

Commit 6155e14

Browse files
davidkyledavidkyle
authored
[TRANSFORM] Remove HLRC from single node tests (#84220)
1 parent 28758b0 commit 6155e14

File tree

6 files changed

+199
-217
lines changed

6 files changed

+199
-217
lines changed

‎x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformGetAndGetStatsIT.java

+67-92
Original file line numberDiff line numberDiff line change
@@ -11,33 +11,8 @@
1111
import org.elasticsearch.client.RequestOptions;
1212
import org.elasticsearch.client.Response;
1313
import org.elasticsearch.client.ResponseException;
14-
import org.elasticsearch.client.RestHighLevelClient;
1514
import org.elasticsearch.client.WarningsHandler;
16-
import org.elasticsearch.client.core.AcknowledgedResponse;
17-
import org.elasticsearch.client.core.PageParams;
18-
import org.elasticsearch.client.transform.DeleteTransformRequest;
19-
import org.elasticsearch.client.transform.GetTransformRequest;
20-
import org.elasticsearch.client.transform.GetTransformResponse;
21-
import org.elasticsearch.client.transform.GetTransformStatsRequest;
22-
import org.elasticsearch.client.transform.GetTransformStatsResponse;
23-
import org.elasticsearch.client.transform.PutTransformRequest;
24-
import org.elasticsearch.client.transform.transforms.DestConfig;
25-
import org.elasticsearch.client.transform.transforms.QueryConfig;
26-
import org.elasticsearch.client.transform.transforms.SourceConfig;
27-
import org.elasticsearch.client.transform.transforms.TransformConfig;
28-
import org.elasticsearch.client.transform.transforms.TransformStats;
29-
import org.elasticsearch.client.transform.transforms.pivot.AggregationConfig;
30-
import org.elasticsearch.client.transform.transforms.pivot.GroupConfig;
31-
import org.elasticsearch.client.transform.transforms.pivot.PivotConfig;
32-
import org.elasticsearch.client.transform.transforms.pivot.TermsGroupSource;
33-
import org.elasticsearch.common.settings.Settings;
3415
import org.elasticsearch.common.xcontent.support.XContentMapValues;
35-
import org.elasticsearch.core.TimeValue;
36-
import org.elasticsearch.index.query.QueryBuilders;
37-
import org.elasticsearch.search.SearchModule;
38-
import org.elasticsearch.search.aggregations.AggregationBuilders;
39-
import org.elasticsearch.search.aggregations.AggregatorFactories;
40-
import org.elasticsearch.xcontent.NamedXContentRegistry;
4116
import org.elasticsearch.xpack.core.transform.TransformField;
4217
import org.elasticsearch.xpack.core.transform.transforms.persistence.TransformInternalIndexConstants;
4318
import org.junit.After;
@@ -50,7 +25,6 @@
5025
import java.util.Map;
5126
import java.util.concurrent.TimeUnit;
5227

53-
import static java.util.Collections.emptyList;
5428
import static java.util.Collections.singletonList;
5529
import static org.hamcrest.Matchers.equalTo;
5630
import static org.hamcrest.Matchers.greaterThan;
@@ -518,83 +492,84 @@ public void testGetStatsWithContinuous() throws Exception {
518492
}, 120, TimeUnit.SECONDS);
519493
}
520494

521-
public void testManyTranformsUsingHlrc() throws IOException {
522-
AggregatorFactories.Builder aggs = AggregatorFactories.builder()
523-
.addAggregator(AggregationBuilders.avg("review_score.avg").field("stars"))
524-
.addAggregator(AggregationBuilders.max("timestamp.max").field("timestamp"));
525-
526-
TransformConfig.Builder configBuilder = TransformConfig.builder()
527-
.setSource(
528-
SourceConfig.builder().setIndex(REVIEWS_INDEX_NAME).setQueryConfig(new QueryConfig(QueryBuilders.matchAllQuery())).build()
529-
)
530-
.setDest(DestConfig.builder().setIndex("dest").build())
531-
.setFrequency(TimeValue.timeValueSeconds(10))
532-
.setDescription("Test 10000 transform configs")
533-
.setPivotConfig(
534-
PivotConfig.builder()
535-
.setGroups(GroupConfig.builder().groupBy("by-user", TermsGroupSource.builder().setField("user_id").build()).build())
536-
.setAggregationConfig(new AggregationConfig(aggs))
537-
.build()
538-
);
539-
540-
try (RestHighLevelClient restClient = new TestRestHighLevelClient()) {
541-
int numberOfTransforms = randomIntBetween(1_500, 4_000);
542-
for (int i = 0; i < numberOfTransforms; ++i) {
543-
AcknowledgedResponse response = restClient.transform()
544-
.putTransform(
545-
new PutTransformRequest(configBuilder.setId(String.format(Locale.ROOT, "t-%05d", i)).build()),
546-
RequestOptions.DEFAULT
547-
);
548-
assertTrue(response.isAcknowledged());
549-
}
550-
551-
for (int i = 0; i < 3; ++i) {
552-
int from = randomIntBetween(0, numberOfTransforms - 1_000);
553-
int size = randomIntBetween(1, 1000);
495+
@SuppressWarnings("unchecked")
496+
public void testManyTransforms() throws IOException {
497+
String config = transformConfig();
498+
499+
int numberOfTransforms = randomIntBetween(1_500, 4_000);
500+
for (int i = 0; i < numberOfTransforms; ++i) {
501+
String transformId = String.format(Locale.ROOT, "t-%05d", i);
502+
final Request createTransformRequest = createRequestWithAuth("PUT", getTransformEndpoint() + transformId, null);
503+
createTransformRequest.setJsonEntity(config);
504+
assertOK(client().performRequest(createTransformRequest));
505+
}
554506

555-
GetTransformRequest request = new GetTransformRequest("*");
556-
request.setPageParams(new PageParams(from, size));
557-
GetTransformStatsRequest statsRequest = new GetTransformStatsRequest("*");
558-
statsRequest.setPageParams(new PageParams(from, size));
507+
for (int i = 0; i < 3; ++i) {
508+
int from = randomIntBetween(0, numberOfTransforms - 1_000);
509+
int size = randomIntBetween(1, 1000);
559510

560-
GetTransformResponse response = restClient.transform().getTransform(request, RequestOptions.DEFAULT);
561-
GetTransformStatsResponse statsResponse = restClient.transform().getTransformStats(statsRequest, RequestOptions.DEFAULT);
511+
var transforms = getTransforms(from, size);
512+
var statsResponse = getTransformsStateAndStats(from, size);
562513

563-
assertEquals(numberOfTransforms, response.getCount());
564-
assertEquals(numberOfTransforms, statsResponse.getCount());
514+
assertEquals(numberOfTransforms, transforms.get("count"));
515+
assertEquals(numberOfTransforms, statsResponse.get("count"));
565516

566-
List<TransformConfig> configs = response.getTransformConfigurations();
567-
List<TransformStats> stats = statsResponse.getTransformsStats();
517+
var configs = (List<Map<String, Object>>) transforms.get("transforms");
518+
var stats = (List<Map<String, Object>>) statsResponse.get("transforms");
568519

569-
assertEquals(size, configs.size());
570-
assertEquals(size, stats.size());
520+
assertEquals(size, configs.size());
521+
assertEquals(size, stats.size());
571522

572-
assertThat(configs.get(0).getId(), equalTo(String.format(Locale.ROOT, "t-%05d", from)));
573-
assertThat(configs.get(configs.size() - 1).getId(), equalTo(String.format(Locale.ROOT, "t-%05d", from + size - 1)));
574-
assertThat(stats.get(0).getId(), equalTo(String.format(Locale.ROOT, "t-%05d", from)));
575-
assertThat(stats.get(stats.size() - 1).getId(), equalTo(String.format(Locale.ROOT, "t-%05d", from + size - 1)));
523+
assertThat(configs.get(0).get("id"), equalTo(String.format(Locale.ROOT, "t-%05d", from)));
524+
assertThat(configs.get(configs.size() - 1).get("id"), equalTo(String.format(Locale.ROOT, "t-%05d", from + size - 1)));
525+
assertThat(stats.get(0).get("id"), equalTo(String.format(Locale.ROOT, "t-%05d", from)));
526+
assertThat(stats.get(stats.size() - 1).get("id"), equalTo(String.format(Locale.ROOT, "t-%05d", from + size - 1)));
576527

577-
if (size > 2) {
578-
int randomElement = randomIntBetween(1, size - 1);
579-
assertThat(configs.get(randomElement).getId(), equalTo(String.format(Locale.ROOT, "t-%05d", from + randomElement)));
580-
assertThat(stats.get(randomElement).getId(), equalTo(String.format(Locale.ROOT, "t-%05d", from + randomElement)));
581-
}
528+
if (size > 2) {
529+
int randomElement = randomIntBetween(1, size - 1);
530+
assertThat(configs.get(randomElement).get("id"), equalTo(String.format(Locale.ROOT, "t-%05d", from + randomElement)));
531+
assertThat(stats.get(randomElement).get("id"), equalTo(String.format(Locale.ROOT, "t-%05d", from + randomElement)));
582532
}
533+
}
583534

584-
for (int i = 0; i < numberOfTransforms; ++i) {
585-
AcknowledgedResponse response = restClient.transform()
586-
.deleteTransform(new DeleteTransformRequest(String.format(Locale.ROOT, "t-%05d", i)), RequestOptions.DEFAULT);
587-
assertTrue(response.isAcknowledged());
588-
}
535+
for (int i = 0; i < numberOfTransforms; ++i) {
536+
deleteTransform(String.format(Locale.ROOT, "t-%05d", i));
589537
}
590538
}
591539

592-
protected static class TestRestHighLevelClient extends RestHighLevelClient {
593-
private static final List<NamedXContentRegistry.Entry> X_CONTENT_ENTRIES = new SearchModule(Settings.EMPTY, emptyList())
594-
.getNamedXContents();
595-
596-
TestRestHighLevelClient() {
597-
super(client(), restClient -> {}, X_CONTENT_ENTRIES);
598-
}
540+
private static String transformConfig() {
541+
return """
542+
{
543+
"description": "Test 10000 transform configs",
544+
"source": {
545+
"index":""" + "\"" + REVIEWS_INDEX_NAME + "\"" + """
546+
},
547+
"pivot": {
548+
"group_by": {
549+
"by-user": {
550+
"terms": {
551+
"field": "user_id"
552+
}
553+
}
554+
},
555+
"aggregations": {
556+
"review_score.avg": {
557+
"avg": {
558+
"field": "stars"
559+
}
560+
},
561+
"timestamp.max": {
562+
"max": {
563+
"field": "timestamp"
564+
}
565+
}
566+
}
567+
},
568+
"dest": {
569+
"index":"dest"
570+
},
571+
"frequency": "10s"
572+
}
573+
""";
599574
}
600575
}

‎x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java

+4-4
Original file line numberDiff line numberDiff line change
@@ -1347,8 +1347,8 @@ public void testPivotWithGeoBoundsAgg() throws Exception {
13471347
)).get(0);
13481348
assertThat(actualObj.get("type"), equalTo("point"));
13491349
List<Double> coordinates = (List<Double>) actualObj.get("coordinates");
1350-
assertEquals((4 + 10), coordinates.get(1), 0.000001);
1351-
assertEquals((4 + 15), coordinates.get(0), 0.000001);
1350+
assertEquals(-76.0, coordinates.get(1), 0.000001);
1351+
assertEquals(-161.0, coordinates.get(0), 0.000001);
13521352
}
13531353

13541354
public void testPivotWithGeoCentroidAgg() throws Exception {
@@ -1411,8 +1411,8 @@ public void testPivotWithGeoCentroidAgg() throws Exception {
14111411
assertEquals(3.878048780, actual.doubleValue(), 0.000001);
14121412
String actualString = (String) ((List<?>) XContentMapValues.extractValue("hits.hits._source.location", searchResult)).get(0);
14131413
String[] latlon = actualString.split(",");
1414-
assertEquals((4 + 10), Double.valueOf(latlon[0]), 0.000001);
1415-
assertEquals((4 + 15), Double.valueOf(latlon[1]), 0.000001);
1414+
assertEquals(-76.0, Double.valueOf(latlon[0]), 0.000001);
1415+
assertEquals(-161.0, Double.valueOf(latlon[1]), 0.000001);
14161416
}
14171417

14181418
@SuppressWarnings("unchecked")

‎x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestSpecialCasesIT.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -248,8 +248,8 @@ public void testRestrictiveBucketSelector() throws Exception {
248248
String indexName = "special_pivot_bucket_selector_reviews";
249249
createReviewsIndex(indexName, 1000, 327, "date", false, 5, "affiliate_id");
250250

251-
verifyDestIndexHitsCount(indexName, "special_pivot_bucket_selector-10", 10, 14);
252-
verifyDestIndexHitsCount(indexName, "special_pivot_bucket_selector-10000", 10000, 14);
251+
verifyDestIndexHitsCount(indexName, "special_pivot_bucket_selector-10", 10, 41);
252+
verifyDestIndexHitsCount(indexName, "special_pivot_bucket_selector-10000", 10000, 41);
253253
}
254254

255255
private void verifyDestIndexHitsCount(String sourceIndex, String transformId, int maxPageSearchSize, long expectedDestIndexCount)

‎x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java

+35-10
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,6 @@
3939
import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
4040
import static org.hamcrest.Matchers.equalTo;
4141
import static org.hamcrest.Matchers.is;
42-
import static org.junit.Assert.assertThat;
4342

4443
public abstract class TransformRestTestCase extends ESRestTestCase {
4544

@@ -88,7 +87,7 @@ protected void createReviewsIndex(
8887
min = 10 + (i % 49);
8988
}
9089
int sec = 10 + (i % 49);
91-
String location = (user + 10) + "," + (user + 15);
90+
String location = (((user + 10) % 180) - 90) + "," + (((user + 15) % 360) - 180);
9291

9392
String date_string = "2017-01-" + day + "T" + hour + ":" + min + ":" + sec;
9493
if (dateType.equals("date_nanos")) {
@@ -123,21 +122,33 @@ protected void createReviewsIndex(
123122

124123
if (i % 50 == 0) {
125124
bulk.append("\r\n");
126-
final Request bulkRequest = new Request("POST", "/_bulk");
127-
bulkRequest.addParameter("refresh", "true");
128-
bulkRequest.setJsonEntity(bulk.toString());
129-
client().performRequest(bulkRequest);
125+
doBulk(bulk.toString(), true);
130126
// clear the builder
131127
bulk.setLength(0);
132128
day += 1;
133129
}
134130
}
131+
135132
bulk.append("\r\n");
133+
doBulk(bulk.toString(), true);
134+
}
136135

137-
final Request bulkRequest = new Request("POST", "/_bulk");
138-
bulkRequest.addParameter("refresh", "true");
139-
bulkRequest.setJsonEntity(bulk.toString());
140-
client().performRequest(bulkRequest);
136+
@SuppressWarnings("unchecked")
137+
protected void doBulk(String bulkDocuments, boolean refresh) throws IOException {
138+
Request bulkRequest = new Request("POST", "/_bulk");
139+
if (refresh) {
140+
bulkRequest.addParameter("refresh", "true");
141+
}
142+
bulkRequest.setJsonEntity(bulkDocuments);
143+
bulkRequest.setOptions(RequestOptions.DEFAULT);
144+
Response bulkResponse = client().performRequest(bulkRequest);
145+
assertOK(bulkResponse);
146+
var bulkMap = entityAsMap(bulkResponse);
147+
var hasErrors = (boolean) bulkMap.get("errors");
148+
if (hasErrors) {
149+
var items = (List<Map<String, Object>>) bulkMap.get("items");
150+
fail("Bulk item failures: " + items.toString());
151+
}
141152
}
142153

143154
protected void putReviewsIndex(String indexName, String dateType, boolean isDataStream) throws IOException {
@@ -463,6 +474,13 @@ protected static List<Map<String, Object>> getTransforms(List<Map<String, String
463474
return transformConfigs == null ? Collections.emptyList() : transformConfigs;
464475
}
465476

477+
@SuppressWarnings("unchecked")
478+
protected static Map<String, Object> getTransforms(int from, int size) throws IOException {
479+
Request request = new Request("GET", getTransformEndpoint() + "_all?from=" + from + "&size=" + size);
480+
Response response = adminClient().performRequest(request);
481+
return entityAsMap(response);
482+
}
483+
466484
protected static String getTransformState(String transformId) throws IOException {
467485
Map<?, ?> transformStatsAsMap = getTransformStateAndStats(transformId);
468486
return transformStatsAsMap == null ? null : (String) XContentMapValues.extractValue("state", transformStatsAsMap);
@@ -477,6 +495,13 @@ protected static String getTransformState(String transformId) throws IOException
477495
return (Map<?, ?>) transforms.get(0);
478496
}
479497

498+
protected static Map<String, Object> getTransformsStateAndStats(int from, int size) throws IOException {
499+
Response statsResponse = client().performRequest(
500+
new Request("GET", getTransformEndpoint() + "_stats?from=" + from + "&size=" + size)
501+
);
502+
return entityAsMap(statsResponse);
503+
}
504+
480505
protected static void deleteTransform(String transformId) throws IOException {
481506
Request request = new Request("DELETE", getTransformEndpoint() + transformId);
482507
request.addParameter("ignore", "404"); // Ignore 404s because they imply someone was racing us to delete this

0 commit comments

Comments
 (0)