Skip to content

Commit 39b4710

Browse files
ywelschywelsch
committed
Add Docker-based tests for proxy connection mode (#51658)
Adds more tests for the new "proxy" remote cluster connection mode, using a Docker-based setup, as well as testing SNI-based routing using HAProxy. - Checks that the new proxy mode can work in situations where the publish host of the nodes in the remote cluster are not routable. - Checks that the new proxy mode can work with clusters where nodes are hidden behind HAProxy. - Checks that the new proxy mode can work with clusters where nodes are hidden behind HAProxy, using SNI to identify the nodes/cluster behind HAProxy. Relates #49067
1 parent 493f77d commit 39b4710

File tree

8 files changed

+696
-0
lines changed

8 files changed

+696
-0
lines changed

‎qa/remote-clusters/build.gradle

+109
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
import org.elasticsearch.gradle.testfixtures.TestFixturesPlugin
20+
21+
apply plugin: 'elasticsearch.standalone-rest-test'
22+
apply plugin: 'elasticsearch.test.fixtures'
23+
24+
testFixtures.useFixture()
25+
26+
configurations {
27+
restSpec
28+
}
29+
30+
dependencies {
31+
restSpec project(':rest-api-spec')
32+
testCompile project(':client:rest-high-level')
33+
}
34+
35+
task copyKeystore(type: Sync) {
36+
from project(':x-pack:plugin:core')
37+
.file('src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks')
38+
into "${buildDir}/certs"
39+
doLast {
40+
file("${buildDir}/certs").setReadable(true, false)
41+
file("${buildDir}/certs/testnode.jks").setReadable(true, false)
42+
}
43+
}
44+
45+
preProcessFixture {
46+
if (TestFixturesPlugin.dockerComposeSupported()) {
47+
if ('default'.equalsIgnoreCase(System.getProperty('tests.distribution', 'default'))) {
48+
dependsOn ":distribution:docker:buildDockerImage"
49+
} else {
50+
dependsOn ":distribution:docker:buildOssDockerImage"
51+
}
52+
}
53+
dependsOn copyKeystore
54+
doLast {
55+
// tests expect to have an empty repo
56+
project.delete(
57+
"${buildDir}/repo",
58+
"${buildDir}/oss-repo"
59+
)
60+
createAndSetWritable(
61+
"${buildDir}/repo",
62+
"${buildDir}/oss-repo",
63+
"${buildDir}/logs/default-1",
64+
"${buildDir}/logs/default-2",
65+
"${buildDir}/logs/oss-1",
66+
"${buildDir}/logs/oss-2"
67+
)
68+
}
69+
}
70+
71+
if (TestFixturesPlugin.dockerComposeSupported()) {
72+
dockerCompose {
73+
tcpPortsToIgnoreWhenWaiting = [9600, 9601]
74+
if ('default'.equalsIgnoreCase(System.getProperty('tests.distribution', 'default'))) {
75+
useComposeFiles = ['docker-compose.yml']
76+
} else {
77+
useComposeFiles = ['docker-compose-oss.yml']
78+
}
79+
}
80+
}
81+
82+
def createAndSetWritable(Object... locations) {
83+
locations.each { location ->
84+
File file = file(location)
85+
file.mkdirs()
86+
file.setWritable(true, false)
87+
}
88+
}
89+
90+
processTestResources {
91+
from({ zipTree(configurations.restSpec.singleFile) }) {
92+
include 'rest-api-spec/api/**'
93+
}
94+
from project(':x-pack:plugin:core')
95+
.file('src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks')
96+
dependsOn configurations.restSpec
97+
}
98+
99+
task integTest(type: Test) {
100+
outputs.doNotCacheIf('Build cache is disabled for Docker tests') { true }
101+
maxParallelForks = '1'
102+
include '**/*IT.class'
103+
// don't add the tasks to build the docker images if we have no way of testing them
104+
if (TestFixturesPlugin.dockerComposeSupported()) {
105+
dependsOn ":distribution:docker:buildDockerImage"
106+
}
107+
}
108+
109+
check.dependsOn integTest

‎qa/remote-clusters/docker-compose-oss.yml

+76
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
# Only used for testing the docker images
2+
version: '3.7'
3+
services:
4+
elasticsearch-oss-1:
5+
image: elasticsearch:test
6+
environment:
7+
- node.name=elasticsearch-oss-1
8+
- cluster.initial_master_nodes=elasticsearch-oss-1
9+
- cluster.name=elasticsearch-oss-1
10+
- bootstrap.memory_lock=true
11+
- network.publish_host=127.0.0.1
12+
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
13+
- path.repo=/tmp/es-repo
14+
- node.attr.testattr=test
15+
- cluster.routing.allocation.disk.watermark.low=1b
16+
- cluster.routing.allocation.disk.watermark.high=1b
17+
- cluster.routing.allocation.disk.watermark.flood_stage=1b
18+
- script.max_compilations_rate=2048/1m
19+
- node.store.allow_mmap=false
20+
volumes:
21+
- ./build/oss-repo:/tmp/es-repo
22+
- ./build/logs/oss-1:/usr/share/elasticsearch/logs
23+
ports:
24+
- "9200"
25+
- "9300"
26+
ulimits:
27+
memlock:
28+
soft: -1
29+
hard: -1
30+
nofile:
31+
soft: 65536
32+
hard: 65536
33+
healthcheck:
34+
start_period: 15s
35+
test: ["CMD", "curl", "-f", "http://localhost:9200"]
36+
interval: 10s
37+
timeout: 2s
38+
retries: 5
39+
elasticsearch-oss-2:
40+
image: elasticsearch:test
41+
environment:
42+
- node.name=elasticsearch-oss-2
43+
- cluster.initial_master_nodes=elasticsearch-oss-2
44+
- cluster.name=elasticsearch-oss-2
45+
- bootstrap.memory_lock=true
46+
- network.publish_host=127.0.0.1
47+
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
48+
- path.repo=/tmp/es-repo
49+
- node.attr.testattr=test
50+
- cluster.routing.allocation.disk.watermark.low=1b
51+
- cluster.routing.allocation.disk.watermark.high=1b
52+
- cluster.routing.allocation.disk.watermark.flood_stage=1b
53+
- script.max_compilations_rate=2048/1m
54+
- node.store.allow_mmap=false
55+
volumes:
56+
- ./build/oss-repo:/tmp/es-repo
57+
- ./build/logs/oss-2:/usr/share/elasticsearch/logs
58+
ports:
59+
- "9200"
60+
- "9300"
61+
ulimits:
62+
memlock:
63+
soft: -1
64+
hard: -1
65+
healthcheck:
66+
start_period: 15s
67+
test: ["CMD", "curl", "-f", "http://localhost:9200"]
68+
interval: 10s
69+
timeout: 2s
70+
retries: 5
71+
haproxy:
72+
image: haproxy:2.1.2
73+
ports:
74+
- "9600"
75+
volumes:
76+
- ./haproxy-oss.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro

‎qa/remote-clusters/docker-compose.yml

+109
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
# Only used for testing the docker images
2+
version: '3.7'
3+
services:
4+
elasticsearch-default-1:
5+
image: elasticsearch:test
6+
environment:
7+
- node.name=elasticsearch-default-1
8+
- cluster.initial_master_nodes=elasticsearch-default-1
9+
- cluster.name=elasticsearch-default-1
10+
- bootstrap.memory_lock=true
11+
- network.publish_host=127.0.0.1
12+
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
13+
- path.repo=/tmp/es-repo
14+
- node.attr.testattr=test
15+
- cluster.routing.allocation.disk.watermark.low=1b
16+
- cluster.routing.allocation.disk.watermark.high=1b
17+
- cluster.routing.allocation.disk.watermark.flood_stage=1b
18+
- script.max_compilations_rate=2048/1m
19+
- node.store.allow_mmap=false
20+
- xpack.security.enabled=true
21+
- xpack.security.transport.ssl.enabled=true
22+
- xpack.security.http.ssl.enabled=true
23+
- xpack.security.authc.token.enabled=true
24+
- xpack.security.audit.enabled=true
25+
- xpack.security.authc.realms.file.file1.order=0
26+
- xpack.security.authc.realms.native.native1.order=1
27+
- xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/testnode.jks
28+
- xpack.security.http.ssl.keystore.path=/usr/share/elasticsearch/config/testnode.jks
29+
- xpack.http.ssl.verification_mode=certificate
30+
- xpack.security.transport.ssl.verification_mode=certificate
31+
- xpack.license.self_generated.type=trial
32+
volumes:
33+
- ./build/repo:/tmp/es-repo
34+
- ./build/certs/testnode.jks:/usr/share/elasticsearch/config/testnode.jks
35+
- ./build/logs/default-1:/usr/share/elasticsearch/logs
36+
- ./docker-test-entrypoint.sh:/docker-test-entrypoint.sh
37+
ports:
38+
- "9200"
39+
- "9300"
40+
ulimits:
41+
memlock:
42+
soft: -1
43+
hard: -1
44+
nofile:
45+
soft: 65536
46+
hard: 65536
47+
entrypoint: /docker-test-entrypoint.sh
48+
healthcheck:
49+
start_period: 15s
50+
test: ["CMD", "curl", "-f", "-u", "x_pack_rest_user:x-pack-test-password", "-k", "https://localhost:9200"]
51+
interval: 10s
52+
timeout: 2s
53+
retries: 5
54+
elasticsearch-default-2:
55+
image: elasticsearch:test
56+
environment:
57+
- node.name=elasticsearch-default-2
58+
- cluster.initial_master_nodes=elasticsearch-default-2
59+
- cluster.name=elasticsearch-default-2
60+
- bootstrap.memory_lock=true
61+
- network.publish_host=127.0.0.1
62+
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
63+
- path.repo=/tmp/es-repo
64+
- node.attr.testattr=test
65+
- cluster.routing.allocation.disk.watermark.low=1b
66+
- cluster.routing.allocation.disk.watermark.high=1b
67+
- cluster.routing.allocation.disk.watermark.flood_stage=1b
68+
- script.max_compilations_rate=2048/1m
69+
- node.store.allow_mmap=false
70+
- xpack.security.enabled=true
71+
- xpack.security.transport.ssl.enabled=true
72+
- xpack.security.http.ssl.enabled=true
73+
- xpack.security.authc.token.enabled=true
74+
- xpack.security.audit.enabled=true
75+
- xpack.security.authc.realms.file.file1.order=0
76+
- xpack.security.authc.realms.native.native1.order=1
77+
- xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/testnode.jks
78+
- xpack.security.http.ssl.keystore.path=/usr/share/elasticsearch/config/testnode.jks
79+
- xpack.http.ssl.verification_mode=certificate
80+
- xpack.security.transport.ssl.verification_mode=certificate
81+
- xpack.license.self_generated.type=trial
82+
volumes:
83+
- ./build/repo:/tmp/es-repo
84+
- ./build/certs/testnode.jks:/usr/share/elasticsearch/config/testnode.jks
85+
- ./build/logs/default-2:/usr/share/elasticsearch/logs
86+
- ./docker-test-entrypoint.sh:/docker-test-entrypoint.sh
87+
ports:
88+
- "9200"
89+
- "9300"
90+
ulimits:
91+
memlock:
92+
soft: -1
93+
hard: -1
94+
nofile:
95+
soft: 65536
96+
hard: 65536
97+
entrypoint: /docker-test-entrypoint.sh
98+
healthcheck:
99+
start_period: 15s
100+
test: ["CMD", "curl", "-f", "-u", "x_pack_rest_user:x-pack-test-password", "-k", "https://localhost:9200"]
101+
interval: 10s
102+
timeout: 2s
103+
retries: 5
104+
haproxy:
105+
image: haproxy:2.1.2
106+
ports:
107+
- "9600"
108+
volumes:
109+
- ./haproxy-default.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro

‎qa/remote-clusters/docker-test-entrypoint.sh

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
#!/bin/bash
2+
cd /usr/share/elasticsearch/bin/
3+
./elasticsearch-users useradd x_pack_rest_user -p x-pack-test-password -r superuser || true
4+
echo "testnode" > /tmp/password
5+
cat /tmp/password | ./elasticsearch-keystore add -x -f -v 'xpack.security.transport.ssl.keystore.secure_password'
6+
cat /tmp/password | ./elasticsearch-keystore add -x -f -v 'xpack.security.http.ssl.keystore.secure_password'
7+
/usr/local/bin/docker-entrypoint.sh | tee > /usr/share/elasticsearch/logs/console.log

‎qa/remote-clusters/haproxy-default.cfg

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
global
2+
log localhost local1 notice
3+
maxconn 2000
4+
daemon
5+
6+
defaults
7+
log global
8+
mode tcp
9+
retries 3
10+
option tcpka
11+
option tcp-check
12+
timeout client 30s
13+
timeout server 30s
14+
timeout connect 5s
15+
16+
frontend ft_ssl
17+
bind *:9600
18+
tcp-request inspect-delay 5s
19+
tcp-request content accept if { req_ssl_hello_type 1 }
20+
default_backend bk_ssl
21+
22+
backend bk_ssl
23+
use-server server1 if { req_ssl_sni -i application1.example.com }
24+
server server1 elasticsearch-default-1:9300 weight 0 check
25+
use-server server2 if { req_ssl_sni -i application2.example.com }
26+
server server2 elasticsearch-default-2:9300 weight 0 check
27+
server default elasticsearch-default-2:9300 check

‎qa/remote-clusters/haproxy-oss.cfg

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
global
2+
log localhost local1 notice
3+
maxconn 2000
4+
daemon
5+
6+
defaults
7+
log global
8+
mode tcp
9+
retries 3
10+
option tcpka
11+
option tcp-check
12+
timeout client 30s
13+
timeout server 30s
14+
timeout connect 5s
15+
16+
frontend ft_reg
17+
bind *:9600
18+
default_backend bk_reg
19+
20+
backend bk_reg
21+
server default elasticsearch-oss-2:9300 check

0 commit comments

Comments
 (0)