Skip to content

Commit 001ff55

Browse files
author
Ravi Sankar Penta
committed
Support for DNS names in egress routes
Introduced dns-proxy egress router mode that allows specifying DNS name for EGRESS_DESTINATION. Currently, dns-proxy egress mode implementation is based on HAProxy. HAProxy 1.6+ version is used to leverage DNS resolution at runtime.
1 parent a35bb2e commit 001ff55

File tree

5 files changed

+239
-1
lines changed

5 files changed

+239
-1
lines changed

hack/build-images.sh

+2-1
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ os::build::image "${tag_prefix}-keepalived-ipfailover" images/ipfailover/keepali
5454
os::build::image "${tag_prefix}-docker-registry" images/dockerregistry
5555
os::build::image "${tag_prefix}-egress-router" images/egress/router
5656
os::build::image "${tag_prefix}-egress-http-proxy" images/egress/http-proxy
57+
os::build::image "${tag_prefix}-egress-dns-proxy" images/egress/dns-proxy
5758
os::build::image "${tag_prefix}-federation" images/federation
5859
# images that depend on "${tag_prefix}
5960
os::build::image "${tag_prefix}-gitserver" examples/gitserver
@@ -67,4 +68,4 @@ os::build::image "openshift/node" images/node
6768
os::build::image "openshift/openvswitch" images/openvswitch
6869

6970
# extra images (not part of infrastructure)
70-
os::build::image "openshift/hello-openshift" examples/hello-openshift
71+
os::build::image "openshift/hello-openshift" examples/hello-openshift

images/egress/dns-proxy/.cccp.yml

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
job-id: origin-egress-dns-proxy

images/egress/dns-proxy/Dockerfile

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
#
2+
# This is the egress router L4 DNS proxy for OpenShift Origin
3+
#
4+
# The standard name for this image is openshift/origin-egress-dns-proxy
5+
6+
FROM openshift/origin-base
7+
8+
# HAProxy 1.6+ version is needed to leverage DNS resolution at runtime.
9+
RUN INSTALL_PKGS="rsyslog gcc make openssl-devel pcre-devel tar wget socat" && \
10+
yum install -y $INSTALL_PKGS && \
11+
rpm -V $INSTALL_PKGS && \
12+
wget http://www.haproxy.org/download/1.7/src/haproxy-1.7.5.tar.gz && \
13+
tar xvzf haproxy-1.7.5.tar.gz && \
14+
groupadd haproxy && \
15+
useradd -g haproxy haproxy && \
16+
cd haproxy-* && make TARGET=linux2628 CPU=native USE_PCRE=1 USE_OPENSSL=1 USE_ZLIB=1 && make install && \
17+
cd .. && rm -rf haproxy-* && \
18+
setcap 'cap_net_bind_service=ep' /usr/local/sbin/haproxy && \
19+
mkdir -p /var/lib/haproxy && \
20+
mkdir -p /etc/haproxy && \
21+
touch /etc/haproxy/haproxy.cfg && \
22+
yum -y remove gcc && \
23+
yum clean all
24+
25+
ADD egress-dns-proxy.sh /bin/egress-dns-proxy.sh
26+
27+
ENTRYPOINT /bin/egress-dns-proxy.sh
28+
+183
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,183 @@
1+
#!/bin/bash
2+
3+
# OpenShift egress DNS proxy setup script
4+
5+
set -o errexit
6+
set -o nounset
7+
set -o pipefail
8+
9+
# Default DNS nameserver port
10+
NS_PORT=53
11+
CONF=/etc/haproxy/haproxy.cfg
12+
13+
BLANK_LINE_OR_COMMENT_REGEX="([[:space:]]*$|#.*)"
14+
IPADDR_REGEX="[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+"
15+
PORT_REGEX="[[:digit:]]+"
16+
DOMAINNAME_REGEX="[[:alnum:]][[:alnum:]-]+?\.[[:alnum:].-]+"
17+
18+
function die() {
19+
echo "$*" 1>&2
20+
exit 1
21+
}
22+
23+
function check_prereqs() {
24+
if [[ -z "${EGRESS_SOURCE}" ]]; then
25+
die "Must specify EGRESS_SOURCE"
26+
fi
27+
if ! [[ "${EGRESS_SOURCE}" =~ ^${IPADDR_REGEX} ]]; then
28+
die "EGRESS_SOURCE must be IPv4 address"
29+
fi
30+
31+
if [[ -z "${EGRESS_DESTINATION}" ]]; then
32+
die "Must specify EGRESS_DESTINATION"
33+
fi
34+
}
35+
36+
function validate_port() {
37+
local port=$1
38+
if [[ "${port}" -lt "1" || "${port}" -gt "65535" ]]; then
39+
die "Invalid port: ${port}, must be in the range 1 to 65535"
40+
fi
41+
}
42+
43+
function generate_haproxy_defaults() {
44+
echo "
45+
global
46+
log 127.0.0.1 local2
47+
48+
chroot /var/lib/haproxy
49+
pidfile /var/run/haproxy.pid
50+
maxconn 4000
51+
user haproxy
52+
group haproxy
53+
54+
defaults
55+
log global
56+
mode tcp
57+
option dontlognull
58+
option tcplog
59+
option redispatch
60+
retries 3
61+
timeout http-request 100s
62+
timeout queue 1m
63+
timeout connect 10s
64+
timeout client 1m
65+
timeout server 1m
66+
timeout http-keep-alive 100s
67+
timeout check 10s
68+
"
69+
}
70+
71+
function generate_dns_resolvers() {
72+
echo "resolvers dns-resolver"
73+
# Fetch nameservers from /etc/resolv.conf
74+
declare -a nameservers=$(cat /etc/resolv.conf |grep nameserver|awk -F" " '{print $2}')
75+
n=0
76+
for ns in ${nameservers[@]}; do
77+
n=$(($n + 1))
78+
echo " nameserver ns$n ${ns}:${NS_PORT}"
79+
done
80+
81+
# Add google DNS servers as fallback
82+
echo " nameserver nsfallback1 8.8.8.8:${NS_PORT}"
83+
echo " nameserver nsfallback2 8.8.4.4:${NS_PORT}"
84+
85+
# Set default optional params
86+
echo " resolve_retries 3"
87+
echo " timeout retry 1s"
88+
echo " hold valid 10s"
89+
echo ""
90+
}
91+
92+
function generate_haproxy_frontends_backends() {
93+
local n=0
94+
declare -A used_ports=()
95+
96+
while read dest; do
97+
local port target targetport resolvers
98+
99+
if [[ "${dest}" =~ ^${BLANK_LINE_OR_COMMENT_REGEX}$ ]]; then
100+
continue
101+
fi
102+
n=$(($n + 1))
103+
resolvers=""
104+
105+
if [[ "${dest}" =~ ^${PORT_REGEX}\ +${IPADDR_REGEX}$ ]]; then
106+
read port target <<< "${dest}"
107+
targetport="${port}"
108+
elif [[ "${dest}" =~ ^${PORT_REGEX}\ +${IPADDR_REGEX}\ +${PORT_REGEX}$ ]]; then
109+
read port target targetport <<< "${dest}"
110+
elif [[ "${dest}" =~ ^${PORT_REGEX}\ +${DOMAINNAME_REGEX}$ ]]; then
111+
read port target <<< "${dest}"
112+
targetport="${port}"
113+
resolvers="resolvers dns-resolver"
114+
elif [[ "${dest}" =~ ^${PORT_REGEX}\ +${DOMAINNAME_REGEX}\ +${PORT_REGEX}$ ]]; then
115+
read port target targetport <<< "${dest}"
116+
resolvers="resolvers dns-resolver"
117+
else
118+
die "Bad destination '${dest}'"
119+
fi
120+
121+
validate_port ${port}
122+
validate_port ${targetport}
123+
124+
if [[ "${used_ports[${port}]:-x}" == "x" ]]; then
125+
used_ports[${port}]=1
126+
else
127+
die "Proxy port $port already used, must be unique for each destination"
128+
fi
129+
130+
echo "
131+
frontend fe$n
132+
bind ${EGRESS_SOURCE}:${port}
133+
default_backend be$n
134+
135+
backend be$n
136+
server dest$n ${target}:${targetport} check $resolvers
137+
"
138+
done <<< "${EGRESS_DESTINATION}"
139+
}
140+
141+
function setup_haproxy_config() {
142+
generate_haproxy_defaults
143+
generate_dns_resolvers
144+
generate_haproxy_frontends_backends
145+
}
146+
147+
function setup_haproxy_syslog() {
148+
cat >> /etc/rsyslog.conf <<EOF
149+
module(load="imudp")
150+
input(type="imudp" port="514")
151+
EOF
152+
153+
echo "local2.* /var/log/haproxy.log" >> /etc/rsyslog.d/haproxy.conf
154+
155+
/usr/sbin/rsyslogd
156+
touch /var/log/haproxy.log
157+
tail -f /var/log/haproxy.log &
158+
}
159+
160+
function run() {
161+
162+
check_prereqs
163+
164+
rm -f ${CONF}
165+
setup_haproxy_config > ${CONF}
166+
167+
setup_haproxy_syslog
168+
169+
echo "Running haproxy with config:"
170+
sed -e 's/^/ /' ${CONF}
171+
echo ""
172+
echo ""
173+
174+
exec haproxy -f ${CONF}
175+
}
176+
177+
if [[ "${EGRESS_DNS_PROXY_MODE:-}" == "unit-test" ]]; then
178+
check_prereqs
179+
setup_haproxy_config
180+
exit 0
181+
fi
182+
183+
run

images/egress/router/egress-router.sh

+25
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,26 @@ function wait_until_killed() {
131131
wait
132132
}
133133

134+
function gen_dnsproxy_iptables_rules() {
135+
pod_ip=$(ip addr show dev eth0 | grep -Po 'inet \K[\d.]+')
136+
if [[ -z "${pod_ip}" ]]; then
137+
die "Failed to fetch Pod IP"
138+
fi
139+
140+
echo -A PREROUTING -p tcp -d "${pod_ip}" -j DNAT --to-destination "${EGRESS_SOURCE}"
141+
}
142+
143+
function setup_dnsproxy_iptables() {
144+
gen_dnsproxy_iptables_rules
145+
146+
iptables -t nat -F
147+
( echo "*nat";
148+
echo ":PREROUTING ACCEPT [0:0]";
149+
echo ":POSTROUTING ACCEPT [0:0]";
150+
gen_dnsproxy_iptables_rules;
151+
echo "COMMIT" ) | iptables-restore --noflush --table nat
152+
}
153+
134154
case "${EGRESS_ROUTER_MODE:=legacy}" in
135155
init)
136156
setup_network
@@ -147,6 +167,11 @@ case "${EGRESS_ROUTER_MODE:=legacy}" in
147167
setup_network
148168
;;
149169

170+
dns-proxy)
171+
setup_network
172+
setup_dnsproxy_iptables
173+
;;
174+
150175
unit-test)
151176
gen_iptables_rules
152177
;;

0 commit comments

Comments
 (0)