diff --git a/hack/lib/constants.sh b/hack/lib/constants.sh index b8ebd940661d..e7123a3d784d 100755 --- a/hack/lib/constants.sh +++ b/hack/lib/constants.sh @@ -325,6 +325,7 @@ readonly OS_ALL_IMAGES=( openshift/origin-f5-router openshift/origin-egress-router openshift/origin-egress-http-proxy + openshift/origin-egress-dns-proxy openshift/origin-recycler openshift/origin-cluster-capacity openshift/origin-service-catalog @@ -366,6 +367,7 @@ function os::build::images() { ( os::build::image "${tag_prefix}" images/origin ) & ( os::build::image "${tag_prefix}-egress-router" images/egress/router ) & ( os::build::image "${tag_prefix}-egress-http-proxy" images/egress/http-proxy ) & + ( os::build::image "${tag_prefix}-egress-dns-proxy" images/egress/dns-proxy ) & ( os::build::image "${tag_prefix}-federation" images/federation ) & for i in `jobs -p`; do wait $i; done diff --git a/images/egress/dns-proxy/.cccp.yml b/images/egress/dns-proxy/.cccp.yml new file mode 100644 index 000000000000..e387f84d1bc7 --- /dev/null +++ b/images/egress/dns-proxy/.cccp.yml @@ -0,0 +1 @@ +job-id: origin-egress-dns-proxy diff --git a/images/egress/dns-proxy/Dockerfile b/images/egress/dns-proxy/Dockerfile new file mode 100644 index 000000000000..94631f5f34f9 --- /dev/null +++ b/images/egress/dns-proxy/Dockerfile @@ -0,0 +1,23 @@ +# +# This is the egress router L4 DNS proxy for OpenShift Origin +# +# The standard name for this image is openshift/origin-egress-dns-proxy + +FROM openshift/origin-base + +# HAProxy 1.6+ version is needed to leverage DNS resolution at runtime. +RUN INSTALL_PKGS="haproxy18 rsyslog" && \ + yum install -y $INSTALL_PKGS && \ + rpm -V $INSTALL_PKGS && \ + yum clean all && \ + mkdir -p /var/lib/haproxy/{run,log} && \ + mkdir -p /etc/haproxy && \ + setcap 'cap_net_bind_service=ep' /usr/sbin/haproxy && \ + chown -R :0 /var/lib/haproxy && \ + chmod -R g+w /var/lib/haproxy && \ + touch /etc/haproxy/haproxy.cfg + +ADD egress-dns-proxy.sh /bin/egress-dns-proxy.sh + +ENTRYPOINT /bin/egress-dns-proxy.sh + diff --git a/images/egress/dns-proxy/egress-dns-proxy.sh b/images/egress/dns-proxy/egress-dns-proxy.sh new file mode 100755 index 000000000000..68a7212aba4c --- /dev/null +++ b/images/egress/dns-proxy/egress-dns-proxy.sh @@ -0,0 +1,178 @@ +#!/bin/bash + +# OpenShift egress DNS proxy setup script + +set -o errexit +set -o nounset +set -o pipefail + +# Default DNS nameserver port +NS_PORT=53 +CONF=/etc/haproxy/haproxy.cfg + +BLANK_LINE_OR_COMMENT_REGEX="([[:space:]]*$|#.*)" +IPADDR_REGEX="[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+" +PORT_REGEX="[[:digit:]]+" +DOMAINNAME_REGEX="[[:alnum:]][[:alnum:]-]+?\.[[:alnum:].-]+" + +function die() { + echo "$*" 1>&2 + exit 1 +} + +function check_prereqs() { + if [[ -z "${EGRESS_DNS_PROXY_DESTINATION}" ]]; then + die "Must specify EGRESS_DNS_PROXY_DESTINATION" + fi +} + +function validate_port() { + local port=$1 + if [[ "${port}" -lt "1" || "${port}" -gt "65535" ]]; then + die "Invalid port: ${port}, must be in the range 1 to 65535" + fi +} + +function generate_haproxy_defaults() { + echo " +global + log 127.0.0.1 local2 + + chroot /var/lib/haproxy + pidfile /var/lib/haproxy/run/haproxy.pid + maxconn 4000 + user haproxy + group haproxy + +defaults + log global + mode tcp + option dontlognull + option tcplog + option redispatch + retries 3 + timeout http-request 100s + timeout queue 1m + timeout connect 10s + timeout client 1m + timeout server 1m + timeout http-keep-alive 100s + timeout check 10s +" +} + +function generate_dns_resolvers() { + echo "resolvers dns-resolver" + + # Fetch nameservers from /etc/resolv.conf + nameservers=() + nameservers=$(awk '/^nameserver/ {print $2}' /etc/resolv.conf) + n=0 + for ns in ${nameservers[@]}; do + n=$(($n + 1)) + echo " nameserver ns$n ${ns}:${NS_PORT}" + done + + # Set default optional params + echo " resolve_retries 3" + echo " timeout retry 1s" + echo " hold valid 10s" + echo "" +} + +function generate_haproxy_frontends_backends() { + local n=0 + declare -A used_ports=() + + while read dest; do + local port target targetport resolvers + + if [[ "${dest}" =~ ^${BLANK_LINE_OR_COMMENT_REGEX}$ ]]; then + continue + fi + n=$(($n + 1)) + resolvers="" + + if [[ "${dest}" =~ ^${PORT_REGEX}\ +${IPADDR_REGEX}$ ]]; then + read port target <<< "${dest}" + targetport="${port}" + elif [[ "${dest}" =~ ^${PORT_REGEX}\ +${IPADDR_REGEX}\ +${PORT_REGEX}$ ]]; then + read port target targetport <<< "${dest}" + elif [[ "${dest}" =~ ^${PORT_REGEX}\ +${DOMAINNAME_REGEX}$ ]]; then + read port target <<< "${dest}" + targetport="${port}" + resolvers="resolvers dns-resolver" + elif [[ "${dest}" =~ ^${PORT_REGEX}\ +${DOMAINNAME_REGEX}\ +${PORT_REGEX}$ ]]; then + read port target targetport <<< "${dest}" + resolvers="resolvers dns-resolver" + else + die "Bad destination '${dest}'" + fi + + validate_port ${port} + validate_port ${targetport} + + if [[ "${used_ports[${port}]:-}" == "" ]]; then + used_ports[${port}]=1 + else + die "Proxy port $port already used, must be unique for each destination" + fi + + echo " +frontend fe$n + bind :${port} + default_backend be$n + +backend be$n + server dest$n ${target}:${targetport} check $resolvers +" + done <<< "${EGRESS_DNS_PROXY_DESTINATION}" +} + +function setup_haproxy_config() { + generate_haproxy_defaults + generate_dns_resolvers + generate_haproxy_frontends_backends +} + +function setup_haproxy_syslog() { + local log_file="/var/lib/haproxy/log/haproxy.log" + + cat >> /etc/rsyslog.conf <> /etc/rsyslog.d/haproxy.conf + + /usr/sbin/rsyslogd + touch "${log_file}" + tail -f "${log_file}" & +} + +function run() { + + check_prereqs + + rm -f "${CONF}" + setup_haproxy_config > "${CONF}" + + if [[ -n "${EGRESS_DNS_PROXY_DEBUG:-}" ]]; then + setup_haproxy_syslog + fi + + echo "Running haproxy with config:" + sed -e 's/^/ /' "${CONF}" + echo "" + echo "" + + exec haproxy -f "${CONF}" + } + +if [[ "${EGRESS_DNS_PROXY_MODE:-}" == "unit-test" ]]; then + check_prereqs + setup_haproxy_config + exit 0 +fi + +run diff --git a/images/egress/dns-proxy/egress_dns_proxy_test.go b/images/egress/dns-proxy/egress_dns_proxy_test.go new file mode 100644 index 000000000000..d57868039dcb --- /dev/null +++ b/images/egress/dns-proxy/egress_dns_proxy_test.go @@ -0,0 +1,260 @@ +package egress_dns_proxy_test + +import ( + "fmt" + "os/exec" + "regexp" + "strings" + "testing" +) + +func TestHAProxyFrontendBackendConf(t *testing.T) { + tests := []struct { + dest string + frontends []string + backends []string + }{ + // Single destination IP + { + dest: "80 11.12.13.14", + frontends: []string{` +frontend fe1 + bind :80 + default_backend be1`}, + backends: []string{` +backend be1 + server dest1 11.12.13.14:80 check`}, + }, + // Multiple destination IPs + { + dest: "80 11.12.13.14\n8080 21.22.23.24 100", + frontends: []string{` +frontend fe1 + bind :80 + default_backend be1`, ` +frontend fe2 + bind :8080 + default_backend be2`}, + backends: []string{` +backend be1 + server dest1 11.12.13.14:80 check`, ` +backend be2 + server dest2 21.22.23.24:100 check`}, + }, + // Single destination domain name + { + dest: "80 example.com", + frontends: []string{` +frontend fe1 + bind :80 + default_backend be1`}, + backends: []string{` +backend be1 + server dest1 example.com:80 check resolvers dns-resolver`}, + }, + // Multiple destination domain names + { + dest: "80 example.com\n8080 foo.com 100", + frontends: []string{` +frontend fe1 + bind :80 + default_backend be1`, ` +frontend fe2 + bind :8080 + default_backend be2`}, + backends: []string{` +backend be1 + server dest1 example.com:80 check resolvers dns-resolver`, ` +backend be2 + server dest2 foo.com:100 check resolvers dns-resolver`}, + }, + // Destination IP and destination domain name + { + dest: "80 11.12.13.14\n8080 example.com 100", + frontends: []string{` +frontend fe1 + bind :80 + default_backend be1`, ` +frontend fe2 + bind :8080 + default_backend be2`}, + backends: []string{` +backend be1 + server dest1 11.12.13.14:80 check`, ` +backend be2 + server dest2 example.com:100 check resolvers dns-resolver`}, + }, + // Destination with comments and blank lines + { + dest: ` +# My DNS proxy egress router rules + +# Port 80 forwards to 11.12.13.14 +80 11.12.13.14 + +# Port 8080 forwards to port 100 on example.com +8080 example.com 100 + +# Skip this rule for now +# 9000 foo.com 200 + +# End +`, + frontends: []string{` +frontend fe1 + bind :80 + default_backend be1`, ` +frontend fe2 + bind :8080 + default_backend be2`}, + backends: []string{` +backend be1 + server dest1 11.12.13.14:80 check`, ` +backend be2 + server dest2 example.com:100 check resolvers dns-resolver`}, + }, + } + + frontendRegex := regexp.MustCompile("\nfrontend ") + backendRegex := regexp.MustCompile("\nbackend ") + + for n, test := range tests { + cmd := exec.Command("./egress-dns-proxy.sh") + cmd.Env = []string{ + fmt.Sprintf("EGRESS_DNS_PROXY_DESTINATION=%s", test.dest), + fmt.Sprintf("EGRESS_DNS_PROXY_MODE=unit-test"), + } + outBytes, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("test %d unexpected error %v, output: %q", n+1, err, string(outBytes)) + } + out := string(outBytes) + for _, frontend := range test.frontends { + if !strings.Contains(out, frontend) { + t.Fatalf("test %d expected frontend in output %q but got %q", n+1, frontend, out) + } + } + matches := frontendRegex.FindAllStringIndex(out, -1) + if len(matches) != len(test.frontends) { + t.Fatalf("test %d number of frontends mismatch, expected %q but got %q", n+1, test.frontends, out) + } + + for _, backend := range test.backends { + if !strings.Contains(out, backend) { + t.Fatalf("test %d expected backend in output %q but got %q", n+1, backend, out) + } + } + matches = backendRegex.FindAllStringIndex(out, -1) + if len(matches) != len(test.backends) { + t.Fatalf("test %d number of backends mismatch, expected %q but got %q", n+1, test.backends, out) + } + } +} + +func TestHAProxyFrontendBackendConfBad(t *testing.T) { + tests := []struct { + dest string + err string + }{ + { + dest: "", + err: "Must specify EGRESS_DNS_PROXY_DESTINATION", + }, + { + dest: "80 11.12.13.14\ninvalid", + err: "Bad destination 'invalid'", + }, + { + dest: "80 11.12.13.14\n8080 invalid", + err: "Bad destination '8080 invalid'", + }, + { + dest: "99999 11.12.13.14", + err: "Invalid port: 99999, must be in the range 1 to 65535", + }, + { + dest: "80 11.12.13.14 88888", + err: "Invalid port: 88888, must be in the range 1 to 65535", + }, + { + dest: "80 11.12.13.14\n80 21.22.23.24 100", + err: "Proxy port 80 already used, must be unique for each destination", + }, + } + + for n, test := range tests { + cmd := exec.Command("./egress-dns-proxy.sh") + cmd.Env = []string{ + "EGRESS_DNS_PROXY_MODE=unit-test", + fmt.Sprintf("EGRESS_DNS_PROXY_DESTINATION=%s", test.dest), + } + out, err := cmd.CombinedOutput() + out_lines := strings.Split(string(out), "\n") + got := out_lines[len(out_lines)-2] + if err == nil { + t.Fatalf("test %d expected error %q but got output %q", n+1, test.err, got) + } + if got != test.err { + t.Fatalf("test %d expected output %q but got %q", n+1, test.err, got) + } + } +} + +func TestHAProxyDefaults(t *testing.T) { + defaults := ` +global + log 127.0.0.1 local2 + + chroot /var/lib/haproxy + pidfile /var/lib/haproxy/run/haproxy.pid + maxconn 4000 + user haproxy + group haproxy + +defaults + log global + mode tcp + option dontlognull + option tcplog + option redispatch + retries 3 + timeout http-request 100s + timeout queue 1m + timeout connect 10s + timeout client 1m + timeout server 1m + timeout http-keep-alive 100s + timeout check 10s +` + cmd := exec.Command("./egress-dns-proxy.sh") + cmd.Env = []string{ + "EGRESS_DNS_PROXY_MODE=unit-test", + "EGRESS_DNS_PROXY_DESTINATION=80 11.12.13.14", + } + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("unexpected error %v", err) + } + if !strings.Contains(string(out), defaults) { + t.Fatalf("expected defaults in output %q but got %q", defaults, string(out)) + } +} + +func TestHAProxyResolver(t *testing.T) { + resolverRegex := "resolvers dns-resolver\n *(nameserver ns.*)+\n +" + + cmd := exec.Command("./egress-dns-proxy.sh") + cmd.Env = []string{ + "EGRESS_DNS_PROXY_MODE=unit-test", + "EGRESS_DNS_PROXY_DESTINATION=80 11.12.13.14", + } + outBytes, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("unexpected error %v", err) + } + out := string(outBytes) + match, er := regexp.MatchString(resolverRegex, out) + if !match || er != nil { + t.Fatalf("dns resolver section not found in output %q", out) + } +} diff --git a/images/egress/router/egress-router.sh b/images/egress/router/egress-router.sh index 04a0bee5f7a5..ebef1bf35e95 100755 --- a/images/egress/router/egress-router.sh +++ b/images/egress/router/egress-router.sh @@ -153,6 +153,10 @@ case "${EGRESS_ROUTER_MODE:=legacy}" in setup_network ;; + dns-proxy) + setup_network + ;; + unit-test) gen_iptables_rules ;;