reload-haproxy

#!/bin/bash

set -o nounset

config_file=/var/lib/haproxy/conf/haproxy.config
pid_file=/var/lib/haproxy/run/haproxy.pid
old_pid=""
haproxy_conf_dir=/var/lib/haproxy/conf
readonly max_wait_time=30
readonly timeout_opts="-m 1 --connect-timeout 1"
readonly numeric_re='^[0-9]+$'

function haproxyHealthCheck() {
  local wait_time=${MAX_RELOAD_WAIT_TIME:-$max_wait_time}
  local port=${ROUTER_SERVICE_HTTP_PORT:-"80"}
  local url="http://localhost:${port}"
  local retries=0
  local start_ts=$(date +"%s")
  local proxy_proto="${ROUTER_USE_PROXY_PROTOCOL-}"

  if ! [[ $wait_time =~ $numeric_re ]]; then
    echo " - Invalid max reload wait time, using default $max_wait_time ..."
    wait_time=$max_wait_time
  fi

  local end_ts=$((start_ts + wait_time))

  # test with proxy protocol on
  if [[ "${proxy_proto}" == "TRUE" || "${proxy_proto}" == "true" ]]; then
    echo " - Proxy protocol on, checking ${url} ..."
    while true; do
      local statusline=$(echo $'PROXY UNKNOWN\r\nGET / HTTP/1.1\r\n' | socat tcp-connect:localhost:${port} stdio | head -1)

      if [[ "$statusline" == *" 503 "* ]]; then
        echo " - Health check ok : $retries retry attempt(s)."
        return 0
      fi

      if [ $(date +"%s") -ge $end_ts ]; then
        echo " - Exceeded max wait time ($wait_time) in health check - $retries retry attempt(s)."
        return 1
      fi

      sleep 0.5
      retries=$((retries + 1))
    done
    return 0
  fi

  echo " - Checking ${url} ..."
  while true; do
    local httpcode=$(curl $timeout_opts -s -o /dev/null -I -H "Host: " -w "%{http_code}" ${url})

    if [ "$httpcode" == "503" ]; then
      echo " - Health check ok : $retries retry attempt(s)."
      return 0
    fi

    if [ $(date +"%s") -ge $end_ts ]; then
      echo " - Exceeded max wait time ($wait_time) in health check - $retries retry attempt(s)."
      return 1
    fi

    sleep 0.5
    retries=$((retries + 1))
  done
}


# How many times to retry removal of the iptables rules (if requested at all)
# It will sleep for 1/2 a second between attempts, so the time is retries / 2 secs
retries=20


# sort the path based map files for the haproxy map_beg function
# Leaving any wildcard route at the end of the resulting file,
# See https://github.com/openshift/origin/issues/16724 for more details.
for mapfile in "$haproxy_conf_dir"/*.map; do
  grep -v -F '^[^\.]*\' "$mapfile" | sort -r -o "/tmp/$mapfile"
  grep -F '^[^\.]*\' "$mapfile" | sort -r >> "/tmp/$mapfile"
  mv -f "/tmp/$mapfile" "$haproxy_conf_dir/$mapfile"
done

old_pids=$(ps -A -opid,args | grep haproxy | egrep -v -e 'grep|reload-haproxy' | awk '{print $1}' | tr '\n' ' ')

reload_status=0
installed_iptables=0
if [ -n "$old_pids" ]; then
  if $(set | grep DROP_SYN_DURING_RESTART= > /dev/null) && [[ "$DROP_SYN_DURING_RESTART" == 'true' || "$DROP_SYN_DURING_RESTART" == '1' ]]; then
    # We install the syn eater so that connections that come in during the restart don't
    # go onto the wrong socket, which is then closed.
    ports=$(grep -E -o '^\s*bind\s+:[[:digit:]]+\w' "$config_file" | cut -f2 -d: | paste -d, -s)
    if [ -n "$ports" ]; then
      # If this doesn't insert, we don't care, we still want to reload
      /usr/sbin/iptables -I INPUT -p tcp -m multiport --dports $ports --syn -j DROP \
			 -m comment --comment "Eat SYNs while reloading haproxy" || :
      installed_iptables=1

      # The sleep is needed to let the socket drain before the new
      # haproxy starts and binds to the same port.  The value was
      # determined by trial and error: I stopped seeing failures at
      # 0.01 under load, so I put in a 10x margin.  At worst, we may
      # leave a connection in the old process' listen buffer that
      # won't get handled, and they'll get a reset.  I didn't want to
      # set it too long, because that affects the overall time a
      # reload takes which means that incoming connections aren't
      # handled while the SYN eater is in place.
      sleep 0.1
    fi
  fi

  /usr/sbin/haproxy -f $config_file -p $pid_file -x /var/lib/haproxy/run/haproxy.sock -sf $old_pids
  reload_status=$?

  if [[ "$installed_iptables" == 1 ]]; then
    # We NEVER want to leave the syn eater in place after the reload or haproxy
    # will never get new connections.  So try to remove it twenty times, and if
    # that fails, log the error and return failure so the pod logs a fatal error.
    i=0
    while (( i++ < retries )) ; do
      /usr/sbin/iptables -D INPUT -p tcp -m multiport --dports $ports --syn -j DROP \
                         -m comment --comment "Eat SYNs while reloading haproxy" || :

      # Test the condition and end the loop if the rule has been removed
      /usr/sbin/iptables -L INPUT | grep -F '/* Eat SYNs while reloading haproxy */' || break

      >&2 echo "Unable to remove SYN eating rule, attempt $i.  Will retry..."

      # But sleep for a bit before retrying
      sleep 0.5
    done
    if (( i >= retries )); then
      # We failed to remove the rule... log failure and exit to signal the caller
      >&2 echo "Unable to remove the iptables SYN eating rule.  Aborting after $retries retries"
      exit 1
    fi
  fi
else
  /usr/sbin/haproxy -f $config_file -p $pid_file
  reload_status=$?
fi

[ $reload_status -ne 0 ] && exit $reload_status
haproxyHealthCheck