run_test

#!/bin/bash
#
# Test the PostgreSQL image.
#
# IMAGE_NAME specifies the name of the candidate image used for testing.
# The image has to be available before this script is executed.
#

set -o nounset
shopt -s nullglob

# library from container-common-scripts
. test/test-lib.sh

# local library
. test/pg-test-lib.sh

TEST_LIST="\
run_container_creation_tests
run_general_tests
run_change_password_test
run_replication_test
run_s2i_test
run_test_cfg_hook
run_s2i_bake_data_test
run_s2i_enable_ssl_test
run_upgrade_test
run_migration_test
run_pgaudit_test
run_env_extension_load_test
run_logging_test
"

test $# -eq 1 -a "${1-}" == --list && echo "$TEST_LIST" && exit 0
test -n "${IMAGE_NAME-}" || false 'make sure $IMAGE_NAME is defined'
test -n "${VERSION-}" || false 'make sure $VERSION is defined'
test -n "${OS-}" || false 'make sure $OS is defined'


####
# HELPER FUNCTIONS
####

DOCKER_EXTRA_ARGS=

volumes_to_clean=
images_to_clean=()
files_to_clean=
test_dir="$(readlink -f "$(dirname "$0")")"

_cleanup_commands_space=
_cleanup_commands=

add_cleanup_command ()
{
    local cmd= space=
    for arg; do
        cmd+="$space$(printf "%q" "$arg")"
        space=' '
    done
    _cleanup_commands+="$_cleanup_commands_space$cmd"
    _cleanup_commands_space='
'
}
function cleanup() {
  # Print a big fat separator to find the error easier
  echo "=================================== Cleanup begins here ============================="

  ct_path_foreach "$volumes_to_clean" cleanup_volume_dir

  if test -n "${images_to_clean-}"; then
    # Workaround for RHEL 7 bash bug:
    # https://bugzilla.redhat.com/show_bug.cgi?id=1636393
    for image in "${images_to_clean[@]}"; do
      docker rmi -f "$image"
    done
  fi

  ct_path_foreach "$files_to_clean" rm

  echo "$_cleanup_commands" | while read -r line; do
    eval "$line"
  done
  echo "================== postgresql-container-specific cleanup ends here ====================="
}

cleanup_volume_dir ()
{
  test ! -d "$1" && : "WARN: cleaned $1 for some reason" && return 0
  # When we run this test script as non-root (we should?), the PostgreSQL server
  # within container is still run under 'postgres' user.  It means that, taking
  # into account 0077 umask of PostgreSQL server, we are unable to remove files
  # created by server.  That's why we need to let docker escalate the privileges
  # again.
  local datadir=/var/lib/pgsql/data
  docker run -v "$1:$datadir:Z" --rm "$IMAGE_NAME" /bin/sh -c "/bin/rm -rf $datadir/userdata"
  rmdir "$1"
}

function get_cid() {
  local id="$1" ; shift || return 1
  echo $(cat "$CID_FILE_DIR/$id")
}

function get_container_ip() {
  local id="$1" ; shift
  docker inspect --format='{{.NetworkSettings.IPAddress}}' $(get_cid "$id")
}

function get_ip_from_cid() {
  local cid="$1"; shift
  docker inspect --format='{{.NetworkSettings.IPAddress}}' $cid
}

function postgresql_cmd() {
  docker run --rm -e PGPASSWORD="$PASS" "$IMAGE_NAME" psql -v ON_ERROR_STOP=1 "postgresql://$PGUSER@$CONTAINER_IP:5432/${DB-db}" "$@"
}

function test_connection() {
  local name=$1 ; shift
  ip=$(get_container_ip $name)
  echo "  Testing PostgreSQL connection to $ip..."
  local max_attempts=20
  local sleep_time=2
  for i in $(seq $max_attempts); do
    echo "    Trying to connect..."
    # Don't let the code come here if neither user nor admin is able to
    # connect.
    if [ -v PGUSER ] && [ -v PASS ]; then
      CONTAINER_IP=$ip postgresql_cmd -At -c "SELECT 1;"
    else
      PGUSER=postgres PASS=$ADMIN_PASS CONTAINER_IP=$ip DB=postgres postgresql_cmd -At -c "SELECT 1;"
    fi
    status=$?
    if [ $status -eq 0 ]; then
      echo "  Success!"
      return 0
    fi
    sleep $sleep_time
  done
  return 1
}

function test_postgresql() {
  local ret=0
  local user=${1:-user}
  echo "  Testing PostgreSQL"
  # test contrib only when having admin privileges
  if [ "$user" == "admin" ] ; then
    postgresql_cmd -At -c "CREATE EXTENSION \"uuid-ossp\";" || ret=1 # to test contrib package
  fi
  postgresql_cmd -At -c "CREATE TABLE tbl (col1 VARCHAR(20), col2 VARCHAR(20));" || ret=2
  postgresql_cmd -At -c "INSERT INTO tbl VALUES ('foo1', 'bar1');" || ret=3
  postgresql_cmd -At -c "INSERT INTO tbl VALUES ('foo2', 'bar2');" || ret=4
  postgresql_cmd -At -c "INSERT INTO tbl VALUES ('foo3', 'bar3');" || ret=5
  postgresql_cmd -At -c "SELECT * FROM tbl;" || ret=6
  # not droping table, other tests depend on having it created after this function is run
  #postgresql_cmd -At -c "DROP TABLE tbl;"
  if [ $ret -eq 0 ]; then
    echo "  Success!"
  fi
  return $ret
}

function create_container() {
  local name=$1 ; shift
  local cargs=${DOCKER_ARGS:-}
  local ret=0
  # TODO: fix all create_container() invocations so that we don't need this,
  # e.g. multiline DOCKER_ARGS var should end by trailing backslashes
  cargs=$(echo "$cargs" | tr  '\n' ' ')
  cidfile="$CID_FILE_DIR/$name"
  # create container with a cidfile in a directory for cleanup
  eval "docker run $cargs --cidfile \$cidfile -d \$IMAGE_NAME \"\$@\""
  ret=$?

  if [ $ret -ne 0 ]; then
    echo "ERROR: Creation of container didn't succeed"
    return 1
  fi

  echo "Created container $(cat $cidfile)"
}


create_volume_dir ()
{
  volume_dir=`mktemp -d --tmpdir pg-testdata.XXXXX`
  setfacl -m u:26:-wx "$volume_dir"
  ct_path_append volumes_to_clean "$volume_dir"
}


create_temp_file ()
{
  temp_file=`mktemp --tmpdir pg-testfile.XXXXX`
  setfacl -m u:26:rw- "$temp_file"
  ct_path_append files_to_clean "$temp_file"
}


function assert_login_access() {
  local PGUSER=$1 ; shift
  local PASS=$1 ; shift
  local success=$1 ; shift

  echo "testing login as $PGUSER:$PASS; should_success=$success"

  if postgresql_cmd -At -c 'SELECT 1;' ; then
    if $success ; then
      echo "    $PGUSER($PASS) access granted as expected"
      return
    fi
  else
    if ! $success ; then
      echo "    $PGUSER($PASS) access denied as expected"
      return
    fi
  fi
  echo "    $PGUSER($PASS) login assertion failed"
  return 1
}

function assert_local_access() {
  local id="$1" ; shift
  docker exec -i $(get_cid "$id") bash -c psql <<< "SELECT 1;"
}


# Make sure the invocation of docker run fails.
function assert_container_creation_fails() {
  local ret=0

  # Time the docker run command. It should fail. If it doesn't fail,
  # postgresql will keep running so we kill it with SIGKILL to make sure
  # timeout returns a non-zero value.
  timeout -s 9 --preserve-status 60s docker run --rm "$@" $IMAGE_NAME
  ret=$?

  # Timeout will exit with a high number.
  if [ $ret -gt 30 ]; then
    return 1
  fi
}


# assert_container_creation_succeeds NAME [ARGS]
# ----------------------------------------------
# Chcek that 'docker run' with IMAGE_NAME succeeds with docker arguments
# specified as ARGS.
assert_container_creation_succeeds ()
{
  local check_env=false
  local name=pg-success-"$(ct_random_string)"
  local PGUSER='' PGPASS=''  DB=''  ADMIN_PASS=
  local docker_args=
  local ret=0

  for arg; do
    docker_args+=" $(printf "%q" "$arg")"
    if $check_env; then
      local env=${arg//=*/}
      local val=${arg//$env=/}
      case $env in
        POSTGRESQL_ADMIN_PASSWORD)  ADMIN_PASS=$val ;;
        POSTGRESQL_USER)            PGUSER=$val ;;
        POSTGRESQL_PASSWORD)        PGPASS=$val ;;
        POSTGRESQL_DATABASE)        DB=$val ;;
      esac
      check_env=false
    elif test "$arg" = -e; then
      check_env=:
    fi
  done

  DOCKER_ARGS=$docker_args create_container "$name" || ret=1

  if test -n "$PGUSER" && test -n "$PGPASS"; then
    PGUSER=$PGUSER PASS=$PGPASS DB=$DB test_connection "$name" || ret=2
  fi

  if test -n "$ADMIN_PASS"; then
    PGUSER=postgres PASS=$ADMIN_PASS DB=$DB test_connection "$name" || ret=3
  fi

  return $ret
}


function try_image_invalid_combinations() {
  local ret=0
  assert_container_creation_fails -e POSTGRESQL_USER=user -e POSTGRESQL_PASSWORD=pass "$@" || ret=1
  assert_container_creation_fails -e POSTGRESQL_USER=user -e POSTGRESQL_DATABASE=db "$@" || ret=2
  assert_container_creation_fails -e POSTGRESQL_PASSWORD=pass -e POSTGRESQL_DATABASE=db "$@" || ret=3
  return $ret
}

function test_config_option() {
  local name=$1 ; shift
  local setting=$1 ; shift
  local value=$1 ; shift

  docker exec $(get_cid ${name}) grep -q "${setting} = ${value}" /var/lib/pgsql/openshift-custom-postgresql.conf
}


# wait_ready
# ----------
# Wait until the PG container becomes ready
wait_ready ()
{
  local max_attempts=20
  for i in $(seq $max_attempts); do
    if docker exec "$(get_cid "$1")" /usr/libexec/check-container; then
      return 0
    fi
    sleep 1
  done
  return 1
}


# assert_runtime_option option value
# ----------------------------------
assert_runtime_option ()
{
  local name=$1 option=$2 value=$3
  wait_ready "$name"
  set -- $(docker exec "$(get_cid "$name")" bash -c "psql -tA -c 'SHOW $option;'")
  test "$value" = "$1"
}


function run_configuration_tests() {
  local name=$1 ; shift
  local ret=0
  echo "  Testing image configuration settings"
  test_config_option ${name} max_connections ${POSTGRESQL_MAX_CONNECTIONS} || ret=1
  test_config_option ${name} max_prepared_transactions ${POSTGRESQL_MAX_PREPARED_TRANSACTIONS} || ret=2
  test_config_option ${name} shared_buffers ${POSTGRESQL_SHARED_BUFFERS} || ret=3
  if [ $ret -eq 0 ]; then
    echo "  Success!"
  fi
  return $ret
}

test_scl_usage() {
  local name="$1"
  local run_cmd="$2"
  local expected="$3"

  echo "  Testing the image SCL enable"
  out=$(docker run --rm ${IMAGE_NAME} /bin/bash -c "${run_cmd}")
  if ! echo "${out}" | grep -q "${expected}"; then
    echo "ERROR[/bin/bash -c "${run_cmd}"] Expected '${expected}', got '${out}'"
    return 1
  fi
  out=$(docker exec $(get_cid $name) /bin/bash -c "${run_cmd}" 2>&1)
  if ! echo "${out}" | grep -q "${expected}"; then
    echo "ERROR[exec /bin/bash -c "${run_cmd}"] Expected '${expected}', got '${out}'"
    return 1
  fi
  out=$(docker exec $(get_cid $name) /bin/sh -ic "${run_cmd}" 2>&1)
  if ! echo "${out}" | grep -q "${expected}"; then
    echo "ERROR[exec /bin/sh -ic "${run_cmd}"] Expected '${expected}', got '${out}'"
    return 1
  fi
}


function run_tests() {
  echo "  Testing general usage (run_tests) with '$1' as argument"
  local name=$1 ; shift
  local ret=0

  user_login=false
  admin_login=false
  envs=
  # NOTE: We work wrongly with variables so please don't try to pass spaces
  # within PGUSER/PASS/ADMIN_PASS variables.
  [ -v PGUSER ] && envs+=" -e POSTGRESQL_USER=$PGUSER"
  [ -v PASS ] && envs+=" -e POSTGRESQL_PASSWORD=$PASS"
  if [ -v PGUSER ] && [ -v PASS ]; then
    envs+=" -e POSTGRESQL_DATABASE=db"
    user_login=:
  fi

  if [ -v ADMIN_PASS ]; then
    envs="$envs -e POSTGRESQL_ADMIN_PASSWORD=$ADMIN_PASS"
    admin_login=:
  fi
  if [ -v POSTGRESQL_MAX_CONNECTIONS ]; then
    envs="$envs -e POSTGRESQL_MAX_CONNECTIONS=$POSTGRESQL_MAX_CONNECTIONS"
  fi
  if [ -v POSTGRESQL_MAX_PREPARED_TRANSACTIONS ]; then
    envs="$envs -e POSTGRESQL_MAX_PREPARED_TRANSACTIONS=$POSTGRESQL_MAX_PREPARED_TRANSACTIONS"
  fi
  if [ -v POSTGRESQL_SHARED_BUFFERS ]; then
    envs="$envs -e POSTGRESQL_SHARED_BUFFERS=$POSTGRESQL_SHARED_BUFFERS"
  fi
  DOCKER_ARGS="${DOCKER_ARGS:-} $envs" create_container $name
  CONTAINER_IP=$(get_container_ip $name)
  test_connection $name || ret=1
  echo "  Testing scl usage"
  test_scl_usage $name 'psql --version' "$VERSION" || ret=2

  echo "  Testing login accesses"
  assert_login_access "${PGUSER:-}"     "${PASS-}"            "$user_login" || ret=3
  assert_login_access "${PGUSER:-}"     "${PASS-}_foo"        false || ret=4

  assert_login_access postgres          "${ADMIN_PASS-}"      "$admin_login" || ret=5
  assert_login_access postgres          "${ADMIN_PASS-}_foo"  false || ret=6

  assert_local_access $name || ret=7
  run_configuration_tests $name || ret=8

  if $user_login; then
    test_postgresql || ret=9
  fi

  if $admin_login; then
    DB=postgres PGUSER=postgres PASS=$ADMIN_PASS test_postgresql admin || ret=10
  fi
  if [ $ret -eq 0 ]; then
    echo "  Success!"
  fi
  return $ret
}

function run_slave() {
  local suffix="$1"; shift
  docker run $cluster_args -e POSTGRESQL_MASTER_IP=${master_hostname} \
    -d --cidfile ${CID_FILE_DIR}/slave-${suffix}.cid $IMAGE_NAME run-postgresql-slave
}

function run_master() {
  local suffix="$1"; shift
  master_args=${master_args-}
  docker run $cluster_args $master_args \
    -d --cidfile ${CID_FILE_DIR}/master-${suffix}.cid $IMAGE_NAME run-postgresql-master >/dev/null
}

function test_slave_visibility() {
  local max_attempts=30

  for slave in $slave_cids; do
    slave_ip=$(get_ip_from_cid $slave)
    if [ -z "$slave_ip" ]; then
        echo "Failed to get IP for slave $slave."
        echo "Dumping logs for $slave"
        docker logs "$slave"
        return 1
    fi
    for i in $(seq $max_attempts); do
      result="$(postgresql_cmd -c "select client_addr from pg_stat_replication;" | grep "$slave_ip" || true)"
      if [[ -n "${result}" ]]; then
        echo "${slave_ip} successfully registered as SLAVE for ${master_ip}"
        break
      fi
      if [[ "${i}" == "${max_attempts}" ]]; then
        echo "The ${slave_ip} failed to register in MASTER"
        echo "Dumping logs for $slave"
        docker logs $slave
        return 1
      fi
      sleep 1
    done
  done
}

function test_value_replication() {
  local max_attempts=30

  # Setup the replication data
  local value
  value=24
  postgresql_cmd -c "CREATE TABLE $table_name (a integer); INSERT INTO $table_name VALUES ($value);"

  # Read value from slaves and check whether it is expected
  for slave in $slave_cids; do
    slave_ip=$(get_ip_from_cid $slave)
    CONTAINER_IP=$slave_ip
    for i in $(seq $max_attempts); do
      result="$(postgresql_cmd -At -c "select * from $table_name" || :)"
      if [[ "$result" == "$value" ]]; then
        echo "${slave_ip} successfully got value from MASTER ${master_ip}"
        break
      fi
      if [[ "${i}" == "${max_attempts}" ]]; then
        echo "The ${slave_ip} failed to see value added on MASTER"
        echo "Dumping logs for $slave"
        docker logs $slave
        return 1
      fi
      sleep 1
    done
  done
}

function setup_replication_cluster() {
  # Run the PostgreSQL master
  run_master "$cid_suffix"

  # Run the PostgreSQL slaves
  local i
  master_ip=$(get_container_ip "master-$cid_suffix.cid")
  local cluster_args="$cluster_args --add-host postgresql-master:$master_ip"
  local master_hostname="postgresql-master"
  for i in $(seq ${slave_num:-1}); do
    slave_cids="$slave_cids $(run_slave $cid_suffix-$i)"
  done

}

test_the_app_image () {
  local container_name=$1
  local mount_opts=$2
  local ret=0
  echo "    Testing s2i app image with invalid configuration"
  assert_container_creation_fails -e POSTGRESQL_PASSWORD=pass -e POSTGRESQL_DATABASE=db || ret=1
  echo "    Testing s2i app image with correct configuration"

    DOCKER_ARGS="
-e POSTGRESQL_DATABASE=db
-e POSTGRESQL_USER=user
-e POSTGRESQL_PASSWORD=password
-e POSTGRESQL_ADMIN_PASSWORD=password
-e POSTGRESQL_BACKUP_USER=backuser
-e POSTGRESQL_BACKUP_PASSWORD=pass
${mount_opts}
" create_container "$container_name" || ret=2

  # need this to wait for the container to start up
  PGUSER=user     PASS=password           test_connection "$container_name" || ret=3
  PGUSER=backuser PASS=pass     DB=backup test_connection "$container_name" || ret=4
  if [ $ret -eq 0 ]; then
    echo "  Success!"
  fi
  return $ret
}


####
# DISABLED TESTS
####

# This test is removed from the test suite
# In our CI is failing sporadically and solution is not available yet.
# Let's remove it from test but the test function remains.
function run_master_restart_test() {
  local DB=postgres
  local PGUSER=master
  local PASS=master

  echo "Testing failed master restart"
  local cluster_args="-e POSTGRESQL_ADMIN_PASSWORD=pass -e POSTGRESQL_MASTER_USER=$PGUSER -e POSTGRESQL_MASTER_PASSWORD=$PASS"
  local cid_suffix="mrestart"
  local table_name="t1"
  local master_ip=
  local slave_cids=
  local ret=0

  create_volume_dir
  local master_args="-v ${volume_dir}:/var/lib/pgsql/data:Z"

  # Setup the cluster
  slave_num=2 setup_replication_cluster

  # Check if the master knows about the slaves
  CONTAINER_IP=$master_ip
  test_slave_visibility

  echo "Kill the master and create a new one"
  local cidfile=$CID_FILE_DIR/master-$cid_suffix.cid
  docker kill $(cat $cidfile)
  docker rm -f $(cat $cidfile)
  # Don't forget to remove its .cid file
  rm $cidfile

  run_master $cid_suffix
  CONTAINER_IP=$(get_container_ip master-$cid_suffix.cid)

  # Update master_ip in slaves
  for slave in $slave_cids; do
    docker exec -u 0 $slave bash -c "sed \"s/$master_ip/$CONTAINER_IP/g\" /etc/hosts >/tmp/hosts && cp /tmp/hosts /etc/hosts" || ret=1
  done
  master_ip=$CONTAINER_IP
  # Check if the new master sees existing slaves
  test_slave_visibility || ret=2

  # Check if the replication works
  table_name="t1" test_value_replication || ret=3
  if [ $ret -eq 0 ]; then
    echo "  Success!"
  fi
  return $ret
}

# run_doc_test
# --------------------
# Checks whether `help.1` file with specific terms is included in default
# working directory in the container image.
run_doc_test() {
  local tmpdir=$(mktemp -d)
  local f
  echo "  Testing documentation in the container image"
  # Extract the help files from the container
  for f in help.1 ; do
    docker run --rm ${IMAGE_NAME} /bin/bash -c "cat /${f}" >${tmpdir}/$(basename ${f})
    # Check whether the files include some important information
    for term in 'POSTGRESQL\\?_ADMIN\\?_PASSWORD' Volume 5432 ; do
      if ! cat ${tmpdir}/$(basename ${f}) | grep -E -q -e "${term}" ; then
        echo "ERROR: File /${f} does not include '${term}'."
        return 1
      fi
    done
  done
  # Check whether the files use the correct format
  if ! file ${tmpdir}/help.1 | grep -q roff ; then
    echo "ERROR: /help.1 is not in troff or groff format"
    return 1
  fi
  echo "  Success!"
  echo
}


####
# TESTING FUNCTIONS
####

# run_container_creation_tests
# --------------------
# Asserts that container image fails to run in certain illegal use of arguments,
# while it correctly runs in other combinations.
# NOTE: Previously the `creation_succeeds` combinations were supposed to fail, but
# these cases are now supposed to work
function run_container_creation_tests() {
  local ret=0
  echo "  Testing image entrypoint usage"
  try_image_invalid_combinations || ret=1
  try_image_invalid_combinations  -e POSTGRESQL_ADMIN_PASSWORD=admin_pass || ret=2

VERY_LONG_IDENTIFIER="very_long_identifier_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
  assert_container_creation_fails -e POSTGRESQL_USER= -e POSTGRESQL_PASSWORD=pass -e POSTGRESQL_DATABASE=db -e POSTGRESQL_ADMIN_PASSWORD=admin_pass || ret=3
  assert_container_creation_fails -e POSTGRESQL_USER=$VERY_LONG_IDENTIFIER -e POSTGRESQL_PASSWORD=pass -e POSTGRESQL_DATABASE=db -e POSTGRESQL_ADMIN_PASSWORD=admin_pass || ret=4
  assert_container_creation_succeeds -e POSTGRESQL_USER=user -e POSTGRESQL_PASSWORD="\"" -e POSTGRESQL_DATABASE=db -e POSTGRESQL_ADMIN_PASSWORD=admin_pass || ret=5
  assert_container_creation_succeeds -e POSTGRESQL_USER=user -e POSTGRESQL_PASSWORD=pass -e POSTGRESQL_DATABASE=9invalid -e POSTGRESQL_ADMIN_PASSWORD=admin_pass || ret=6
  assert_container_creation_fails -e POSTGRESQL_USER=user -e POSTGRESQL_PASSWORD=pass -e POSTGRESQL_DATABASE=$VERY_LONG_IDENTIFIER -e POSTGRESQL_ADMIN_PASSWORD=admin_pass || ret=7
  assert_container_creation_succeeds -e POSTGRESQL_USER=user -e POSTGRESQL_PASSWORD=pass -e POSTGRESQL_DATABASE=db -e POSTGRESQL_ADMIN_PASSWORD="\"" || ret=8

  assert_container_creation_succeeds -e POSTGRESQL_ADMIN_PASSWORD="the @password" || ret=9
  assert_container_creation_succeeds -e POSTGRESQL_PASSWORD="the pass" -e POSTGRESQL_USER="the user" -e POSTGRESQL_DATABASE="the db" || ret=10

  if [ $ret -eq 0 ]; then
    echo "  Success!"
  fi
  return $ret
}

# run_general_tests
# --------------------
# Tests different combinations of parameters and that they work. Also tests SCL
# usage. Tries to actually SELECT something from the database.
function run_general_tests() {
  local ret=0
  PGUSER=user PASS=pass POSTGRESQL_MAX_CONNECTIONS=42 POSTGRESQL_MAX_PREPARED_TRANSACTIONS=42 POSTGRESQL_SHARED_BUFFERS=64MB run_tests no_admin || ret=1
  PGUSER=user1 PASS=pass1 ADMIN_PASS=r00t run_tests admin || ret=2
  DB=postgres ADMIN_PASS=r00t run_tests only_admin || ret=3
  # Test with arbitrary uid for the container
  DOCKER_ARGS="-u 12345" PGUSER=user2 PASS=pass run_tests no_admin_altuid || ret=4
  DOCKER_ARGS="-u 12345" PGUSER=user3 PASS=pass1 ADMIN_PASS=r00t run_tests admin_altuid || ret=5
  DB=postgres DOCKER_ARGS="-u 12345" ADMIN_PASS=rOOt run_tests only_admin_altuid || ret=6
  if [ $ret -eq 0 ]; then
    echo "  Success!"
  fi
  return $ret
}

# run_change_password_test
# --------------------
# Check whether starting container with different password argument correctly
# changes the password and allows connection to old already used db.
function run_change_password_test() {
  echo "  Testing password change"
  local name="change_password"

  local database='db'
  local user='user'
  local password='password'
  local admin_password='adminPassword'
  local ret=0

  create_volume_dir || ret=1
  local volume_options="-v ${volume_dir}:/var/lib/pgsql/data:Z"

  DOCKER_ARGS="
-e POSTGRESQL_DATABASE=${database}
-e POSTGRESQL_USER=${user}
-e POSTGRESQL_PASSWORD=${password}
-e POSTGRESQL_ADMIN_PASSWORD=${admin_password}
$volume_options
" create_container ${name} || ret=2

  # need to set these because `postgresql_cmd` relies on global variables
  PGUSER=${user}
  PASS=${password}

  # need this to wait for the container to start up
  CONTAINER_IP=$(get_container_ip ${name})
  test_connection ${name} || ret=3

  echo "  Testing login"

  assert_login_access ${user} ${password} true || ret=4
  assert_login_access 'postgres' ${admin_password} true || ret=5
  test_postgresql || ret 5

  echo "  Changing passwords"

  echo "Kill the previous container and create a new one"
  local cidfile=$CID_FILE_DIR/"${name}"
  docker kill $(cat $cidfile)
  docker rm -f $(cat $cidfile)
  # Don't forget to remove its .cid file
  rm $cidfile

  DOCKER_ARGS="
-e POSTGRESQL_DATABASE=${database}
-e POSTGRESQL_USER=${user}
-e POSTGRESQL_PASSWORD=NEW_${password}
-e POSTGRESQL_ADMIN_PASSWORD=NEW_${admin_password}
$volume_options
" create_container "${name}_NEW" || ret=6

  # need to set this because `postgresql_cmd` relies on global variables
  PASS="NEW_${password}"

  # need this to wait for the container to start up
  CONTAINER_IP=$(get_container_ip "${name}_NEW")
  test_connection "${name}_NEW" || ret=7

  echo "  Testing login with new passwords"

  assert_login_access ${user} "NEW_${password}" true || ret=8
  assert_login_access ${user} ${password} false || ret=9

  assert_login_access 'postgres' "NEW_${admin_password}" true || ret=10
  assert_login_access 'postgres' ${admin_password} false || ret=11

  # check that we still work with the original volume
  postgresql_cmd -At -c  "SELECT * FROM tbl;" | grep bar3 || ret=12

  if [ $ret -eq 0 ]; then
    echo "  Success!"
  fi
  return $ret
}

# run_replication_test
# --------------------
# Start two containers one as main, one as secondary. Checks whether the main
# sees secondary server connected. Tries to create data on main server and
# checks whether they are correctly replicated to secondary server.
function run_replication_test() {
  local DB=postgres
  local PGUSER=master
  local PASS=master
  local ret=0

  echo "Testing master-slave replication"
  local cluster_args="-e POSTGRESQL_ADMIN_PASSWORD=pass -e POSTGRESQL_MASTER_USER=$PGUSER -e POSTGRESQL_MASTER_PASSWORD=$PASS"
  local cid_suffix="basic"
  local master_ip=
  local slave_cids=

  # Setup the cluster
  setup_replication_cluster || ret=1

  # Check if the master knows about the slaves
  CONTAINER_IP=$master_ip
  test_slave_visibility || ret=2

  # Do some real work to test replication in practice
  table_name="t1" test_value_replication || ret=3
  if [ $ret -eq 0 ]; then
    echo "  Success!"
  fi
  return $ret
}

# run_s2i_test
# --------------------
# Build S2I image with basic script. Try running the container with usage of the
# script that we built into that S2I image. Compare with original image.
run_s2i_test() {
  local temp_file
  local ret=0
  echo "  Testing s2i build"

  local s2i_image_name=$IMAGE_NAME-testapp_$(ct_random_string)
  images_to_clean+=( "$s2i_image_name" )

  ct_s2i_build_as_df "file://${test_dir}/test-app" "${IMAGE_NAME}" "$s2i_image_name" 1>/dev/null || ret=2
  IMAGE_NAME=$s2i_image_name test_the_app_image s2i_config_build "" || ret=3

  echo "  Testing s2i mount"
  create_temp_file
  cat "$test_dir"/test-app/postgresql-init/backup_user.sh >> "$temp_file"

  # Test against original image, not the s2i one.  But even if so, we expect
  # user mouns the directory under "s2i" direcetory $APP_DATA/src.
  local mount_point=/opt/app-root/src/postgresql-init/add_backup_user.sh
  test_the_app_image _s2i_test_mount "-v ${temp_file}:$mount_point:z,ro" || ret=4
  if [ $ret -eq 0 ]; then
    echo "  Success!"
  fi
  return $ret
}

# run_test_cfg_hook
# --------------------
# Checks whether using config files in persistent mounted volume works. Also
# checks that the config file has priority over env variables.
run_test_cfg_hook()
{
  local ret=0
  local volume_dir name=pg-test-cfg-dir
  volume_dir=$(mktemp -d --tmpdir pg-hook-volume.XXXXX)
  add_cleanup_command /bin/rm -rf "$volume_dir"
  setfacl -R -m u:26:rwx "$volume_dir"
  cp -r "$test_dir"/examples/custom-config/* "$volume_dir"
  setfacl -R -m u:26:rwx "$volume_dir"

  DOCKER_ARGS="
-e POSTGRESQL_ADMIN_PASSWORD=password
-v $volume_dir:/opt/app-root/src:Z
  " create_container "$name"
  assert_runtime_option "$name" shared_buffers 111MB || ret=1

  # Check that POSTGRESQL_SHARED_BUFFERS has effect.
  DOCKER_ARGS="
-e POSTGRESQL_ADMIN_PASSWORD=password
-e POSTGRESQL_SHARED_BUFFERS=113MB
  " create_container "$name-2"
  assert_runtime_option "$name-2" shared_buffers 113MB || ret=2

  # Check that volume has priority over POSTGRESQL_SHARED_BUFFERS.
  DOCKER_ARGS="
-e POSTGRESQL_ADMIN_PASSWORD=password
-e POSTGRESQL_SHARED_BUFFERS=113MB
-v $volume_dir:/opt/app-root/src:Z
  " create_container "$name-3"
  assert_runtime_option "$name-3" shared_buffers 111MB || ret=3
  if [ $ret -eq 0 ]; then
    echo "  Success!"
  fi
  return $ret
}

# run_s2i_bake_data_test
# --------------------
# Testing pre-start script and `init.sql` file with prefilled data placed in S2I
# resulting image.
run_s2i_bake_data_test ()
{
  local s2i_image_name="$IMAGE_NAME-bake_$(ct_random_string)"
  ct_s2i_build_as_df "file://$test_dir/examples/s2i-dump-data" "${IMAGE_NAME}" "$s2i_image_name" 1>/dev/null
  images_to_clean+=( "$s2i_image_name" )

  local container_name=bake-data-test

  DOCKER_ARGS="-e POSTGRESQL_ADMIN_PASSWORD=password" \
    IMAGE_NAME="$s2i_image_name" create_container "$container_name"

  wait_ready "$container_name" || \
    false "FAIL: Container did not start up properly."

  test "hello world" == "$(docker exec "$(get_cid "$container_name")" \
                           bash -c "psql -tA -c 'SELECT * FROM test;'")"
}

# run_s2i_enable_ssl_test
# --------------------
# Creates S2I image with SSL config and certificates. Tries to connect with
# required SSL connection.
run_s2i_enable_ssl_test()
{
  local s2i_image_name="$IMAGE_NAME-ssl_$(ct_random_string)"
  ct_s2i_build_as_df "file://$test_dir/examples/enable-ssl" "${IMAGE_NAME}" "$s2i_image_name" 1>/dev/null
  images_to_clean+=( "$s2i_image_name" )

  local container_name=enable-ssl-test

  DOCKER_ARGS="-e POSTGRESQL_ADMIN_PASSWORD=password" \
    IMAGE_NAME="$s2i_image_name" create_container "$container_name"

  wait_ready "$container_name" || \
    false "FAIL: Container did not start up properly."
  CONTAINER_IP=$(get_container_ip $container_name)

  DB=postgres assert_login_access postgres password true

  docker run --rm -e PGPASSWORD="password" "$IMAGE_NAME" psql "postgresql://postgres@$CONTAINER_IP:5432/postgres?sslmode=require" || \
    false "FAIL: Did not manage to connect using SSL only."
}

# run_upgrade_test
# --------------------
# Testing upgrade from previous version to current version using
# POSTGRESQL_UPGRADE env variable. Checks that upgrade is successfull using two
# demo databases (simple and pagila)
run_upgrade_test ()
{
  local ret=0

    local upgrade_path= prev= act=
    case $OS in
        rhel8)
            upgrade_path="none 12 13 15 16 none"
            ;;
        rhel9|c9s)
            upgrade_path="none 13 15 16 none"
            ;;
        rhel10|c10s)
            upgrade_path="none 13 15 16 none"
            ;;
        fedora)
            upgrade_path="none 12 13 14 15 16 none"
            ;;
        *)
            echo "unsupported OS variable" >&2
            return 1
            ;;
    esac

    for act in $upgrade_path; do
        if test "$act" = $VERSION; then
            break
        fi
        prev=$act
    done
    test "$prev" != none || return 0
    # Check if the previous image is available in the registry
    docker pull "$(get_image_id "$prev:remote")" || return 0

    # TODO: We run this script from $VERSION directory, through test/run symlink.
    test/run_upgrade_test "$prev:remote" "$VERSION:local" || ret=1
  if [ $ret -eq 0 ]; then
    echo "  Success!"
  fi
  return $ret
}

# run_migration_test
# --------------------
# Testing migration from each supported version to currently tested one. Pagila
# is used as a demo database - data integrity is checked. Asserts that no
# invalid options can be passed without container failing to start.
run_migration_test ()
{
  local ret=0
    [ "${OS}" == "fedora" ] && return 0

    local from_version
    local upgrade_path="12 13 15 16"

    for from_version in $upgrade_path; do
        # Do not test migration from $VERSION:remote to $VERSION:local
        test $(version2number $from_version) -lt $(version2number "$VERSION") \
            || break
        # Skip if the previous image is not available in the registry
        docker pull "$(get_image_id "$from_version:remote")" || continue
        test/run_migration_test $from_version:remote $VERSION:local || ret=1
    done
  if [ $ret -eq 0 ]; then
    echo "  Success!"
  fi
  return $ret
}

# run_pgaudit_test
# --------------------
# Configure container to include pgaudit using config directory volumes. Check
# that pgaudit is loaded. Set pgaudit logging and check that logs are present in
# log folder.
run_pgaudit_test()
{
  local ret=0
  # extension pgaudit is not available for older versions
  case ${VERSION} in
    9.6|10|11) echo "pgaudit not expected, test skipped."; return ;;
    *) ;;
  esac

  local config_dir data_dir name=pg-test-pgaudit

  # create a dir for config
  config_dir=$(mktemp -d --tmpdir pg-hook-volume.XXXXX)
  add_cleanup_command /bin/rm -rf "$config_dir"
  test -n "$DOCKER_EXTRA_ARGS" || cp -r "$test_dir"/examples/pgaudit/* "$config_dir"
  setfacl -R -m u:26:rwx "$config_dir"

  # create a dir for data
  create_volume_dir
  data_dir="${volume_dir}"

  DOCKER_ARGS="
-e POSTGRESQL_ADMIN_PASSWORD=password
$DOCKER_EXTRA_ARGS
-v ${config_dir}:/opt/app-root/src:Z
-v ${data_dir}:/var/lib/pgsql/data:Z
  " create_container "$name"

  assert_runtime_option "$name" shared_preload_libraries pgaudit || ret=1
  wait_ready "$name" || \
    false "FAIL: Container did not start up properly."

  # enable the pgaudit extension
  # Deliberately moving heredoc into the container, otherwise it does not work
  # in podman 1.6.x due to https://bugzilla.redhat.com/show_bug.cgi?id=1827324
  docker exec -i $(get_cid "$name") bash -c "psql <<EOSQL
SET pgaudit.log = 'read, ddl';
CREATE DATABASE pgaudittest;
EOSQL" || ret=2

  # simulate some trafic that should be audited
  docker exec -i $(get_cid "$name") bash -c "psql pgaudittest <<EOSQL
SET pgaudit.log = 'read, ddl';
CREATE TABLE account (id int, name text, password text, description text);
INSERT INTO account (id, name, password, description) VALUES (1, 'user1', 'HASH1', 'blah, blah');
SELECT * FROM account;
EOSQL" || ret=3

  # give server some time for write all audit messages
  sleep 1

  grep -E 'AUDIT: SESSION,.*,.*,DDL,CREATE DATABASE,,,CREATE DATABASE pgaudittest' "${data_dir}"/userdata/log/postgresql-*.log || ret=4
  grep -E 'AUDIT: SESSION,.*,.*,READ,SELECT,,,SELECT' "${data_dir}"/userdata/log/postgresql-*.log || ret=5
  if [ $ret -eq 0 ]; then
    echo "  Success!"
  fi
  return $ret
}

# run_env_extension_load_test
# --------------------
# Tries to load pgaudit extension using environment variables
# `POSTGRESQL_EXTENSIONS` and `POSTGRESQL_LIBRARIES`
run_env_extension_load_test() {
    DOCKER_EXTRA_ARGS="
-e POSTGRESQL_EXTENSIONS=pgaudit
-e POSTGRESQL_LIBRARIES=pgaudit"
    run_pgaudit_test
}

# run_logging_test
# --------------------
# Checks that changing log location via `POSTGRESQL_LOG_DESTINATION` env
# variable works correctly.
run_logging_test()
{
  local data_dir name=pg-test-logging

  # create a dir for data
  create_volume_dir
  data_dir="${volume_dir}"

  DOCKER_ARGS="
-e POSTGRESQL_ADMIN_PASSWORD=password
-e POSTGRESQL_LOG_DESTINATION=/dev/stderr
-v ${data_dir}:/var/lib/pgsql/data:Z
  " create_container "$name"

  assert_runtime_option "$name" log_destination logging_collector log_directory log_filename
  wait_ready "$name" || \
    false "FAIL: Container did not start up properly."

  # try loggin in as a user that does not exist to trigger an error log message
  echo 'psql -U nonexistent' | docker exec -i $(get_cid "$name") bash || :

  # give server some time for write all audit messages
  sleep 1

  # check whether we have a correct output in the container log
  if docker logs $(get_cid "$name") 2>&1 | grep -q 'FATAL:  role "nonexistent" does not exist' ; then
    echo "    PASS: the container log does include expected error message"
  else
    echo "ERROR: the container log does not include expected error message"
    return 1
  fi

  # the traditional log file should not exist
  for f in "${data_dir}"/userdata/log/postgresql-*.log ; do
    if test -f "$f" ; then
      echo "ERROR: the traditional log file $f should not exist"
      return 1
    fi
  done
  echo "    PASS: the traditional log file under ${data_dir}/userdata/log/postgresql-*.log does not exist"
  echo "  Success!"
}

# configuration defaults
POSTGRESQL_MAX_CONNECTIONS=100
POSTGRESQL_MAX_PREPARED_TRANSACTIONS=0
POSTGRESQL_SHARED_BUFFERS=32MB

ct_init
TEST_SET=${TESTS:-$TEST_LIST} ct_run_tests_from_testset 'postgresql-container_tests'