Remove legacy shell from test scripts (mongodb#1404)

The legacy shell was only used in AWS authentication tests, so updating
those gets rid of the last remaining use of the legacy shell.

JAVA-4791

Author: jyemin

.evergreen/.evg.yml

@@ -150,7 +150,7 @@ functions:
           ${PREPARE_SHELL}
           REQUIRE_API_VERSION=${REQUIRE_API_VERSION} LOAD_BALANCER=${LOAD_BALANCER} MONGODB_VERSION=${VERSION} TOPOLOGY=${TOPOLOGY} \
           AUTH=${AUTH} SSL=${SSL} STORAGE_ENGINE=${STORAGE_ENGINE} ORCHESTRATION_FILE=${ORCHESTRATION_FILE} \
-          INSTALL_LEGACY_SHELL=${INSTALL_LEGACY_SHELL} bash ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh
+          bash ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh
     # run-orchestration generates expansion file with the MONGODB_URI for the cluster
     - command: expansions.update
       params:
@@ -344,241 +344,108 @@ functions:
           JAVA_VERSION="8" MONGODB_URI="${plain_auth_mongodb_uri}" .evergreen/run-plain-auth-test.sh
 
   "add aws auth variables to file":
+    - command: ec2.assume_role
+      params:
+        role_arn: ${aws_test_secrets_role}
     - command: shell.exec
       type: test
       params:
+        include_expansions_in_env: [ "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN" ]
+        shell: "bash"
         working_dir: "src"
-        silent: true
         script: |
-          cat <<EOF > ${DRIVERS_TOOLS}/.evergreen/auth_aws/aws_e2e_setup.json
-          {
-              "iam_auth_ecs_account" : "${iam_auth_ecs_account}",
-              "iam_auth_ecs_secret_access_key" : "${iam_auth_ecs_secret_access_key}",
-              "iam_auth_ecs_account_arn": "arn:aws:iam::557821124784:user/authtest_fargate_user",
-              "iam_auth_ecs_cluster": "${iam_auth_ecs_cluster}",
-              "iam_auth_ecs_task_definition": "${iam_auth_ecs_task_definition}",
-              "iam_auth_ecs_subnet_a": "${iam_auth_ecs_subnet_a}",
-              "iam_auth_ecs_subnet_b": "${iam_auth_ecs_subnet_b}",
-              "iam_auth_ecs_security_group": "${iam_auth_ecs_security_group}",
-
-              "iam_auth_assume_aws_account" : "${iam_auth_assume_aws_account}",
-              "iam_auth_assume_aws_secret_access_key" : "${iam_auth_assume_aws_secret_access_key}",
-              "iam_auth_assume_role_name" : "${iam_auth_assume_role_name}",
-
-              "iam_auth_ec2_instance_account" : "${iam_auth_ec2_instance_account}",
-              "iam_auth_ec2_instance_secret_access_key" : "${iam_auth_ec2_instance_secret_access_key}",
-              "iam_auth_ec2_instance_profile" : "${iam_auth_ec2_instance_profile}",
-
-              "iam_auth_assume_web_role_name": "${iam_auth_assume_web_role_name}",
-              "iam_web_identity_issuer": "${iam_web_identity_issuer}",
-              "iam_web_identity_rsa_key": "${iam_web_identity_rsa_key}",
-              "iam_web_identity_jwks_uri": "${iam_web_identity_jwks_uri}",
-              "iam_web_identity_token_file": "${iam_web_identity_token_file}"
-          }
-          EOF
+          ${PREPARE_SHELL}
+          cd $DRIVERS_TOOLS/.evergreen/auth_aws
+          ./setup_secrets.sh drivers/aws_auth
 
   "run aws auth test with regular aws credentials":
     - command: shell.exec
       type: test
       params:
-        working_dir: "src"
         shell: "bash"
-        script: |
-          ${PREPARE_SHELL}
-          cd ${DRIVERS_TOOLS}/.evergreen/auth_aws
-          . ./activate-authawsvenv.sh
-          mongo aws_e2e_regular_aws.js
-    - command: shell.exec
-      type: test
-      params:
         working_dir: "src"
-        silent: true
         script: |
-          cat <<'EOF' > "${PROJECT_DIRECTORY}/prepare_mongodb_aws.sh"
-            alias urlencode='python -c "import sys, urllib as ul; print ul.quote_plus(sys.argv[1])"'
-            USER=$(urlencode ${iam_auth_ecs_account})
-            PASS=$(urlencode ${iam_auth_ecs_secret_access_key})
-            MONGODB_URI="mongodb://$USER:$PASS@localhost"
-          EOF
-          JAVA_VERSION=${JAVA_VERSION} PROJECT_DIRECTORY=${PROJECT_DIRECTORY} \
-          AWS_CREDENTIAL_PROVIDER=${AWS_CREDENTIAL_PROVIDER} \
-          .evergreen/run-mongodb-aws-test.sh
+          ${PREPARE_SHELL}
+          JAVA_VERSION=${JAVA_VERSION} AWS_CREDENTIAL_PROVIDER=${AWS_CREDENTIAL_PROVIDER} .evergreen/run-mongodb-aws-test.sh regular
 
   "run aws auth test with assume role credentials":
     - command: shell.exec
       type: test
       params:
-        working_dir: "src"
         shell: "bash"
-        script: |
-          ${PREPARE_SHELL}
-          cd ${DRIVERS_TOOLS}/.evergreen/auth_aws
-          . ./activate-authawsvenv.sh
-          mongo aws_e2e_assume_role.js
-    - command: shell.exec
-      type: test
-      params:
         working_dir: "src"
-        silent: true
         script: |
-          # DO NOT ECHO WITH XTRACE (which PREPARE_SHELL does)
-          cat <<'EOF' > "${PROJECT_DIRECTORY}/prepare_mongodb_aws.sh"
-            alias urlencode='python -c "import sys, urllib as ul; print ul.quote_plus(sys.argv[1])"'
-            USER=$(jq -r '.AccessKeyId' ${DRIVERS_TOOLS}/.evergreen/auth_aws/creds.json)
-            USER=$(urlencode $USER)
-            PASS=$(jq -r '.SecretAccessKey' ${DRIVERS_TOOLS}/.evergreen/auth_aws/creds.json)
-            PASS=$(urlencode $PASS)
-            SESSION_TOKEN=$(jq -r '.SessionToken' ${DRIVERS_TOOLS}/.evergreen/auth_aws/creds.json)
-            SESSION_TOKEN=$(urlencode $SESSION_TOKEN)
-            MONGODB_URI="mongodb://$USER:$PASS@localhost"
-          EOF
-          JAVA_VERSION=${JAVA_VERSION} PROJECT_DIRECTORY=${PROJECT_DIRECTORY} DRIVERS_TOOLS=${DRIVERS_TOOLS} \
-          AWS_CREDENTIAL_PROVIDER=${AWS_CREDENTIAL_PROVIDER} \
-          .evergreen/run-mongodb-aws-test.sh
+          ${PREPARE_SHELL}
+          JAVA_VERSION=${JAVA_VERSION} AWS_CREDENTIAL_PROVIDER=${AWS_CREDENTIAL_PROVIDER} .evergreen/run-mongodb-aws-test.sh assume-role
 
   "run aws auth test with aws EC2 credentials":
     - command: shell.exec
       type: test
       params:
-        working_dir: "src"
         shell: "bash"
-        script: |
-          ${PREPARE_SHELL}
-          cd ${DRIVERS_TOOLS}/.evergreen/auth_aws
-          . ./activate-authawsvenv.sh
-          mongo aws_e2e_ec2.js
-    - command: shell.exec
-      type: test
-      params:
         working_dir: "src"
-        shell: "bash"
         script: |
           ${PREPARE_SHELL}
-          # Write an empty prepare_mongodb_aws so no auth environment variables are set.
-          echo "" > "${PROJECT_DIRECTORY}/prepare_mongodb_aws.sh"
-          JAVA_VERSION=${JAVA_VERSION} AWS_CREDENTIAL_PROVIDER=${AWS_CREDENTIAL_PROVIDER} .evergreen/run-mongodb-aws-test.sh
+          if [ "${SKIP_EC2_AUTH_TEST}" = "true" ]; then
+            echo "This platform does not support the EC2 auth test, skipping..."
+            exit 0
+          fi
+          JAVA_VERSION=${JAVA_VERSION} AWS_CREDENTIAL_PROVIDER=${AWS_CREDENTIAL_PROVIDER} .evergreen/run-mongodb-aws-test.sh ec2
 
   "run aws auth test with web identity credentials":
     - command: shell.exec
       type: test
       params:
-        working_dir: "src"
         shell: "bash"
-        script: |
-          ${PREPARE_SHELL}
-          cd ${DRIVERS_TOOLS}/.evergreen/auth_aws
-          . ./activate-authawsvenv.sh
-          mongo aws_e2e_web_identity.js
-    - command: shell.exec
-      type: test
-      params:
         working_dir: "src"
-        shell: "bash"
-        silent: true
-        script: |
-          # DO NOT ECHO WITH XTRACE (which PREPARE_SHELL does)
-          cat <<'EOF' > "${PROJECT_DIRECTORY}/prepare_mongodb_aws.sh"
-            export AWS_ROLE_ARN="${iam_auth_assume_web_role_name}"
-            export AWS_WEB_IDENTITY_TOKEN_FILE="${iam_web_identity_token_file}"
-          EOF
-    - command: shell.exec
-      type: test
-      params:
-        working_dir: "src"
-        shell: "bash"
         script: |
           ${PREPARE_SHELL}
           if [ "${AWS_CREDENTIAL_PROVIDER}" = "builtIn" ]; then
              echo "Built-in AWS credential provider does not support the web identity auth test, skipping..."
              exit 0
           fi
-          JAVA_VERSION=${JAVA_VERSION} AWS_CREDENTIAL_PROVIDER=${AWS_CREDENTIAL_PROVIDER} ASSERT_NO_URI_CREDS=true .evergreen/run-mongodb-aws-test.sh
+          if [ "${SKIP_WEB_IDENTITY_AUTH_TEST}" = "true" ]; then
+             echo "This platform does not support the web identity auth test, skipping..."
+             exit 0
+          fi
+          JAVA_VERSION=${JAVA_VERSION} AWS_CREDENTIAL_PROVIDER=${AWS_CREDENTIAL_PROVIDER} .evergreen/run-mongodb-aws-test.sh web-identity
     - command: shell.exec
       type: test
       params:
-        working_dir: "src"
         shell: "bash"
-        silent: true
-        script: |
-          # DO NOT ECHO WITH XTRACE (which PREPARE_SHELL does)
-          cat <<'EOF' > "${PROJECT_DIRECTORY}/prepare_mongodb_aws.sh"
-            export AWS_ROLE_ARN="${iam_auth_assume_web_role_name}"
-            export AWS_WEB_IDENTITY_TOKEN_FILE="${iam_web_identity_token_file}"
-            export AWS_ROLE_SESSION_NAME="test"
-          EOF
-    - command: shell.exec
-      type: test
-      params:
         working_dir: "src"
-        shell: "bash"
         script: |
           ${PREPARE_SHELL}
           if [ "${AWS_CREDENTIAL_PROVIDER}" = "builtIn" ]; then
              echo "Built-in AWS credential provider does not support the web identity auth test, skipping..."
              exit 0
           fi
-          JAVA_VERSION=${JAVA_VERSION} AWS_CREDENTIAL_PROVIDER=${AWS_CREDENTIAL_PROVIDER} ASSERT_NO_URI_CREDS=true .evergreen/run-mongodb-aws-test.sh
+          if [ "${SKIP_WEB_IDENTITY_AUTH_TEST}" = "true" ]; then
+             echo "This platform does not support the web identity auth test, skipping..."
+             exit 0
+          fi
+          export AWS_ROLE_SESSION_NAME="test"
+          JAVA_VERSION=${JAVA_VERSION} AWS_CREDENTIAL_PROVIDER=${AWS_CREDENTIAL_PROVIDER} .evergreen/run-mongodb-aws-test.sh web-identity
 
   "run aws auth test with aws credentials as environment variables":
     - command: shell.exec
       type: test
       params:
-        working_dir: "src"
         shell: "bash"
-        script: |
-          ${PREPARE_SHELL}
-          cd ${DRIVERS_TOOLS}/.evergreen/auth_aws
-          . ./activate-authawsvenv.sh
-          mongo aws_e2e_regular_aws.js
-    - command: shell.exec
-      type: test
-      params:
-        working_dir: "src"
-        silent: true
-        script: |
-          # DO NOT ECHO WITH XTRACE (which PREPARE_SHELL does)
-          cat <<'EOF' > "${PROJECT_DIRECTORY}/prepare_mongodb_aws.sh"
-            export AWS_ACCESS_KEY_ID=${iam_auth_ecs_account}
-            export AWS_SECRET_ACCESS_KEY=${iam_auth_ecs_secret_access_key}
-          EOF
-    - command: shell.exec
-      type: test
-      params:
         working_dir: "src"
         script: |
           ${PREPARE_SHELL}
-          JAVA_VERSION=${JAVA_VERSION} AWS_CREDENTIAL_PROVIDER=${AWS_CREDENTIAL_PROVIDER} .evergreen/run-mongodb-aws-test.sh
+          JAVA_VERSION=${JAVA_VERSION} AWS_CREDENTIAL_PROVIDER=${AWS_CREDENTIAL_PROVIDER} .evergreen/run-mongodb-aws-test.sh env-creds
 
   "run aws auth test with aws credentials and session token as environment variables":
     - command: shell.exec
       type: test
       params:
-        working_dir: "src"
         shell: "bash"
-        script: |
-          ${PREPARE_SHELL}
-          cd ${DRIVERS_TOOLS}/.evergreen/auth_aws
-          . ./activate-authawsvenv.sh
-          mongo aws_e2e_assume_role.js
-    - command: shell.exec
-      type: test
-      params:
-        working_dir: "src"
-        silent: true
-        script: |
-          # DO NOT ECHO WITH XTRACE (which PREPARE_SHELL does)
-          cat <<'EOF' > "${PROJECT_DIRECTORY}/prepare_mongodb_aws.sh"
-            export AWS_ACCESS_KEY_ID=$(jq -r '.AccessKeyId' ${DRIVERS_TOOLS}/.evergreen/auth_aws/creds.json)
-            export AWS_SECRET_ACCESS_KEY=$(jq -r '.SecretAccessKey' ${DRIVERS_TOOLS}/.evergreen/auth_aws/creds.json)
-            export AWS_SESSION_TOKEN=$(jq -r '.SessionToken' ${DRIVERS_TOOLS}/.evergreen/auth_aws/creds.json)
-          EOF
-    - command: shell.exec
-      type: test
-      params:
         working_dir: "src"
         script: |
           ${PREPARE_SHELL}
-          JAVA_VERSION=${JAVA_VERSION} AWS_CREDENTIAL_PROVIDER=${AWS_CREDENTIAL_PROVIDER} .evergreen/run-mongodb-aws-test.sh
+          JAVA_VERSION=${JAVA_VERSION} AWS_CREDENTIAL_PROVIDER=${AWS_CREDENTIAL_PROVIDER} .evergreen/run-mongodb-aws-test.sh session-creds
 
   "run aws ECS auth test":
     - command: shell.exec
@@ -951,7 +818,6 @@ tasks:
             AUTH: "auth"
             ORCHESTRATION_FILE: "auth-aws.json"
             TOPOLOGY: "server"
-            INSTALL_LEGACY_SHELL: "true"
         - func: "add aws auth variables to file"
         - func: "run aws auth test with regular aws credentials"
 
@@ -962,7 +828,6 @@ tasks:
             AUTH: "auth"
             ORCHESTRATION_FILE: "auth-aws.json"
             TOPOLOGY: "server"
-            INSTALL_LEGACY_SHELL: "true"
         - func: "add aws auth variables to file"
         - func: "run aws auth test with assume role credentials"
 
@@ -973,7 +838,6 @@ tasks:
             AUTH: "auth"
             ORCHESTRATION_FILE: "auth-aws.json"
             TOPOLOGY: "server"
-            INSTALL_LEGACY_SHELL: "true"
         - func: "add aws auth variables to file"
         - func: "run aws auth test with aws credentials as environment variables"
 
@@ -984,7 +848,6 @@ tasks:
             AUTH: "auth"
             ORCHESTRATION_FILE: "auth-aws.json"
             TOPOLOGY: "server"
-            INSTALL_LEGACY_SHELL: "true"
         - func: "add aws auth variables to file"
         - func: "run aws auth test with aws credentials and session token as environment variables"
 
@@ -995,7 +858,6 @@ tasks:
             AUTH: "auth"
             ORCHESTRATION_FILE: "auth-aws.json"
             TOPOLOGY: "server"
-            INSTALL_LEGACY_SHELL: "true"
         - func: "add aws auth variables to file"
         - func: "run aws auth test with aws EC2 credentials"
 
@@ -1006,7 +868,6 @@ tasks:
             AUTH: "auth"
             ORCHESTRATION_FILE: "auth-aws.json"
             TOPOLOGY: "server"
-            INSTALL_LEGACY_SHELL: "true"
         - func: "add aws auth variables to file"
         - func: "run aws auth test with web identity credentials"
 
@@ -1017,7 +878,6 @@ tasks:
             AUTH: "auth"
             ORCHESTRATION_FILE: "auth-aws.json"
             TOPOLOGY: "server"
-            INSTALL_LEGACY_SHELL: "true"
         - func: "add aws auth variables to file"
         - func: "run aws ECS auth test"
 

.evergreen/run-mongodb-aws-test.sh

@@ -15,19 +15,8 @@ RELATIVE_DIR_PATH="$(dirname "${BASH_SOURCE:-$0}")"
 
 echo "Running MONGODB-AWS authentication tests"
 
-
-# ensure no secrets are printed in log files
-set +x
-
-# load the script
-shopt -s expand_aliases # needed for `urlencode` alias
-[ -s "${PROJECT_DIRECTORY}/prepare_mongodb_aws.sh" ] && source "${PROJECT_DIRECTORY}/prepare_mongodb_aws.sh"
-
-MONGODB_URI=${MONGODB_URI:-"mongodb://localhost"}
-MONGODB_URI="${MONGODB_URI}/aws?authMechanism=MONGODB-AWS"
-if [[ -n ${SESSION_TOKEN} ]]; then
-    MONGODB_URI="${MONGODB_URI}&authMechanismProperties=AWS_SESSION_TOKEN:${SESSION_TOKEN}"
-fi
+# Handle credentials and environment setup.
+. $DRIVERS_TOOLS/.evergreen/auth_aws/aws_setup.sh $1
 
 # show test output
 set -x