[7.x][ML] Retry getting AWS credentials in CI builds (#1764)

It's very frustrating if CI builds fail because of
a transient failure obtaining AWS credentials for
uploading the final output.

Backport of #1762

Author: droberts195

dev-tools/jenkins_ci.ps1

@@ -12,8 +12,6 @@
 # 3. If this is not a PR build, upload the build to the artifacts directory on
 #    S3 that subsequent Java builds will download the C++ components from
 
-$ErrorActionPreference="Stop"
-
 # If this isn't a PR build then obtain credentials from Vault
 if (!(Test-Path Env:PR_AUTHOR)) {
     # Generate a Vault token
@@ -22,19 +20,31 @@ if (!(Test-Path Env:PR_AUTHOR)) {
         Exit $LastExitCode
     }
 
-    $AwsCreds=& vault read -format=json -field=data aws-dev/creds/prelertartifacts
-    if ($LastExitCode -ne 0) {
-        Exit $LastExitCode
-    }
-    $Env:ML_AWS_ACCESS_KEY=(echo $AwsCreds | jq -r ".access_key")
-    $Env:ML_AWS_SECRET_KEY=(echo $AwsCreds | jq -r ".secret_key")
+    $Failures=0
+    do {
+        $AwsCreds=& vault read -format=json -field=data aws-dev/creds/prelertartifacts
+        if ($LastExitCode -eq 0) {
+            $Env:ML_AWS_ACCESS_KEY=(echo $AwsCreds | jq -r ".access_key")
+            $Env:ML_AWS_SECRET_KEY=(echo $AwsCreds | jq -r ".secret_key")
+        } else {
+            $Failures++
+            Write-Output "Attempt $Failures to get AWS credentials failed"
+        }
+    } while (($Failures -lt 3) -and [string]::IsNullOrEmpty($Env:ML_AWS_ACCESS_KEY))
 
     # Remove VAULT_* environment variables
     Remove-Item Env:VAULT_TOKEN
     Remove-Item Env:VAULT_ROLE_ID
     Remove-Item Env:VAULT_SECRET_ID
+
+    if ([string]::IsNullOrEmpty($Env:ML_AWS_ACCESS_KEY) -or [string]::IsNullOrEmpty($Env:ML_AWS_SECRET_KEY)) {
+        Write-Output "Exiting after failing to get AWS credentials $Failures times"
+        Exit 1
+    }
 }
 
+$ErrorActionPreference="Stop"
+
 # Change directory to the top level of the repo
 Set-Location -Path "$PSScriptRoot\.."
 

dev-tools/jenkins_ci.sh

@@ -35,11 +35,25 @@ if [ -z "$PR_AUTHOR" ] ; then
     set +x
     export VAULT_TOKEN=$(vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID")
 
-    AWS_CREDS=$(vault read -format=json -field=data aws-dev/creds/prelertartifacts)
-    export ML_AWS_ACCESS_KEY=$(echo $AWS_CREDS | jq -r '.access_key')
-    export ML_AWS_SECRET_KEY=$(echo $AWS_CREDS | jq -r '.secret_key')
+    unset ML_AWS_ACCESS_KEY ML_AWS_SECRET_KEY
+    FAILURES=0
+    while [ $FAILURES -lt 3 -a -z "$ML_AWS_ACCESS_KEY" ] ; do
+        AWS_CREDS=$(vault read -format=json -field=data aws-dev/creds/prelertartifacts)
+        if [ $? -eq 0 ] ; then
+            export ML_AWS_ACCESS_KEY=$(echo $AWS_CREDS | jq -r '.access_key')
+            export ML_AWS_SECRET_KEY=$(echo $AWS_CREDS | jq -r '.secret_key')
+        else
+            let FAILURES++
+            echo "Attempt $FAILURES to get AWS credentials failed"
+        fi
+    done
 
     unset VAULT_TOKEN VAULT_ROLE_ID VAULT_SECRET_ID
+
+    if [ -z "$ML_AWS_ACCESS_KEY" -o -z "$ML_AWS_SECRET_KEY" ] ; then
+        echo "Exiting after failing to get AWS credentials $FAILURES times"
+        exit 1
+    fi
     set -x
 fi
 
@@ -124,7 +138,7 @@ case `uname` in
 
     *)
         echo `uname 2>&1` "- unsupported operating system"
-        exit 1
+        exit 2
         ;;
 esac