Skip to content

Commit 49896f9

Browse files
Priyanka-MicrosoftPriyanka-Microsoft
authored
ci: implement quota check in CI Pipeline to ensure resource availability in supported region (#375)
* Add quota check in CI pipeline to verify availability in supported region and prevent failures due to insufficient quota. * updated capacity according to bicep * updated capacity according to bicep
1 parent 549ad7c commit 49896f9

File tree

2 files changed

+143
-2
lines changed

2 files changed

+143
-2
lines changed

‎.github/workflows/CAdeploy.yml

Lines changed: 46 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,50 @@ jobs:
1616
steps:
1717
- name: Checkout Code
1818
uses: actions/checkout@v3
19+
20+
- name: Run Quota Check
21+
id: quota-check
22+
run: |
23+
export AZURE_CLIENT_ID=${{ secrets.AZURE_CLIENT_ID }}
24+
export AZURE_TENANT_ID=${{ secrets.AZURE_TENANT_ID }}
25+
export AZURE_CLIENT_SECRET=${{ secrets.AZURE_CLIENT_SECRET }}
26+
export AZURE_SUBSCRIPTION_ID="${{ secrets.AZURE_SUBSCRIPTION_ID }}"
27+
export GPT_MIN_CAPACITY="11"
28+
export TEXT_EMBEDDING_MIN_CAPACITY="45"
29+
30+
chmod +x ClientAdvisor/Deployment/scripts/checkquota.sh
31+
if ! ClientAdvisor/Deployment/scripts/checkquota.sh; then
32+
# If quota check fails due to insufficient quota, set the flag
33+
if grep -q "No region with sufficient quota found" ClientAdvisor/Deployment/scripts/checkquota.sh; then
34+
echo "QUOTA_FAILED=true" >> $GITHUB_ENV
35+
fi
36+
exit 1 # Fail the pipeline if any other failure occurs
37+
fi
38+
39+
40+
- name: Send Notification on Quota Failure
41+
if: env.QUOTA_FAILED == 'true'
42+
run: |
43+
RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
44+
EMAIL_BODY=$(cat <<EOF
45+
{
46+
"body": "<p>Dear Team,</p><p>The quota check has failed, and the pipeline cannot proceed.</p><p><strong>Build URL:</strong> ${RUN_URL}</p><p>Please take necessary action.</p><p>Best regards,<br>Your Automation Team</p>"
47+
}
48+
EOF
49+
)
50+
51+
curl -X POST "${{ secrets.LOGIC_APP_URL }}" \
52+
-H "Content-Type: application/json" \
53+
-d "$EMAIL_BODY" || echo "Failed to send notification"
54+
55+
- name: Fail Pipeline if Quota Check Fails
56+
if: env.QUOTA_FAILED == 'true'
57+
run: exit 1
58+
59+
- name: Set Deployment Region
60+
run: |
61+
echo "Deployment Region: $VALID_REGION"
62+
echo "AZURE_LOCATION=$VALID_REGION" >> $GITHUB_ENV
1963
2064
- name: Setup Azure CLI
2165
run: |
@@ -48,7 +92,7 @@ jobs:
4892
rg_exists=$(az group exists --name ${{ env.RESOURCE_GROUP_NAME }})
4993
if [ "$rg_exists" = "false" ]; then
5094
echo "Resource group does not exist. Creating..."
51-
az group create --name ${{ env.RESOURCE_GROUP_NAME }} --location uksouth || { echo "Error creating resource group"; exit 1; }
95+
az group create --name ${{ env.RESOURCE_GROUP_NAME }} --location ${{ env.AZURE_LOCATION }} || { echo "Error creating resource group"; exit 1; }
5296
else
5397
echo "Resource group already exists."
5498
fi
@@ -71,7 +115,7 @@ jobs:
71115
az deployment group create \
72116
--resource-group ${{ env.RESOURCE_GROUP_NAME }} \
73117
--template-file ClientAdvisor/Deployment/bicep/main.bicep \
74-
--parameters solutionPrefix=${{ env.SOLUTION_PREFIX }} cosmosLocation=eastus2
118+
--parameters solutionPrefix=${{ env.SOLUTION_PREFIX }} cosmosLocation=${{ env.AZURE_LOCATION }}
75119
76120
- name: List KeyVaults and Store in Array
77121
id: list_keyvaults

‎ClientAdvisor/Deployment/scripts/checkquota.sh

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
#!/bin/bash
2+
3+
# List of Azure regions to check for quota (update as needed)
4+
REGIONS=("eastus" "westus" "northcentralus" "uksouth" "swedencentral")
5+
6+
SUBSCRIPTION_ID="${AZURE_SUBSCRIPTION_ID}"
7+
GPT_MIN_CAPACITY="${GPT_MIN_CAPACITY}"
8+
TEXT_EMBEDDING_MIN_CAPACITY="${TEXT_EMBEDDING_MIN_CAPACITY}"
9+
AZURE_CLIENT_ID="${AZURE_CLIENT_ID}"
10+
AZURE_TENANT_ID="${AZURE_TENANT_ID}"
11+
AZURE_CLIENT_SECRET="${AZURE_CLIENT_SECRET}"
12+
13+
# Authenticate using Managed Identity
14+
echo "Authentication using Managed Identity..."
15+
if ! az login --service-principal -u "$AZURE_CLIENT_ID" -p "$AZURE_CLIENT_SECRET" --tenant "$AZURE_TENANT_ID"; then
16+
echo "❌ Error: Failed to login using Managed Identity."
17+
exit 1
18+
fi
19+
20+
echo "🔄 Validating required environment variables..."
21+
if [[ -z "$SUBSCRIPTION_ID" || -z "$GPT_MIN_CAPACITY" || -z "$TEXT_EMBEDDING_MIN_CAPACITY" ]]; then
22+
echo "❌ ERROR: Missing required environment variables."
23+
exit 1
24+
fi
25+
26+
echo "🔄 Setting Azure subscription..."
27+
if ! az account set --subscription "$SUBSCRIPTION_ID"; then
28+
echo "❌ ERROR: Invalid subscription ID or insufficient permissions."
29+
exit 1
30+
fi
31+
echo "✅ Azure subscription set successfully."
32+
33+
# Define models and their minimum required capacities
34+
declare -A MIN_CAPACITY=(
35+
["OpenAI.Standard.gpt-4"]=$GPT_MIN_CAPACITY
36+
["OpenAI.Standard.text-embedding-ada-002"]=$TEXT_EMBEDDING_MIN_CAPACITY
37+
)
38+
39+
VALID_REGION=""
40+
for REGION in "${REGIONS[@]}"; do
41+
echo "----------------------------------------"
42+
echo "🔍 Checking region: $REGION"
43+
44+
QUOTA_INFO=$(az cognitiveservices usage list --location "$REGION" --output json)
45+
if [ -z "$QUOTA_INFO" ]; then
46+
echo "⚠️ WARNING: Failed to retrieve quota for region $REGION. Skipping."
47+
continue
48+
fi
49+
50+
INSUFFICIENT_QUOTA=false
51+
for MODEL in "${!MIN_CAPACITY[@]}"; do
52+
MODEL_INFO=$(echo "$QUOTA_INFO" | awk -v model="\"value\": \"$MODEL\"" '
53+
BEGIN { RS="},"; FS="," }
54+
$0 ~ model { print $0 }
55+
')
56+
57+
if [ -z "$MODEL_INFO" ]; then
58+
echo "⚠️ WARNING: No quota information found for model: $MODEL in $REGION. Skipping."
59+
continue
60+
fi
61+
62+
CURRENT_VALUE=$(echo "$MODEL_INFO" | awk -F': ' '/"currentValue"/ {print $2}' | tr -d ',' | tr -d ' ')
63+
LIMIT=$(echo "$MODEL_INFO" | awk -F': ' '/"limit"/ {print $2}' | tr -d ',' | tr -d ' ')
64+
65+
CURRENT_VALUE=${CURRENT_VALUE:-0}
66+
LIMIT=${LIMIT:-0}
67+
68+
CURRENT_VALUE=$(echo "$CURRENT_VALUE" | cut -d'.' -f1)
69+
LIMIT=$(echo "$LIMIT" | cut -d'.' -f1)
70+
71+
AVAILABLE=$((LIMIT - CURRENT_VALUE))
72+
73+
echo "✅ Model: $MODEL | Used: $CURRENT_VALUE | Limit: $LIMIT | Available: $AVAILABLE"
74+
75+
if [ "$AVAILABLE" -lt "${MIN_CAPACITY[$MODEL]}" ]; then
76+
echo "❌ ERROR: $MODEL in $REGION has insufficient quota."
77+
INSUFFICIENT_QUOTA=true
78+
break
79+
fi
80+
done
81+
82+
if [ "$INSUFFICIENT_QUOTA" = false ]; then
83+
VALID_REGION="$REGION"
84+
break
85+
fi
86+
87+
done
88+
89+
if [ -z "$VALID_REGION" ]; then
90+
echo "❌ No region with sufficient quota found. Blocking deployment."
91+
echo "QUOTA_FAILED=true" >> "$GITHUB_ENV"
92+
exit 0
93+
else
94+
echo "✅ Deployment Region: $VALID_REGION"
95+
echo "VALID_REGION=$VALID_REGION" >> "$GITHUB_ENV"
96+
exit 0
97+
fi

0 commit comments

Comments
 (0)