diff --git a/internal/controller/ols_app_server_assets.go b/internal/controller/ols_app_server_assets.go
index a23d6e06..7444cf72 100644
--- a/internal/controller/ols_app_server_assets.go
+++ b/internal/controller/ols_app_server_assets.go
@@ -120,9 +120,21 @@ func (r *OLSConfigReconciler) checkLLMCredentials(ctx context.Context, cr *olsv1
 			}
 			return fmt.Errorf("failed to get LLM provider %s credential secret %s: %w", provider.Name, provider.CredentialsSecretRef.Name, err)
 		}
-		// secret must contain a key named "apitoken"
-		if _, ok := secret.Data["apitoken"]; !ok {
-			return fmt.Errorf("LLM provider %s credential secret %s missing key 'apitoken'", provider.Name, provider.CredentialsSecretRef.Name)
+		if provider.Type == AzureOpenAIType {
+			// Azure OpenAI secret must contain "apitoken" or 3 keys named "client_id", "tenant_id", "client_secret"
+			if _, ok := secret.Data["apitoken"]; ok {
+				continue
+			}
+			for _, key := range []string{"client_id", "tenant_id", "client_secret"} {
+				if _, ok := secret.Data[key]; !ok {
+					return fmt.Errorf("LLM provider %s credential secret %s missing key '%s'", provider.Name, provider.CredentialsSecretRef.Name, key)
+				}
+			}
+		} else {
+			// Other providers (e.g. WatsonX, OpenAI) must contain a key named "apikey"
+			if _, ok := secret.Data["apitoken"]; !ok {
+				return fmt.Errorf("LLM provider %s credential secret %s missing key 'apitoken'", provider.Name, provider.CredentialsSecretRef.Name)
+			}
 		}
 	}
 	return nil
diff --git a/internal/controller/ols_app_server_reconciliator_test.go b/internal/controller/ols_app_server_reconciliator_test.go
index d726c2a6..31a98330 100644
--- a/internal/controller/ols_app_server_reconciliator_test.go
+++ b/internal/controller/ols_app_server_reconciliator_test.go
@@ -419,7 +419,7 @@ var _ = Describe("App server reconciliator", Ordered, func() {
 		})
 
 		It("should return error when the LLM provider token secret does not have required keys", func() {
-			By("Reconcile after modifying the token secret")
+			By("General provider: the token secret miss 'apitoken' key")
 			secret, _ := generateRandomSecret()
 			// delete the required key "apitoken"
 			delete(secret.Data, "apitoken")
@@ -428,6 +428,41 @@ var _ = Describe("App server reconciliator", Ordered, func() {
 			err = reconciler.reconcileAppServer(ctx, cr)
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("missing key 'apitoken'"))
+
+			By("AzureOpenAI provider: the token secret miss 'clientid', 'tenantid', 'client_secret' key")
+			secret, _ = generateRandomSecret()
+			delete(secret.Data, "client_id")
+			delete(secret.Data, "tenant_id")
+			delete(secret.Data, "client_secret")
+			err = k8sClient.Update(ctx, secret)
+			Expect(err).NotTo(HaveOccurred())
+			crAzure := cr.DeepCopy()
+			crAzure.Spec.LLMConfig.Providers[0].Type = AzureOpenAIType
+			err = reconciler.reconcileAppServer(ctx, crAzure)
+			Expect(err).NotTo(HaveOccurred())
+			delete(secret.Data, "apitoken")
+			err = k8sClient.Update(ctx, secret)
+			Expect(err).NotTo(HaveOccurred())
+			err = reconciler.reconcileAppServer(ctx, crAzure)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("missing key 'client_id'"))
+			secret.Data["client_id"] = []byte("test-client-id")
+			err = k8sClient.Update(ctx, secret)
+			Expect(err).NotTo(HaveOccurred())
+			err = reconciler.reconcileAppServer(ctx, crAzure)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("missing key 'tenant_id'"))
+			secret.Data["tenant_id"] = []byte("test-tenant-id")
+			err = k8sClient.Update(ctx, secret)
+			Expect(err).NotTo(HaveOccurred())
+			err = reconciler.reconcileAppServer(ctx, crAzure)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("missing key 'client_secret'"))
+			secret.Data["client_secret"] = []byte("test-client-secret")
+			err = k8sClient.Update(ctx, secret)
+			Expect(err).NotTo(HaveOccurred())
+			err = reconciler.reconcileAppServer(ctx, crAzure)
+			Expect(err).NotTo(HaveOccurred())
 		})
 
 	})