feat: Verbose logging with flag

.github/ISSUE_TEMPLATE/bug_report.md

@@ -20,8 +20,10 @@ Steps to reproduce the behavior:
 **Expected behavior**
 A clear and concise description of what you expected to happen.
 
-**Screenshots**
+**Screenshots/Verbose output**
 If applicable, add screenshots to help explain your problem.
 
+If you've tried running `argocd-vault-plugin generate` with `--verbose-sensitive-output` to help debug, please include that output here after redacting any secrets.
+
 **Additional context**
 Add any other context about the problem here.

cmd/generate.go

@@ -10,6 +10,7 @@ import (
 	"github.com/argoproj-labs/argocd-vault-plugin/pkg/config"
 	"github.com/argoproj-labs/argocd-vault-plugin/pkg/kube"
 	"github.com/argoproj-labs/argocd-vault-plugin/pkg/types"
+	"github.com/argoproj-labs/argocd-vault-plugin/pkg/utils"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
@@ -18,6 +19,7 @@ import (
 func NewGenerateCommand() *cobra.Command {
 	const StdIn = "-"
 	var configPath, secretName string
+	var verboseOutput bool
 
 	var command = &cobra.Command{
 		Use:   "generate <path>",
@@ -59,6 +61,7 @@ func NewGenerateCommand() *cobra.Command {
 			}
 
 			v := viper.New()
+			viper.Set("verboseOutput", verboseOutput)
 			cmdConfig, err := config.New(v, &config.Options{
 				SecretName: secretName,
 				ConfigPath: configPath,
@@ -90,6 +93,8 @@ func NewGenerateCommand() *cobra.Command {
 					if err != nil {
 						return err
 					}
+				} else {
+					utils.VerboseToStdErr("skipping %s.%s because %s annotation is present", manifest.GetNamespace(), manifest.GetName(), types.AVPIgnoreAnnotation)
 				}
 
 				output, err := template.ToYAML()
@@ -106,5 +111,6 @@ func NewGenerateCommand() *cobra.Command {
 
 	command.Flags().StringVarP(&configPath, "config-path", "c", "", "path to a file containing Vault configuration (YAML, JSON, envfile) to use")
 	command.Flags().StringVarP(&secretName, "secret-name", "s", "", "name of a Kubernetes Secret in the argocd namespace containing Vault configuration data in the argocd namespace of your ArgoCD host (Only available when used in ArgoCD). The namespace can be overridden by using the format <namespace>:<name>")
+	command.Flags().BoolVar(&verboseOutput, "verbose-sensitive-output", false, "enable verbose mode for detailed info to help with debugging. Includes sensitive data (credentials), logged to stderr")
 	return command
 }

docs/cmd/generate.md

@@ -6,9 +6,10 @@ argocd-vault-plugin generate PATH [flags]
 
 ### Options
 ```
-  -c, --config-path string   path to a file containing Vault configuration (YAML, JSON, envfile) to use
-  -h, --help                 help for generate
-  -s, --secret-name string   name of a Kubernetes Secret containing Vault configuration data in the argocd namespace of your ArgoCD host (Only available when used in ArgoCD)
+  -c, --config-path string         path to a file containing Vault configuration (YAML, JSON, envfile) to use
+  -h, --help                       help for generate
+  -s, --secret-name string         name of a Kubernetes Secret in the argocd namespace containing Vault configuration data in the argocd namespace of your ArgoCD host (Only available when used in ArgoCD). The namespace can be overridden by using the format <namespace>:<name>
+      --verbose-sensitive-output   enable verbose mode for detailed info to help with debugging. Includes sensitive data (credentials), logged to stderr
 ```
 
 ### SEE ALSO

pkg/auth/vault/approle.go

@@ -37,15 +37,19 @@ func (a *AppRoleAuth) Authenticate(vaultClient *api.Client) error {
 		"role_id":   a.RoleID,
 		"secret_id": a.SecretID,
 	}
+
+	utils.VerboseToStdErr("Hashicorp Vault authenticating with role ID %s and secret ID %s", a.RoleID, a.SecretID)
 	data, err := vaultClient.Logical().Write(fmt.Sprintf("%s/login", a.MountPath), payload)
 	if err != nil {
 		return err
 	}
 
+	utils.VerboseToStdErr("Hashicorp Vault authentication response: %v", data)
+
 	// If we cannot write the Vault token, we'll just have to login next time. Nothing showstopping.
 	err = utils.SetToken(vaultClient, data.Auth.ClientToken)
 	if err != nil {
-		print(err)
+		utils.VerboseToStdErr("Hashicorp Vault cannot cache token for future runs: %v", err)
 	}
 
 	return nil

pkg/auth/vault/github.go

@@ -36,14 +36,18 @@ func (g *GithubAuth) Authenticate(vaultClient *api.Client) error {
 		"token": g.AccessToken,
 	}
 
+	utils.VerboseToStdErr("Hashicorp Vault authenticating with Github token %s", g.AccessToken)
 	data, err := vaultClient.Logical().Write(fmt.Sprintf("%s/login", g.MountPath), payload)
 	if err != nil {
 		return err
 	}
+
+	utils.VerboseToStdErr("Hashicorp Vault authentication response: %v", data)
+
 	// If we cannot write the Vault token, we'll just have to login next time. Nothing showstopping.
 	err = utils.SetToken(vaultClient, data.Auth.ClientToken)
 	if err != nil {
-		print(err)
+		utils.VerboseToStdErr("Hashicorp Vault cannot cache token for future runs: %v", err)
 	}
 
 	return nil

pkg/auth/vault/kubernetes.go

@@ -54,15 +54,18 @@ func (k *K8sAuth) Authenticate(vaultClient *api.Client) error {
 		kubeAuthPath = k.MountPath
 	}
 
+	utils.VerboseToStdErr("Hashicorp Vault authenticating with Vault role %s using Kubernetes service account token %s read from %s", k.Role, serviceAccountFile, token)
 	data, err := vaultClient.Logical().Write(fmt.Sprintf("%s/login", kubeAuthPath), payload)
 	if err != nil {
 		return err
 	}
 
+	utils.VerboseToStdErr("Hashicorp Vault authentication response: %v", data)
+
 	// If we cannot write the Vault token, we'll just have to login next time. Nothing showstopping.
 	err = utils.SetToken(vaultClient, data.Auth.ClientToken)
 	if err != nil {
-		print(err)
+		utils.VerboseToStdErr("Hashicorp Vault cannot cache token for future runs: %v", err)
 	}
 
 	return nil

pkg/backends/awssecretsmanager.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 
+	"github.com/argoproj-labs/argocd-vault-plugin/pkg/utils"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/secretsmanager"
 	"github.com/aws/aws-sdk-go/service/secretsmanager/secretsmanageriface"
@@ -36,11 +37,14 @@ func (a *AWSSecretsManager) GetSecrets(path string, version string, annotations
 		input.SetVersionId(version)
 	}
 
+	utils.VerboseToStdErr("AWS Secrets Manager getting secret %s at version %s", path, version)
 	result, err := a.Client.GetSecretValue(input)
 	if err != nil {
 		return nil, err
 	}
 
+	utils.VerboseToStdErr("AWS Secrets Manager get secret response %v", result)
+
 	var dat map[string]interface{}
 
 	if result.SecretString != nil {

pkg/backends/azurekeyvault.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"github.com/Azure/azure-sdk-for-go/profiles/latest/keyvault/keyvault"
+	"github.com/argoproj-labs/argocd-vault-plugin/pkg/utils"
 	"path"
 	"strings"
 	"time"
@@ -36,18 +37,25 @@ func (a *AzureKeyVault) GetSecrets(kvpath string, version string, _ map[string]s
 
 	data := make(map[string]interface{})
 
+	utils.VerboseToStdErr("Azure Key Vault listing secrets in vault %v", kvpath)
 	secretList, err := a.Client.GetSecretsComplete(ctx, kvpath, nil)
 	if err != nil {
 		return nil, err
 	}
+
+	utils.VerboseToStdErr("Azure Key Vault list secrets response %v", secretList)
 	// Gather all secrets in Key Vault
+
 	for ; secretList.NotDone(); secretList.NextWithContext(ctx) {
 		secret := path.Base(*secretList.Value().ID)
 		if version == "" {
+			utils.VerboseToStdErr("Azure Key Vault getting secret %s from vault %s", secret, kvpath)
 			secretResp, err := a.Client.GetSecret(ctx, kvpath, secret, "")
 			if err != nil {
 				return nil, err
 			}
+
+			utils.VerboseToStdErr("Azure Key Vault get unversioned secret response %v", secretResp)
 			data[secret] = *secretResp.Value
 			continue
 		}
@@ -62,10 +70,13 @@ func (a *AzureKeyVault) GetSecrets(kvpath string, version string, _ map[string]s
 			}
 			// Secret version matched given version
 			if strings.Contains(*secretVersion.ID, version) {
+				utils.VerboseToStdErr("Azure Key Vault getting secret %s from vault %s at version %s", secret, kvpath, version)
 				secretResp, err := a.Client.GetSecret(ctx, kvpath, secret, version)
 				if err != nil {
 					return nil, err
 				}
+
+				utils.VerboseToStdErr("Azure Key Vault get versioned secret response %v", secretResp)
 				data[secret] = *secretResp.Value
 			}
 		}
@@ -81,11 +92,15 @@ func (a *AzureKeyVault) GetIndividualSecret(kvpath, secret, version string, anno
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
 
+	utils.VerboseToStdErr("Azure Key Vault getting secret %s from vault %s at version %s", secret, kvpath, version)
+
 	kvpath = fmt.Sprintf("https://%s.vault.azure.net", kvpath)
 	data, err := a.Client.GetSecret(ctx, kvpath, secret, version)
 	if err != nil {
 		return nil, err
 	}
 
+	utils.VerboseToStdErr("Azure Key Vault get versioned secret response %v", data)
+
 	return *data.Value, nil
 }

pkg/backends/gcpsecretmanager.go

@@ -6,6 +6,7 @@ import (
 	"regexp"
 
 	"github.com/argoproj-labs/argocd-vault-plugin/pkg/types"
+	"github.com/argoproj-labs/argocd-vault-plugin/pkg/utils"
 	"github.com/googleapis/gax-go/v2"
 	secretmanagerpb "google.golang.org/genproto/googleapis/cloud/secretmanager/v1"
 )
@@ -50,11 +51,14 @@ func (a *GCPSecretManager) GetSecrets(path string, version string, annotations m
 		Name: fmt.Sprintf("%s/versions/%s", path, version),
 	}
 
+	utils.VerboseToStdErr("GCP Secret Manager accessing secret at path %s at version  %v", path, version)
 	result, err := a.Client.AccessSecretVersion(a.Context, req)
 	if err != nil {
 		return nil, fmt.Errorf("Could not find secret: %v", err)
 	}
 
+	utils.VerboseToStdErr("GCP Secret Manager access secret version response %v", result)
+
 	data := make(map[string]interface{})
 
 	secretName := matches[GCPPath.SubexpIndex("secretid")]

pkg/backends/ibmsecretsmanager.go

@@ -8,6 +8,7 @@ import (
 	"github.com/IBM/go-sdk-core/v5/core"
 	ibmsm "github.com/IBM/secrets-manager-go-sdk/secretsmanagerv1"
 	"github.com/argoproj-labs/argocd-vault-plugin/pkg/types"
+	"github.com/argoproj-labs/argocd-vault-plugin/pkg/utils"
 )
 
 var IBMPath, _ = regexp.Compile(`ibmcloud/(?P<type>.+)/secrets/groups/(?P<groupId>.+)`)
@@ -120,6 +121,8 @@ func (i *IBMSecretsManager) getSecretVersionedOrNot(secret *ibmsm.SecretResource
 			return nil, fmt.Errorf("Could not retrieve secret %s after %d retries, statuscode %d", *secret.ID, types.IBMMaxRetries, httpResponse.GetStatusCode())
 		}
 
+		utils.VerboseToStdErr("IBM Cloud Secrets Manager get versioned secret %s HTTP response: %v", *secret.ID, httpResponse)
+
 		result = (secretVersion.Resources[0].(*ibmsm.SecretVersion)).SecretData.(map[string]interface{})
 	} else {
 		secretRes, httpResponse, err := i.Client.GetSecret(&ibmsm.GetSecretOptions{
@@ -133,6 +136,8 @@ func (i *IBMSecretsManager) getSecretVersionedOrNot(secret *ibmsm.SecretResource
 			return nil, fmt.Errorf("Could not retrieve secret %s after %d retries, statuscode %d", *secret.ID, types.IBMMaxRetries, httpResponse.GetStatusCode())
 		}
 
+		utils.VerboseToStdErr("IBM Cloud Secrets Manager get unversioned secret %s HTTP response: %v", *secret.ID, httpResponse)
+
 		// APIKey secrets don't come from `SecretData`
 		if *secret.SecretType == types.IBMIAMCredentialsType {
 			result = map[string]interface{}{
@@ -171,9 +176,10 @@ func (i *IBMSecretsManager) getSecret(secret *ibmsm.SecretResource, version stri
 
 	// Bypass the cache when explicit version is requested
 	if cacheResult != nil && version == "" {
+		utils.VerboseToStdErr("IBM Cloud Secrets Manager get secret: cache hit for %s of type %s from group %s", secretName, secretType, groupId)
 		result["payload"] = cacheResult
 	} else {
-
+		utils.VerboseToStdErr("IBM Cloud Secrets Manager get secret: getting secret %s of type %s from group %s", secretName, secretType, groupId)
 		secretData, err := i.getSecretVersionedOrNot(secret, version)
 		var payload interface{}
 		if err != nil {
@@ -211,11 +217,13 @@ func (i *IBMSecretsManager) listSecretsInGroup(groupId, secretType string) (map[
 	ckey := cacheKey{groupId, secretType}
 	cachedData := i.listAllSecretsCache[ckey]
 	if cachedData != nil {
+		utils.VerboseToStdErr("IBM Cloud Secrets Manager list secrets in group: cache hit group %s", groupId)
 		return cachedData, nil
 	}
 
 	var offset int64 = 0
 	for {
+		utils.VerboseToStdErr("IBM Cloud Secrets Manager listing secrets of from group %s starting at offset %d", groupId, offset)
 		res, details, err := i.Client.ListAllSecrets(&ibmsm.ListAllSecretsOptions{
 			Groups: []string{groupId},
 			Offset: &offset,
@@ -227,6 +235,8 @@ func (i *IBMSecretsManager) listSecretsInGroup(groupId, secretType string) (map[
 			return nil, fmt.Errorf("Could not list secrets for secret group %s: %d\n%s", groupId, details.GetStatusCode(), details.String())
 		}
 
+		utils.VerboseToStdErr("IBM Cloud Secrets Manager list secrets in group HTTP response: %v", details)
+
 		for _, secret := range res.Resources {
 			name := *(secret.(*ibmsm.SecretResource).Name)
 			ttype := *(secret.(*ibmsm.SecretResource).SecretType)

pkg/backends/localsecretmanager.go

@@ -3,6 +3,7 @@ package backends
 import (
 	"fmt"
 
+	"github.com/argoproj-labs/argocd-vault-plugin/pkg/utils"
 	"k8s.io/apimachinery/pkg/util/yaml"
 )
 
@@ -28,9 +29,11 @@ func (a *LocalSecretManager) Login() error {
 
 // GetSecrets gets secrets using decrypt function and returns the formatted data
 func (a *LocalSecretManager) GetSecrets(path string, version string, annotations map[string]string) (map[string]interface{}, error) {
-
+	utils.VerboseToStdErr("Local secret manager getting secret %s at version %s", path, version)
 	cleartext, err := a.Decrypt(path, "yaml")
 
+	utils.VerboseToStdErr("Local secret manager get secret response: %v", cleartext)
+
 	var dat map[string]interface{}
 
 	if err != nil {

pkg/backends/onepasswordconnect.go

@@ -4,6 +4,7 @@ import (
 	"strings"
 
 	"github.com/1Password/connect-sdk-go/connect"
+	"github.com/argoproj-labs/argocd-vault-plugin/pkg/utils"
 )
 
 // OnePassword is a struct for working with a 1Password Connect backend
@@ -30,11 +31,14 @@ func (a *OnePasswordConnect) GetSecrets(path string, version string, annotations
 	vaultUUID := splits[1]
 	itemUUID := splits[3]
 
+	utils.VerboseToStdErr("OnePassword Connect getting item %s from vault %s", itemUUID, vaultUUID)
 	result, err := a.Client.GetItem(itemUUID, vaultUUID)
 	if err != nil {
 		return nil, err
 	}
 
+	utils.VerboseToStdErr("OnePassword Connect get secret response: %v", result)
+
 	data := make(map[string]interface{})
 
 	for _, field := range result.Fields {

pkg/backends/vault.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 
 	"github.com/argoproj-labs/argocd-vault-plugin/pkg/types"
+	"github.com/argoproj-labs/argocd-vault-plugin/pkg/utils"
 	"github.com/hashicorp/vault/api"
 )
 
@@ -47,17 +48,21 @@ func (v *Vault) GetSecrets(path string, version string, annotations map[string]s
 
 	// Vault KV-V1 doesn't support versioning so we only honor `version` if KV-V2 is used
 	if version != "" && kvVersion == "2" {
+		utils.VerboseToStdErr("Hashicorp Vault getting kv pairs from KV-V2 path %s at version %s", path, version)
 		secret, err = v.VaultClient.Logical().ReadWithData(path, map[string][]string{
 			"version": {version},
 		})
 	} else {
+		utils.VerboseToStdErr("Hashicorp Vault getting kv pairs from KV-V1 path %s", path)
 		secret, err = v.VaultClient.Logical().Read(path)
 	}
 
 	if err != nil {
 		return nil, err
 	}
 
+	utils.VerboseToStdErr("Hashicorp Vault get kv pairs response: %v", secret)
+
 	if secret == nil {
 		// Do not mention `version` in error message when it's not honored (KV-V1)
 		if version == "" || kvVersion == "1" {

pkg/backends/yandexcloudlockbox.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/argoproj-labs/argocd-vault-plugin/pkg/utils"
 	"github.com/yandex-cloud/go-genproto/yandex/cloud/lockbox/v1"
 )
 
@@ -34,11 +35,14 @@ func (ycl *YandexCloudLockbox) GetSecrets(secretID string, version string, _ map
 		req.SetVersionId(version)
 	}
 
+	utils.VerboseToStdErr("Yandex Cloud Lockbox getting secret %s at version %s", secretID, version)
 	resp, err := ycl.client.Get(context.Background(), req)
 	if err != nil {
 		return nil, err
 	}
 
+	utils.VerboseToStdErr("Yandex Cloud Lockbox get secret response %v", resp)
+
 	result := make(map[string]interface{}, len(resp.GetEntries()))
 	for _, v := range resp.GetEntries() {
 		result[v.GetKey()] = v.GetTextValue()