feat[ibmsm]: Secret group name resolution and simpler key interpolation (#609)

* feat[ibmsm]: Secret group name resolution and simpler key interpolation

Signed-off-by: Jarek Gawor <[email protected]>

* add docs

Signed-off-by: Jarek Gawor <[email protected]>

* address comments

Signed-off-by: Jarek Gawor <[email protected]>

---------

Signed-off-by: Jarek Gawor <[email protected]>

Author: jgawor

docs/backends.md

@@ -170,14 +170,28 @@ data:
 **Note**: Only Vault KV-V2 backends support versioning. Versions specified with a KV-V1 Vault will be ignored and the latest version will be retrieved.
 
 ### IBM Cloud Secrets Manager
-For IBM Cloud Secret Manager we only support using IAM authentication at this time.
 
-We support all types of secrets that can be retrieved from IBM Cloud Secret Manager. Please note:
+The path for IBM Cloud Secret Manager secrets can be specified in two ways:
+1. `ibmcloud/<SECRET_TYPE>/secrets/groups/<GROUP>#<SECRET_NAME>`, or
+2. `ibmcloud/<SECRET_TYPE>/secrets/groups/<GROUP>/<SECRET_NAME>#<SECRET_KEY>`
 
-- Secrets that are JSON data (i.e, non `arbitrary` secrets or an `arbitrary` secret with JSON `payload`) can have the select keys (i.e, the `username` in a `username_password` type secret) interpolated with the [jsonPath](./howitworks.md#jsonPath) modifier. Not all keys are available for extraction with `jsonPath`. Refer to the [IBM Cloud Secret Manager API docs](https://cloud.ibm.com/apidocs/secrets-manager#get-secret) for more details
+Where:
+* `<SECRET_TYPE>` can be one of the following: `arbitrary`, `iam_credentials`, `imported_cert`, `kv`, `private_cert`, `public_cert`, or `username_password`.
+* `<GROUP>` can be a secret group ID or name.
+* `<SECRET_NAME>` is the name of the secret.
+* `<SECRET_KEY>` is the key name within the secret. Specifically, the following keys are available for extraction:
+  * `api_key` for the `iam_credentials` secret type
+  * `username` and `password` for the `username_password` secret type
+  * `certificate`, `private_key`, `intermediate` for the `imported_cert` or `public_cert` secret types
+  * `certificate`, `private_key`, `issuing_ca`, `ca_chain` for the `private_cert` secret type
+  * any key of the `kv` secret type
+  `<SECRET_KEY>` is not supported for the `arbitrary` secret type.
 
-##### IAM Authentication
-For IAM Authentication, these are the required parameters:
+When using the first path syntax, secrets that are JSON data (i.e, non `arbitrary` secrets or an `arbitrary` secret with JSON `payload`) can have select keys (listed under `<SECRET_KEY>` above) interpolated with the [jsonPath](./howitworks.md#jsonPath) modifier. With the second path syntax, the interpolation with the `jsonPath` modifier is not necessary. 
+
+##### Authentication
+
+IAM authentication is only supported at this time. The following parameters are required for IAM authentication:
 ```
 AVP_IBM_INSTANCE_URL or VAULT_ADDR: Your IBM Cloud Secret Manager Endpoint
 AVP_TYPE: ibmsecretsmanager
@@ -233,6 +247,25 @@ stringData:
     <my-cert-secret | jsonPath {.private_key}>
 ```
 
+###### Non-arbitrary secrets (alternative path syntax)
+
+```yaml
+kind: Secret
+apiVersion: v1
+metadata:
+  name: ibm-example
+  annotations:
+    avp.kubernetes.io/path: "ibmcloud/imported_cert/secrets/groups/myGroup/my-cert-secret"
+type: Opaque
+stringData:
+  PUBLIC_CRT: |
+    <certificate>
+  PRIVATE_KEY: |
+    <private_key>
+  USERNAME: <path:ibmcloud/username_password/secrets/groups/myGroup/basic-auth#username>
+  PASSWORD: <path:ibmcloud/username_password/secrets/groups/myGroup/basic-auth#password>
+```
+
 ### AWS Secrets Manager
 
 ##### AWS Authentication

pkg/backends/ibmsecretsmanager.go

@@ -11,7 +11,8 @@ import (
 	"github.com/argoproj-labs/argocd-vault-plugin/pkg/utils"
 )
 
-var IBMPath, _ = regexp.Compile(`ibmcloud/(?P<type>.+)/secrets/groups/(?P<groupId>.+)`)
+var IBMPath, _ = regexp.Compile(`ibmcloud/(?P<type>.+)/secrets/groups/(?P<groupId>[^/\n]+)(/(?P<secretName>.+))?`)
+var GroupId, _ = regexp.Compile(`[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}`)
 
 // IBMSecretMetadata wraps the SecretMetadataIntf provided by the SDK
 // It provides a generic method for accessing the metadata regardless of secret type
@@ -282,6 +283,7 @@ type IBMSecretsManagerClient interface {
 	ListSecrets(listAllSecretsOptions *ibmsm.ListSecretsOptions) (result *ibmsm.SecretMetadataPaginatedCollection, response *core.DetailedResponse, err error)
 	GetSecret(getSecretOptions *ibmsm.GetSecretOptions) (result ibmsm.SecretIntf, response *core.DetailedResponse, err error)
 	GetSecretVersion(getSecretOptions *ibmsm.GetSecretVersionOptions) (result ibmsm.SecretVersionIntf, response *core.DetailedResponse, err error)
+	ListSecretGroups(listSecretGroupsOptions *ibmsm.ListSecretGroupsOptions) (result *ibmsm.SecretGroupCollection, response *core.DetailedResponse, err error)
 }
 
 // Used as the key into the several caches for IBM SM API calls
@@ -314,6 +316,8 @@ type IBMSecretsManager struct {
 	// Keeps track of whether GetSecrets has been called for a given group and secret type
 	// Only read/written to by the main goroutine, no synchronized access needed
 	retrievedAllSecrets map[cacheKey]bool
+
+	secretGroups map[string]string
 }
 
 // NewIBMSecretsManagerBackend initializes a new IBM Secret Manager backend
@@ -323,17 +327,18 @@ func NewIBMSecretsManagerBackend(client IBMSecretsManagerClient) *IBMSecretsMana
 		listAllSecretsCache: make(map[cacheKey]map[string]*IBMSecretMetadata),
 		getSecretsCache:     make(map[cacheKey]map[string]interface{}),
 		retrievedAllSecrets: make(map[cacheKey]bool),
+		secretGroups:        make(map[string]string),
 	}
 	return ibmSecretsManager
 }
 
 // parsePath returns the groupId, secretType represented by the path
-func parsePath(path string) (string, string, error) {
+func parsePath(path string) (string, string, string, error) {
 	matches := IBMPath.FindStringSubmatch(path)
 	if len(matches) == 0 {
-		return "", "", fmt.Errorf("Path is not in the correct format (ibmcloud/$TYPE/secrets/groups/$GROUP_ID) for IBM Secrets Manager: %s", path)
+		return "", "", "", fmt.Errorf("Path is not in the correct format (ibmcloud/$TYPE/secrets/groups/$GROUP_ID) for IBM Secrets Manager: %s", path)
 	}
-	return matches[IBMPath.SubexpIndex("type")], matches[IBMPath.SubexpIndex("groupId")], nil
+	return matches[IBMPath.SubexpIndex("type")], matches[IBMPath.SubexpIndex("groupId")], matches[IBMPath.SubexpIndex("secretName")], nil
 }
 
 func (i *IBMSecretsManager) readSecretFromCache(groupId, secretType, secretName string) interface{} {
@@ -536,18 +541,64 @@ func storeSecret(secrets *map[string]interface{}, result map[string]interface{})
 	return nil
 }
 
+func (i *IBMSecretsManager) resolveGroup(group string) (string, error) {
+	// no need to resolve default or groupIds
+	if group == "default" || GroupId.MatchString(group) {
+		return group, nil
+	}
+
+	// list groups
+	if len(i.secretGroups) == 0 {
+		opts := &ibmsm.ListSecretGroupsOptions{}
+		secretGroupCollection, _, err := i.Client.ListSecretGroups(opts)
+		if err != nil {
+			return "", fmt.Errorf("Could not list secret groups: %s", err)
+		}
+		for _, group := range secretGroupCollection.SecretGroups {
+			i.secretGroups[*group.Name] = *group.ID
+		}
+	}
+
+	// look up group id for group name
+	groupId := i.secretGroups[group]
+	if groupId == "" {
+		return "", fmt.Errorf("No such secret group %s", group)
+	} else {
+		return groupId, nil
+	}
+}
+
 // GetSecrets returns the data for all secrets of a specific type of a group in IBM Secrets Manager
 func (i *IBMSecretsManager) GetSecrets(path string, version string, annotations map[string]string) (map[string]interface{}, error) {
-	secretType, groupId, err := parsePath(path)
+	secretType, group, secretName, err := parsePath(path)
+	if err != nil {
+		return nil, fmt.Errorf("Path is not in the correct format (ibmcloud/$TYPE/secrets/groups/$GROUP) for IBM Secrets Manager: %s", path)
+	}
+	if secretType == "arbitrary" && secretName != "" {
+		return nil, fmt.Errorf("The 'ibmcloud/$TYPE/secrets/groups/$GROUP/$SECRET' path format is not supported for arbitrary secrets: %s", path)
+	}
+
+	groupId, err := i.resolveGroup(group)
 	if err != nil {
-		return nil, fmt.Errorf("Path is not in the correct format (ibmcloud/$TYPE/secrets/groups/$GROUP_ID) for IBM Secrets Manager: %s", path)
+		return nil, err
 	}
+
 	ckey := cacheKey{groupId, secretType}
 
 	// Bypass the cache when explicit version is requested
 	// Otherwise, use it if applicable
 	if version == "" && i.retrievedAllSecrets[ckey] {
-		return i.getSecretsCache[ckey], nil
+		secrets := i.getSecretsCache[ckey]
+		if secretName != "" {
+			secretData, ok := secrets[secretName].(map[string]interface{})
+			if ok {
+				return secretData, nil
+			} else {
+				return nil, nil
+			}
+		} else {
+			return secrets, nil
+		}
 	}
 
 	// So we query the group to enumerate the secret ids, and retrieve each one to return a complete map of them
@@ -598,22 +649,57 @@ func (i *IBMSecretsManager) GetSecrets(path string, version string, annotations
 
 	i.retrievedAllSecrets[ckey] = true
 
-	return secrets, nil
+	if secretName != "" {
+		secretData, ok := secrets[secretName].(map[string]interface{})
+		if ok {
+			return secretData, nil
+		} else {
+			return nil, nil
+		}
+	} else {
+		return secrets, nil
+	}
 }
 
 // GetIndividualSecret will get the specific secret (placeholder) from the SM backend
 // This requires listing the secrets of the group to obtain the id, and then using that to grab the one secret's payload
-func (i *IBMSecretsManager) GetIndividualSecret(kvpath, secretName, version string, annotations map[string]string) (interface{}, error) {
-	secretType, groupId, err := parsePath(kvpath)
+func (i *IBMSecretsManager) GetIndividualSecret(kvpath, secretRef, version string, annotations map[string]string) (interface{}, error) {
+	secretType, group, secretName, err := parsePath(kvpath)
 	if err != nil {
-		return nil, fmt.Errorf("Path is not in the correct format (ibmcloud/$TYPE/secrets/groups/$GROUP_ID) for IBM Secrets Manager: %s", kvpath)
+		return nil, fmt.Errorf("Path is not in the correct format (ibmcloud/$TYPE/secrets/groups/$GROUP) for IBM Secrets Manager: %s", kvpath)
+	}
+	if secretType == "arbitrary" && secretName != "" {
+		return nil, fmt.Errorf("The 'ibmcloud/$TYPE/secrets/groups/$GROUP/$SECRET' path format is not supported for arbitrary secrets: %s", kvpath)
 	}
+
+	groupId, err := i.resolveGroup(group)
+	if err != nil {
+		return nil, err
+	}
+
+	var secretKey string
+	if secretName == "" {
+		secretName = secretRef
+	} else {
+		secretKey = secretRef
+	}
+
 	ckey := cacheKey{groupId, secretType}
 
 	// Bypass the cache when explicit version is requested
 	// If we have already retrieved all the secrets for the requested secret's group and type, we have a cache hit
 	if version == "" && i.retrievedAllSecrets[ckey] {
-		return i.getSecretsCache[ckey][secretName], nil
+		secretData := i.getSecretsCache[ckey][secretName]
+		if secretKey != "" {
+			secretValue, ok := secretData.(map[string]interface{})
+			if ok {
+				return secretValue[secretKey], nil
+			} else {
+				return nil, nil
+			}
+		} else {
+			return secretData, nil
+		}
 	}
 
 	// Grab the *ibmsm.SecretMetadata corresponding to the secret
@@ -644,5 +730,15 @@ func (i *IBMSecretsManager) GetIndividualSecret(kvpath, secretName, version stri
 		return nil, err
 	}
 
-	return secrets[secretName], nil
+	secretData := secrets[secretName]
+	if secretKey != "" {
+		secretValue, ok := secretData.(map[string]interface{})
+		if ok {
+			return secretValue[secretKey], nil
+		} else {
+			return nil, nil
+		}
+	} else {
+		return secretData, nil
+	}
 }