Skip to content

Commit e83dd62

Browse files
julienduchesnejulienduchesne
authored
Config Generation: Generate resources with sensitive attributes (#1715)
* Config Generation: Generate resources with sensitive attributes This is a fairly complex one but it's the last missing piece to full config generation The `terraform plan -generate-config-out` command is overly aggressive in redacting sensitive values Whenever a child attribute of a block is sensitive, the whole block is redacted, meaning that the generated resources are invalid if the block was required In this PR: - Add listers for `grafana_user` and `grafana_contact_point` - Replace nulled sensitive attributes with a placeholder (example: `password` in `grafana_user`_) - Generate "sensitive" blocks by making all attribute non-sensitive. This is a bit hacky but it's integrated well enough and I haven't found a better way (tried lots of things) * Fix integration test * Fix lint
1 parent 4bc50ca commit e83dd62

20 files changed

+380
-71
lines changed

‎internal/resources/grafana/resource_alerting_contact_point.go

+30-40
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"time"
99

1010
"github.com/go-openapi/runtime"
11+
goapi "github.com/grafana/grafana-openapi-client-go/client"
1112
"github.com/grafana/grafana-openapi-client-go/client/provisioning"
1213
"github.com/grafana/grafana-openapi-client-go/models"
1314
"github.com/hashicorp/terraform-plugin-sdk/v2/diag"
@@ -103,48 +104,37 @@ This resource requires Grafana 9.1.0 or later.
103104
"grafana_contact_point",
104105
orgResourceIDString("name"),
105106
resource,
106-
)
107+
).WithLister(listerFunctionOrgResource(listContactPoints))
107108
}
108109

109-
// TODO: Fix contact points lister. Terraform doesn't read any of the sensitive fields (or their container)
110-
// It outputs an empty `email {}` block for example, which is not valid.
111-
// func listContactPoints(ctx context.Context, client *goapi.GrafanaHTTPAPI, data *ListerData) ([]string, error) {
112-
// orgIDs, err := data.OrgIDs(client)
113-
// if err != nil {
114-
// return nil, err
115-
// }
116-
117-
// idMap := map[string]bool{}
118-
// for _, orgID := range orgIDs {
119-
// client = client.Clone().WithOrgID(orgID)
120-
121-
// // Retry if the API returns 500 because it may be that the alertmanager is not ready in the org yet.
122-
// // The alertmanager is provisioned asynchronously when the org is created.
123-
// if err := retry.RetryContext(ctx, 2*time.Minute, func() *retry.RetryError {
124-
// resp, err := client.Provisioning.GetContactpoints(provisioning.NewGetContactpointsParams())
125-
// if err != nil {
126-
// if orgID > 1 && (err.(*runtime.APIError).IsCode(500) || err.(*runtime.APIError).IsCode(403)) {
127-
// return retry.RetryableError(err)
128-
// }
129-
// return retry.NonRetryableError(err)
130-
// }
131-
132-
// for _, contactPoint := range resp.Payload {
133-
// idMap[MakeOrgResourceID(orgID, contactPoint.Name)] = true
134-
// }
135-
// return nil
136-
// }); err != nil {
137-
// return nil, err
138-
// }
139-
// }
140-
141-
// var ids []string
142-
// for id := range idMap {
143-
// ids = append(ids, id)
144-
// }
145-
146-
// return ids, nil
147-
// }
110+
func listContactPoints(ctx context.Context, client *goapi.GrafanaHTTPAPI, orgID int64) ([]string, error) {
111+
idMap := map[string]bool{}
112+
// Retry if the API returns 500 because it may be that the alertmanager is not ready in the org yet.
113+
// The alertmanager is provisioned asynchronously when the org is created.
114+
if err := retry.RetryContext(ctx, 2*time.Minute, func() *retry.RetryError {
115+
resp, err := client.Provisioning.GetContactpoints(provisioning.NewGetContactpointsParams())
116+
if err != nil {
117+
if orgID > 1 && (err.(*runtime.APIError).IsCode(500) || err.(*runtime.APIError).IsCode(403)) {
118+
return retry.RetryableError(err)
119+
}
120+
return retry.NonRetryableError(err)
121+
}
122+
123+
for _, contactPoint := range resp.Payload {
124+
idMap[MakeOrgResourceID(orgID, contactPoint.Name)] = true
125+
}
126+
return nil
127+
}); err != nil {
128+
return nil, err
129+
}
130+
131+
var ids []string
132+
for id := range idMap {
133+
ids = append(ids, id)
134+
}
135+
136+
return ids, nil
137+
}
148138

149139
func readContactPoint(ctx context.Context, data *schema.ResourceData, meta interface{}) diag.Diagnostics {
150140
client, orgID, name := OAPIClientFromExistingOrgResource(meta, data.Id())

‎internal/resources/grafana/resource_user.go

+26-26
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ import (
55
"strconv"
66
"strings"
77

8+
goapi "github.com/grafana/grafana-openapi-client-go/client"
9+
"github.com/grafana/grafana-openapi-client-go/client/users"
810
"github.com/grafana/grafana-openapi-client-go/models"
911
"github.com/grafana/terraform-provider-grafana/v3/internal/common"
1012
"github.com/hashicorp/terraform-plugin-sdk/v2/diag"
@@ -75,34 +77,32 @@ You must use basic auth.
7577
"grafana_user",
7678
resourceUserID,
7779
schema,
78-
)
79-
// ).WithLister(listerFunction(listUsers))
80+
).WithLister(listerFunction(listUsers))
8081
}
8182

82-
// TODO: Fix issues with password
83-
// func listUsers(ctx context.Context, client *goapi.GrafanaHTTPAPI, data *ListerData) ([]string, error) {
84-
// var ids []string
85-
// var page int64 = 1
86-
// for {
87-
// params := users.NewSearchUsersParams().WithPage(&page)
88-
// resp, err := client.Users.SearchUsers(params)
89-
// if err != nil {
90-
// return nil, err
91-
// }
92-
93-
// for _, user := range resp.Payload {
94-
// ids = append(ids, strconv.FormatInt(user.ID, 10))
95-
// }
96-
97-
// if len(resp.Payload) == 0 {
98-
// break
99-
// }
100-
101-
// page++
102-
// }
103-
104-
// return ids, nil
105-
// }
83+
func listUsers(ctx context.Context, client *goapi.GrafanaHTTPAPI, data *ListerData) ([]string, error) {
84+
var ids []string
85+
var page int64 = 1
86+
for {
87+
params := users.NewSearchUsersParams().WithPage(&page)
88+
resp, err := client.Users.SearchUsers(params)
89+
if err != nil {
90+
return nil, err
91+
}
92+
93+
for _, user := range resp.Payload {
94+
ids = append(ids, strconv.FormatInt(user.ID, 10))
95+
}
96+
97+
if len(resp.Payload) == 0 {
98+
break
99+
}
100+
101+
page++
102+
}
103+
104+
return ids, nil
105+
}
106106

107107
func CreateUser(ctx context.Context, d *schema.ResourceData, meta interface{}) diag.Diagnostics {
108108
client, err := OAPIGlobalClient(meta)

‎internal/testutils/provider.go

+6-1
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package testutils
22

33
import (
44
"context"
5+
"errors"
56
"fmt"
67
"os"
78
"path/filepath"
@@ -50,7 +51,11 @@ var (
5051
configureResp, err := server.ConfigureProvider(context.Background(), &tfprotov5.ConfigureProviderRequest{Config: &testDynamicValue})
5152
if err != nil || len(configureResp.Diagnostics) > 0 {
5253
if err == nil {
53-
err = fmt.Errorf("provider configuration failed: %v", configureResp.Diagnostics)
54+
errs := []error{}
55+
for _, diag := range configureResp.Diagnostics {
56+
errs = append(errs, fmt.Errorf("%s %s: %s", diag.Severity, diag.Summary, diag.Detail))
57+
}
58+
err = errors.Join(errs...)
5459
}
5560
return nil, fmt.Errorf("failed to configure provider: %v", err)
5661
}

‎pkg/generate/generate.go

+15-2
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414
"github.com/grafana/terraform-provider-grafana/v3/internal/common"
1515
"github.com/grafana/terraform-provider-grafana/v3/pkg/generate/postprocessing"
1616
"github.com/grafana/terraform-provider-grafana/v3/pkg/generate/utils"
17+
"github.com/grafana/terraform-provider-grafana/v3/pkg/provider"
1718
"github.com/hashicorp/hcl/v2/hclwrite"
1819
"github.com/hashicorp/terraform-exec/tfexec"
1920
"github.com/zclconf/go-cty/cty"
@@ -95,6 +96,14 @@ func Generate(ctx context.Context, cfg *Config) GenerationResult {
9596
return failuref("failed to create output directory %s: %s", cfg.OutputDir, err)
9697
}
9798

99+
// Enable "unsensitive" mode for the provider
100+
os.Setenv(provider.EnableGenerateEnvVar, "true")
101+
defer os.Unsetenv(provider.EnableGenerateEnvVar)
102+
if err := os.WriteFile(filepath.Join(cfg.OutputDir, provider.EnableGenerateMarkerFile), []byte("unsensitive!"), 0600); err != nil {
103+
return failuref("failed to write marker file: %w", err)
104+
}
105+
defer os.Remove(filepath.Join(cfg.OutputDir, provider.EnableGenerateMarkerFile))
106+
98107
// Generate provider installation block
99108
providerBlock := hclwrite.NewBlock("terraform", nil)
100109
requiredProvidersBlock := hclwrite.NewBlock("required_providers", nil)
@@ -104,7 +113,7 @@ func Generate(ctx context.Context, cfg *Config) GenerationResult {
104113
}))
105114
providerBlock.Body().AppendBlock(requiredProvidersBlock)
106115
if err := writeBlocks(filepath.Join(cfg.OutputDir, "provider.tf"), providerBlock); err != nil {
107-
log.Fatal(err)
116+
return failure(err)
108117
}
109118

110119
tf, err := setupTerraform(cfg)
@@ -304,7 +313,7 @@ func generateImportBlocks(ctx context.Context, client *common.Client, listerData
304313
return failure(err)
305314
}
306315
_, err = cfg.Terraform.Plan(ctx, tfexec.GenerateConfigOut(generatedFilename("resources.tf")))
307-
if err != nil {
316+
if err != nil && !strings.Contains(err.Error(), "Missing required argument") {
308317
// If resources.tf was created and is not empty, return the error as a "non-critical" error
309318
if stat, statErr := os.Stat(generatedFilename("resources.tf")); statErr == nil && stat.Size() > 0 {
310319
returnResult.Errors = append(returnResult.Errors, NonCriticalGenerationFailure{err})
@@ -313,6 +322,10 @@ func generateImportBlocks(ctx context.Context, client *common.Client, listerData
313322
}
314323
}
315324

325+
if err := postprocessing.ReplaceNullSensitiveAttributes(generatedFilename("resources.tf")); err != nil {
326+
return failure(err)
327+
}
328+
316329
if err := removeOrphanedImports(generatedFilename("imports.tf"), generatedFilename("resources.tf")); err != nil {
317330
return failure(err)
318331
}

‎pkg/generate/postprocessing/postprocessing_test.go

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
package postprocessing
2+
3+
import (
4+
"os"
5+
"path/filepath"
6+
"strings"
7+
"testing"
8+
9+
"github.com/stretchr/testify/require"
10+
)
11+
12+
func postprocessingTest(t *testing.T, testFile string, fn func(fpath string)) {
13+
t.Helper()
14+
15+
t.Run(testFile, func(t *testing.T) {
16+
goldenFilepath := strings.Replace(testFile, ".tf", ".golden.tf", 1)
17+
18+
// Copy the file to a temporary location
19+
tmpFilepath := filepath.Join(t.TempDir(), filepath.Base(testFile))
20+
file, err := os.ReadFile(testFile)
21+
require.NoError(t, err)
22+
require.NoError(t, os.WriteFile(tmpFilepath, file, 0600))
23+
24+
// Run the postprocessing function
25+
fn(tmpFilepath)
26+
27+
// Compare the file with the golden file
28+
got, err := os.ReadFile(tmpFilepath)
29+
require.NoError(t, err)
30+
want, err := os.ReadFile(goldenFilepath)
31+
require.NoError(t, err)
32+
33+
require.Equal(t, string(want), string(got))
34+
})
35+
}

‎pkg/generate/postprocessing/replace_null_sensitive.go

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
package postprocessing
2+
3+
import (
4+
"log"
5+
6+
"github.com/grafana/terraform-provider-grafana/v3/internal/common"
7+
"github.com/grafana/terraform-provider-grafana/v3/pkg/provider"
8+
"github.com/hashicorp/hcl/v2/hclwrite"
9+
"github.com/zclconf/go-cty/cty"
10+
)
11+
12+
func ReplaceNullSensitiveAttributes(fpath string) error {
13+
providerResources := map[string]*common.Resource{}
14+
for _, r := range provider.Resources() {
15+
providerResources[r.Name] = r
16+
}
17+
18+
return postprocessFile(fpath, func(file *hclwrite.File) error {
19+
for _, block := range file.Body().Blocks() {
20+
if block.Type() != "resource" {
21+
continue
22+
}
23+
24+
resourceType := block.Labels()[0]
25+
resourceInfo := providerResources[resourceType]
26+
resourceSchema := resourceInfo.Schema
27+
if resourceSchema == nil {
28+
// Plugin Framework schema not implemented because we have no resources with sensitive attributes in it yet
29+
log.Printf("resource %s doesn't use the legacy SDK", resourceType)
30+
continue
31+
}
32+
33+
for key := range block.Body().Attributes() {
34+
attrSchema := resourceSchema.Schema[key]
35+
if attrSchema == nil {
36+
// Attribute not found in schema
37+
continue
38+
}
39+
if attrSchema.Sensitive && attrSchema.Required {
40+
block.Body().SetAttributeValue(key, cty.StringVal("SENSITIVE_VALUE_TO_REPLACE"))
41+
}
42+
}
43+
}
44+
return nil
45+
})
46+
}

‎pkg/generate/postprocessing/replace_null_sensitive_test.go

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
package postprocessing
2+
3+
import "testing"
4+
5+
func TestReplaceNullSensitiveAttributes(t *testing.T) {
6+
for _, testFile := range []string{
7+
"testdata/replace-user-password.tf",
8+
} {
9+
postprocessingTest(t, testFile, func(fpath string) {
10+
ReplaceNullSensitiveAttributes(fpath)
11+
})
12+
}
13+
}

‎pkg/generate/postprocessing/testdata/replace-user-password.golden.tf

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
# __generated__ by Terraform
2+
resource "grafana_user" "_1" {
3+
email = "admin@localhost"
4+
is_admin = true
5+
login = "admin"
6+
name = null
7+
password = "SENSITIVE_VALUE_TO_REPLACE"
8+
}

‎pkg/generate/postprocessing/testdata/replace-user-password.tf

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
# __generated__ by Terraform
2+
resource "grafana_user" "_1" {
3+
email = "admin@localhost"
4+
is_admin = true
5+
login = "admin"
6+
name = null
7+
password = null
8+
}

‎pkg/generate/terraform.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ func setupTerraform(cfg *Config) (*tfexec.Terraform, error) {
7474

7575
err = tf.Init(context.Background(), initOptions...)
7676
if err != nil {
77-
return nil, fmt.Errorf("error running Init: %s", err)
77+
return nil, fmt.Errorf("error running Init: %w", err)
7878
}
7979

8080
return tf, nil

‎pkg/generate/testdata/generate/alerting-in-org/imports.tf

+15
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,18 @@
1+
import {
2+
to = grafana_contact_point._1_email_receiver
3+
id = "1:email receiver"
4+
}
5+
6+
import {
7+
to = grafana_contact_point._2_email_receiver
8+
id = "2:email receiver"
9+
}
10+
11+
import {
12+
to = grafana_contact_point._2_my-contact-point
13+
id = "2:my-contact-point"
14+
}
15+
116
import {
217
to = grafana_folder._2_alert-rule-folder
318
id = "2:alert-rule-folder"

0 commit comments

Comments
 (0)