Skip to content

Commit 341efcb

Browse files
julienduchesnejulienduchesne
authored
Config Generation: Add type and name filter (#1590)
Allows generating only a subset of resources, or all resources containing a certain string in their ID (or a combination of that)
1 parent 6cb8900 commit 341efcb

File tree

9 files changed

+210
-39
lines changed

9 files changed

+210
-39
lines changed

‎cmd/generate/main.go

+12
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,17 @@ func run() error {
5757
EnvVars: []string{"TFGEN_TERRAFORM_PROVIDER_VERSION"},
5858
Value: version,
5959
},
60+
&cli.StringSliceFlag{
61+
Name: "include-resources",
62+
Usage: `List of resources to include in the "resourceType.resourceName" format. If not set, all resources will be included
63+
This supports a glob format. Examples:
64+
* Generate all dashboards and folders: --resource-names 'grafana_dashboard.*' --resource-names 'grafana_folder.*'
65+
* Generate all resources with "hello" in their ID (this is usually the resource UIDs): --resource-names '*.*hello*'
66+
* Generate all resources (same as default behaviour): --resource-names '*.*'
67+
`,
68+
EnvVars: []string{"TFGEN_INCLUDE_RESOURCES"},
69+
Required: false,
70+
},
6071

6172
// Grafana OSS flags
6273
&cli.StringFlag{
@@ -130,6 +141,7 @@ func parseFlags(ctx *cli.Context) (*generate.Config, error) {
130141
CreateStackServiceAccount: ctx.Bool("cloud-create-stack-service-account"),
131142
StackServiceAccountName: ctx.String("cloud-stack-service-account-name"),
132143
},
144+
IncludeResources: ctx.StringSlice("include-resources"),
133145
}
134146

135147
if config.ProviderVersion == "" {

‎pkg/generate/cloud.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ func generateCloudResources(ctx context.Context, cfg *Config) ([]stack, error) {
6666
}
6767

6868
data := cloud.NewListerData(cfg.Cloud.Org)
69-
if err := generateImportBlocks(ctx, client, data, cloud.Resources, cfg.OutputDir, "cloud"); err != nil {
69+
if err := generateImportBlocks(ctx, client, data, cloud.Resources, cfg.OutputDir, "cloud", cfg.IncludeResources); err != nil {
7070
return nil, err
7171
}
7272

‎pkg/generate/config.go

+7-1
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,13 @@ type CloudConfig struct {
2323
}
2424

2525
type Config struct {
26-
OutputDir string
26+
// IncludeResources is a list of patterns to filter resources by.
27+
// If a resource name matches any of the patterns, it will be included in the output.
28+
// Patterns are in the form of `resourceType.resourceName` and support * as a wildcard.
29+
IncludeResources []string
30+
// OutputDir is the directory to write the generated files to.
31+
OutputDir string
32+
// Clobber will overwrite existing files in the output directory.
2733
Clobber bool
2834
Format OutputFormat
2935
ProviderVersion string

‎pkg/generate/generate.go

+71-8
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ func Generate(ctx context.Context, cfg *Config) error {
2828
return fmt.Errorf("failed to delete %s: %s", cfg.OutputDir, err)
2929
}
3030
} else if err == nil && !cfg.Clobber {
31-
return fmt.Errorf("output dir %q already exists. Use --clobber to delete it", cfg.OutputDir)
31+
return fmt.Errorf("output dir %q already exists. Use the clobber option to delete it", cfg.OutputDir)
3232
}
3333

3434
log.Printf("Generating resources to %s", cfg.OutputDir)
@@ -60,14 +60,14 @@ func Generate(ctx context.Context, cfg *Config) error {
6060
}
6161

6262
for _, stack := range stacks {
63-
if err := generateGrafanaResources(ctx, stack.managementKey, stack.url, "stack-"+stack.slug, false, cfg.OutputDir, stack.smURL, stack.smToken); err != nil {
63+
if err := generateGrafanaResources(ctx, stack.managementKey, stack.url, "stack-"+stack.slug, false, cfg.OutputDir, stack.smURL, stack.smToken, cfg.IncludeResources); err != nil {
6464
return err
6565
}
6666
}
6767
}
6868

6969
if cfg.Grafana != nil {
70-
if err := generateGrafanaResources(ctx, cfg.Grafana.Auth, cfg.Grafana.URL, "", true, cfg.OutputDir, "", ""); err != nil {
70+
if err := generateGrafanaResources(ctx, cfg.Grafana.Auth, cfg.Grafana.URL, "", true, cfg.OutputDir, "", "", cfg.IncludeResources); err != nil {
7171
return err
7272
}
7373
}
@@ -82,7 +82,7 @@ func Generate(ctx context.Context, cfg *Config) error {
8282
return nil
8383
}
8484

85-
func generateImportBlocks(ctx context.Context, client *common.Client, listerData any, resources []*common.Resource, outPath, provider string) error {
85+
func generateImportBlocks(ctx context.Context, client *common.Client, listerData any, resources []*common.Resource, outPath, provider string, includedResources []string) error {
8686
generatedFilename := func(suffix string) string {
8787
if provider == "" {
8888
return filepath.Join(outPath, suffix)
@@ -91,6 +91,11 @@ func generateImportBlocks(ctx context.Context, client *common.Client, listerData
9191
return filepath.Join(outPath, provider+"-"+suffix)
9292
}
9393

94+
resources, err := filterResources(resources, includedResources)
95+
if err != nil {
96+
return err
97+
}
98+
9499
// Generate HCL blocks in parallel with a wait group
95100
wg := sync.WaitGroup{}
96101
wg.Add(len(resources))
@@ -129,22 +134,34 @@ func generateImportBlocks(ctx context.Context, client *common.Client, listerData
129134
// to = aws_iot_thing.bar
130135
// id = "foo"
131136
// }
132-
blocks := make([]*hclwrite.Block, len(ids))
133-
for i, id := range ids {
137+
var blocks []*hclwrite.Block
138+
for _, id := range ids {
134139
cleanedID := allowedTerraformChars.ReplaceAllString(id, "_")
135140
if provider != "cloud" {
136141
cleanedID = strings.ReplaceAll(provider, "-", "_") + "_" + cleanedID
137142
}
138143

144+
matched, err := filterResourceByName(resource.Name, cleanedID, includedResources)
145+
if err != nil {
146+
wg.Done()
147+
results <- result{
148+
resource: resource,
149+
err: err,
150+
}
151+
return
152+
}
153+
if !matched {
154+
continue
155+
}
156+
139157
b := hclwrite.NewBlock("import", nil)
140158
b.Body().SetAttributeTraversal("to", traversal(resource.Name, cleanedID))
141159
b.Body().SetAttributeValue("id", cty.StringVal(id))
142160
if provider != "" {
143161
b.Body().SetAttributeTraversal("provider", traversal("grafana", provider))
144162
}
145163

146-
blocks[i] = b
147-
// TODO: Match and update existing import blocks
164+
blocks = append(blocks, b)
148165
}
149166

150167
wg.Done()
@@ -187,3 +204,49 @@ func generateImportBlocks(ctx context.Context, client *common.Client, listerData
187204

188205
return sortResourcesFile(generatedFilename("resources.tf"))
189206
}
207+
208+
func filterResources(resources []*common.Resource, includedResources []string) ([]*common.Resource, error) {
209+
if len(includedResources) == 0 {
210+
return resources, nil
211+
}
212+
213+
filteredResources := []*common.Resource{}
214+
allowedResourceTypes := []string{}
215+
for _, included := range includedResources {
216+
if !strings.Contains(included, ".") {
217+
return nil, fmt.Errorf("included resource %q is not in the format <type>.<name>", included)
218+
}
219+
allowedResourceTypes = append(allowedResourceTypes, strings.Split(included, ".")[0])
220+
}
221+
222+
for _, resource := range resources {
223+
for _, allowedResourceType := range allowedResourceTypes {
224+
matched, err := filepath.Match(allowedResourceType, resource.Name)
225+
if err != nil {
226+
return nil, err
227+
}
228+
if matched {
229+
filteredResources = append(filteredResources, resource)
230+
break
231+
}
232+
}
233+
}
234+
return filteredResources, nil
235+
}
236+
237+
func filterResourceByName(resourceType, resourceName string, includedResources []string) (bool, error) {
238+
if len(includedResources) == 0 {
239+
return true, nil
240+
}
241+
242+
for _, included := range includedResources {
243+
matched, err := filepath.Match(included, resourceType+"."+resourceName)
244+
if err != nil {
245+
return false, err
246+
}
247+
if matched {
248+
return true, nil
249+
}
250+
}
251+
return false, nil
252+
}

‎pkg/generate/generate_test.go

+88-27
Original file line numberDiff line numberDiff line change
@@ -16,38 +16,99 @@ import (
1616
"golang.org/x/exp/slices"
1717
)
1818

19-
func TestAccGenerate_Dashboard(t *testing.T) {
19+
func TestAccGenerate(t *testing.T) {
2020
testutils.CheckOSSTestsEnabled(t)
2121

22-
resource.Test(t, resource.TestCase{
23-
ProtoV5ProviderFactories: testutils.ProtoV5ProviderFactories,
24-
Steps: []resource.TestStep{
25-
{
26-
Config: testutils.TestAccExample(t, "resources/grafana_dashboard/resource.tf"),
27-
Check: func(s *terraform.State) error {
28-
tempDir := t.TempDir()
29-
config := generate.Config{
30-
OutputDir: tempDir,
31-
Clobber: true,
32-
Format: generate.OutputFormatHCL,
33-
ProviderVersion: "v3.0.0",
34-
Grafana: &generate.GrafanaConfig{
35-
URL: "http://localhost:3000",
36-
Auth: "admin:admin",
37-
},
38-
}
22+
cases := []struct {
23+
name string
24+
config string
25+
generateConfig func(cfg *generate.Config)
26+
check func(t *testing.T, tempDir string)
27+
}{
28+
{
29+
name: "dashboard",
30+
config: testutils.TestAccExample(t, "resources/grafana_dashboard/resource.tf"),
31+
check: func(t *testing.T, tempDir string) {
32+
assertFiles(t, tempDir, "testdata/generate/dashboard-expected", "", []string{
33+
".terraform",
34+
".terraform.lock.hcl",
35+
})
36+
},
37+
},
38+
{
39+
name: "dashboard-filter-strict",
40+
config: testutils.TestAccExample(t, "resources/grafana_dashboard/resource.tf"),
41+
generateConfig: func(cfg *generate.Config) {
42+
cfg.IncludeResources = []string{"grafana_dashboard._1_my-dashboard-uid"}
43+
},
44+
check: func(t *testing.T, tempDir string) {
45+
assertFiles(t, tempDir, "testdata/generate/dashboard-filtered", "", []string{
46+
".terraform",
47+
".terraform.lock.hcl",
48+
})
49+
},
50+
},
51+
{
52+
name: "dashboard-filter-wildcard-on-resource-type",
53+
config: testutils.TestAccExample(t, "resources/grafana_dashboard/resource.tf"),
54+
generateConfig: func(cfg *generate.Config) {
55+
cfg.IncludeResources = []string{"*._1_my-dashboard-uid"}
56+
},
57+
check: func(t *testing.T, tempDir string) {
58+
assertFiles(t, tempDir, "testdata/generate/dashboard-filtered", "", []string{
59+
".terraform",
60+
".terraform.lock.hcl",
61+
})
62+
},
63+
},
64+
{
65+
name: "dashboard-filter-wildcard-on-resource-name",
66+
config: testutils.TestAccExample(t, "resources/grafana_dashboard/resource.tf"),
67+
generateConfig: func(cfg *generate.Config) {
68+
cfg.IncludeResources = []string{"grafana_dashboard.*"}
69+
},
70+
check: func(t *testing.T, tempDir string) {
71+
assertFiles(t, tempDir, "testdata/generate/dashboard-filtered", "", []string{
72+
".terraform",
73+
".terraform.lock.hcl",
74+
})
75+
},
76+
},
77+
}
78+
79+
for _, tc := range cases {
80+
t.Run(tc.name, func(t *testing.T) {
81+
resource.Test(t, resource.TestCase{
82+
ProtoV5ProviderFactories: testutils.ProtoV5ProviderFactories,
83+
Steps: []resource.TestStep{
84+
{
85+
Config: tc.config,
86+
Check: func(s *terraform.State) error {
87+
tempDir := t.TempDir()
88+
config := generate.Config{
89+
OutputDir: tempDir,
90+
Clobber: true,
91+
Format: generate.OutputFormatHCL,
92+
ProviderVersion: "v3.0.0",
93+
Grafana: &generate.GrafanaConfig{
94+
URL: "http://localhost:3000",
95+
Auth: "admin:admin",
96+
},
97+
}
98+
if tc.generateConfig != nil {
99+
tc.generateConfig(&config)
100+
}
39101

40-
require.NoError(t, generate.Generate(context.Background(), &config))
41-
assertFiles(t, tempDir, "testdata/generate/dashboard-expected", "", []string{
42-
".terraform",
43-
".terraform.lock.hcl",
44-
})
102+
require.NoError(t, generate.Generate(context.Background(), &config))
103+
tc.check(t, tempDir)
45104

46-
return nil
105+
return nil
106+
},
107+
},
47108
},
48-
},
49-
},
50-
})
109+
})
110+
})
111+
}
51112
}
52113

53114
// assertFiles checks that all files in the "expectedFilesDir" directory match the files in the "gotFilesDir" directory.

‎pkg/generate/grafana.go

+3-2
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,8 @@ import (
1515
"github.com/zclconf/go-cty/cty"
1616
)
1717

18-
func generateGrafanaResources(ctx context.Context, auth, url, stackName string, genProvider bool, outPath, smURL, smToken string) error {
18+
// TODO: Refactor this sig
19+
func generateGrafanaResources(ctx context.Context, auth, url, stackName string, genProvider bool, outPath, smURL, smToken string, includedResources []string) error {
1920
generatedFilename := func(suffix string) string {
2021
if stackName == "" {
2122
return filepath.Join(outPath, suffix)
@@ -65,7 +66,7 @@ func generateGrafanaResources(ctx context.Context, auth, url, stackName string,
6566
resources = append(resources, machinelearning.Resources...)
6667
resources = append(resources, syntheticmonitoring.Resources...)
6768
}
68-
if err := generateImportBlocks(ctx, client, listerData, resources, outPath, stackName); err != nil {
69+
if err := generateImportBlocks(ctx, client, listerData, resources, outPath, stackName, includedResources); err != nil {
6970
return err
7071
}
7172

‎pkg/generate/testdata/generate/dashboard-filtered/imports.tf

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
import {
2+
to = grafana_dashboard._1_my-dashboard-uid
3+
id = "1:my-dashboard-uid"
4+
}

‎pkg/generate/testdata/generate/dashboard-filtered/provider.tf

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
terraform {
2+
required_providers {
3+
grafana = {
4+
source = "grafana/grafana"
5+
version = "3.0.0"
6+
}
7+
}
8+
}
9+
10+
provider "grafana" {
11+
url = "http://localhost:3000"
12+
auth = "admin:admin"
13+
}

‎pkg/generate/testdata/generate/dashboard-filtered/resources.tf

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
# __generated__ by Terraform
2+
# Please review these resources and move them into your main configuration files.
3+
4+
# __generated__ by Terraform from "1:my-dashboard-uid"
5+
resource "grafana_dashboard" "_1_my-dashboard-uid" {
6+
config_json = jsonencode({
7+
title = "My Dashboard"
8+
uid = "my-dashboard-uid"
9+
})
10+
folder = "my-folder-uid"
11+
}

0 commit comments

Comments
 (0)