Skip to content

Commit c254e8c

Browse files
julienduchesnejulienduchesne
committed
Config Generation: Add type and name filter
Allows generating only a subset of resources, or all resources containing a certain string in their ID (or a combination of that)
1 parent b88a3d6 commit c254e8c

File tree

10 files changed

+212
-39
lines changed

10 files changed

+212
-39
lines changed

‎cmd/generate/main.go

+12
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,17 @@ func run() error {
5757
EnvVars: []string{"TFGEN_TERRAFORM_PROVIDER_VERSION"},
5858
Value: version,
5959
},
60+
&cli.StringSliceFlag{
61+
Name: "include-resources",
62+
Usage: `List of resources to include in the "resourceType.resourceName" format. If not set, all resources will be included
63+
This supports a glob format. Examples:
64+
* Generate all dashboards and folders: --resource-names 'grafana_dashboard.*' --resource-names 'grafana_folder.*'
65+
* Generate all resources with "hello" in their ID (this is usually the resource UIDs): --resource-names '*.*hello*'
66+
* Generate all resources (same as default behaviour): --resource-names '*.*'
67+
`,
68+
EnvVars: []string{"TFGEN_INCLUDE_RESOURCES"},
69+
Required: false,
70+
},
6071

6172
// Grafana OSS flags
6273
&cli.StringFlag{
@@ -130,6 +141,7 @@ func parseFlags(ctx *cli.Context) (*generate.Config, error) {
130141
CreateStackServiceAccount: ctx.Bool("cloud-create-stack-service-account"),
131142
StackServiceAccountName: ctx.String("cloud-stack-service-account-name"),
132143
},
144+
IncludeResources: ctx.StringSlice("include-resources"),
133145
}
134146

135147
if config.ProviderVersion == "" {

‎pkg/generate/cloud.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ func generateCloudResources(ctx context.Context, cfg *Config) ([]stack, error) {
6666
}
6767

6868
data := cloud.NewListerData(cfg.Cloud.Org)
69-
if err := generateImportBlocks(ctx, client, data, cloud.Resources, cfg.OutputDir, "cloud"); err != nil {
69+
if err := generateImportBlocks(ctx, client, data, cloud.Resources, cfg.OutputDir, "cloud", cfg.IncludeResources); err != nil {
7070
return nil, err
7171
}
7272

‎pkg/generate/config.go

+7-1
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,13 @@ type CloudConfig struct {
2323
}
2424

2525
type Config struct {
26-
OutputDir string
26+
// IncludeResources is a list of patterns to filter resources by.
27+
// If a resource name matches any of the patterns, it will be included in the output.
28+
// Patterns are in the form of `resourceType.resourceName` and support * as a wildcard.
29+
IncludeResources []string
30+
// OutputDir is the directory to write the generated files to.
31+
OutputDir string
32+
// Clobber will overwrite existing files in the output directory.
2733
Clobber bool
2834
Format OutputFormat
2935
ProviderVersion string

‎pkg/generate/generate.go

+71-8
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ func Generate(ctx context.Context, cfg *Config) error {
3030
return fmt.Errorf("failed to delete %s: %s", cfg.OutputDir, err)
3131
}
3232
} else if err == nil && !cfg.Clobber {
33-
return fmt.Errorf("output dir %q already exists. Use --clobber to delete it", cfg.OutputDir)
33+
return fmt.Errorf("output dir %q already exists. Use the clobber option to delete it", cfg.OutputDir)
3434
}
3535

3636
log.Printf("Generating resources to %s", cfg.OutputDir)
@@ -62,7 +62,7 @@ func Generate(ctx context.Context, cfg *Config) error {
6262
}
6363

6464
for _, stack := range stacks {
65-
if err := generateGrafanaResources(ctx, stack.managementKey, stack.url, "stack-"+stack.slug, false, cfg.OutputDir, stack.smURL, stack.smToken); err != nil {
65+
if err := generateGrafanaResources(ctx, stack.managementKey, stack.url, "stack-"+stack.slug, false, cfg.OutputDir, stack.smURL, stack.smToken, cfg.IncludeResources); err != nil {
6666
return err
6767
}
6868
}
@@ -78,7 +78,7 @@ func Generate(ctx context.Context, cfg *Config) error {
7878
if net.ParseIP(stackName) != nil {
7979
stackName = "ip_" + strings.ReplaceAll(stackName, ".", "_")
8080
}
81-
if err := generateGrafanaResources(ctx, cfg.Grafana.Auth, cfg.Grafana.URL, stackName, true, cfg.OutputDir, "", ""); err != nil {
81+
if err := generateGrafanaResources(ctx, cfg.Grafana.Auth, cfg.Grafana.URL, stackName, true, cfg.OutputDir, "", "", cfg.IncludeResources); err != nil {
8282
return err
8383
}
8484
}
@@ -93,7 +93,12 @@ func Generate(ctx context.Context, cfg *Config) error {
9393
return nil
9494
}
9595

96-
func generateImportBlocks(ctx context.Context, client *common.Client, listerData any, resources []*common.Resource, outPath, provider string) error {
96+
func generateImportBlocks(ctx context.Context, client *common.Client, listerData any, resources []*common.Resource, outPath, provider string, includedResources []string) error {
97+
resources, err := filterResources(resources, includedResources)
98+
if err != nil {
99+
return err
100+
}
101+
97102
// Generate HCL blocks in parallel with a wait group
98103
wg := sync.WaitGroup{}
99104
wg.Add(len(resources))
@@ -132,20 +137,32 @@ func generateImportBlocks(ctx context.Context, client *common.Client, listerData
132137
// to = aws_iot_thing.bar
133138
// id = "foo"
134139
// }
135-
blocks := make([]*hclwrite.Block, len(ids))
136-
for i, id := range ids {
140+
var blocks []*hclwrite.Block
141+
for _, id := range ids {
137142
cleanedID := allowedTerraformChars.ReplaceAllString(id, "_")
138143
if provider != "cloud" {
139144
cleanedID = strings.ReplaceAll(provider, "-", "_") + "_" + cleanedID
140145
}
141146

147+
matched, err := filterResourceByName(resource.Name, cleanedID, includedResources)
148+
if err != nil {
149+
wg.Done()
150+
results <- result{
151+
resource: resource,
152+
err: err,
153+
}
154+
return
155+
}
156+
if !matched {
157+
continue
158+
}
159+
142160
b := hclwrite.NewBlock("import", nil)
143161
b.Body().SetAttributeTraversal("provider", traversal("grafana", provider))
144162
b.Body().SetAttributeTraversal("to", traversal(resource.Name, cleanedID))
145163
b.Body().SetAttributeValue("id", cty.StringVal(id))
146164

147-
blocks[i] = b
148-
// TODO: Match and update existing import blocks
165+
blocks = append(blocks, b)
149166
}
150167

151168
wg.Done()
@@ -189,3 +206,49 @@ func generateImportBlocks(ctx context.Context, client *common.Client, listerData
189206

190207
return sortResourcesFile(filepath.Join(outPath, generatedFilename))
191208
}
209+
210+
func filterResources(resources []*common.Resource, includedResources []string) ([]*common.Resource, error) {
211+
if len(includedResources) == 0 {
212+
return resources, nil
213+
}
214+
215+
filteredResources := []*common.Resource{}
216+
allowedResourceTypes := []string{}
217+
for _, included := range includedResources {
218+
if !strings.Contains(included, ".") {
219+
return nil, fmt.Errorf("included resource %q is not in the format <type>.<name>", included)
220+
}
221+
allowedResourceTypes = append(allowedResourceTypes, strings.Split(included, ".")[0])
222+
}
223+
224+
for _, resource := range resources {
225+
for _, allowedResourceType := range allowedResourceTypes {
226+
matched, err := filepath.Match(allowedResourceType, resource.Name)
227+
if err != nil {
228+
return nil, err
229+
}
230+
if matched {
231+
filteredResources = append(filteredResources, resource)
232+
break
233+
}
234+
}
235+
}
236+
return filteredResources, nil
237+
}
238+
239+
func filterResourceByName(resourceType, resourceName string, includedResources []string) (bool, error) {
240+
if len(includedResources) == 0 {
241+
return true, nil
242+
}
243+
244+
for _, included := range includedResources {
245+
matched, err := filepath.Match(included, resourceType+"."+resourceName)
246+
if err != nil {
247+
return false, err
248+
}
249+
if matched {
250+
return true, nil
251+
}
252+
}
253+
return false, nil
254+
}

‎pkg/generate/generate_test.go

+88-27
Original file line numberDiff line numberDiff line change
@@ -16,38 +16,99 @@ import (
1616
"golang.org/x/exp/slices"
1717
)
1818

19-
func TestAccGenerate_Dashboard(t *testing.T) {
19+
func TestAccGenerate(t *testing.T) {
2020
testutils.CheckOSSTestsEnabled(t)
2121

22-
resource.Test(t, resource.TestCase{
23-
ProtoV5ProviderFactories: testutils.ProtoV5ProviderFactories,
24-
Steps: []resource.TestStep{
25-
{
26-
Config: testutils.TestAccExample(t, "resources/grafana_dashboard/resource.tf"),
27-
Check: func(s *terraform.State) error {
28-
tempDir := t.TempDir()
29-
config := generate.Config{
30-
OutputDir: tempDir,
31-
Clobber: true,
32-
Format: generate.OutputFormatHCL,
33-
ProviderVersion: "v3.0.0",
34-
Grafana: &generate.GrafanaConfig{
35-
URL: "http://localhost:3000",
36-
Auth: "admin:admin",
37-
},
38-
}
22+
cases := []struct {
23+
name string
24+
config string
25+
generateConfig func(cfg *generate.Config)
26+
check func(t *testing.T, tempDir string)
27+
}{
28+
{
29+
name: "dashboard",
30+
config: testutils.TestAccExample(t, "resources/grafana_dashboard/resource.tf"),
31+
check: func(t *testing.T, tempDir string) {
32+
assertFiles(t, tempDir, "testdata/generate/dashboard-expected", "", []string{
33+
".terraform",
34+
".terraform.lock.hcl",
35+
})
36+
},
37+
},
38+
{
39+
name: "dashboard-filter-strict",
40+
config: testutils.TestAccExample(t, "resources/grafana_dashboard/resource.tf"),
41+
generateConfig: func(cfg *generate.Config) {
42+
cfg.IncludeResources = []string{"grafana_dashboard.localhost_1_my-dashboard-uid"}
43+
},
44+
check: func(t *testing.T, tempDir string) {
45+
assertFiles(t, tempDir, "testdata/generate/dashboard-filtered", "", []string{
46+
".terraform",
47+
".terraform.lock.hcl",
48+
})
49+
},
50+
},
51+
{
52+
name: "dashboard-filter-wildcard-on-resource-type",
53+
config: testutils.TestAccExample(t, "resources/grafana_dashboard/resource.tf"),
54+
generateConfig: func(cfg *generate.Config) {
55+
cfg.IncludeResources = []string{"*.localhost_1_my-dashboard-uid"}
56+
},
57+
check: func(t *testing.T, tempDir string) {
58+
assertFiles(t, tempDir, "testdata/generate/dashboard-filtered", "", []string{
59+
".terraform",
60+
".terraform.lock.hcl",
61+
})
62+
},
63+
},
64+
{
65+
name: "dashboard-filter-wildcard-on-resource-name",
66+
config: testutils.TestAccExample(t, "resources/grafana_dashboard/resource.tf"),
67+
generateConfig: func(cfg *generate.Config) {
68+
cfg.IncludeResources = []string{"grafana_dashboard.*"}
69+
},
70+
check: func(t *testing.T, tempDir string) {
71+
assertFiles(t, tempDir, "testdata/generate/dashboard-filtered", "", []string{
72+
".terraform",
73+
".terraform.lock.hcl",
74+
})
75+
},
76+
},
77+
}
78+
79+
for _, tc := range cases {
80+
t.Run(tc.name, func(t *testing.T) {
81+
resource.Test(t, resource.TestCase{
82+
ProtoV5ProviderFactories: testutils.ProtoV5ProviderFactories,
83+
Steps: []resource.TestStep{
84+
{
85+
Config: tc.config,
86+
Check: func(s *terraform.State) error {
87+
tempDir := t.TempDir()
88+
config := generate.Config{
89+
OutputDir: tempDir,
90+
Clobber: true,
91+
Format: generate.OutputFormatHCL,
92+
ProviderVersion: "v3.0.0",
93+
Grafana: &generate.GrafanaConfig{
94+
URL: "http://localhost:3000",
95+
Auth: "admin:admin",
96+
},
97+
}
98+
if tc.generateConfig != nil {
99+
tc.generateConfig(&config)
100+
}
39101

40-
require.NoError(t, generate.Generate(context.Background(), &config))
41-
assertFiles(t, tempDir, "testdata/generate/dashboard-expected", "", []string{
42-
".terraform",
43-
".terraform.lock.hcl",
44-
})
102+
require.NoError(t, generate.Generate(context.Background(), &config))
103+
tc.check(t, tempDir)
45104

46-
return nil
105+
return nil
106+
},
107+
},
47108
},
48-
},
49-
},
50-
})
109+
})
110+
})
111+
}
51112
}
52113

53114
// assertFiles checks that all files in the "expectedFilesDir" directory match the files in the "gotFilesDir" directory.

‎pkg/generate/grafana.go

+3-2
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,8 @@ import (
1616
"github.com/zclconf/go-cty/cty"
1717
)
1818

19-
func generateGrafanaResources(ctx context.Context, auth, url, stackName string, genProvider bool, outPath, smURL, smToken string) error {
19+
// TODO: Refactor this sig
20+
func generateGrafanaResources(ctx context.Context, auth, url, stackName string, genProvider bool, outPath, smURL, smToken string, includedResources []string) error {
2021
if genProvider {
2122
providerBlock := hclwrite.NewBlock("provider", []string{"grafana"})
2223
providerBlock.Body().SetAttributeValue("alias", cty.StringVal(stackName))
@@ -56,7 +57,7 @@ func generateGrafanaResources(ctx context.Context, auth, url, stackName string,
5657
resources = append(resources, machinelearning.Resources...)
5758
resources = append(resources, syntheticmonitoring.Resources...)
5859
}
59-
if err := generateImportBlocks(ctx, client, listerData, resources, outPath, stackName); err != nil {
60+
if err := generateImportBlocks(ctx, client, listerData, resources, outPath, stackName, includedResources); err != nil {
6061
return err
6162
}
6263

‎pkg/generate/testdata/generate/dashboard-filtered/localhost-imports.tf

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
import {
2+
provider = grafana.localhost
3+
to = grafana_dashboard.localhost_1_my-dashboard-uid
4+
id = "1:my-dashboard-uid"
5+
}

‎pkg/generate/testdata/generate/dashboard-filtered/localhost-provider.tf

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
provider "grafana" {
2+
alias = "localhost"
3+
url = "http://localhost:3000"
4+
auth = "admin:admin"
5+
}

‎pkg/generate/testdata/generate/dashboard-filtered/localhost-resources.tf

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
# __generated__ by Terraform
2+
# Please review these resources and move them into your main configuration files.
3+
4+
# __generated__ by Terraform from "1:my-dashboard-uid"
5+
resource "grafana_dashboard" "localhost_1_my-dashboard-uid" {
6+
provider = grafana.localhost
7+
config_json = jsonencode({
8+
title = "My Dashboard"
9+
uid = "my-dashboard-uid"
10+
})
11+
folder = "my-folder-uid"
12+
}

‎pkg/generate/testdata/generate/dashboard-filtered/provider.tf

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
terraform {
2+
required_providers {
3+
grafana = {
4+
source = "grafana/grafana"
5+
version = "3.0.0"
6+
}
7+
}
8+
}

0 commit comments

Comments
 (0)