diff --git a/cmd/generate/main.go b/cmd/generate/main.go index 47aa28ce7..3647c60f2 100644 --- a/cmd/generate/main.go +++ b/cmd/generate/main.go @@ -57,6 +57,17 @@ func run() error { EnvVars: []string{"TFGEN_TERRAFORM_PROVIDER_VERSION"}, Value: version, }, + &cli.StringSliceFlag{ + Name: "include-resources", + Usage: `List of resources to include in the "resourceType.resourceName" format. If not set, all resources will be included +This supports a glob format. Examples: + * Generate all dashboards and folders: --resource-names 'grafana_dashboard.*' --resource-names 'grafana_folder.*' + * Generate all resources with "hello" in their ID (this is usually the resource UIDs): --resource-names '*.*hello*' + * Generate all resources (same as default behaviour): --resource-names '*.*' +`, + EnvVars: []string{"TFGEN_INCLUDE_RESOURCES"}, + Required: false, + }, // Grafana OSS flags &cli.StringFlag{ @@ -130,6 +141,7 @@ func parseFlags(ctx *cli.Context) (*generate.Config, error) { CreateStackServiceAccount: ctx.Bool("cloud-create-stack-service-account"), StackServiceAccountName: ctx.String("cloud-stack-service-account-name"), }, + IncludeResources: ctx.StringSlice("include-resources"), } if config.ProviderVersion == "" { diff --git a/pkg/generate/cloud.go b/pkg/generate/cloud.go index e15339a9d..25feb1067 100644 --- a/pkg/generate/cloud.go +++ b/pkg/generate/cloud.go @@ -66,7 +66,7 @@ func generateCloudResources(ctx context.Context, cfg *Config) ([]stack, error) { } data := cloud.NewListerData(cfg.Cloud.Org) - if err := generateImportBlocks(ctx, client, data, cloud.Resources, cfg.OutputDir, "cloud"); err != nil { + if err := generateImportBlocks(ctx, client, data, cloud.Resources, cfg.OutputDir, "cloud", cfg.IncludeResources); err != nil { return nil, err } diff --git a/pkg/generate/config.go b/pkg/generate/config.go index a3f33f9a0..b4173f70a 100644 --- a/pkg/generate/config.go +++ b/pkg/generate/config.go @@ -23,7 +23,13 @@ type CloudConfig struct { } type Config struct { - OutputDir string + // IncludeResources is a list of patterns to filter resources by. + // If a resource name matches any of the patterns, it will be included in the output. + // Patterns are in the form of `resourceType.resourceName` and support * as a wildcard. + IncludeResources []string + // OutputDir is the directory to write the generated files to. + OutputDir string + // Clobber will overwrite existing files in the output directory. Clobber bool Format OutputFormat ProviderVersion string diff --git a/pkg/generate/generate.go b/pkg/generate/generate.go index 7190e3af5..9d9bfcfc1 100644 --- a/pkg/generate/generate.go +++ b/pkg/generate/generate.go @@ -28,7 +28,7 @@ func Generate(ctx context.Context, cfg *Config) error { return fmt.Errorf("failed to delete %s: %s", cfg.OutputDir, err) } } else if err == nil && !cfg.Clobber { - return fmt.Errorf("output dir %q already exists. Use --clobber to delete it", cfg.OutputDir) + return fmt.Errorf("output dir %q already exists. Use the clobber option to delete it", cfg.OutputDir) } log.Printf("Generating resources to %s", cfg.OutputDir) @@ -60,14 +60,14 @@ func Generate(ctx context.Context, cfg *Config) error { } for _, stack := range stacks { - if err := generateGrafanaResources(ctx, stack.managementKey, stack.url, "stack-"+stack.slug, false, cfg.OutputDir, stack.smURL, stack.smToken); err != nil { + if err := generateGrafanaResources(ctx, stack.managementKey, stack.url, "stack-"+stack.slug, false, cfg.OutputDir, stack.smURL, stack.smToken, cfg.IncludeResources); err != nil { return err } } } if cfg.Grafana != nil { - if err := generateGrafanaResources(ctx, cfg.Grafana.Auth, cfg.Grafana.URL, "", true, cfg.OutputDir, "", ""); err != nil { + if err := generateGrafanaResources(ctx, cfg.Grafana.Auth, cfg.Grafana.URL, "", true, cfg.OutputDir, "", "", cfg.IncludeResources); err != nil { return err } } @@ -82,7 +82,7 @@ func Generate(ctx context.Context, cfg *Config) error { return nil } -func generateImportBlocks(ctx context.Context, client *common.Client, listerData any, resources []*common.Resource, outPath, provider string) error { +func generateImportBlocks(ctx context.Context, client *common.Client, listerData any, resources []*common.Resource, outPath, provider string, includedResources []string) error { generatedFilename := func(suffix string) string { if provider == "" { return filepath.Join(outPath, suffix) @@ -91,6 +91,11 @@ func generateImportBlocks(ctx context.Context, client *common.Client, listerData return filepath.Join(outPath, provider+"-"+suffix) } + resources, err := filterResources(resources, includedResources) + if err != nil { + return err + } + // Generate HCL blocks in parallel with a wait group wg := sync.WaitGroup{} wg.Add(len(resources)) @@ -129,13 +134,26 @@ func generateImportBlocks(ctx context.Context, client *common.Client, listerData // to = aws_iot_thing.bar // id = "foo" // } - blocks := make([]*hclwrite.Block, len(ids)) - for i, id := range ids { + var blocks []*hclwrite.Block + for _, id := range ids { cleanedID := allowedTerraformChars.ReplaceAllString(id, "_") if provider != "cloud" { cleanedID = strings.ReplaceAll(provider, "-", "_") + "_" + cleanedID } + matched, err := filterResourceByName(resource.Name, cleanedID, includedResources) + if err != nil { + wg.Done() + results <- result{ + resource: resource, + err: err, + } + return + } + if !matched { + continue + } + b := hclwrite.NewBlock("import", nil) b.Body().SetAttributeTraversal("to", traversal(resource.Name, cleanedID)) b.Body().SetAttributeValue("id", cty.StringVal(id)) @@ -143,8 +161,7 @@ func generateImportBlocks(ctx context.Context, client *common.Client, listerData b.Body().SetAttributeTraversal("provider", traversal("grafana", provider)) } - blocks[i] = b - // TODO: Match and update existing import blocks + blocks = append(blocks, b) } wg.Done() @@ -187,3 +204,49 @@ func generateImportBlocks(ctx context.Context, client *common.Client, listerData return sortResourcesFile(generatedFilename("resources.tf")) } + +func filterResources(resources []*common.Resource, includedResources []string) ([]*common.Resource, error) { + if len(includedResources) == 0 { + return resources, nil + } + + filteredResources := []*common.Resource{} + allowedResourceTypes := []string{} + for _, included := range includedResources { + if !strings.Contains(included, ".") { + return nil, fmt.Errorf("included resource %q is not in the format .", included) + } + allowedResourceTypes = append(allowedResourceTypes, strings.Split(included, ".")[0]) + } + + for _, resource := range resources { + for _, allowedResourceType := range allowedResourceTypes { + matched, err := filepath.Match(allowedResourceType, resource.Name) + if err != nil { + return nil, err + } + if matched { + filteredResources = append(filteredResources, resource) + break + } + } + } + return filteredResources, nil +} + +func filterResourceByName(resourceType, resourceName string, includedResources []string) (bool, error) { + if len(includedResources) == 0 { + return true, nil + } + + for _, included := range includedResources { + matched, err := filepath.Match(included, resourceType+"."+resourceName) + if err != nil { + return false, err + } + if matched { + return true, nil + } + } + return false, nil +} diff --git a/pkg/generate/generate_test.go b/pkg/generate/generate_test.go index 71b702e6c..7c2c90b20 100644 --- a/pkg/generate/generate_test.go +++ b/pkg/generate/generate_test.go @@ -16,38 +16,99 @@ import ( "golang.org/x/exp/slices" ) -func TestAccGenerate_Dashboard(t *testing.T) { +func TestAccGenerate(t *testing.T) { testutils.CheckOSSTestsEnabled(t) - resource.Test(t, resource.TestCase{ - ProtoV5ProviderFactories: testutils.ProtoV5ProviderFactories, - Steps: []resource.TestStep{ - { - Config: testutils.TestAccExample(t, "resources/grafana_dashboard/resource.tf"), - Check: func(s *terraform.State) error { - tempDir := t.TempDir() - config := generate.Config{ - OutputDir: tempDir, - Clobber: true, - Format: generate.OutputFormatHCL, - ProviderVersion: "v3.0.0", - Grafana: &generate.GrafanaConfig{ - URL: "http://localhost:3000", - Auth: "admin:admin", - }, - } + cases := []struct { + name string + config string + generateConfig func(cfg *generate.Config) + check func(t *testing.T, tempDir string) + }{ + { + name: "dashboard", + config: testutils.TestAccExample(t, "resources/grafana_dashboard/resource.tf"), + check: func(t *testing.T, tempDir string) { + assertFiles(t, tempDir, "testdata/generate/dashboard-expected", "", []string{ + ".terraform", + ".terraform.lock.hcl", + }) + }, + }, + { + name: "dashboard-filter-strict", + config: testutils.TestAccExample(t, "resources/grafana_dashboard/resource.tf"), + generateConfig: func(cfg *generate.Config) { + cfg.IncludeResources = []string{"grafana_dashboard._1_my-dashboard-uid"} + }, + check: func(t *testing.T, tempDir string) { + assertFiles(t, tempDir, "testdata/generate/dashboard-filtered", "", []string{ + ".terraform", + ".terraform.lock.hcl", + }) + }, + }, + { + name: "dashboard-filter-wildcard-on-resource-type", + config: testutils.TestAccExample(t, "resources/grafana_dashboard/resource.tf"), + generateConfig: func(cfg *generate.Config) { + cfg.IncludeResources = []string{"*._1_my-dashboard-uid"} + }, + check: func(t *testing.T, tempDir string) { + assertFiles(t, tempDir, "testdata/generate/dashboard-filtered", "", []string{ + ".terraform", + ".terraform.lock.hcl", + }) + }, + }, + { + name: "dashboard-filter-wildcard-on-resource-name", + config: testutils.TestAccExample(t, "resources/grafana_dashboard/resource.tf"), + generateConfig: func(cfg *generate.Config) { + cfg.IncludeResources = []string{"grafana_dashboard.*"} + }, + check: func(t *testing.T, tempDir string) { + assertFiles(t, tempDir, "testdata/generate/dashboard-filtered", "", []string{ + ".terraform", + ".terraform.lock.hcl", + }) + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + resource.Test(t, resource.TestCase{ + ProtoV5ProviderFactories: testutils.ProtoV5ProviderFactories, + Steps: []resource.TestStep{ + { + Config: tc.config, + Check: func(s *terraform.State) error { + tempDir := t.TempDir() + config := generate.Config{ + OutputDir: tempDir, + Clobber: true, + Format: generate.OutputFormatHCL, + ProviderVersion: "v3.0.0", + Grafana: &generate.GrafanaConfig{ + URL: "http://localhost:3000", + Auth: "admin:admin", + }, + } + if tc.generateConfig != nil { + tc.generateConfig(&config) + } - require.NoError(t, generate.Generate(context.Background(), &config)) - assertFiles(t, tempDir, "testdata/generate/dashboard-expected", "", []string{ - ".terraform", - ".terraform.lock.hcl", - }) + require.NoError(t, generate.Generate(context.Background(), &config)) + tc.check(t, tempDir) - return nil + return nil + }, + }, }, - }, - }, - }) + }) + }) + } } // assertFiles checks that all files in the "expectedFilesDir" directory match the files in the "gotFilesDir" directory. diff --git a/pkg/generate/grafana.go b/pkg/generate/grafana.go index 11216f1f2..5f09a9363 100644 --- a/pkg/generate/grafana.go +++ b/pkg/generate/grafana.go @@ -15,7 +15,8 @@ import ( "github.com/zclconf/go-cty/cty" ) -func generateGrafanaResources(ctx context.Context, auth, url, stackName string, genProvider bool, outPath, smURL, smToken string) error { +// TODO: Refactor this sig +func generateGrafanaResources(ctx context.Context, auth, url, stackName string, genProvider bool, outPath, smURL, smToken string, includedResources []string) error { generatedFilename := func(suffix string) string { if stackName == "" { return filepath.Join(outPath, suffix) @@ -65,7 +66,7 @@ func generateGrafanaResources(ctx context.Context, auth, url, stackName string, resources = append(resources, machinelearning.Resources...) resources = append(resources, syntheticmonitoring.Resources...) } - if err := generateImportBlocks(ctx, client, listerData, resources, outPath, stackName); err != nil { + if err := generateImportBlocks(ctx, client, listerData, resources, outPath, stackName, includedResources); err != nil { return err } diff --git a/pkg/generate/testdata/generate/dashboard-filtered/imports.tf b/pkg/generate/testdata/generate/dashboard-filtered/imports.tf new file mode 100644 index 000000000..7a41c7fb1 --- /dev/null +++ b/pkg/generate/testdata/generate/dashboard-filtered/imports.tf @@ -0,0 +1,4 @@ +import { + to = grafana_dashboard._1_my-dashboard-uid + id = "1:my-dashboard-uid" +} diff --git a/pkg/generate/testdata/generate/dashboard-filtered/provider.tf b/pkg/generate/testdata/generate/dashboard-filtered/provider.tf new file mode 100644 index 000000000..0538588ae --- /dev/null +++ b/pkg/generate/testdata/generate/dashboard-filtered/provider.tf @@ -0,0 +1,13 @@ +terraform { + required_providers { + grafana = { + source = "grafana/grafana" + version = "3.0.0" + } + } +} + +provider "grafana" { + url = "http://localhost:3000" + auth = "admin:admin" +} diff --git a/pkg/generate/testdata/generate/dashboard-filtered/resources.tf b/pkg/generate/testdata/generate/dashboard-filtered/resources.tf new file mode 100644 index 000000000..43c89b738 --- /dev/null +++ b/pkg/generate/testdata/generate/dashboard-filtered/resources.tf @@ -0,0 +1,11 @@ +# __generated__ by Terraform +# Please review these resources and move them into your main configuration files. + +# __generated__ by Terraform from "1:my-dashboard-uid" +resource "grafana_dashboard" "_1_my-dashboard-uid" { + config_json = jsonencode({ + title = "My Dashboard" + uid = "my-dashboard-uid" + }) + folder = "my-folder-uid" +}