Config Generation: Add type and name filter

cmd/generate/main.go

@@ -57,6 +57,17 @@ func run() error {
 				EnvVars: []string{"TFGEN_TERRAFORM_PROVIDER_VERSION"},
 				Value:   version,
 			},
+			&cli.StringSliceFlag{
+				Name: "include-resources",
+				Usage: `List of resources to include in the "resourceType.resourceName" format. If not set, all resources will be included
+This supports a glob format. Examples:
+  * Generate all dashboards and folders: --resource-names 'grafana_dashboard.*' --resource-names 'grafana_folder.*'
+  * Generate all resources with "hello" in their ID (this is usually the resource UIDs): --resource-names '*.*hello*'
+  * Generate all resources (same as default behaviour): --resource-names '*.*'
+`,
+				EnvVars:  []string{"TFGEN_INCLUDE_RESOURCES"},
+				Required: false,
+			},
 
 			// Grafana OSS flags
 			&cli.StringFlag{
@@ -130,6 +141,7 @@ func parseFlags(ctx *cli.Context) (*generate.Config, error) {
 			CreateStackServiceAccount: ctx.Bool("cloud-create-stack-service-account"),
 			StackServiceAccountName:   ctx.String("cloud-stack-service-account-name"),
 		},
+		IncludeResources: ctx.StringSlice("include-resources"),
 	}
 
 	if config.ProviderVersion == "" {

pkg/generate/cloud.go

@@ -66,7 +66,7 @@ func generateCloudResources(ctx context.Context, cfg *Config) ([]stack, error) {
 	}
 
 	data := cloud.NewListerData(cfg.Cloud.Org)
-	if err := generateImportBlocks(ctx, client, data, cloud.Resources, cfg.OutputDir, "cloud"); err != nil {
+	if err := generateImportBlocks(ctx, client, data, cloud.Resources, cfg.OutputDir, "cloud", cfg.IncludeResources); err != nil {
 		return nil, err
 	}
 

pkg/generate/config.go

@@ -23,7 +23,13 @@ type CloudConfig struct {
 }
 
 type Config struct {
-	OutputDir       string
+	// IncludeResources is a list of patterns to filter resources by.
+	// If a resource name matches any of the patterns, it will be included in the output.
+	// Patterns are in the form of `resourceType.resourceName` and support * as a wildcard.
+	IncludeResources []string
+	// OutputDir is the directory to write the generated files to.
+	OutputDir string
+	// Clobber will overwrite existing files in the output directory.
 	Clobber         bool
 	Format          OutputFormat
 	ProviderVersion string

pkg/generate/generate.go

@@ -28,7 +28,7 @@ func Generate(ctx context.Context, cfg *Config) error {
 			return fmt.Errorf("failed to delete %s: %s", cfg.OutputDir, err)
 		}
 	} else if err == nil && !cfg.Clobber {
-		return fmt.Errorf("output dir %q already exists. Use --clobber to delete it", cfg.OutputDir)
+		return fmt.Errorf("output dir %q already exists. Use the clobber option to delete it", cfg.OutputDir)
 	}
 
 	log.Printf("Generating resources to %s", cfg.OutputDir)
@@ -60,14 +60,14 @@ func Generate(ctx context.Context, cfg *Config) error {
 		}
 
 		for _, stack := range stacks {
-			if err := generateGrafanaResources(ctx, stack.managementKey, stack.url, "stack-"+stack.slug, false, cfg.OutputDir, stack.smURL, stack.smToken); err != nil {
+			if err := generateGrafanaResources(ctx, stack.managementKey, stack.url, "stack-"+stack.slug, false, cfg.OutputDir, stack.smURL, stack.smToken, cfg.IncludeResources); err != nil {
 				return err
 			}
 		}
 	}
 
 	if cfg.Grafana != nil {
-		if err := generateGrafanaResources(ctx, cfg.Grafana.Auth, cfg.Grafana.URL, "", true, cfg.OutputDir, "", ""); err != nil {
+		if err := generateGrafanaResources(ctx, cfg.Grafana.Auth, cfg.Grafana.URL, "", true, cfg.OutputDir, "", "", cfg.IncludeResources); err != nil {
 			return err
 		}
 	}
@@ -82,7 +82,7 @@ func Generate(ctx context.Context, cfg *Config) error {
 	return nil
 }
 
-func generateImportBlocks(ctx context.Context, client *common.Client, listerData any, resources []*common.Resource, outPath, provider string) error {
+func generateImportBlocks(ctx context.Context, client *common.Client, listerData any, resources []*common.Resource, outPath, provider string, includedResources []string) error {
 	generatedFilename := func(suffix string) string {
 		if provider == "" {
 			return filepath.Join(outPath, suffix)
@@ -91,6 +91,11 @@ func generateImportBlocks(ctx context.Context, client *common.Client, listerData
 		return filepath.Join(outPath, provider+"-"+suffix)
 	}
 
+	resources, err := filterResources(resources, includedResources)
+	if err != nil {
+		return err
+	}
+
 	// Generate HCL blocks in parallel with a wait group
 	wg := sync.WaitGroup{}
 	wg.Add(len(resources))
@@ -129,22 +134,34 @@ func generateImportBlocks(ctx context.Context, client *common.Client, listerData
 			//   to = aws_iot_thing.bar
 			//   id = "foo"
 			// }
-			blocks := make([]*hclwrite.Block, len(ids))
-			for i, id := range ids {
+			var blocks []*hclwrite.Block
+			for _, id := range ids {
 				cleanedID := allowedTerraformChars.ReplaceAllString(id, "_")
 				if provider != "cloud" {
 					cleanedID = strings.ReplaceAll(provider, "-", "_") + "_" + cleanedID
 				}
 
+				matched, err := filterResourceByName(resource.Name, cleanedID, includedResources)
+				if err != nil {
+					wg.Done()
+					results <- result{
+						resource: resource,
+						err:      err,
+					}
+					return
+				}
+				if !matched {
+					continue
+				}
+
 				b := hclwrite.NewBlock("import", nil)
 				b.Body().SetAttributeTraversal("to", traversal(resource.Name, cleanedID))
 				b.Body().SetAttributeValue("id", cty.StringVal(id))
 				if provider != "" {
 					b.Body().SetAttributeTraversal("provider", traversal("grafana", provider))
 				}
 
-				blocks[i] = b
-				// TODO: Match and update existing import blocks
+				blocks = append(blocks, b)
 			}
 
 			wg.Done()
@@ -187,3 +204,49 @@ func generateImportBlocks(ctx context.Context, client *common.Client, listerData
 
 	return sortResourcesFile(generatedFilename("resources.tf"))
 }
+
+func filterResources(resources []*common.Resource, includedResources []string) ([]*common.Resource, error) {
+	if len(includedResources) == 0 {
+		return resources, nil
+	}
+
+	filteredResources := []*common.Resource{}
+	allowedResourceTypes := []string{}
+	for _, included := range includedResources {
+		if !strings.Contains(included, ".") {
+			return nil, fmt.Errorf("included resource %q is not in the format <type>.<name>", included)
+		}
+		allowedResourceTypes = append(allowedResourceTypes, strings.Split(included, ".")[0])
+	}
+
+	for _, resource := range resources {
+		for _, allowedResourceType := range allowedResourceTypes {
+			matched, err := filepath.Match(allowedResourceType, resource.Name)
+			if err != nil {
+				return nil, err
+			}
+			if matched {
+				filteredResources = append(filteredResources, resource)
+				break
+			}
+		}
+	}
+	return filteredResources, nil
+}
+
+func filterResourceByName(resourceType, resourceName string, includedResources []string) (bool, error) {
+	if len(includedResources) == 0 {
+		return true, nil
+	}
+
+	for _, included := range includedResources {
+		matched, err := filepath.Match(included, resourceType+"."+resourceName)
+		if err != nil {
+			return false, err
+		}
+		if matched {
+			return true, nil
+		}
+	}
+	return false, nil
+}

pkg/generate/generate_test.go

@@ -16,38 +16,99 @@ import (
 	"golang.org/x/exp/slices"
 )
 
-func TestAccGenerate_Dashboard(t *testing.T) {
+func TestAccGenerate(t *testing.T) {
 	testutils.CheckOSSTestsEnabled(t)
 
-	resource.Test(t, resource.TestCase{
-		ProtoV5ProviderFactories: testutils.ProtoV5ProviderFactories,
-		Steps: []resource.TestStep{
-			{
-				Config: testutils.TestAccExample(t, "resources/grafana_dashboard/resource.tf"),
-				Check: func(s *terraform.State) error {
-					tempDir := t.TempDir()
-					config := generate.Config{
-						OutputDir:       tempDir,
-						Clobber:         true,
-						Format:          generate.OutputFormatHCL,
-						ProviderVersion: "v3.0.0",
-						Grafana: &generate.GrafanaConfig{
-							URL:  "http://localhost:3000",
-							Auth: "admin:admin",
-						},
-					}
+	cases := []struct {
+		name           string
+		config         string
+		generateConfig func(cfg *generate.Config)
+		check          func(t *testing.T, tempDir string)
+	}{
+		{
+			name:   "dashboard",
+			config: testutils.TestAccExample(t, "resources/grafana_dashboard/resource.tf"),
+			check: func(t *testing.T, tempDir string) {
+				assertFiles(t, tempDir, "testdata/generate/dashboard-expected", "", []string{
+					".terraform",
+					".terraform.lock.hcl",
+				})
+			},
+		},
+		{
+			name:   "dashboard-filter-strict",
+			config: testutils.TestAccExample(t, "resources/grafana_dashboard/resource.tf"),
+			generateConfig: func(cfg *generate.Config) {
+				cfg.IncludeResources = []string{"grafana_dashboard._1_my-dashboard-uid"}
+			},
+			check: func(t *testing.T, tempDir string) {
+				assertFiles(t, tempDir, "testdata/generate/dashboard-filtered", "", []string{
+					".terraform",
+					".terraform.lock.hcl",
+				})
+			},
+		},
+		{
+			name:   "dashboard-filter-wildcard-on-resource-type",
+			config: testutils.TestAccExample(t, "resources/grafana_dashboard/resource.tf"),
+			generateConfig: func(cfg *generate.Config) {
+				cfg.IncludeResources = []string{"*._1_my-dashboard-uid"}
+			},
+			check: func(t *testing.T, tempDir string) {
+				assertFiles(t, tempDir, "testdata/generate/dashboard-filtered", "", []string{
+					".terraform",
+					".terraform.lock.hcl",
+				})
+			},
+		},
+		{
+			name:   "dashboard-filter-wildcard-on-resource-name",
+			config: testutils.TestAccExample(t, "resources/grafana_dashboard/resource.tf"),
+			generateConfig: func(cfg *generate.Config) {
+				cfg.IncludeResources = []string{"grafana_dashboard.*"}
+			},
+			check: func(t *testing.T, tempDir string) {
+				assertFiles(t, tempDir, "testdata/generate/dashboard-filtered", "", []string{
+					".terraform",
+					".terraform.lock.hcl",
+				})
+			},
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			resource.Test(t, resource.TestCase{
+				ProtoV5ProviderFactories: testutils.ProtoV5ProviderFactories,
+				Steps: []resource.TestStep{
+					{
+						Config: tc.config,
+						Check: func(s *terraform.State) error {
+							tempDir := t.TempDir()
+							config := generate.Config{
+								OutputDir:       tempDir,
+								Clobber:         true,
+								Format:          generate.OutputFormatHCL,
+								ProviderVersion: "v3.0.0",
+								Grafana: &generate.GrafanaConfig{
+									URL:  "http://localhost:3000",
+									Auth: "admin:admin",
+								},
+							}
+							if tc.generateConfig != nil {
+								tc.generateConfig(&config)
+							}
 
-					require.NoError(t, generate.Generate(context.Background(), &config))
-					assertFiles(t, tempDir, "testdata/generate/dashboard-expected", "", []string{
-						".terraform",
-						".terraform.lock.hcl",
-					})
+							require.NoError(t, generate.Generate(context.Background(), &config))
+							tc.check(t, tempDir)
 
-					return nil
+							return nil
+						},
+					},
 				},
-			},
-		},
-	})
+			})
+		})
+	}
 }
 
 // assertFiles checks that all files in the "expectedFilesDir" directory match the files in the "gotFilesDir" directory.

pkg/generate/grafana.go

@@ -15,7 +15,8 @@ import (
 	"github.com/zclconf/go-cty/cty"
 )
 
-func generateGrafanaResources(ctx context.Context, auth, url, stackName string, genProvider bool, outPath, smURL, smToken string) error {
+// TODO: Refactor this sig
+func generateGrafanaResources(ctx context.Context, auth, url, stackName string, genProvider bool, outPath, smURL, smToken string, includedResources []string) error {
 	generatedFilename := func(suffix string) string {
 		if stackName == "" {
 			return filepath.Join(outPath, suffix)
@@ -65,7 +66,7 @@ func generateGrafanaResources(ctx context.Context, auth, url, stackName string,
 		resources = append(resources, machinelearning.Resources...)
 		resources = append(resources, syntheticmonitoring.Resources...)
 	}
-	if err := generateImportBlocks(ctx, client, listerData, resources, outPath, stackName); err != nil {
+	if err := generateImportBlocks(ctx, client, listerData, resources, outPath, stackName, includedResources); err != nil {
 		return err
 	}
 

pkg/generate/testdata/generate/dashboard-filtered/imports.tf

@@ -0,0 +1,4 @@
+import {
+  to = grafana_dashboard._1_my-dashboard-uid
+  id = "1:my-dashboard-uid"
+}

pkg/generate/testdata/generate/dashboard-filtered/provider.tf

@@ -0,0 +1,13 @@
+terraform {
+  required_providers {
+    grafana = {
+      source  = "grafana/grafana"
+      version = "3.0.0"
+    }
+  }
+}
+
+provider "grafana" {
+  url  = "http://localhost:3000"
+  auth = "admin:admin"
+}

pkg/generate/testdata/generate/dashboard-filtered/resources.tf

@@ -0,0 +1,11 @@
+# __generated__ by Terraform
+# Please review these resources and move them into your main configuration files.
+
+# __generated__ by Terraform from "1:my-dashboard-uid"
+resource "grafana_dashboard" "_1_my-dashboard-uid" {
+  config_json = jsonencode({
+    title = "My Dashboard"
+    uid   = "my-dashboard-uid"
+  })
+  folder = "my-folder-uid"
+}