Code Generation: Working prototype for Grafana

cmd/generate/generate.go

@@ -4,10 +4,14 @@ import (
 	"context"
 	"fmt"
 	"log"
+	"net/url"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strings"
+	"sync"
 
+	"github.com/grafana/terraform-provider-grafana/v2/internal/common"
 	"github.com/hashicorp/hcl/v2/hclwrite"
 	"github.com/zclconf/go-cty/cty"
 )
@@ -20,7 +24,10 @@ const (
 	outputFormatCrossplane outputFormat = "crossplane"
 )
 
-var outputFormats = []outputFormat{outputFormatJSON, outputFormatHCL, outputFormatCrossplane}
+var (
+	outputFormats         = []outputFormat{outputFormatJSON, outputFormatHCL, outputFormatCrossplane}
+	allowedTerraformChars = regexp.MustCompile(`[^a-zA-Z0-9_-]`)
+)
 
 type config struct {
 	outputDir       string
@@ -73,8 +80,15 @@ func generate(ctx context.Context, cfg *config) error {
 		if err := generateCloudResources(ctx, cfg.cloudAccessPolicyToken, cfg.cloudOrg); err != nil {
 			return err
 		}
-	} else {
-		if err := generateGrafanaResources(ctx, cfg.grafanaURL, cfg.grafanaAuth); err != nil {
+	}
+
+	if cfg.grafanaAuth != "" {
+		grafanaURLParsed, err := url.Parse(cfg.grafanaURL)
+		if err != nil {
+			return err
+		}
+
+		if err := generateGrafanaResources(ctx, cfg.grafanaAuth, cfg.grafanaURL, grafanaURLParsed.Hostname(), true, cfg.outputDir, "", ""); err != nil {
 			return err
 		}
 	}
@@ -85,3 +99,88 @@ func generate(ctx context.Context, cfg *config) error {
 
 	return nil
 }
+
+func generateImportBlocks(ctx context.Context, client *common.Client, listerData any, resources []*common.Resource, outPath, provider string) error {
+	// Generate HCL blocks in parallel with a wait group
+	wg := sync.WaitGroup{}
+	wg.Add(len(resources))
+	type result struct {
+		resource *common.Resource
+		blocks   []*hclwrite.Block
+		err      error
+	}
+	results := make(chan result, len(resources))
+
+	for _, resource := range resources {
+		go func(resource *common.Resource) {
+			lister := resource.ListIDsFunc
+			if lister == nil {
+				log.Printf("skipping %s because it does not have a lister\n", resource.Name)
+				wg.Done()
+				results <- result{
+					resource: resource,
+				}
+				return
+			}
+
+			log.Printf("generating %s resources\n", resource.Name)
+			ids, err := lister(ctx, client, listerData)
+			if err != nil {
+				wg.Done()
+				results <- result{
+					resource: resource,
+					err:      err,
+				}
+				return
+			}
+
+			// Write blocks like these
+			// import {
+			//   to = aws_iot_thing.bar
+			//   id = "foo"
+			// }
+			blocks := make([]*hclwrite.Block, len(ids))
+			for i, id := range ids {
+				cleanedID := allowedTerraformChars.ReplaceAllString(id, "_")
+				if provider != "cloud" {
+					cleanedID = strings.ReplaceAll(provider, "-", "_") + "_" + cleanedID
+				}
+
+				b := hclwrite.NewBlock("import", nil)
+				b.Body().SetAttributeTraversal("provider", traversal("grafana", provider))
+				b.Body().SetAttributeTraversal("to", traversal(resource.Name, cleanedID))
+				b.Body().SetAttributeValue("id", cty.StringVal(id))
+
+				blocks[i] = b
+				// TODO: Match and update existing import blocks
+			}
+
+			wg.Done()
+			results <- result{
+				resource: resource,
+				blocks:   blocks,
+			}
+			log.Printf("finished generating blocks for %s resources\n", resource.Name)
+		}(resource)
+	}
+
+	// Wait for all results
+	wg.Wait()
+	close(results)
+
+	// Collect results
+	allBlocks := []*hclwrite.Block{}
+	for r := range results {
+		if r.err != nil {
+			return fmt.Errorf("failed to generate %s resources: %w", r.resource.Name, r.err)
+		}
+		allBlocks = append(allBlocks, r.blocks...)
+	}
+
+	if err := writeBlocks(filepath.Join(outPath, provider+"-imports.tf"), allBlocks...); err != nil {
+		return err
+	}
+
+	generatedFilename := fmt.Sprintf("%s-resources.tf", provider)
+	return runTerraform(outPath, "plan", "-generate-config-out="+generatedFilename)
+}

cmd/generate/grafana.go

@@ -1,8 +1,63 @@
 package main
 
-import "context"
+import (
+	"context"
+	"path/filepath"
+	"strings"
+
+	"github.com/grafana/terraform-provider-grafana/v2/internal/resources/grafana"
+	"github.com/grafana/terraform-provider-grafana/v2/internal/resources/machinelearning"
+	"github.com/grafana/terraform-provider-grafana/v2/internal/resources/slo"
+	"github.com/grafana/terraform-provider-grafana/v2/internal/resources/syntheticmonitoring"
+	"github.com/grafana/terraform-provider-grafana/v2/pkg/provider"
+	"github.com/hashicorp/hcl/v2/hclwrite"
+	"github.com/hashicorp/terraform-plugin-framework/types"
+	"github.com/zclconf/go-cty/cty"
+)
+
+func generateGrafanaResources(ctx context.Context, auth, url, stackName string, genProvider bool, outPath, smURL, smToken string) error {
+	if genProvider {
+		providerBlock := hclwrite.NewBlock("provider", []string{"grafana"})
+		providerBlock.Body().SetAttributeValue("alias", cty.StringVal(stackName))
+		providerBlock.Body().SetAttributeValue("url", cty.StringVal(url))
+		providerBlock.Body().SetAttributeValue("auth", cty.StringVal(auth))
+		if err := writeBlocks(filepath.Join(outPath, stackName+"-provider.tf"), providerBlock); err != nil {
+			return err
+		}
+	}
+
+	singleOrg := !strings.Contains(auth, ":")
+	listerData := grafana.NewListerData(singleOrg)
+
+	// Generate resources
+	config := provider.ProviderConfig{
+		URL:  types.StringValue(url),
+		Auth: types.StringValue(auth),
+	}
+	if smToken != "" {
+		config.SMAccessToken = types.StringValue(smToken)
+	}
+	if smURL != "" {
+		config.SMURL = types.StringValue(smURL)
+	}
+	if err := config.SetDefaults(); err != nil {
+		return err
+	}
+
+	client, err := provider.CreateClients(config)
+	if err != nil {
+		return err
+	}
+
+	resources := grafana.Resources
+	if strings.HasPrefix(stackName, "stack-") { // TODO: is cloud. Find a better way to detect this
+		resources = append(resources, slo.Resources...)
+		resources = append(resources, machinelearning.Resources...)
+		resources = append(resources, syntheticmonitoring.Resources...)
+	}
+	if err := generateImportBlocks(ctx, client, listerData, resources, outPath, stackName); err != nil {
+		return err
+	}
 
-func generateGrafanaResources(ctx context.Context, url, auth string) error {
-	// TODO: Implement
 	return nil
 }

cmd/generate/terraform.go

@@ -7,6 +7,7 @@ import (
 	"os/exec"
 	"path/filepath"
 
+	"github.com/hashicorp/hcl/v2"
 	"github.com/hashicorp/hcl/v2/hclparse"
 	"github.com/hashicorp/hcl/v2/hclwrite"
 	"github.com/tmccombs/hcl2json/convert"
@@ -73,3 +74,11 @@ func convertToTFJSON(dir string) error {
 
 	return nil
 }
+
+func traversal(root string, attrs ...string) hcl.Traversal {
+	tr := hcl.Traversal{hcl.TraverseRoot{Name: root}}
+	for _, attr := range attrs {
+		tr = append(tr, hcl.TraverseAttr{Name: attr})
+	}
+	return tr
+}

internal/resources/grafana/common_lister.go

@@ -25,14 +25,14 @@ func NewListerData(singleOrg bool) *ListerData {
 
 func (ld *ListerData) OrgIDs(client *goapi.GrafanaHTTPAPI) ([]int64, error) {
 	if ld.singleOrg {
-		return nil, nil
+		return []int64{0}, nil
 	}
 
 	var err error
 	ld.orgsInit.Do(func() {
 		client = client.Clone().WithOrgID(0)
 
-		var page int64 = 1
+		var page int64 = 0
 		for {
 			var resp *orgs.SearchOrgsOK
 			if resp, err = client.Orgs.SearchOrgs(orgs.NewSearchOrgsParams().WithPage(&page)); err != nil {