Config Generation: Go implementation for executing Terraform binary

go.mod

@@ -19,7 +19,11 @@ require (
 	github.com/hashicorp/go-cty v1.4.1-0.20200414143053-d3edf31b6320
 	github.com/hashicorp/go-retryablehttp v0.7.6
 	github.com/hashicorp/go-uuid v1.0.3
+	github.com/hashicorp/go-version v1.6.0
+	github.com/hashicorp/hc-install v0.6.4
 	github.com/hashicorp/hcl/v2 v2.20.1
+	github.com/hashicorp/terraform-exec v0.21.0
+	github.com/hashicorp/terraform-json v0.22.1
 	github.com/hashicorp/terraform-plugin-docs v0.19.2
 	github.com/hashicorp/terraform-plugin-framework v1.8.0
 	github.com/hashicorp/terraform-plugin-framework-validators v0.12.0
@@ -30,6 +34,7 @@ require (
 	github.com/tmccombs/hcl2json v0.6.3
 	github.com/urfave/cli/v2 v2.27.2
 	github.com/zclconf/go-cty v1.14.4
+	golang.org/x/exp v0.0.0-20240119083558-1b970713d09a
 	golang.org/x/text v0.15.0
 )
 
@@ -70,11 +75,7 @@ require (
 	github.com/hashicorp/go-hclog v1.6.3 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/go-plugin v1.6.0 // indirect
-	github.com/hashicorp/go-version v1.6.0 // indirect
-	github.com/hashicorp/hc-install v0.6.4 // indirect
 	github.com/hashicorp/logutils v1.0.0 // indirect
-	github.com/hashicorp/terraform-exec v0.21.0 // indirect
-	github.com/hashicorp/terraform-json v0.22.1 // indirect
 	github.com/hashicorp/terraform-plugin-log v0.9.0 // indirect
 	github.com/hashicorp/terraform-registry-address v0.2.3 // indirect
 	github.com/hashicorp/terraform-svchost v0.1.1 // indirect
@@ -111,7 +112,6 @@ require (
 	go.opentelemetry.io/otel/metric v1.24.0 // indirect
 	go.opentelemetry.io/otel/trace v1.24.0 // indirect
 	golang.org/x/crypto v0.23.0 // indirect
-	golang.org/x/exp v0.0.0-20240119083558-1b970713d09a // indirect
 	golang.org/x/mod v0.16.0 // indirect
 	golang.org/x/net v0.24.0 // indirect
 	golang.org/x/sync v0.7.0 // indirect
@@ -125,3 +125,5 @@ require (
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
+
+replace github.com/hashicorp/terraform-exec v0.21.0 => github.com/hrmsk66/terraform-exec v0.21.0

go.sum

@@ -147,8 +147,6 @@ github.com/hashicorp/hcl/v2 v2.20.1 h1:M6hgdyz7HYt1UN9e61j+qKJBqR3orTWbI1HKBJEdx
 github.com/hashicorp/hcl/v2 v2.20.1/go.mod h1:TZDqQ4kNKCbh1iJp99FdPiUaVDDUPivbqxZulxDYqL4=
 github.com/hashicorp/logutils v1.0.0 h1:dLEQVugN8vlakKOUE3ihGLTZJRB4j+M2cdTm/ORI65Y=
 github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
-github.com/hashicorp/terraform-exec v0.21.0 h1:uNkLAe95ey5Uux6KJdua6+cv8asgILFVWkd/RG0D2XQ=
-github.com/hashicorp/terraform-exec v0.21.0/go.mod h1:1PPeMYou+KDUSSeRE9szMZ/oHf4fYUmB923Wzbq1ICg=
 github.com/hashicorp/terraform-json v0.22.1 h1:xft84GZR0QzjPVWs4lRUwvTcPnegqlyS7orfb5Ltvec=
 github.com/hashicorp/terraform-json v0.22.1/go.mod h1:JbWSQCLFSXFFhg42T7l9iJwdGXBYV8fmmD6o/ML4p3A=
 github.com/hashicorp/terraform-plugin-docs v0.19.2 h1:YjdKa1vuqt9EnPYkkrv9HnGZz175HhSJ7Vsn8yZeWus=
@@ -171,6 +169,8 @@ github.com/hashicorp/terraform-svchost v0.1.1 h1:EZZimZ1GxdqFRinZ1tpJwVxxt49xc/S
 github.com/hashicorp/terraform-svchost v0.1.1/go.mod h1:mNsjQfZyf/Jhz35v6/0LWcv26+X7JPS+buii2c9/ctc=
 github.com/hashicorp/yamux v0.1.1 h1:yrQxtgseBDrq9Y652vSRDvsKCJKOUD+GzTS4Y0Y8pvE=
 github.com/hashicorp/yamux v0.1.1/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ=
+github.com/hrmsk66/terraform-exec v0.21.0 h1:k/hnRAZULf6rkzJW7v2fRKAA1wAshyiUv1clw3RklrQ=
+github.com/hrmsk66/terraform-exec v0.21.0/go.mod h1:rHqaL9Y7oPlRDZffl2xr/UkSrIhKL2l9G5P81Sza7Ts=
 github.com/huandu/xstrings v1.3.3 h1:/Gcsuc1x8JVbJ9/rlye4xZnVAbEkGauT8lbebqcQws4=
 github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
 github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=

pkg/generate/cloud.go

@@ -14,11 +14,13 @@ import (
 	"github.com/grafana/terraform-provider-grafana/v3/internal/resources/cloud"
 	"github.com/grafana/terraform-provider-grafana/v3/pkg/provider"
 	"github.com/hashicorp/hcl/v2/hclwrite"
+	"github.com/hashicorp/terraform-exec/tfexec"
 	"github.com/hashicorp/terraform-plugin-framework/types"
 	"github.com/zclconf/go-cty/cty"
 )
 
 type stack struct {
+	name          string
 	slug          string
 	url           string
 	managementKey string
@@ -66,7 +68,7 @@ func generateCloudResources(ctx context.Context, cfg *Config) ([]stack, error) {
 	}
 
 	data := cloud.NewListerData(cfg.Cloud.Org)
-	if err := generateImportBlocks(ctx, client, data, cloud.Resources, cfg.OutputDir, "cloud", cfg.IncludeResources); err != nil {
+	if err := generateImportBlocks(ctx, client, data, cloud.Resources, cfg, "cloud"); err != nil {
 		return nil, err
 	}
 
@@ -140,42 +142,42 @@ func generateCloudResources(ctx context.Context, cfg *Config) ([]stack, error) {
 		}
 
 		// Apply then go into the state and find the management key
-		err := runTerraform(cfg.OutputDir, "apply", "-auto-approve",
-			"-target=grafana_cloud_stack_service_account."+stack.Slug,
-			"-target=grafana_cloud_stack_service_account_token."+stack.Slug,
-			"-target=grafana_cloud_access_policy."+policyResourceName,
-			"-target=grafana_cloud_access_policy_token."+policyResourceName,
-			"-target=grafana_synthetic_monitoring_installation."+stack.Slug,
+		err := cfg.Terraform.Apply(ctx,
+			tfexec.Target("grafana_cloud_stack_service_account."+stack.Slug),
+			tfexec.Target("grafana_cloud_stack_service_account_token."+stack.Slug),
+			tfexec.Target("grafana_cloud_access_policy."+policyResourceName),
+			tfexec.Target("grafana_cloud_access_policy_token."+policyResourceName),
+			tfexec.Target("grafana_synthetic_monitoring_installation."+stack.Slug),
 		)
 		if err != nil {
 			return nil, fmt.Errorf("failed to apply management service account blocks for stack %q: %w", stack.Slug, err)
 		}
 	}
 
 	managedStacks := []stack{}
-	state, err := getState(cfg.OutputDir)
+	state, err := getState(ctx, cfg)
 	if err != nil {
 		return nil, err
 	}
 	stacksMap := map[string]stack{}
-	for _, resource := range state.resources() {
-		if resource.resourceType() == "grafana_cloud_stack_service_account_token" {
-			slug := resource.values()["stack_slug"].(string)
+	for _, resource := range state.Values.RootModule.Resources {
+		if resource.Type == "grafana_cloud_stack_service_account_token" {
+			slug := resource.AttributeValues["stack_slug"].(string)
 			stack := stacksMap[slug]
 			stack.slug = slug
 			stack.url = stacksBySlug[slug].Url
-			stack.managementKey = resource.values()["key"].(string)
+			stack.managementKey = resource.AttributeValues["key"].(string)
 			stacksMap[slug] = stack
 		}
-		if resource.resourceType() == "grafana_synthetic_monitoring_installation" {
-			idStr := resource.values()["stack_id"].(string)
+		if resource.Type == "grafana_synthetic_monitoring_installation" {
+			idStr := resource.AttributeValues["stack_id"].(string)
 			slug := idStr
 			if id, err := strconv.Atoi(idStr); err == nil {
 				slug = stacksByID[id].Slug
 			}
 			stack := stacksMap[slug]
-			stack.smToken = resource.values()["sm_access_token"].(string)
-			stack.smURL = resource.values()["stack_sm_api_url"].(string)
+			stack.smToken = resource.AttributeValues["sm_access_token"].(string)
+			stack.smURL = resource.AttributeValues["stack_sm_api_url"].(string)
 			stacksMap[slug] = stack
 		}
 	}

pkg/generate/config.go

@@ -1,5 +1,7 @@
 package generate
 
+import "github.com/hashicorp/terraform-exec/tfexec"
+
 type OutputFormat string
 
 const (
@@ -35,4 +37,5 @@ type Config struct {
 	ProviderVersion string
 	Grafana         *GrafanaConfig
 	Cloud           *CloudConfig
+	Terraform       *tfexec.Terraform
 }

pkg/generate/generate.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/grafana/terraform-provider-grafana/v3/internal/common"
 	"github.com/hashicorp/hcl/v2/hclwrite"
+	"github.com/hashicorp/terraform-exec/tfexec"
 	"github.com/zclconf/go-cty/cty"
 )
 
@@ -48,10 +49,12 @@ func Generate(ctx context.Context, cfg *Config) error {
 		log.Fatal(err)
 	}
 
+	tf, err := setupTerraform(cfg)
 	// Terraform init to download the provider
-	if err := runTerraform(cfg.OutputDir, "init"); err != nil {
+	if err != nil {
 		return fmt.Errorf("failed to run terraform init: %w", err)
 	}
+	cfg.Terraform = tf
 
 	if cfg.Cloud != nil {
 		stacks, err := generateCloudResources(ctx, cfg)
@@ -60,14 +63,21 @@ func Generate(ctx context.Context, cfg *Config) error {
 		}
 
 		for _, stack := range stacks {
-			if err := generateGrafanaResources(ctx, stack.managementKey, stack.url, "stack-"+stack.slug, false, cfg.OutputDir, stack.smURL, stack.smToken, cfg.IncludeResources); err != nil {
+			stack.name = "stack-" + stack.slug
+			if err := generateGrafanaResources(ctx, cfg, stack, false); err != nil {
 				return err
 			}
 		}
 	}
 
 	if cfg.Grafana != nil {
-		if err := generateGrafanaResources(ctx, cfg.Grafana.Auth, cfg.Grafana.URL, "", true, cfg.OutputDir, "", "", cfg.IncludeResources); err != nil {
+		stack := stack{
+			managementKey: cfg.Grafana.Auth,
+			url:           cfg.Grafana.URL,
+			smToken:       "",
+			smURL:         "",
+		}
+		if err := generateGrafanaResources(ctx, cfg, stack, true); err != nil {
 			return err
 		}
 	}
@@ -82,16 +92,16 @@ func Generate(ctx context.Context, cfg *Config) error {
 	return nil
 }
 
-func generateImportBlocks(ctx context.Context, client *common.Client, listerData any, resources []*common.Resource, outPath, provider string, includedResources []string) error {
+func generateImportBlocks(ctx context.Context, client *common.Client, listerData any, resources []*common.Resource, cfg *Config, provider string) error {
 	generatedFilename := func(suffix string) string {
 		if provider == "" {
-			return filepath.Join(outPath, suffix)
+			return filepath.Join(cfg.OutputDir, suffix)
 		}
 
-		return filepath.Join(outPath, provider+"-"+suffix)
+		return filepath.Join(cfg.OutputDir, provider+"-"+suffix)
 	}
 
-	resources, err := filterResources(resources, includedResources)
+	resources, err := filterResources(resources, cfg.IncludeResources)
 	if err != nil {
 		return err
 	}
@@ -141,7 +151,7 @@ func generateImportBlocks(ctx context.Context, client *common.Client, listerData
 					cleanedID = strings.ReplaceAll(provider, "-", "_") + "_" + cleanedID
 				}
 
-				matched, err := filterResourceByName(resource.Name, cleanedID, includedResources)
+				matched, err := filterResourceByName(resource.Name, cleanedID, cfg.IncludeResources)
 				if err != nil {
 					wg.Done()
 					results <- result{
@@ -198,7 +208,8 @@ func generateImportBlocks(ctx context.Context, client *common.Client, listerData
 		return err
 	}
 
-	if err := runTerraform(outPath, "plan", "-generate-config-out="+generatedFilename("resources.tf")); err != nil {
+	_, err = cfg.Terraform.Plan(ctx, tfexec.GenerateConfigOut(generatedFilename("resources.tf")))
+	if err != nil {
 		return fmt.Errorf("failed to generate resources: %w", err)
 	}
 

pkg/generate/grafana.go

@@ -15,41 +15,40 @@ import (
 	"github.com/zclconf/go-cty/cty"
 )
 
-// TODO: Refactor this sig
-func generateGrafanaResources(ctx context.Context, auth, url, stackName string, genProvider bool, outPath, smURL, smToken string, includedResources []string) error {
+func generateGrafanaResources(ctx context.Context, cfg *Config, stack stack, genProvider bool) error {
 	generatedFilename := func(suffix string) string {
-		if stackName == "" {
-			return filepath.Join(outPath, suffix)
+		if stack.name == "" {
+			return filepath.Join(cfg.OutputDir, suffix)
 		}
 
-		return filepath.Join(outPath, stackName+"-"+suffix)
+		return filepath.Join(cfg.OutputDir, stack.name+"-"+suffix)
 	}
 
 	if genProvider {
 		providerBlock := hclwrite.NewBlock("provider", []string{"grafana"})
-		providerBlock.Body().SetAttributeValue("url", cty.StringVal(url))
-		providerBlock.Body().SetAttributeValue("auth", cty.StringVal(auth))
-		if stackName != "" {
-			providerBlock.Body().SetAttributeValue("alias", cty.StringVal(stackName))
+		providerBlock.Body().SetAttributeValue("url", cty.StringVal(stack.url))
+		providerBlock.Body().SetAttributeValue("auth", cty.StringVal(stack.managementKey))
+		if stack.name != "" {
+			providerBlock.Body().SetAttributeValue("alias", cty.StringVal(stack.name))
 		}
 		if err := writeBlocks(generatedFilename("provider.tf"), providerBlock); err != nil {
 			return err
 		}
 	}
 
-	singleOrg := !strings.Contains(auth, ":")
+	singleOrg := !strings.Contains(stack.managementKey, ":")
 	listerData := grafana.NewListerData(singleOrg)
 
 	// Generate resources
 	config := provider.ProviderConfig{
-		URL:  types.StringValue(url),
-		Auth: types.StringValue(auth),
+		URL:  types.StringValue(stack.url),
+		Auth: types.StringValue(stack.managementKey),
 	}
-	if smToken != "" {
-		config.SMAccessToken = types.StringValue(smToken)
+	if stack.smToken != "" {
+		config.SMAccessToken = types.StringValue(stack.smToken)
 	}
-	if smURL != "" {
-		config.SMURL = types.StringValue(smURL)
+	if stack.smURL != "" {
+		config.SMURL = types.StringValue(stack.smURL)
 	}
 	if err := config.SetDefaults(); err != nil {
 		return err
@@ -61,12 +60,12 @@ func generateGrafanaResources(ctx context.Context, auth, url, stackName string,
 	}
 
 	resources := grafana.Resources
-	if strings.HasPrefix(stackName, "stack-") { // TODO: is cloud. Find a better way to detect this
+	if strings.HasPrefix(stack.name, "stack-") { // TODO: is cloud. Find a better way to detect this
 		resources = append(resources, slo.Resources...)
 		resources = append(resources, machinelearning.Resources...)
 		resources = append(resources, syntheticmonitoring.Resources...)
 	}
-	if err := generateImportBlocks(ctx, client, listerData, resources, outPath, stackName, includedResources); err != nil {
+	if err := generateImportBlocks(ctx, client, listerData, resources, cfg, stack.name); err != nil {
 		return err
 	}
 

pkg/generate/terraform.go

@@ -1,31 +1,46 @@
 package generate
 
 import (
+	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"strings"
 
+	"github.com/hashicorp/go-version"
+	"github.com/hashicorp/hc-install/product"
+	"github.com/hashicorp/hc-install/releases"
 	"github.com/hashicorp/hcl/v2"
 	"github.com/hashicorp/hcl/v2/hclparse"
 	"github.com/hashicorp/hcl/v2/hclwrite"
+	"github.com/hashicorp/terraform-exec/tfexec"
 	"github.com/tmccombs/hcl2json/convert"
 )
 
-func runTerraformWithOutput(dir string, command ...string) ([]byte, error) {
-	cmd := exec.Command("terraform", command...)
-	cmd.Dir = dir
-	cmd.Stderr = os.Stderr
-	return cmd.Output()
-}
+func setupTerraform(cfg *Config) (*tfexec.Terraform, error) {
+	installer := &releases.ExactVersion{
+		Product: product.Terraform,
+		Version: version.Must(version.NewVersion("1.8.4")),
+	}
+
+	execPath, err := installer.Install(context.Background())
+	if err != nil {
+		return nil, fmt.Errorf("error installing Terraform: %s", err)
+	}
+
+	tf, err := tfexec.NewTerraform(cfg.OutputDir, execPath)
+	if err != nil {
+		return nil, fmt.Errorf("error running NewTerraform: %s", err)
+	}
+
+	err = tf.Init(context.Background(), tfexec.Upgrade(true))
+	if err != nil {
+		return nil, fmt.Errorf("error running Init: %s", err)
+	}
 
-func runTerraform(dir string, command ...string) error {
-	out, err := runTerraformWithOutput(dir, command...)
-	fmt.Println(string(out))
-	return err
+	return tf, nil
 }
 
 func writeBlocks(filepath string, blocks ...*hclwrite.Block) error {