Config Generation: Various small fixes (#1596)

* Config Generation: Various small fixes
- More robust alerting resource listing. Sometimes they fail to list in new orgs because the alertmanager hasn't been init yet
- Run tests on a different runner (to avoid conflicts and ID increments)

* Oops, rename testdata

Author: julienduchesne

.github/workflows/acc-tests.yml

@@ -55,7 +55,7 @@ jobs:
         # OSS tests, run on all versions
         version: ['11.0.0', '10.4.3', '9.5.18']
         type: ['oss']
-        subset: ['basic', 'other', 'long']
+        subset: ['basic', 'other', 'long', 'generate']
         include:
           - version: '11.0.0'
             type: 'oss'
@@ -108,3 +108,4 @@ jobs:
             ${{ matrix.subset == 'other' && '-skip=".*_basic" -short -parallel 2' || '' }}
             ${{ matrix.subset == 'long' && '-run=".*longtest" -parallel 1' || '' }}
             ${{ matrix.subset == 'examples' && '-run=".*Examples" -parallel 1' || '' }}
+            ${{ matrix.subset == 'generate' && '-run="TestAccGenerate" -parallel 1' || '' }}

internal/resources/grafana/resource_alerting_message_template.go

@@ -78,13 +78,23 @@ func listMessageTemplate(ctx context.Context, client *goapi.GrafanaHTTPAPI, data
 	for _, orgID := range orgIDs {
 		client = client.Clone().WithOrgID(orgID)
 
-		resp, err := client.Provisioning.GetTemplates()
-		if err != nil {
-			return nil, err
-		}
+		// Retry if the API returns 500 because it may be that the alertmanager is not ready in the org yet.
+		// The alertmanager is provisioned asynchronously when the org is created.
+		if err := retry.RetryContext(ctx, 2*time.Minute, func() *retry.RetryError {
+			resp, err := client.Provisioning.GetTemplates()
+			if err != nil {
+				if orgID > 1 && (err.(*runtime.APIError).IsCode(500) || err.(*runtime.APIError).IsCode(403)) {
+					return retry.RetryableError(err)
+				}
+				return retry.NonRetryableError(err)
+			}
 
-		for _, template := range resp.Payload {
-			ids = append(ids, MakeOrgResourceID(orgID, template.Name))
+			for _, template := range resp.Payload {
+				ids = append(ids, MakeOrgResourceID(orgID, template.Name))
+			}
+			return nil
+		}); err != nil {
+			return nil, err
 		}
 	}
 

internal/resources/grafana/resource_alerting_mute_timing.go

@@ -5,12 +5,15 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
+	"time"
 
+	"github.com/go-openapi/runtime"
 	goapi "github.com/grafana/grafana-openapi-client-go/client"
 	"github.com/grafana/grafana-openapi-client-go/client/provisioning"
 	"github.com/grafana/grafana-openapi-client-go/models"
 	"github.com/grafana/terraform-provider-grafana/v3/internal/common"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/diag"
+	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/retry"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
 )
 
@@ -142,13 +145,23 @@ func listMuteTimings(ctx context.Context, client *goapi.GrafanaHTTPAPI, data *Li
 	for _, orgID := range orgIDs {
 		client = client.Clone().WithOrgID(orgID)
 
-		resp, err := client.Provisioning.GetMuteTimings()
-		if err != nil {
-			return nil, err
-		}
+		// Retry if the API returns 500 because it may be that the alertmanager is not ready in the org yet.
+		// The alertmanager is provisioned asynchronously when the org is created.
+		if err := retry.RetryContext(ctx, 2*time.Minute, func() *retry.RetryError {
+			resp, err := client.Provisioning.GetMuteTimings()
+			if err != nil {
+				if orgID > 1 && (err.(*runtime.APIError).IsCode(500) || err.(*runtime.APIError).IsCode(403)) {
+					return retry.RetryableError(err)
+				}
+				return retry.NonRetryableError(err)
+			}
 
-		for _, muteTiming := range resp.Payload {
-			ids = append(ids, MakeOrgResourceID(orgID, muteTiming.Name))
+			for _, muteTiming := range resp.Payload {
+				ids = append(ids, MakeOrgResourceID(orgID, muteTiming.Name))
+			}
+			return nil
+		}); err != nil {
+			return nil, err
 		}
 	}
 

internal/resources/grafana/resource_report.go

@@ -262,7 +262,7 @@ func listReports(ctx context.Context, client *goapi.GrafanaHTTPAPI, data *Lister
 		client = client.Clone().WithOrgID(orgID)
 
 		resp, err := client.Reports.GetReports()
-		if common.IsNotFoundError(err) {
+		if err != nil && common.IsNotFoundError(err) {
 			return nil, nil // Reports are not available in the current Grafana version (Probably OSS)
 		}
 		if err != nil {

pkg/generate/generate_test.go

@@ -17,6 +17,9 @@ import (
 )
 
 func TestAccGenerate(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping long test")
+	}
 	testutils.CheckOSSTestsEnabled(t)
 
 	cases := []struct {
@@ -29,7 +32,7 @@ func TestAccGenerate(t *testing.T) {
 			name:   "dashboard",
 			config: testutils.TestAccExample(t, "resources/grafana_dashboard/resource.tf"),
 			check: func(t *testing.T, tempDir string) {
-				assertFiles(t, tempDir, "testdata/generate/dashboard-expected", "", []string{
+				assertFiles(t, tempDir, "testdata/generate/dashboard", []string{
 					".terraform",
 					".terraform.lock.hcl",
 				})
@@ -42,7 +45,7 @@ func TestAccGenerate(t *testing.T) {
 				cfg.Format = generate.OutputFormatJSON
 			},
 			check: func(t *testing.T, tempDir string) {
-				assertFiles(t, tempDir, "testdata/generate/dashboard-json", "", []string{
+				assertFiles(t, tempDir, "testdata/generate/dashboard-json", []string{
 					".terraform",
 					".terraform.lock.hcl",
 				})
@@ -55,7 +58,7 @@ func TestAccGenerate(t *testing.T) {
 				cfg.IncludeResources = []string{"grafana_dashboard._1_my-dashboard-uid"}
 			},
 			check: func(t *testing.T, tempDir string) {
-				assertFiles(t, tempDir, "testdata/generate/dashboard-filtered", "", []string{
+				assertFiles(t, tempDir, "testdata/generate/dashboard-filtered", []string{
 					".terraform",
 					".terraform.lock.hcl",
 				})
@@ -68,7 +71,7 @@ func TestAccGenerate(t *testing.T) {
 				cfg.IncludeResources = []string{"*._1_my-dashboard-uid"}
 			},
 			check: func(t *testing.T, tempDir string) {
-				assertFiles(t, tempDir, "testdata/generate/dashboard-filtered", "", []string{
+				assertFiles(t, tempDir, "testdata/generate/dashboard-filtered", []string{
 					".terraform",
 					".terraform.lock.hcl",
 				})
@@ -81,7 +84,7 @@ func TestAccGenerate(t *testing.T) {
 				cfg.IncludeResources = []string{"grafana_dashboard.*"}
 			},
 			check: func(t *testing.T, tempDir string) {
-				assertFiles(t, tempDir, "testdata/generate/dashboard-filtered", "", []string{
+				assertFiles(t, tempDir, "testdata/generate/dashboard-filtered", []string{
 					".terraform",
 					".terraform.lock.hcl",
 				})
@@ -125,7 +128,12 @@ func TestAccGenerate(t *testing.T) {
 }
 
 // assertFiles checks that all files in the "expectedFilesDir" directory match the files in the "gotFilesDir" directory.
-func assertFiles(t *testing.T, gotFilesDir, expectedFilesDir, subdir string, ignoreDirEntries []string) {
+func assertFiles(t *testing.T, gotFilesDir, expectedFilesDir string, ignoreDirEntries []string) {
+	t.Helper()
+	assertFilesSubdir(t, gotFilesDir, expectedFilesDir, "", ignoreDirEntries)
+}
+
+func assertFilesSubdir(t *testing.T, gotFilesDir, expectedFilesDir, subdir string, ignoreDirEntries []string) {
 	t.Helper()
 
 	originalGotFilesDir := gotFilesDir
@@ -149,7 +157,7 @@ func assertFiles(t *testing.T, gotFilesDir, expectedFilesDir, subdir string, ign
 		}
 
 		if gotFile.IsDir() {
-			assertFiles(t, originalGotFilesDir, originalExpectedFilesDir, filepath.Join(subdir, gotFile.Name()), ignoreDirEntries)
+			assertFilesSubdir(t, originalGotFilesDir, originalExpectedFilesDir, filepath.Join(subdir, gotFile.Name()), ignoreDirEntries)
 			continue
 		}
 
@@ -169,7 +177,7 @@ func assertFiles(t *testing.T, gotFilesDir, expectedFilesDir, subdir string, ign
 	}
 	for _, expectedFile := range expectedFiles {
 		if expectedFile.IsDir() {
-			assertFiles(t, originalGotFilesDir, originalExpectedFilesDir, filepath.Join(subdir, expectedFile.Name()), ignoreDirEntries)
+			assertFilesSubdir(t, originalGotFilesDir, originalExpectedFilesDir, filepath.Join(subdir, expectedFile.Name()), ignoreDirEntries)
 			continue
 		}
 		expectedContent, err := os.ReadFile(filepath.Join(expectedFilesDir, expectedFile.Name()))