Skip to content

Commit 155af86

Browse files
julienduchesnejulienduchesne
authored
Config Generation: Find references between resources (#1585)
- Step 1: Find all possible references (ex: `folder` attribute from dashboard can be a `grafana_folder.uid` attribute) from tests and examples - Step 2: Use those known references to match values between resources TODO next: Also use the plan state to find refs. Computed fields are not going to be in the configuration
1 parent 7dafddf commit 155af86

File tree

7 files changed

+332
-3
lines changed

7 files changed

+332
-3
lines changed

‎pkg/generate/cloud.go

+3
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,9 @@ func generateCloudResources(ctx context.Context, cfg *Config) ([]stack, error) {
7979
if err := wrapJSONFieldsInFunction(filepath.Join(cfg.OutputDir, "cloud-resources.tf")); err != nil {
8080
return nil, err
8181
}
82+
if err := replaceReferences(filepath.Join(cfg.OutputDir, "cloud-resources.tf"), nil); err != nil {
83+
return nil, err
84+
}
8285

8386
if !cfg.Cloud.CreateStackServiceAccount {
8487
return nil, nil

‎pkg/generate/genreferences/main.go

+131
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
package main
2+
3+
import (
4+
"flag"
5+
"fmt"
6+
"go/format"
7+
"log"
8+
"os"
9+
"path/filepath"
10+
"regexp"
11+
"sort"
12+
"strings"
13+
)
14+
15+
// Find all references in the examples directory and in test files and generate a map of known references.
16+
// Write that map in the specified file (in args).
17+
func main() {
18+
// Parse flags
19+
var (
20+
walkDir string
21+
fileToWrite string
22+
)
23+
flag.StringVar(&walkDir, "walk-dir", "", "directory to walk and find references in")
24+
flag.StringVar(&fileToWrite, "file", "", "file to write the known references to")
25+
flag.Parse()
26+
27+
if walkDir == "" || fileToWrite == "" {
28+
log.Fatal("examples-dir and file flags are required")
29+
}
30+
31+
exampleFiles := []string{}
32+
if err := filepath.Walk(walkDir, func(path string, info os.FileInfo, err error) error {
33+
if err != nil {
34+
return err
35+
}
36+
if filepath.Ext(path) == ".tf" {
37+
exampleFiles = append(exampleFiles, path)
38+
}
39+
if strings.HasSuffix(path, "_test.go") {
40+
exampleFiles = append(exampleFiles, path)
41+
}
42+
return nil
43+
}); err != nil {
44+
log.Fatal(err)
45+
}
46+
47+
resourceRe := regexp.MustCompile(`resource\s+"(\w+)"\s+"([\w-]+)"\s+\{`)
48+
assignmentRe := regexp.MustCompile(`\s*(\w+)\s*=\s*(?:\[\s*)?(\w+)\.(\w+)\.(\w+)`)
49+
knownReferencesMap := map[string]struct{}{}
50+
for _, file := range exampleFiles {
51+
log.Printf("Processing file: %s\n", file)
52+
53+
bytes, err := os.ReadFile(file)
54+
if err != nil {
55+
log.Fatal(err)
56+
}
57+
58+
lines := strings.Split(string(bytes), "\n")
59+
var currentResource string
60+
for _, line := range lines {
61+
trimmedLine := strings.TrimSpace(line)
62+
if strings.HasPrefix(trimmedLine, "output ") || strings.HasPrefix(trimmedLine, "data ") {
63+
currentResource = ""
64+
continue
65+
}
66+
67+
resourceMatch := resourceRe.FindStringSubmatch(line)
68+
if resourceMatch != nil {
69+
currentResource = resourceMatch[1]
70+
continue
71+
}
72+
73+
if currentResource == "" {
74+
continue
75+
}
76+
77+
assignmentMatch := assignmentRe.FindStringSubmatch(line)
78+
if assignmentMatch != nil {
79+
refToResource, refToAttribute := assignmentMatch[2], assignmentMatch[4]
80+
if !strings.HasPrefix(refToResource, "grafana_") { // TODO: Enable data resources
81+
continue
82+
}
83+
entry := fmt.Sprintf("%s.%s=%s.%s", currentResource, assignmentMatch[1], refToResource, refToAttribute)
84+
knownReferencesMap[entry] = struct{}{}
85+
}
86+
}
87+
}
88+
var knownReferences []string
89+
for k := range knownReferencesMap {
90+
knownReferences = append(knownReferences, k)
91+
}
92+
sort.Strings(knownReferences)
93+
94+
// Write the known references to the specified file
95+
// Find the knownReferences var and replace it
96+
log.Printf("Writing known references to %s\n", fileToWrite)
97+
bytes, err := os.ReadFile(fileToWrite)
98+
if err != nil {
99+
log.Fatal(err)
100+
}
101+
stat, err := os.Stat(fileToWrite)
102+
if err != nil {
103+
log.Fatal(err)
104+
}
105+
106+
content := string(bytes)
107+
start := strings.Index(content, "var knownReferences = []string{")
108+
if start == -1 {
109+
log.Fatal("Could not find knownReferences var")
110+
}
111+
end := strings.Index(content[start:], "}")
112+
113+
knownReferencesStr := "var knownReferences = []string{\n"
114+
for _, v := range knownReferences {
115+
knownReferencesStr += fmt.Sprintf("%q,\n", v)
116+
}
117+
knownReferencesStr += "}"
118+
fmt.Println(knownReferencesStr)
119+
120+
content = content[:start] + knownReferencesStr + content[start+end+1:]
121+
122+
// Run gofmt on the content
123+
bytesToWrite, err := format.Source([]byte(content))
124+
if err != nil {
125+
log.Fatal(err)
126+
}
127+
128+
if err := os.WriteFile(fileToWrite, bytesToWrite, stat.Mode()); err != nil {
129+
log.Fatal(err)
130+
}
131+
}

‎pkg/generate/grafana.go

+5
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,11 @@ func generateGrafanaResources(ctx context.Context, cfg *Config, stack stack, gen
8585
if err := wrapJSONFieldsInFunction(generatedFilename("resources.tf")); err != nil {
8686
return err
8787
}
88+
if err := replaceReferences(generatedFilename("resources.tf"), []string{
89+
"*.org_id=grafana_organization.id",
90+
}); err != nil {
91+
return err
92+
}
8893

8994
return nil
9095
}

‎pkg/generate/postprocessing.go

+190
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,196 @@ import (
1616
"github.com/zclconf/go-cty/cty"
1717
)
1818

19+
// knownReferences is a map of all resource fields that can be referenced from another resource.
20+
// For example, the `folder` field of a `grafana_dashboard` resource can be a `grafana_folder` reference.
21+
//
22+
//go:generate go run ./genreferences --file=$GOFILE --walk-dir=../..
23+
var knownReferences = []string{
24+
"grafana_annotation.dashboard_uid=grafana_dashboard.uid",
25+
"grafana_annotation.org_id=grafana_organization.id",
26+
"grafana_api_key.auth=grafana_api_key.key",
27+
"grafana_cloud_access_policy.identifier=grafana_cloud_stack.id",
28+
"grafana_cloud_access_policy_token.access_policy_id=grafana_cloud_access_policy.policy_id",
29+
"grafana_cloud_plugin_installation.stack_slug=grafana_cloud_stack.slug",
30+
"grafana_cloud_stack_service_account.stack_slug=grafana_cloud_stack.slug",
31+
"grafana_cloud_stack_service_account_token.auth=grafana_cloud_stack_service_account_token.key",
32+
"grafana_cloud_stack_service_account_token.service_account_id=grafana_cloud_stack_service_account.id",
33+
"grafana_cloud_stack_service_account_token.stack_slug=grafana_cloud_stack.slug",
34+
"grafana_cloud_stack_service_account_token.url=grafana_cloud_stack.url",
35+
"grafana_contact_point.org_id=grafana_organization.id",
36+
"grafana_dashboard.folder=grafana_folder.id",
37+
"grafana_dashboard.folder=grafana_folder.uid",
38+
"grafana_dashboard.name=grafana_library_panel.name",
39+
"grafana_dashboard.org_id=grafana_organization.id",
40+
"grafana_dashboard.org_id=grafana_organization.org_id",
41+
"grafana_dashboard.uid=grafana_library_panel.uid",
42+
"grafana_dashboard_permission.dashboard_uid=grafana_dashboard.uid",
43+
"grafana_dashboard_permission.team_id=grafana_team.id",
44+
"grafana_dashboard_permission.user_id=grafana_user.id",
45+
"grafana_dashboard_permission_item.dashboard_uid=grafana_dashboard.uid",
46+
"grafana_dashboard_permission_item.team=grafana_team.id",
47+
"grafana_dashboard_permission_item.user=grafana_service_account.id",
48+
"grafana_dashboard_permission_item.user=grafana_user.id",
49+
"grafana_dashboard_public.dashboard_uid=grafana_dashboard.uid",
50+
"grafana_dashboard_public.org_id=grafana_organization.org_id",
51+
"grafana_data_source.datasourceUid=grafana_data_source.uid",
52+
"grafana_data_source.org_id=grafana_organization.id",
53+
"grafana_data_source_config.datasourceUid=grafana_data_source.uid",
54+
"grafana_data_source_config.uid=grafana_data_source.uid",
55+
"grafana_data_source_permission.datasource_uid=grafana_data_source.uid",
56+
"grafana_data_source_permission.team_id=grafana_team.id",
57+
"grafana_data_source_permission.user_id=grafana_service_account.id",
58+
"grafana_data_source_permission.user_id=grafana_user.id",
59+
"grafana_data_source_permission_item.datasource_uid=grafana_data_source.uid",
60+
"grafana_data_source_permission_item.team=grafana_team.id",
61+
"grafana_data_source_permission_item.user=grafana_service_account.id",
62+
"grafana_data_source_permission_item.user=grafana_user.id",
63+
"grafana_folder.org_id=grafana_organization.id",
64+
"grafana_folder.org_id=grafana_organization.org_id",
65+
"grafana_folder.parent_folder_uid=grafana_folder.uid",
66+
"grafana_folder_permission.folder_uid=grafana_folder.uid",
67+
"grafana_folder_permission.team_id=grafana_team.id",
68+
"grafana_folder_permission.user_id=grafana_service_account.id",
69+
"grafana_folder_permission.user_id=grafana_user.id",
70+
"grafana_folder_permission_item.folder_uid=grafana_folder.uid",
71+
"grafana_folder_permission_item.team=grafana_team.id",
72+
"grafana_folder_permission_item.user=grafana_service_account.id",
73+
"grafana_folder_permission_item.user=grafana_user.id",
74+
"grafana_library_panel.folder_uid=grafana_folder.uid",
75+
"grafana_library_panel.org_id=grafana_organization.id",
76+
"grafana_machine_learning_job.datasource_uid=grafana_data_source.uid",
77+
"grafana_message_template.org_id=grafana_organization.id",
78+
"grafana_notification_policy.contact_point=grafana_contact_point.name",
79+
"grafana_notification_policy.mute_timings=grafana_mute_timing.name",
80+
"grafana_notification_policy.org_id=grafana_organization.id",
81+
"grafana_oncall_escalation.escalation_chain_id=grafana_oncall_escalation_chain.id",
82+
"grafana_oncall_integration.escalation_chain_id=grafana_oncall_escalation_chain.id",
83+
"grafana_oncall_route.escalation_chain_id=grafana_oncall_escalation_chain.id",
84+
"grafana_oncall_route.integration_id=grafana_oncall_integration.id",
85+
"grafana_organization.org_id=grafana_organization.id",
86+
"grafana_organization_preferences.home_dashboard_uid=grafana_dashboard.uid",
87+
"grafana_organization_preferences.org_id=grafana_organization.id",
88+
"grafana_playlist.org_id=grafana_organization.id",
89+
"grafana_report.dashboard_id=grafana_dashboard.dashboard_id",
90+
"grafana_report.org_id=grafana_organization.id",
91+
"grafana_report.uid=grafana_dashboard.uid",
92+
"grafana_role.org_id=grafana_organization.id",
93+
"grafana_role_assignment.auth=grafana_cloud_stack_service_account_token.key",
94+
"grafana_role_assignment.org_id=grafana_organization.id",
95+
"grafana_role_assignment.role_uid=grafana_role.uid",
96+
"grafana_role_assignment.service_accounts=grafana_cloud_stack_service_account.id",
97+
"grafana_role_assignment.service_accounts=grafana_service_account.id",
98+
"grafana_role_assignment.teams=grafana_team.id",
99+
"grafana_role_assignment.url=grafana_cloud_stack.url",
100+
"grafana_role_assignment.users=grafana_user.id",
101+
"grafana_role_assignment_item.role_uid=grafana_role.uid",
102+
"grafana_role_assignment_item.service_account_id=grafana_service_account.id",
103+
"grafana_role_assignment_item.team_id=grafana_team.id",
104+
"grafana_role_assignment_item.user_id=grafana_user.id",
105+
"grafana_rule_group.folder_uid=grafana_folder.uid",
106+
"grafana_rule_group.org_id=grafana_organization.id",
107+
"grafana_service_account.org_id=grafana_organization.id",
108+
"grafana_service_account.role_uid=grafana_role.uid",
109+
"grafana_service_account.service_account_id=grafana_service_account.id",
110+
"grafana_service_account.team_id=grafana_team.id",
111+
"grafana_service_account.user_id=grafana_user.id",
112+
"grafana_service_account_permission.org_id=grafana_organization.id",
113+
"grafana_service_account_permission.service_account_id=grafana_cloud_stack_service_account.id",
114+
"grafana_service_account_permission.service_account_id=grafana_service_account.id",
115+
"grafana_service_account_permission.team_id=grafana_team.id",
116+
"grafana_service_account_permission.user_id=grafana_user.id",
117+
"grafana_service_account_permission_item.auth=grafana_cloud_stack_service_account_token.key",
118+
"grafana_service_account_permission_item.org_id=grafana_organization.id",
119+
"grafana_service_account_permission_item.service_account_id=grafana_cloud_stack_service_account.id",
120+
"grafana_service_account_permission_item.service_account_id=grafana_service_account.id",
121+
"grafana_service_account_permission_item.team=grafana_team.id",
122+
"grafana_service_account_permission_item.url=grafana_cloud_stack.url",
123+
"grafana_service_account_permission_item.user=grafana_user.id",
124+
"grafana_service_account_token.auth=grafana_service_account_token.key",
125+
"grafana_service_account_token.service_account_id=grafana_service_account.id",
126+
"grafana_slo.folder_uid=grafana_folder.uid",
127+
"grafana_synthetic_monitoring_installation.logs_instance_id=grafana_cloud_stack.logs_user_id",
128+
"grafana_synthetic_monitoring_installation.metrics_instance_id=grafana_cloud_stack.prometheus_user_id",
129+
"grafana_synthetic_monitoring_installation.metrics_publisher_key=grafana_cloud_access_policy_token.token",
130+
"grafana_synthetic_monitoring_installation.metrics_publisher_key=grafana_cloud_api_key.key",
131+
"grafana_synthetic_monitoring_installation.sm_access_token=grafana_synthetic_monitoring_installation.sm_access_token",
132+
"grafana_synthetic_monitoring_installation.sm_url=grafana_synthetic_monitoring_installation.stack_sm_api_url",
133+
"grafana_synthetic_monitoring_installation.stack_id=grafana_cloud_stack.id",
134+
"grafana_team.home_dashboard_uid=grafana_dashboard.uid",
135+
"grafana_team.org_id=grafana_organization.id",
136+
"grafana_team_external_group.team_id=grafana_team.id",
137+
"grafana_team_preferences.home_dashboard_uid=grafana_dashboard.uid",
138+
"grafana_team_preferences.team_id=grafana_team.id",
139+
}
140+
141+
// TODO: Also find references from the state (computed fields, like ID)
142+
func replaceReferences(fpath string, extraKnownReferences []string) error {
143+
file, err := readHCLFile(fpath)
144+
if err != nil {
145+
return err
146+
}
147+
148+
hasChanges := false
149+
150+
knownReferences := knownReferences
151+
knownReferences = append(knownReferences, extraKnownReferences...)
152+
// Find all resources. This map will be used to search for references
153+
resourcesBlocks := map[string]*hclwrite.Block{}
154+
for _, block := range file.Body().Blocks() {
155+
if block.Type() == "resource" {
156+
resourcesBlocks[block.Labels()[0]+"."+block.Labels()[1]] = block
157+
}
158+
}
159+
160+
for _, block := range file.Body().Blocks() {
161+
for attrName, attr := range block.Body().Attributes() {
162+
attrValue := string(attr.Expr().BuildTokens(nil).Bytes())
163+
attrReplaced := false
164+
165+
// Check the field name. If it has a possible reference, we have to search for it in the resources
166+
for _, ref := range knownReferences {
167+
if attrReplaced {
168+
break
169+
}
170+
171+
refFrom := strings.Split(ref, "=")[0]
172+
refTo := strings.Split(ref, "=")[1]
173+
hasPossibleReference := refFrom == fmt.Sprintf("%s.%s", block.Labels()[0], attrName) || (strings.HasPrefix(refFrom, "*.") && strings.HasSuffix(refFrom, fmt.Sprintf(".%s", attrName)))
174+
if !hasPossibleReference {
175+
continue
176+
}
177+
178+
refToResource := strings.Split(refTo, ".")[0]
179+
refToAttr := strings.Split(refTo, ".")[1]
180+
181+
for possibleResourceRefName, possibleResourceRef := range resourcesBlocks {
182+
if strings.HasPrefix(possibleResourceRefName, refToResource+".") {
183+
valueFromRef := ""
184+
if possibleResourceRef.Body().GetAttribute(refToAttr) != nil {
185+
valueFromRef = string(possibleResourceRef.Body().GetAttribute(refToAttr).Expr().BuildTokens(nil).Bytes())
186+
}
187+
// If the value from the first block matches the value from the second block, we have a reference
188+
if attrValue == valueFromRef {
189+
// Replace the value with the reference
190+
block.Body().SetAttributeTraversal(attrName, traversal(possibleResourceRefName, refToAttr))
191+
hasChanges = true
192+
attrReplaced = true
193+
break
194+
}
195+
}
196+
}
197+
}
198+
}
199+
}
200+
201+
if hasChanges {
202+
log.Printf("Updating file: %s\n", fpath)
203+
return os.WriteFile(fpath, file.Bytes(), 0600)
204+
}
205+
206+
return nil
207+
}
208+
19209
func stripDefaults(fpath string, extraFieldsToRemove map[string]any) error {
20210
file, err := readHCLFile(fpath)
21211
if err != nil {

‎pkg/generate/testdata/generate/alerting-in-org/resources.tf

+1-1
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ resource "grafana_organization_preferences" "_2" {
6565
# __generated__ by Terraform from "2:alert-rule-folder:My Rule Group"
6666
resource "grafana_rule_group" "_2_alert-rule-folder_My_Rule_Group" {
6767
disable_provenance = false
68-
folder_uid = "alert-rule-folder"
68+
folder_uid = grafana_folder._2_alert-rule-folder.uid
6969
interval_seconds = 240
7070
name = "My Rule Group"
7171
org_id = jsonencode(2)

‎pkg/generate/testdata/generate/dashboard-json/resources.tf.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
"_1_my-dashboard-uid": [
55
{
66
"config_json": "${jsonencode({\n title = \"My Dashboard\"\n uid = \"my-dashboard-uid\"\n })}",
7-
"folder": "my-folder-uid"
7+
"folder": "${grafana_folder._1_my-folder-uid.uid}"
88
}
99
]
1010
},

‎pkg/generate/testdata/generate/dashboard/resources.tf

+1-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ resource "grafana_dashboard" "_1_my-dashboard-uid" {
77
title = "My Dashboard"
88
uid = "my-dashboard-uid"
99
})
10-
folder = "my-folder-uid"
10+
folder = grafana_folder._1_my-folder-uid.uid
1111
}
1212

1313
# __generated__ by Terraform from "1:my-folder-uid"

0 commit comments

Comments
 (0)