diff --git a/pkg/generate/generate_test.go b/pkg/generate/generate_test.go
index 61d9cce74..51546fbb7 100644
--- a/pkg/generate/generate_test.go
+++ b/pkg/generate/generate_test.go
@@ -120,6 +120,20 @@ func TestAccGenerate(t *testing.T) {
 				require.NoError(t, err)
 				return string(content)
 			}(),
+			generateConfig: func(cfg *generate.Config) {
+				// The alerting rule group sometimes also creates an annotation.
+				// It seems to be async so it makes the test flaky.
+				// We can include only the resources we care about to avoid this.x
+				cfg.IncludeResources = []string{
+					"grafana_contact_point.*",
+					"grafana_folder.*",
+					"grafana_message_template.*",
+					"grafana_mute_timing.*",
+					"grafana_notification_policy.*",
+					"grafana_organization.*",
+					"grafana_rule_group.*",
+				}
+			},
 			check: func(t *testing.T, tempDir string) {
 				assertFiles(t, tempDir, "testdata/generate/alerting-in-org", []string{
 					".terraform",
diff --git a/pkg/generate/testdata/generate/alerting-in-org/imports.tf b/pkg/generate/testdata/generate/alerting-in-org/imports.tf
index 5ee2a6ee3..e444ba111 100644
--- a/pkg/generate/testdata/generate/alerting-in-org/imports.tf
+++ b/pkg/generate/testdata/generate/alerting-in-org/imports.tf
@@ -28,16 +28,6 @@ import {
   id = "2"
 }
 
-import {
-  to = grafana_organization_preferences._1
-  id = "1"
-}
-
-import {
-  to = grafana_organization_preferences._2
-  id = "2"
-}
-
 import {
   to = grafana_rule_group._2_alert-rule-folder_My_Rule_Group
   id = "2:alert-rule-folder:My Rule Group"
diff --git a/pkg/generate/testdata/generate/alerting-in-org/resources.tf b/pkg/generate/testdata/generate/alerting-in-org/resources.tf
index c21b97235..dbdab1ed7 100644
--- a/pkg/generate/testdata/generate/alerting-in-org/resources.tf
+++ b/pkg/generate/testdata/generate/alerting-in-org/resources.tf
@@ -53,15 +53,6 @@ resource "grafana_organization" "_2" {
   name   = "alerting-org"
 }
 
-# __generated__ by Terraform from "1"
-resource "grafana_organization_preferences" "_1" {
-}
-
-# __generated__ by Terraform from "2"
-resource "grafana_organization_preferences" "_2" {
-  org_id = grafana_organization._2.id
-}
-
 # __generated__ by Terraform from "2:alert-rule-folder:My Rule Group"
 resource "grafana_rule_group" "_2_alert-rule-folder_My_Rule_Group" {
   disable_provenance = false