diff --git a/ovh/data_cloud_project_kube.go b/ovh/data_cloud_project_kube.go index c23de8a70..51073c6e6 100644 --- a/ovh/data_cloud_project_kube.go +++ b/ovh/data_cloud_project_kube.go @@ -6,6 +6,7 @@ import ( "net/url" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + "github.com/ovh/terraform-provider-ovh/ovh/helpers" ) func dataSourceCloudProjectKube() *schema.Resource { @@ -29,20 +30,70 @@ func dataSourceCloudProjectKube() *schema.Resource { Type: schema.TypeString, Optional: true, }, - kubeClusterCustomizationKey: { + kubeClusterProxyModeKey: { + Type: schema.TypeString, + Optional: true, + ForceNew: true, + ValidateFunc: helpers.ValidateEnum([]string{"iptables", "ipvs"}), + }, + kubeClusterCustomizationApiServerKey: { Type: schema.TypeSet, Computed: true, Optional: true, + // Required: true, ForceNew: false, - MaxItems: 1, + // MaxItems: 1, + Set: CustomSchemaSetFunc(), Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ - "apiserver": { + "admissionplugins": { Type: schema.TypeSet, Computed: true, Optional: true, + // Required: true, ForceNew: false, - MaxItems: 1, + // MaxItems: 1, + Set: CustomSchemaSetFunc(), + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "enabled": { + Type: schema.TypeList, + Computed: true, + Optional: true, + // Required: true, + ForceNew: false, + Elem: &schema.Schema{Type: schema.TypeString}, + }, + "disabled": { + Type: schema.TypeList, + Computed: true, + Optional: true, + // Required: true, + ForceNew: false, + Elem: &schema.Schema{Type: schema.TypeString}, + }, + }, + }, + }, + }, + }, + }, + kubeClusterCustomization: { + Type: schema.TypeSet, + Computed: true, + Optional: true, + ForceNew: false, + Set: CustomSchemaSetFunc(), + Deprecated: fmt.Sprintf("Use %s instead", kubeClusterCustomizationApiServerKey), + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "apiserver": { + Type: schema.TypeSet, + Computed: true, + Optional: true, + ForceNew: false, + Set: CustomSchemaSetFunc(), + Deprecated: fmt.Sprintf("Use %s instead", kubeClusterCustomizationApiServerKey), Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ "admissionplugins": { @@ -50,7 +101,7 @@ func dataSourceCloudProjectKube() *schema.Resource { Computed: true, Optional: true, ForceNew: false, - MaxItems: 1, + Set: CustomSchemaSetFunc(), Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ "enabled": { @@ -76,6 +127,98 @@ func dataSourceCloudProjectKube() *schema.Resource { }, }, }, + kubeClusterCustomizationKubeProxyKey: { + Type: schema.TypeSet, + Computed: false, + Optional: true, + ForceNew: false, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "iptables": { + Type: schema.TypeSet, + Computed: false, + Optional: true, + ForceNew: false, + MaxItems: 1, + Set: CustomIPVSIPTablesSchemaSetFunc(), + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "min_sync_period": { + Type: schema.TypeString, + Computed: false, + Optional: true, + ForceNew: false, + ValidateFunc: helpers.ValidateRFC3339Duration, + }, + "sync_period": { + Type: schema.TypeString, + Computed: false, + Optional: true, + ForceNew: false, + ValidateFunc: helpers.ValidateRFC3339Duration, + }, + }, + }, + }, + "ipvs": { + Type: schema.TypeSet, + Computed: false, + Optional: true, + ForceNew: false, + MaxItems: 1, + Set: CustomIPVSIPTablesSchemaSetFunc(), + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "min_sync_period": { + Type: schema.TypeString, + Computed: false, + Optional: true, + ForceNew: false, + ValidateFunc: helpers.ValidateRFC3339Duration, + }, + "sync_period": { + Type: schema.TypeString, + Computed: false, + Optional: true, + ForceNew: false, + ValidateFunc: helpers.ValidateRFC3339Duration, + }, + "scheduler": { + Type: schema.TypeString, + Computed: false, + Optional: true, + ForceNew: false, + ValidateFunc: helpers.ValidateEnum([]string{"rr", "lc", "dh", "sh", "sed", "nq"}), + }, + "tcp_fin_timeout": { + Type: schema.TypeString, + Computed: false, + Optional: true, + ForceNew: false, + ValidateFunc: helpers.ValidateRFC3339Duration, + }, + "tcp_timeout": { + Type: schema.TypeString, + Computed: false, + Optional: true, + ForceNew: false, + ValidateFunc: helpers.ValidateRFC3339Duration, + }, + "udp_timeout": { + Type: schema.TypeString, + Computed: false, + Optional: true, + ForceNew: false, + ValidateFunc: helpers.ValidateRFC3339Duration, + }, + }, + }, + }, + }, + }, + }, + "private_network_id": { Type: schema.TypeString, Computed: true, @@ -132,12 +275,11 @@ func dataSourceCloudProjectKubeRead(d *schema.ResourceData, meta interface{}) er url.PathEscape(serviceName), url.PathEscape(kubeId), ) - err := config.OVHClient.Get(endpoint, res) - if err != nil { + if err := config.OVHClient.Get(endpoint, res); err != nil { return fmt.Errorf("Error calling %s:\n\t %q", endpoint, err) } - for k, v := range res.ToMap() { + for k, v := range res.ToMap(d) { if k != "id" { d.Set(k, v) } else { diff --git a/ovh/data_cloud_project_kube_test.go b/ovh/data_cloud_project_kube_test.go index 6ab18b1a5..126460bd2 100644 --- a/ovh/data_cloud_project_kube_test.go +++ b/ovh/data_cloud_project_kube_test.go @@ -25,6 +25,8 @@ func TestAccCloudProjectKubeDataSource_basic(t *testing.T) { resource.Test(t, resource.TestCase{ PreCheck: func() { + testAccPreCheckCloud(t) + testAccCheckCloudProjectExists(t) testAccPreCheckKubernetes(t) }, Providers: testAccProviders, @@ -32,12 +34,49 @@ func TestAccCloudProjectKubeDataSource_basic(t *testing.T) { { Config: config, Check: resource.ComposeTestCheckFunc( - resource.TestCheckResourceAttr( - "data.ovh_cloud_project_kube.cluster", "region", region), - resource.TestCheckResourceAttr( - "data.ovh_cloud_project_kube.cluster", "name", name), - resource.TestMatchResourceAttr( - "data.ovh_cloud_project_kube.cluster", "version", matchVersion), + resource.TestCheckResourceAttr("data.ovh_cloud_project_kube.cluster", "region", region), + resource.TestCheckResourceAttr("data.ovh_cloud_project_kube.cluster", "name", name), + resource.TestMatchResourceAttr("data.ovh_cloud_project_kube.cluster", "version", matchVersion), + ), + }, + }, + }) +} + +func TestAccCloudProjectKubeDataSource_kubeProxy(t *testing.T) { + name := acctest.RandomWithPrefix(test_prefix) + region := os.Getenv("OVH_CLOUD_PROJECT_KUBE_REGION_TEST") + config := fmt.Sprintf( + testAccCloudProjectKubeDatasourceKubeProxyConfig, + os.Getenv("OVH_CLOUD_PROJECT_SERVICE_TEST"), + name, + region, + ) + + resource.Test(t, resource.TestCase{ + PreCheck: func() { + testAccPreCheckCloud(t) + testAccCheckCloudProjectExists(t) + testAccPreCheckKubernetes(t) + }, + Providers: testAccProviders, + Steps: []resource.TestStep{ + { + Config: config, + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr("data.ovh_cloud_project_kube.cluster", "region", region), + resource.TestCheckResourceAttr("data.ovh_cloud_project_kube.cluster", "name", name), + resource.TestCheckResourceAttr("data.ovh_cloud_project_kube.cluster", "kube_proxy_mode", "ipvs"), + + resource.TestCheckResourceAttr("data.ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.iptables.0.min_sync_period", "PT30S"), + resource.TestCheckResourceAttr("data.ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.iptables.0.sync_period", "PT30S"), + + resource.TestCheckResourceAttr("data.ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.0.min_sync_period", "PT30S"), + resource.TestCheckResourceAttr("data.ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.0.sync_period", "PT30S"), + resource.TestCheckResourceAttr("data.ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.0.scheduler", "rr"), + resource.TestCheckResourceAttr("data.ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.0.tcp_fin_timeout", "PT30S"), + resource.TestCheckResourceAttr("data.ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.0.tcp_timeout", "PT30S"), + resource.TestCheckResourceAttr("data.ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.0.udp_timeout", "PT30S"), ), }, }, @@ -57,3 +96,33 @@ data "ovh_cloud_project_kube" "cluster" { kube_id = ovh_cloud_project_kube.cluster.id } ` + +var testAccCloudProjectKubeDatasourceKubeProxyConfig = ` +resource "ovh_cloud_project_kube" "cluster" { + service_name = "%s" + name = "%s" + region = "%s" + + kube_proxy_mode = "ipvs" + customization_kube_proxy { + iptables { + min_sync_period = "PT30S" + sync_period = "PT30S" + } + + ipvs { + min_sync_period = "PT30S" + sync_period = "PT30S" + scheduler = "rr" + tcp_fin_timeout = "PT30S" + tcp_timeout = "PT30S" + udp_timeout = "PT30S" + } + } +} + +data "ovh_cloud_project_kube" "cluster" { + service_name = ovh_cloud_project_kube.cluster.service_name + kube_id = ovh_cloud_project_kube.cluster.id +} +` diff --git a/ovh/helpers/helpers.go b/ovh/helpers/helpers.go index 35811dcb8..93a71b51b 100644 --- a/ovh/helpers/helpers.go +++ b/ovh/helpers/helpers.go @@ -10,6 +10,7 @@ import ( "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" "github.com/ovh/go-ovh/ovh" + "github.com/ybriffa/rfc3339" ) func ValidateIpBlock(value string) error { @@ -186,6 +187,14 @@ func ValidateDedicatedCephStatus(value string) error { }) } +// ValidateRFC3339Duration implements schema.SchemaValidateFunc for RFC3339 durations. +func ValidateRFC3339Duration(i interface{}, _ string) (_ []string, errors []error) { + if _, err := rfc3339.ParseDuration(i.(string)); err != nil { + errors = append(errors, err) + } + return +} + func ValidateDedicatedCephACLFamily(value string) error { return ValidateStringEnum(value, []string{ "IPv4", diff --git a/ovh/resource_cloud_project_kube.go b/ovh/resource_cloud_project_kube.go index 999e36282..8eab918cf 100644 --- a/ovh/resource_cloud_project_kube.go +++ b/ovh/resource_cloud_project_kube.go @@ -3,6 +3,7 @@ package ovh import ( "fmt" "log" + "sort" "strings" "time" @@ -19,7 +20,12 @@ const ( kubeClusterPrivateNetworkConfigurationKey = "private_network_configuration" kubeClusterUpdatePolicyKey = "update_policy" kubeClusterVersionKey = "version" - kubeClusterCustomizationKey = "customization" + + kubeClusterProxyModeKey = "kube_proxy_mode" + + kubeClusterCustomization = "customization" // Deprecated + kubeClusterCustomizationApiServerKey = "customization_apiserver" + kubeClusterCustomizationKubeProxyKey = "customization_kube_proxy" ) func resourceCloudProjectKube() *schema.Resource { @@ -59,20 +65,60 @@ func resourceCloudProjectKube() *schema.Resource { Optional: true, ForceNew: false, }, - kubeClusterCustomizationKey: { - Type: schema.TypeSet, - Computed: true, - Optional: true, - ForceNew: false, - MaxItems: 1, + kubeClusterCustomizationApiServerKey: { + Type: schema.TypeSet, + Computed: true, + Optional: true, + ForceNew: false, + Set: CustomSchemaSetFunc(), + ConflictsWith: []string{kubeClusterCustomization}, Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ - "apiserver": { + "admissionplugins": { Type: schema.TypeSet, Computed: true, Optional: true, ForceNew: false, - MaxItems: 1, + Set: CustomApiServerAdmissionPluginsSchemaSetFunc(), + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "enabled": { + Type: schema.TypeList, + Computed: true, + Optional: true, + ForceNew: false, + Elem: &schema.Schema{Type: schema.TypeString}, + }, + "disabled": { + Type: schema.TypeList, + Computed: true, + Optional: true, + ForceNew: false, + Elem: &schema.Schema{Type: schema.TypeString}, + }, + }, + }, + }, + }, + }, + }, + kubeClusterCustomization: { + Type: schema.TypeSet, + Computed: true, + Optional: true, + ForceNew: false, + Set: CustomSchemaSetFunc(), + ConflictsWith: []string{kubeClusterCustomizationApiServerKey}, + Deprecated: fmt.Sprintf("Use %s instead", kubeClusterCustomizationApiServerKey), + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "apiserver": { + Type: schema.TypeSet, + Computed: true, + Optional: true, + ForceNew: false, + Set: CustomSchemaSetFunc(), + Deprecated: fmt.Sprintf("Use %s instead", kubeClusterCustomizationApiServerKey), Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ "admissionplugins": { @@ -80,7 +126,7 @@ func resourceCloudProjectKube() *schema.Resource { Computed: true, Optional: true, ForceNew: false, - MaxItems: 1, + Set: CustomApiServerAdmissionPluginsSchemaSetFunc(), Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ "enabled": { @@ -106,11 +152,110 @@ func resourceCloudProjectKube() *schema.Resource { }, }, }, + kubeClusterCustomizationKubeProxyKey: { + Type: schema.TypeSet, + Computed: false, + Optional: true, + ForceNew: false, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "iptables": { + Type: schema.TypeSet, + Computed: false, + Optional: true, + ForceNew: false, + MaxItems: 1, + Set: CustomIPVSIPTablesSchemaSetFunc(), + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "min_sync_period": { + Type: schema.TypeString, + Computed: false, + Optional: true, + ForceNew: false, + ValidateFunc: helpers.ValidateRFC3339Duration, + }, + "sync_period": { + Type: schema.TypeString, + Computed: false, + Optional: true, + ForceNew: false, + ValidateFunc: helpers.ValidateRFC3339Duration, + }, + }, + }, + }, + "ipvs": { + Type: schema.TypeSet, + Computed: false, + Optional: true, + ForceNew: false, + MaxItems: 1, + Set: CustomIPVSIPTablesSchemaSetFunc(), + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "min_sync_period": { + Type: schema.TypeString, + Computed: false, + Optional: true, + ForceNew: false, + ValidateFunc: helpers.ValidateRFC3339Duration, + }, + "sync_period": { + Type: schema.TypeString, + Computed: false, + Optional: true, + ForceNew: false, + ValidateFunc: helpers.ValidateRFC3339Duration, + }, + "scheduler": { + Type: schema.TypeString, + Computed: false, + Optional: true, + ForceNew: false, + ValidateFunc: helpers.ValidateEnum([]string{"rr", "lc", "dh", "sh", "sed", "nq"}), + }, + "tcp_fin_timeout": { + Type: schema.TypeString, + Computed: false, + Optional: true, + ForceNew: false, + ValidateFunc: helpers.ValidateRFC3339Duration, + }, + "tcp_timeout": { + Type: schema.TypeString, + Computed: false, + Optional: true, + ForceNew: false, + ValidateFunc: helpers.ValidateRFC3339Duration, + }, + "udp_timeout": { + Type: schema.TypeString, + Computed: false, + Optional: true, + ForceNew: false, + ValidateFunc: helpers.ValidateRFC3339Duration, + }, + }, + }, + }, + }, + }, + }, + kubeClusterPrivateNetworkIDKey: { Type: schema.TypeString, Optional: true, ForceNew: true, }, + kubeClusterProxyModeKey: { + Type: schema.TypeString, + Optional: true, + ForceNew: true, + Computed: true, + ValidateFunc: helpers.ValidateEnum([]string{"iptables", "ipvs"}), + }, kubeClusterPrivateNetworkConfigurationKey: { Type: schema.TypeSet, Optional: true, @@ -207,6 +352,60 @@ func resourceCloudProjectKube() *schema.Resource { } } +// CustomIPVSIPTablesSchemaSetFunc is a custom schema.SchemaSetFunc for IPVS and IPTables +// block configuration. +// +// Even if setting in the API `PT0S`, it returns `P0D` which is exactly the same duration but +// induce issue when calculating hashset. +// +// Moreover, we cannot use DiffSuppressFunc because even if the diff is removed the hashset is still different. +// +// Using schema.StateFunc does not help because of internal terraform execution diff calculation +// order. +func CustomIPVSIPTablesSchemaSetFunc() schema.SchemaSetFunc { + return func(i interface{}) int { + for k, v := range i.(map[string]interface{}) { + if v == "P0D" { + i.(map[string]interface{})[k] = "PT0S" + } + } + + out := fmt.Sprintf("%#v", i) + return schema.HashString(out) + } +} + +func CustomSchemaSetFunc() schema.SchemaSetFunc { + return func(i interface{}) int { + out := fmt.Sprintf("%#v", i) + return schema.HashString(out) + } +} + +// CustomApiServerAdmissionPluginsSchemaSetFunc is a custom schema.SchemaSetFunc for api_server.admission_plugins +// It orders plugins by alphabetical order to avoid hashset diff +func CustomApiServerAdmissionPluginsSchemaSetFunc() schema.SchemaSetFunc { + return func(i interface{}) int { + enabled := i.(map[string]interface{})["enabled"].([]interface{}) + disabled := i.(map[string]interface{})["disabled"].([]interface{}) + + orderSliceByAlphabeticalOrder := func(s []interface{}) { + sort.Slice(s, func(i, j int) bool { + return s[i].(string) < s[j].(string) + }) + } + + orderSliceByAlphabeticalOrder(enabled) + orderSliceByAlphabeticalOrder(disabled) + + i.(map[string]interface{})["enabled"] = enabled + i.(map[string]interface{})["disabled"] = disabled + + out := fmt.Sprintf("%#v", i) + return schema.HashString(out) + } +} + func resourceCloudProjectKubeImportState(d *schema.ResourceData, meta interface{}) ([]*schema.ResourceData, error) { givenId := d.Id() splitId := strings.SplitN(givenId, "/", 2) @@ -232,31 +431,29 @@ func resourceCloudProjectKubeCreate(d *schema.ResourceData, meta interface{}) er config := meta.(*Config) serviceName := d.Get("service_name").(string) - endpoint := fmt.Sprintf("/cloud/project/%s/kube", serviceName) - params := (&CloudProjectKubeCreateOpts{}).FromResource(d) + params := new(CloudProjectKubeCreateOpts) + params.FromResource(d) + res := &CloudProjectKubeResponse{} - log.Printf("[DEBUG] Will create kube: %+v", params) - err := config.OVHClient.Post(endpoint, params, res) - if err != nil { + log.Printf("[DEBUG] Will create kube: %s", params) + endpoint := fmt.Sprintf("/cloud/project/%s/kube", serviceName) + if err := config.OVHClient.Post(endpoint, params, res); err != nil { return fmt.Errorf("calling Post %s with params %s:\n\t %w", endpoint, params, err) } - // This is a fix for a weird bug where the kube is not immediately available on API log.Printf("[DEBUG] Waiting for kube %s to be available", res.Id) endpoint = fmt.Sprintf("/cloud/project/%s/kube/%s", serviceName, res.Id) - err = helpers.WaitAvailable(config.OVHClient, endpoint, 2*time.Minute) - if err != nil { + if err := helpers.WaitAvailable(config.OVHClient, endpoint, d.Timeout(schema.TimeoutCreate)); err != nil { return err } log.Printf("[DEBUG] Waiting for kube %s to be READY", res.Id) - err = waitForCloudProjectKubeReady(config.OVHClient, serviceName, res.Id, []string{"INSTALLING"}, []string{"READY"}, d.Timeout(schema.TimeoutCreate)) - if err != nil { + if err := waitForCloudProjectKubeReady(config.OVHClient, serviceName, res.Id, []string{"INSTALLING"}, []string{"READY"}, d.Timeout(schema.TimeoutCreate)); err != nil { return fmt.Errorf("timeout while waiting kube %s to be READY: %w", res.Id, err) } - log.Printf("[DEBUG] kube %s is READY", res.Id) + log.Printf("[DEBUG] kube %s is READY", res.Id) d.SetId(res.Id) return resourceCloudProjectKubeRead(d, meta) @@ -273,7 +470,8 @@ func resourceCloudProjectKubeRead(d *schema.ResourceData, meta interface{}) erro if err := config.OVHClient.Get(endpoint, res); err != nil { return helpers.CheckDeleted(d, err, endpoint) } - for k, v := range res.ToMap() { + for k, v := range res.ToMap(d) { + log.Printf("[DEBUG] Will set %s to %v", k, v) if k != "id" { d.Set(k, v) } else { @@ -320,23 +518,40 @@ func resourceCloudProjectKubeUpdate(d *schema.ResourceData, meta interface{}) er config := meta.(*Config) serviceName := d.Get("service_name").(string) - if d.HasChange(kubeClusterCustomizationKey) { - _, newValueI := d.GetChange(kubeClusterCustomizationKey) - customization := loadCustomization(newValueI) + // if customization has changed, update it + if d.HasChange(kubeClusterCustomizationApiServerKey) || d.HasChange(kubeClusterCustomization) || d.HasChange(kubeClusterCustomizationKubeProxyKey) { + customization := new(Customization) - endpoint := fmt.Sprintf("/cloud/project/%s/kube/%s/customization", serviceName, d.Id()) - err := config.OVHClient.Put(endpoint, CloudProjectKubeUpdateCustomizationOpts{ + if d.HasChange(kubeClusterCustomizationKubeProxyKey) { + customization.KubeProxy = loadKubeProxyCustomization(d.Get(kubeClusterCustomizationKubeProxyKey)) + } + + if d.HasChange(kubeClusterCustomizationApiServerKey) { + _, apiServerCustomization := d.GetChange(kubeClusterCustomizationApiServerKey) + customization.APIServer = loadApiServerCustomization(apiServerCustomization) + } + + // deprecated api server customization + if d.HasChange(kubeClusterCustomization) { + _, oldApiServerCustomization := d.GetChange(kubeClusterCustomization) + customization.APIServer = loadDeprecatedApiServerCustomization(oldApiServerCustomization) + } + + params := &CloudProjectKubeUpdateCustomizationOpts{ APIServer: customization.APIServer, - }, nil) - if err != nil { + KubeProxy: customization.KubeProxy, + } + + endpoint := fmt.Sprintf("/cloud/project/%s/kube/%s/customization", serviceName, d.Id()) + if err := config.OVHClient.Put(endpoint, params, nil); err != nil { return err } log.Printf("[DEBUG] Waiting for kube %s to be READY", d.Id()) - err = waitForCloudProjectKubeReady(config.OVHClient, serviceName, d.Id(), []string{"REDEPLOYING", "RESETTING"}, []string{"READY"}, d.Timeout(schema.TimeoutUpdate)) - if err != nil { + if err := waitForCloudProjectKubeReady(config.OVHClient, serviceName, d.Id(), []string{"REDEPLOYING", "RESETTING"}, []string{"READY"}, d.Timeout(schema.TimeoutUpdate)); err != nil { return fmt.Errorf("timeout while waiting kube %s to be READY: %w", d.Id(), err) } + log.Printf("[DEBUG] kube %s is READY", d.Id()) } diff --git a/ovh/resource_cloud_project_kube_nodepool.go b/ovh/resource_cloud_project_kube_nodepool.go index dc20e10f8..0027c7d0d 100644 --- a/ovh/resource_cloud_project_kube_nodepool.go +++ b/ovh/resource_cloud_project_kube_nodepool.go @@ -149,11 +149,7 @@ func resourceCloudProjectKubeNodePool() *schema.Resource { Optional: true, Type: schema.TypeSet, MaxItems: 1, - Set: func(i interface{}) int { - out := fmt.Sprintf("%#v", i) - hash := int(schema.HashString(out)) - return hash - }, + Set: CustomSchemaSetFunc(), Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ "metadata": { @@ -161,11 +157,7 @@ func resourceCloudProjectKubeNodePool() *schema.Resource { Optional: true, Type: schema.TypeSet, MaxItems: 1, - Set: func(i interface{}) int { - out := fmt.Sprintf("%#v", i) - hash := int(schema.HashString(out)) - return hash - }, + Set: CustomSchemaSetFunc(), Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ "finalizers": { @@ -196,11 +188,7 @@ func resourceCloudProjectKubeNodePool() *schema.Resource { Optional: true, Type: schema.TypeSet, MaxItems: 1, - Set: func(i interface{}) int { - out := fmt.Sprintf("%#v", i) - hash := int(schema.HashString(out)) - return hash - }, + Set: CustomSchemaSetFunc(), Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ "unschedulable": { @@ -322,7 +310,7 @@ func resourceCloudProjectKubeNodePoolUpdate(d *schema.ResourceData, meta interfa log.Printf("[DEBUG] Will update nodepool: %#v", *params) err = config.OVHClient.Put(endpoint, params, nil) if err != nil { - return fmt.Errorf("calling Put %s with params %#v:\n\t %w", endpoint, *params, err) + return fmt.Errorf("calling Put %s with params %v:\n\t %w", endpoint, *params, err) } log.Printf("[DEBUG] Waiting for nodepool %s to be READY", d.Id()) diff --git a/ovh/resource_cloud_project_kube_test.go b/ovh/resource_cloud_project_kube_test.go index 379f5b903..35f03e0b0 100644 --- a/ovh/resource_cloud_project_kube_test.go +++ b/ovh/resource_cloud_project_kube_test.go @@ -5,6 +5,7 @@ import ( "fmt" "log" "os" + "regexp" "strconv" "strings" "testing" @@ -13,6 +14,7 @@ import ( "github.com/hashicorp/terraform-plugin-sdk/v2/helper/acctest" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" ) func init() { @@ -141,23 +143,29 @@ resource "ovh_cloud_project_kube" "cluster" { } ` -var testAccCloudProjectKubeCustomizationApiServerAdmissionPluginsCreateConfig = ` +var testAccCloudProjectKubeCustomizationApiServerAdmissionPluginsCreateConfigDefaultValues = ` resource "ovh_cloud_project_kube" "cluster" { service_name = "%s" name = "%s" region = "%s" - customization { - apiserver { - admissionplugins { - enabled = ["NodeRestriction"] - disabled = ["AlwaysPullImages"] - } +} +` + +var testAccCloudProjectKubeCustomizationApiServerAdmissionPluginsUpdateConfigEnabledAndDisabled = ` +resource "ovh_cloud_project_kube" "cluster" { + service_name = "%s" + name = "%s" + region = "%s" + customization_apiserver { + admissionplugins { + enabled = ["NodeRestriction"] + disabled = ["AlwaysPullImages"] } } } ` -var testAccCloudProjectKubeCustomizationApiServerAdmissionPluginsUpdateConfig = ` +var testAccCloudProjectKubeDeprecatedCustomizationApiServerAdmissionPluginsUpdateConfigEnabledAndDisabled = ` resource "ovh_cloud_project_kube" "cluster" { service_name = "%s" name = "%s" @@ -165,7 +173,8 @@ resource "ovh_cloud_project_kube" "cluster" { customization { apiserver { admissionplugins { - enabled = ["AlwaysPullImages","NodeRestriction"] + enabled = ["NodeRestriction"] + disabled = ["AlwaysPullImages"] } } } @@ -187,14 +196,311 @@ func TestAccCloudProjectKubeCustomizationApiServerAdmissionPlugins(t *testing.T) serviceName := os.Getenv("OVH_CLOUD_PROJECT_SERVICE_TEST") name := acctest.RandomWithPrefix(test_prefix) - config := fmt.Sprintf( - testAccCloudProjectKubeCustomizationApiServerAdmissionPluginsCreateConfig, + createConfig := fmt.Sprintf( + testAccCloudProjectKubeCustomizationApiServerAdmissionPluginsCreateConfigDefaultValues, serviceName, name, region, ) - updatedConfig := fmt.Sprintf( - testAccCloudProjectKubeCustomizationApiServerAdmissionPluginsUpdateConfig, + + updatedConfigEnabledAndDisabled := fmt.Sprintf( + testAccCloudProjectKubeCustomizationApiServerAdmissionPluginsUpdateConfigEnabledAndDisabled, + serviceName, + name, + region, + ) + + resource.Test(t, resource.TestCase{ + PreCheck: func() { + testAccPreCheckCloud(t) + testAccCheckCloudProjectExists(t) + testAccPreCheckKubernetes(t) + }, + Providers: testAccProviders, + Steps: []resource.TestStep{ + { + // no apiserver customization, should contain default values from API + Config: createConfig, + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", kubeClusterNameKey, name), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "region", region), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "service_name", serviceName), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_apiserver.0.admissionplugins.0.disabled.#", "0"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_apiserver.0.admissionplugins.0.enabled.0", "AlwaysPullImages"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_apiserver.0.admissionplugins.0.enabled.1", "NodeRestriction"), + + // Conflicts with the old schema + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization.#", "0"), + ), + }, + { + Config: updatedConfigEnabledAndDisabled, + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", kubeClusterNameKey, name), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "region", region), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "service_name", serviceName), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_apiserver.0.admissionplugins.0.enabled.0", "NodeRestriction"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_apiserver.0.admissionplugins.0.disabled.0", "AlwaysPullImages"), + + // Conflicts with the old schema + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization.#", "0"), + ), + }, + }, + }) +} + +// TestAccCloudProjectKubeDeprecatedCustomizationApiServerAdmissionPlugins aims to test that +// values are the same between customization_apiserver.admissionplugins are the same and customization.apiserver.admissionplugins. +// This is deprecated and will be removed in the future. +func TestAccCloudProjectKubeDeprecatedCustomizationApiServerAdmissionPlugins(t *testing.T) { + region := os.Getenv("OVH_CLOUD_PROJECT_KUBE_REGION_TEST") + serviceName := os.Getenv("OVH_CLOUD_PROJECT_SERVICE_TEST") + name := acctest.RandomWithPrefix(test_prefix) + + createConfig := fmt.Sprintf( + testAccCloudProjectKubeDeprecatedCustomizationApiServerAdmissionPluginsUpdateConfigEnabledAndDisabled, + serviceName, + name, + region, + ) + + resource.Test(t, resource.TestCase{ + PreCheck: func() { + testAccPreCheckCloud(t) + testAccCheckCloudProjectExists(t) + testAccPreCheckKubernetes(t) + }, + Providers: testAccProviders, + Steps: []resource.TestStep{ + { + Config: createConfig, + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", kubeClusterNameKey, name), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "region", region), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "service_name", serviceName), + + // Deprecated configuration + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization.0.apiserver.0.admissionplugins.0.enabled.0", "NodeRestriction"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization.0.apiserver.0.admissionplugins.0.disabled.0", "AlwaysPullImages"), + + // Conflicts with the new schema + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_apiserver.#", "0"), + ), + }, + }, + }) +} + +func TestAccCloudProjectKube_kube_proxy_iptables(t *testing.T) { + region := os.Getenv("OVH_CLOUD_PROJECT_KUBE_REGION_TEST") + serviceName := os.Getenv("OVH_CLOUD_PROJECT_SERVICE_TEST") + name := acctest.RandomWithPrefix(test_prefix) + + config := fmt.Sprintf(` +resource "ovh_cloud_project_kube" "cluster" { + service_name = "%s" + name = "%s" + region = "%s" +} +`, + serviceName, + name, + region, + ) + + updatedConfig := fmt.Sprintf(` +resource "ovh_cloud_project_kube" "cluster" { + service_name = "%s" + name = "%s" + region = "%s" + + kube_proxy_mode = "iptables" + customization_kube_proxy { + iptables { + min_sync_period = "PT0S" + } + } +} +`, + serviceName, + name, + region, + ) + + updatedConfigWithDifferentTime := fmt.Sprintf(` +resource "ovh_cloud_project_kube" "cluster" { + service_name = "%s" + name = "%s" + region = "%s" + + kube_proxy_mode = "iptables" + customization_kube_proxy { + iptables { + min_sync_period = "P0D" + } + } +} +`, + serviceName, + name, + region, + ) + + updatedConfigNewArgument := fmt.Sprintf(` +resource "ovh_cloud_project_kube" "cluster" { + service_name = "%s" + name = "%s" + region = "%s" + + kube_proxy_mode = "iptables" + customization_kube_proxy { + iptables { + min_sync_period = "PT30S" + sync_period = "PT30S" + } + } +} +`, + serviceName, + name, + region, + ) + + resource.Test(t, resource.TestCase{ + PreCheck: func() { + testAccPreCheckCloud(t) + testAccCheckCloudProjectExists(t) + testAccPreCheckKubernetes(t) + }, + Providers: testAccProviders, + Steps: []resource.TestStep{ + { + // no kube proxy mode specified, should contain default values from API + Config: config, + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", kubeClusterNameKey, name), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "region", region), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "service_name", serviceName), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "kube_proxy_mode", "iptables"), + resource.TestCheckNoResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy"), + ), + }, + { + Config: updatedConfig, + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", kubeClusterNameKey, name), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "region", region), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "service_name", serviceName), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "kube_proxy_mode", "iptables"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.iptables.0.min_sync_period", "PT0S"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.iptables.0.sync_period", ""), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.#", "0"), + ), + }, + { + Config: updatedConfigWithDifferentTime, + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", kubeClusterNameKey, name), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "region", region), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "service_name", serviceName), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "kube_proxy_mode", "iptables"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.iptables.0.min_sync_period", "PT0S"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.iptables.0.sync_period", ""), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.#", "0"), + ), + }, + { + Config: updatedConfigNewArgument, + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", kubeClusterNameKey, name), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "region", region), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "service_name", serviceName), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "kube_proxy_mode", "iptables"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.iptables.0.min_sync_period", "PT30S"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.iptables.0.sync_period", "PT30S"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.#", "0"), + ), + }, + }, + }) +} + +func TestAccCloudProjectKube_kube_proxy_ipvs(t *testing.T) { + region := os.Getenv("OVH_CLOUD_PROJECT_KUBE_REGION_TEST") + serviceName := os.Getenv("OVH_CLOUD_PROJECT_SERVICE_TEST") + name := acctest.RandomWithPrefix(test_prefix) + + config := fmt.Sprintf(` +resource "ovh_cloud_project_kube" "cluster" { + service_name = "%s" + name = "%s" + region = "%s" + kube_proxy_mode = "ipvs" +} +`, + serviceName, + name, + region, + ) + + updatedConfig := fmt.Sprintf(` +resource "ovh_cloud_project_kube" "cluster" { + service_name = "%s" + name = "%s" + region = "%s" + + kube_proxy_mode = "ipvs" + customization_kube_proxy { + ipvs { + min_sync_period = "PT0S" + } + } +} +`, + serviceName, + name, + region, + ) + + updatedConfigWithDifferentTime := fmt.Sprintf(` +resource "ovh_cloud_project_kube" "cluster" { + service_name = "%s" + name = "%s" + region = "%s" + + kube_proxy_mode = "ipvs" + customization_kube_proxy { + ipvs { + min_sync_period = "P0D" + } + } +} +`, + serviceName, + name, + region, + ) + + updatedConfigAllArguments := fmt.Sprintf(` +resource "ovh_cloud_project_kube" "cluster" { + service_name = "%s" + name = "%s" + region = "%s" + + kube_proxy_mode = "ipvs" + customization_kube_proxy { + ipvs { + min_sync_period = "PT30S" + sync_period = "PT30S" + scheduler = "rr" + tcp_fin_timeout = "PT30S" + tcp_timeout = "PT30S" + udp_timeout = "PT30S" + } + } +} +`, serviceName, name, region, @@ -208,11 +514,250 @@ func TestAccCloudProjectKubeCustomizationApiServerAdmissionPlugins(t *testing.T) }, Providers: testAccProviders, Steps: []resource.TestStep{ + { + // no kube proxy mode specified, should contain default values from API + Config: config, + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", kubeClusterNameKey, name), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "region", region), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "service_name", serviceName), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "kube_proxy_mode", "ipvs"), + resource.TestCheckNoResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy"), + ), + }, + { + Config: updatedConfig, + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", kubeClusterNameKey, name), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "region", region), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "service_name", serviceName), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "kube_proxy_mode", "ipvs"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.0.min_sync_period", "PT0S"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.0.sync_period", ""), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.0.scheduler", ""), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.0.tcp_fin_timeout", ""), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.0.tcp_timeout", ""), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.0.udp_timeout", ""), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.iptables.#", "0"), + ), + }, + { + Config: updatedConfigWithDifferentTime, + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", kubeClusterNameKey, name), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "region", region), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "service_name", serviceName), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "kube_proxy_mode", "ipvs"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.0.min_sync_period", "PT0S"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.0.sync_period", ""), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.0.scheduler", ""), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.0.tcp_fin_timeout", ""), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.0.tcp_timeout", ""), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.0.udp_timeout", ""), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.iptables.#", "0"), + ), + }, + { + Config: updatedConfigAllArguments, + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", kubeClusterNameKey, name), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "region", region), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "service_name", serviceName), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "kube_proxy_mode", "ipvs"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.0.min_sync_period", "PT30S"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.0.sync_period", "PT30S"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.0.scheduler", "rr"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.0.tcp_fin_timeout", "PT30S"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.0.tcp_timeout", "PT30S"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.0.udp_timeout", "PT30S"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.iptables.#", "0"), + ), + }, + }, + }) +} + +func TestAccCloudProjectKube_customization_full_deprecated(t *testing.T) { + region := os.Getenv("OVH_CLOUD_PROJECT_KUBE_REGION_TEST") + serviceName := os.Getenv("OVH_CLOUD_PROJECT_SERVICE_TEST") + name := acctest.RandomWithPrefix(test_prefix) + + erroredConfigKubeProxyMode := fmt.Sprintf(` +resource "ovh_cloud_project_kube" "cluster" { + service_name = "%s" + name = "%s" + region = "%s" + kube_proxy_mode = "foo" +} +`, + serviceName, + name, + region, + ) + + erroredConfigInvalidRFC3339Duration := fmt.Sprintf(` +resource "ovh_cloud_project_kube" "cluster" { + service_name = "%s" + name = "%s" + region = "%s" + + customization_kube_proxy { + iptables { + min_sync_period = "foo" + sync_period = "foo" + } + ipvs { + min_sync_period = "foo" + scheduler = "rr" + sync_period = "foo" + tcp_fin_timeout = "foo" + tcp_timeout = "foo" + udp_timeout = "foo" + } + } +} +`, + serviceName, + name, + region, + ) + + erroredConfigInvalidScheduler := fmt.Sprintf(` +resource "ovh_cloud_project_kube" "cluster" { + service_name = "%s" + name = "%s" + region = "%s" + + customization_kube_proxy { + ipvs { + scheduler = "foo" + } + } +} +`, + serviceName, + name, + region, + ) + + config := fmt.Sprintf(` +resource "ovh_cloud_project_kube" "cluster" { + service_name = "%s" + name = "%s" + region = "%s" + kube_proxy_mode = "iptables" + + customization { + apiserver { + admissionplugins { + enabled = ["NodeRestriction"] + disabled = ["AlwaysPullImages"] + } + } + } + + customization_kube_proxy { + iptables { + min_sync_period = "PT0S" + sync_period = "PT0S" + } + ipvs { + min_sync_period = "PT0S" + scheduler = "rr" + sync_period = "PT0S" + tcp_fin_timeout = "PT0S" + tcp_timeout = "PT0S" + udp_timeout = "PT0S" + } + } +} +`, + serviceName, + name, + region, + ) + + updatedConfig := fmt.Sprintf(` +resource "ovh_cloud_project_kube" "cluster" { + service_name = "%s" + name = "%s" + region = "%s" + kube_proxy_mode = "iptables" + + customization { + apiserver { + admissionplugins { + enabled = ["AlwaysPullImages", "NodeRestriction"] + disabled = [] + } + } + } + + customization_kube_proxy { + iptables { + min_sync_period = "PT30S" + sync_period = "PT30S" + } + ipvs { + min_sync_period = "PT30S" + scheduler = "rr" + sync_period = "PT30S" + tcp_fin_timeout = "PT30S" + tcp_timeout = "PT30S" + udp_timeout = "PT30S" + } + } +} +`, + serviceName, + name, + region, + ) + + resource.Test(t, resource.TestCase{ + PreCheck: func() { + testAccPreCheckCloud(t) + testAccCheckCloudProjectExists(t) + testAccPreCheckKubernetes(t) + }, + Providers: testAccProviders, + Steps: []resource.TestStep{ + { + Config: erroredConfigKubeProxyMode, + ExpectError: regexp.MustCompile(`is not among valid values`), + }, + { + Config: erroredConfigInvalidRFC3339Duration, + ExpectError: regexp.MustCompile(`does not match RFC3339 duration`), + }, + { + Config: erroredConfigInvalidScheduler, + ExpectError: regexp.MustCompile(`is not among valid values`), + }, { Config: config, Check: resource.ComposeTestCheckFunc( resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", kubeClusterNameKey, name), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "region", region), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "service_name", serviceName), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "kube_proxy_mode", "iptables"), + + // customization_kube_proxy - ipvs + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.0.min_sync_period", "PT0S"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.0.scheduler", "rr"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.0.sync_period", "PT0S"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.0.tcp_fin_timeout", "PT0S"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.0.tcp_timeout", "PT0S"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.0.udp_timeout", "PT0S"), + + // customization_kube_proxy - iptables + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.iptables.0.min_sync_period", "PT0S"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.iptables.0.sync_period", "PT0S"), + + // customization - apiserver + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization.0.apiserver.0.admissionplugins.0.enabled.#", "1"), resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization.0.apiserver.0.admissionplugins.0.enabled.0", "NodeRestriction"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization.0.apiserver.0.admissionplugins.0.disabled.#", "1"), resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization.0.apiserver.0.admissionplugins.0.disabled.0", "AlwaysPullImages"), ), }, @@ -220,9 +765,168 @@ func TestAccCloudProjectKubeCustomizationApiServerAdmissionPlugins(t *testing.T) Config: updatedConfig, Check: resource.ComposeTestCheckFunc( resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", kubeClusterNameKey, name), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "region", region), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "service_name", serviceName), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "kube_proxy_mode", "iptables"), + + // customization_kube_proxy - ipvs + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.0.min_sync_period", "PT30S"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.0.scheduler", "rr"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.0.sync_period", "PT30S"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.0.tcp_fin_timeout", "PT30S"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.0.tcp_timeout", "PT30S"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.0.udp_timeout", "PT30S"), + + // customization_kube_proxy - iptables + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.iptables.0.min_sync_period", "PT30S"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.iptables.0.sync_period", "PT30S"), + + // customization - apiserver + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization.0.apiserver.0.admissionplugins.0.disabled.#", "0"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization.0.apiserver.0.admissionplugins.0.enabled.#", "2"), resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization.0.apiserver.0.admissionplugins.0.enabled.0", "AlwaysPullImages"), resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization.0.apiserver.0.admissionplugins.0.enabled.1", "NodeRestriction"), - resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization.0.apiserver.0.admissionplugins.0.disabled.#", "0"), + ), + }, + }, + }) +} + +func TestAccCloudProjectKube_customization_full(t *testing.T) { + region := os.Getenv("OVH_CLOUD_PROJECT_KUBE_REGION_TEST") + serviceName := os.Getenv("OVH_CLOUD_PROJECT_SERVICE_TEST") + name := acctest.RandomWithPrefix(test_prefix) + + config := fmt.Sprintf(` +resource "ovh_cloud_project_kube" "cluster" { + service_name = "%s" + name = "%s" + region = "%s" + kube_proxy_mode = "iptables" + + customization_apiserver { + admissionplugins { + enabled = ["NodeRestriction"] + disabled = ["AlwaysPullImages"] + } + } + + customization_kube_proxy { + iptables { + min_sync_period = "PT0S" + sync_period = "PT0S" + } + ipvs { + min_sync_period = "PT0S" + scheduler = "rr" + sync_period = "PT0S" + tcp_fin_timeout = "PT0S" + tcp_timeout = "PT0S" + udp_timeout = "PT0S" + } + } +} +`, + serviceName, + name, + region, + ) + + updatedConfig := fmt.Sprintf(` +resource "ovh_cloud_project_kube" "cluster" { + service_name = "%s" + name = "%s" + region = "%s" + kube_proxy_mode = "iptables" + + customization_apiserver { + admissionplugins { + enabled = ["AlwaysPullImages", "NodeRestriction"] + disabled = [] + } + } + + customization_kube_proxy { + iptables { + min_sync_period = "PT30S" + sync_period = "PT30S" + } + ipvs { + min_sync_period = "PT30S" + scheduler = "rr" + sync_period = "PT30S" + tcp_fin_timeout = "PT30S" + tcp_timeout = "PT30S" + udp_timeout = "PT30S" + } + } +} +`, + serviceName, + name, + region, + ) + + resource.Test(t, resource.TestCase{ + PreCheck: func() { + testAccPreCheckCloud(t) + testAccCheckCloudProjectExists(t) + testAccPreCheckKubernetes(t) + }, + Providers: testAccProviders, + Steps: []resource.TestStep{ + { + Config: config, + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", kubeClusterNameKey, name), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "region", region), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "service_name", serviceName), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "kube_proxy_mode", "iptables"), + + // customization_kube_proxy - ipvs + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.0.min_sync_period", "PT0S"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.0.scheduler", "rr"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.0.sync_period", "PT0S"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.0.tcp_fin_timeout", "PT0S"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.0.tcp_timeout", "PT0S"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.0.udp_timeout", "PT0S"), + + // customization_kube_proxy - iptables + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.iptables.0.min_sync_period", "PT0S"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.iptables.0.sync_period", "PT0S"), + + // customization - apiserver + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_apiserver.0.admissionplugins.0.enabled.#", "1"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_apiserver.0.admissionplugins.0.enabled.0", "NodeRestriction"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_apiserver.0.admissionplugins.0.disabled.#", "1"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_apiserver.0.admissionplugins.0.disabled.0", "AlwaysPullImages"), + ), + }, + { + Config: updatedConfig, + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", kubeClusterNameKey, name), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "region", region), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "service_name", serviceName), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "kube_proxy_mode", "iptables"), + + // customization_kube_proxy - ipvs + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.0.min_sync_period", "PT30S"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.0.scheduler", "rr"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.0.sync_period", "PT30S"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.0.tcp_fin_timeout", "PT30S"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.0.tcp_timeout", "PT30S"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.ipvs.0.udp_timeout", "PT30S"), + + // customization_kube_proxy - iptables + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.iptables.0.min_sync_period", "PT30S"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_kube_proxy.0.iptables.0.sync_period", "PT30S"), + + // customization - apiserver + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_apiserver.0.admissionplugins.0.disabled.#", "0"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_apiserver.0.admissionplugins.0.enabled.#", "2"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_apiserver.0.admissionplugins.0.enabled.0", "AlwaysPullImages"), + resource.TestCheckResourceAttr("ovh_cloud_project_kube.cluster", "customization_apiserver.0.admissionplugins.0.enabled.1", "NodeRestriction"), ), }, }, @@ -498,3 +1202,82 @@ func TestAccCloudProjectKubeUpdateVersion_basic(t *testing.T) { }, }) } + +func TestCustomIPVSIPTablesSchemaSetFunc(t *testing.T) { + tests := []struct { + name string + input map[string]interface{} + expectedHex string + }{ + { + name: "Input with P0D value", + input: map[string]interface{}{ + "key1": "P0D", + "key2": "value2", + }, + expectedHex: fmt.Sprintf("%#v", map[string]interface{}{ + "key1": "PT0S", + "key2": "value2", + }), + }, + { + name: "Input without P0D value", + input: map[string]interface{}{ + "key1": "value1", + "key2": "value2", + }, + expectedHex: fmt.Sprintf("%#v", map[string]interface{}{ + "key1": "value1", + "key2": "value2", + }), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + expectedHash := schema.HashString(tt.expectedHex) + if got := CustomIPVSIPTablesSchemaSetFunc()(tt.input); got != expectedHash { + t.Errorf("CustomIPVSIPTablesSchemaSetFunc() = %v, want %v", got, expectedHash) + } + }) + } +} + +func TestCustomApiServerAdmissionPluginsSchemaSetFunc(t *testing.T) { + tests := []struct { + name string + input map[string]interface{} + expectedHex string + }{ + { + name: "No plugins", + input: map[string]interface{}{ + "enabled": []interface{}{}, + "disabled": []interface{}{}, + }, + expectedHex: fmt.Sprintf("%#v", map[string]interface{}{ + "enabled": []interface{}{}, + "disabled": []interface{}{}, + }), + }, + { + name: "Should reorder plugins", + input: map[string]interface{}{ + "enabled": []interface{}{"foo", "bar"}, + "disabled": []interface{}{"bar", "foo"}, + }, + expectedHex: fmt.Sprintf("%#v", map[string]interface{}{ + "enabled": []interface{}{"bar", "foo"}, + "disabled": []interface{}{"bar", "foo"}, + }), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + expectedHash := schema.HashString(tt.expectedHex) + if got := CustomApiServerAdmissionPluginsSchemaSetFunc()(tt.input); got != expectedHash { + t.Errorf("CustomApiServerAdmissionPluginsSchemaSetFunc() = %v, want %v", got, expectedHash) + } + }) + } +} diff --git a/ovh/types_cloud_project_kube.go b/ovh/types_cloud_project_kube.go index 8c3279372..542de4ba8 100644 --- a/ovh/types_cloud_project_kube.go +++ b/ovh/types_cloud_project_kube.go @@ -2,6 +2,7 @@ package ovh import ( "fmt" + "log" "strings" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" @@ -30,69 +31,195 @@ type CloudProjectKubeCreateOpts struct { Version *string `json:"version,omitempty"` UpdatePolicy *string `json:"updatePolicy,omitempty"` Customization *Customization `json:"customization,omitempty"` + KubeProxyMode *string `json:"kubeProxyMode,omitempty"` } type Customization struct { - APIServer *APIServer `json:"apiServer,omitempty"` + APIServer *APIServer `json:"apiServer,omitempty"` + KubeProxy *kubeProxyCustomization `json:"kubeProxy,omitempty"` } type APIServer struct { AdmissionPlugins *AdmissionPlugins `json:"admissionPlugins,omitempty"` } +type kubeProxyCustomization struct { + IPTables *kubeProxyCustomizationIPTables `json:"iptables,omitempty"` + IPVS *kubeProxyCustomizationIPVS `json:"ipvs,omitempty"` +} + +type kubeProxyCustomizationIPTables struct { + MinSyncPeriod *string `json:"minSyncPeriod,omitempty"` + SyncPeriod *string `json:"syncPeriod,omitempty"` +} + +type kubeProxyCustomizationIPVS struct { + MinSyncPeriod *string `json:"minSyncPeriod,omitempty"` + Scheduler *string `json:"scheduler,omitempty"` + SyncPeriod *string `json:"syncPeriod,omitempty"` + TCPFinTimeout *string `json:"tcpFinTimeout,omitempty"` + TCPTimeout *string `json:"tcpTimeout,omitempty"` + UDPTimeout *string `json:"udpTimeout,omitempty"` +} + type AdmissionPlugins struct { Enabled *[]string `json:"enabled,omitempty"` Disabled *[]string `json:"disabled,omitempty"` } -func (opts *CloudProjectKubeCreateOpts) FromResource(d *schema.ResourceData) *CloudProjectKubeCreateOpts { +func (opts *CloudProjectKubeCreateOpts) FromResource(d *schema.ResourceData) { opts.Region = d.Get("region").(string) opts.Version = helpers.GetNilStringPointerFromData(d, "version") opts.Name = helpers.GetNilStringPointerFromData(d, "name") opts.UpdatePolicy = helpers.GetNilStringPointerFromData(d, "update_policy") opts.PrivateNetworkId = helpers.GetNilStringPointerFromData(d, "private_network_id") opts.PrivateNetworkConfiguration = loadPrivateNetworkConfiguration(d.Get("private_network_configuration")) - opts.Customization = loadCustomization(d.Get("customization")) - return opts + opts.KubeProxyMode = helpers.GetNilStringPointerFromData(d, kubeClusterProxyModeKey) + + opts.Customization = &Customization{ + APIServer: nil, + KubeProxy: loadKubeProxyCustomization(d.Get(kubeClusterCustomizationKubeProxyKey)), + } + + // load the filled api server customization + // both the new and the deprecated syntax are supported, but they are mutual exclusive + if userIsUsingDeprecatedCustomizationSyntax(d) { + log.Printf("[DEBUG] Using DEPRECATED syntax for api server customization") + opts.Customization.APIServer = loadDeprecatedApiServerCustomization(d.Get(kubeClusterCustomization)) + } else { + log.Printf("[DEBUG] Using new syntax for api server customization") + opts.Customization.APIServer = loadApiServerCustomization(d.Get(kubeClusterCustomizationApiServerKey)) + } } -func loadCustomization(i interface{}) *Customization { - if i == nil { +func userIsUsingDeprecatedCustomizationSyntax(d *schema.ResourceData) bool { + funcTypeSetNotNilAndNotEmpty := func(d *schema.ResourceData, key string) bool { + return d.Get(key) != nil && len(d.Get(key).(*schema.Set).List()) > 0 + } + + return funcTypeSetNotNilAndNotEmpty(d, kubeClusterCustomization) +} + +// loadApiServerCustomization reads the api server customization +func loadApiServerCustomization(apiServerAdmissionPlugins interface{}) *APIServer { + if apiServerAdmissionPlugins == nil { return nil } - customizationOutput := Customization{ - APIServer: &APIServer{ - AdmissionPlugins: &AdmissionPlugins{}, - }, + apiServerOutput := &APIServer{ + AdmissionPlugins: &AdmissionPlugins{}, + } + + // Customization + customizationSet := apiServerAdmissionPlugins.(*schema.Set).List() + if len(customizationSet) > 0 { + customization := customizationSet[0].(map[string]interface{}) + admissionPluginsSet := customization["admissionplugins"].(*schema.Set).List() + admissionPlugins := admissionPluginsSet[0].(map[string]interface{}) + + readApiServerAdmissionPlugins(admissionPlugins, apiServerOutput) + + log.Printf("[DEBUG] Enabled admission plugins from new syntax: %v", apiServerOutput.AdmissionPlugins.Enabled) + log.Printf("[DEBUG] Disabled admission plugins from new syntax: %v", apiServerOutput.AdmissionPlugins.Disabled) + } + + return apiServerOutput +} + +func readApiServerAdmissionPlugins(admissionPlugins map[string]interface{}, apiServerOutput *APIServer) { + // Enabled admission plugins + { + stringArray := admissionPlugins["enabled"].([]interface{}) + enabled := make([]string, 0, len(stringArray)) + for _, s := range stringArray { + enabled = append(enabled, s.(string)) + } + apiServerOutput.AdmissionPlugins.Enabled = &enabled } - customizationSet := i.(*schema.Set).List() - for _, customization := range customizationSet { - apiServerSet := customization.(map[string]interface{})["apiserver"].(*schema.Set).List() - for _, apiServer := range apiServerSet { - admissionPluginsSet := apiServer.(map[string]interface{})["admissionplugins"].(*schema.Set).List() - for _, admissionPlugins := range admissionPluginsSet { + // Disabled admission plugins + { + stringArray := admissionPlugins["disabled"].([]interface{}) + disabled := make([]string, 0, len(stringArray)) + for _, s := range stringArray { + disabled = append(disabled, s.(string)) + } + apiServerOutput.AdmissionPlugins.Disabled = &disabled + } +} - stringArray := admissionPlugins.(map[string]interface{})["enabled"].([]interface{}) - enabled := []string{} - for _, s := range stringArray { - enabled = append(enabled, s.(string)) - } - customizationOutput.APIServer.AdmissionPlugins.Enabled = &enabled +// loadDeprecatedApiServerCustomization reads the deprecated api server customization +// Deprecated, should be removed in the future +func loadDeprecatedApiServerCustomization(deprecatedApiServerCustomizationInterface interface{}) *APIServer { + if deprecatedApiServerCustomizationInterface == nil { + return nil + } + + apiServerOutput := &APIServer{ + AdmissionPlugins: &AdmissionPlugins{}, + } - stringArray = admissionPlugins.(map[string]interface{})["disabled"].([]interface{}) - disabled := []string{} - for _, s := range stringArray { - disabled = append(disabled, s.(string)) - } - customizationOutput.APIServer.AdmissionPlugins.Disabled = &disabled + oldCustomizationSet := deprecatedApiServerCustomizationInterface.(*schema.Set).List() + if len(oldCustomizationSet) > 0 { + oldApiServerCustomization := oldCustomizationSet[0].(map[string]interface{}) + oldApiServerCustomizationSet := oldApiServerCustomization["apiserver"].(*schema.Set).List() + if len(oldApiServerCustomizationSet) > 0 { + oldApiServerCustomizationAdmissionPlugins := oldApiServerCustomizationSet[0].(map[string]interface{}) + oldApiServerCustomizationAdmissionPluginsSet := oldApiServerCustomizationAdmissionPlugins["admissionplugins"].(*schema.Set).List() + admissionPlugins := oldApiServerCustomizationAdmissionPluginsSet[0].(map[string]interface{}) + + readApiServerAdmissionPlugins(admissionPlugins, apiServerOutput) + } + } + + log.Printf("[DEBUG] Enabled admission plugins from DEPRECATED syntax: %v", apiServerOutput.AdmissionPlugins.Enabled) + log.Printf("[DEBUG] Disabled admission plugins from DEPRECATED syntax: %v", apiServerOutput.AdmissionPlugins.Disabled) + + return apiServerOutput +} + +// loadKubeProxyCustomization reads the kube proxy customization +func loadKubeProxyCustomization(kubeProxyCustomizationInterface interface{}) *kubeProxyCustomization { + if kubeProxyCustomizationInterface == nil { + return nil + } + + kubeProxyOutput := &kubeProxyCustomization{ + IPTables: &kubeProxyCustomizationIPTables{}, + IPVS: &kubeProxyCustomizationIPVS{}, + } + + kubeProxySet := kubeProxyCustomizationInterface.(*schema.Set).List() + if len(kubeProxySet) > 0 { + kubeProxy := kubeProxySet[0].(map[string]interface{}) + + // Nested IPTables customization + { + ipTablesSet := kubeProxy["iptables"].(*schema.Set).List() + if len(ipTablesSet) > 0 { + ipTables := ipTablesSet[0].(map[string]interface{}) + kubeProxyOutput.IPTables.MinSyncPeriod = helpers.GetNilStringPointerFromData(ipTables, "min_sync_period") + kubeProxyOutput.IPTables.SyncPeriod = helpers.GetNilStringPointerFromData(ipTables, "sync_period") + } + } + + // Nested IPVS customization + { + ipvsSet := kubeProxy["ipvs"].(*schema.Set).List() + if len(ipvsSet) > 0 { + ipvs := ipvsSet[0].(map[string]interface{}) + kubeProxyOutput.IPVS.MinSyncPeriod = helpers.GetNilStringPointerFromData(ipvs, "min_sync_period") + kubeProxyOutput.IPVS.Scheduler = helpers.GetNilStringPointerFromData(ipvs, "scheduler") + kubeProxyOutput.IPVS.SyncPeriod = helpers.GetNilStringPointerFromData(ipvs, "sync_period") + kubeProxyOutput.IPVS.TCPFinTimeout = helpers.GetNilStringPointerFromData(ipvs, "tcp_fin_timeout") + kubeProxyOutput.IPVS.TCPTimeout = helpers.GetNilStringPointerFromData(ipvs, "tcp_timeout") + kubeProxyOutput.IPVS.UDPTimeout = helpers.GetNilStringPointerFromData(ipvs, "udp_timeout") } } } - return &customizationOutput + return kubeProxyOutput } func loadPrivateNetworkConfiguration(i interface{}) *privateNetworkConfiguration { @@ -110,8 +237,19 @@ func loadPrivateNetworkConfiguration(i interface{}) *privateNetworkConfiguration return &pncOutput } -func (s *CloudProjectKubeCreateOpts) String() string { - return fmt.Sprintf("%s(%s): %s", *s.Name, s.Region, *s.Version) +func (opts *CloudProjectKubeCreateOpts) String() string { + var str string + if opts.Name != nil { + str = *opts.Name + } + + str += fmt.Sprintf(" (%s)", opts.Region) + + if opts.Version != nil { + str += fmt.Sprintf(": %s", *opts.Version) + } + + return str } type CloudProjectKubeResponse struct { @@ -128,9 +266,10 @@ type CloudProjectKubeResponse struct { Url string `json:"url"` Version string `json:"version"` Customization Customization `json:"customization"` + KubeProxyMode string `json:"kubeProxyMode"` } -func (v CloudProjectKubeResponse) ToMap() map[string]interface{} { +func (v *CloudProjectKubeResponse) ToMap(d *schema.ResourceData) map[string]interface{} { obj := make(map[string]interface{}) obj["control_plane_is_up_to_date"] = v.ControlPlaneIsUpToDate obj["id"] = v.Id @@ -144,7 +283,81 @@ func (v CloudProjectKubeResponse) ToMap() map[string]interface{} { obj["update_policy"] = v.UpdatePolicy obj["url"] = v.Url obj["version"] = v.Version[:strings.LastIndex(v.Version, ".")] - obj["customization"] = []map[string]interface{}{ + obj[kubeClusterProxyModeKey] = v.KubeProxyMode + + if v.Customization.APIServer != nil { + if userIsUsingDeprecatedCustomizationSyntax(d) { + loadDeprecatedApiServerCustomizationToMap(obj, v) + } else { + loadApiServerCustomizationToMap(obj, v) + } + } + + if v.Customization.KubeProxy != nil { + loadKubeProxyCustomizationToMap(obj, v) + } + + return obj +} + +func loadKubeProxyCustomizationToMap(obj map[string]interface{}, v *CloudProjectKubeResponse) { + obj[kubeClusterCustomizationKubeProxyKey] = []map[string]interface{}{{}} + + if v.Customization.KubeProxy.IPTables != nil { + data := make(map[string]interface{}) + if vv := v.Customization.KubeProxy.IPTables.MinSyncPeriod; vv != nil && *vv != "" { + data["min_sync_period"] = vv + } + + if vv := v.Customization.KubeProxy.IPTables.SyncPeriod; vv != nil && *vv != "" { + data["sync_period"] = vv + } + + if len(data) > 0 { + obj[kubeClusterCustomizationKubeProxyKey].([]map[string]interface{})[0]["iptables"] = []map[string]interface{}{data} + } + } + + if v.Customization.KubeProxy.IPVS != nil { + data := make(map[string]interface{}) + if vv := v.Customization.KubeProxy.IPVS.MinSyncPeriod; vv != nil && *vv != "" { + data["min_sync_period"] = vv + } + + if vv := v.Customization.KubeProxy.IPVS.Scheduler; vv != nil && *vv != "" { + data["scheduler"] = vv + } + + if vv := v.Customization.KubeProxy.IPVS.SyncPeriod; vv != nil && *vv != "" { + data["sync_period"] = vv + } + + if vv := v.Customization.KubeProxy.IPVS.TCPFinTimeout; vv != nil && *vv != "" { + data["tcp_fin_timeout"] = vv + } + + if vv := v.Customization.KubeProxy.IPVS.TCPTimeout; vv != nil && *vv != "" { + data["tcp_timeout"] = vv + } + + if vv := v.Customization.KubeProxy.IPVS.UDPTimeout; vv != nil && *vv != "" { + data["udp_timeout"] = vv + } + + if len(data) > 0 { + obj[kubeClusterCustomizationKubeProxyKey].([]map[string]interface{})[0]["ipvs"] = []map[string]interface{}{data} + } + } + + // Delete entire customization_kube_proxy if empty + if len(obj[kubeClusterCustomizationKubeProxyKey].([]map[string]interface{})[0]) == 0 { + delete(obj, kubeClusterCustomizationKubeProxyKey) + } +} + +// Deprecated: use loadApiServerCustomizationToMap instead +func loadDeprecatedApiServerCustomizationToMap(obj map[string]interface{}, v *CloudProjectKubeResponse) { + obj[kubeClusterCustomization] = []map[string]interface{}{ { "apiserver": []map[string]interface{}{ { @@ -158,11 +371,23 @@ func (v CloudProjectKubeResponse) ToMap() map[string]interface{} { }, }, } - return obj } -func (s *CloudProjectKubeResponse) String() string { - return fmt.Sprintf("%s(%s): %s", s.Name, s.Id, s.Status) +func loadApiServerCustomizationToMap(obj map[string]interface{}, v *CloudProjectKubeResponse) { + obj[kubeClusterCustomizationApiServerKey] = []map[string]interface{}{ + { + "admissionplugins": []map[string]interface{}{ + { + "enabled": v.Customization.APIServer.AdmissionPlugins.Enabled, + "disabled": v.Customization.APIServer.AdmissionPlugins.Disabled, + }, + }, + }, + } +} + +func (v *CloudProjectKubeResponse) String() string { + return fmt.Sprintf("%s(%s): %s", v.Name, v.Id, v.Status) } type CloudProjectKubeKubeConfigResponse struct { @@ -183,7 +408,8 @@ type CloudProjectKubeUpdatePNCOpts struct { } type CloudProjectKubeUpdateCustomizationOpts struct { - APIServer *APIServer `json:"apiServer"` + APIServer *APIServer `json:"apiServer,omitempty"` + KubeProxy *kubeProxyCustomization `json:"kubeProxy,omitempty"` } type CloudProjectKubeNodeResponse struct { diff --git a/ovh/types_cloud_project_kube_test.go b/ovh/types_cloud_project_kube_test.go new file mode 100644 index 000000000..28068d70a --- /dev/null +++ b/ovh/types_cloud_project_kube_test.go @@ -0,0 +1,475 @@ +package ovh + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" +) + +func TestCloudProjectKubeResponse_ToMap(t *testing.T) { + t.Skipf("Skipped as we need to create a *schema.ResourceData to test this function.") + + type fields struct { + ControlPlaneIsUpToDate bool + Id string + IsUpToDate bool + Name string + NextUpgradeVersions []string + NodesUrl string + PrivateNetworkId string + Region string + Status string + UpdatePolicy string + Url string + Version string + Customization Customization + KubeProxyMode string + } + + type args struct { + d *schema.ResourceData + } + + pointerArray := func(s []string) *[]string { return &s } + + tests := []struct { + name string + fields fields + args args + want map[string]interface{} + }{ + { + name: "No customization", + fields: fields{ + ControlPlaneIsUpToDate: false, + Id: "", + IsUpToDate: false, + Name: "", + NextUpgradeVersions: nil, + NodesUrl: "", + PrivateNetworkId: "", + Region: "", + Status: "", + UpdatePolicy: "", + Url: "", + Version: "1.0.0", + Customization: Customization{}, + KubeProxyMode: "", + }, + args: args{}, + want: map[string]interface{}{ + "control_plane_is_up_to_date": false, + "id": "", + "is_up_to_date": false, + "name": "", + "next_upgrade_versions": []string(nil), + "nodes_url": "", + "private_network_id": "", + "region": "", + "status": "", + "update_policy": "", + "url": "", + "version": "1.0", + kubeClusterProxyModeKey: "", + }, + }, + { + name: "Deprecated expected apiserver customization", + fields: fields{ + ControlPlaneIsUpToDate: false, + Id: "", + IsUpToDate: false, + Name: "", + NextUpgradeVersions: nil, + NodesUrl: "", + PrivateNetworkId: "", + Region: "", + Status: "", + UpdatePolicy: "", + Url: "", + Version: "1.0.0", + Customization: Customization{ + APIServer: &APIServer{ + AdmissionPlugins: &AdmissionPlugins{ + Enabled: pointerArray([]string{"foo"}), + Disabled: pointerArray([]string{"bar"}), + }, + }, + KubeProxy: nil, + }, + KubeProxyMode: "", + }, + args: args{}, + want: map[string]interface{}{ + "control_plane_is_up_to_date": false, + "id": "", + "is_up_to_date": false, + "name": "", + "next_upgrade_versions": []string(nil), + "nodes_url": "", + "private_network_id": "", + "region": "", + "status": "", + "update_policy": "", + "url": "", + "version": "1.0", + kubeClusterProxyModeKey: "", + "customization": []map[string]interface{}{ + { + "apiserver": []map[string]interface{}{ + { + "admissionplugins": []map[string]interface{}{ + { + "enabled": pointerArray([]string{"foo"}), + "disabled": pointerArray([]string{"bar"}), + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "Expected apiserver customization", + fields: fields{ + ControlPlaneIsUpToDate: false, + Id: "", + IsUpToDate: false, + Name: "", + NextUpgradeVersions: nil, + NodesUrl: "", + PrivateNetworkId: "", + Region: "", + Status: "", + UpdatePolicy: "", + Url: "", + Version: "1.0.0", + Customization: Customization{ + APIServer: &APIServer{ + AdmissionPlugins: &AdmissionPlugins{ + Enabled: pointerArray([]string{"foo"}), + Disabled: pointerArray([]string{"bar"}), + }, + }, + KubeProxy: nil, + }, + KubeProxyMode: "", + }, + args: args{}, + want: map[string]interface{}{ + "control_plane_is_up_to_date": false, + "id": "", + "is_up_to_date": false, + "name": "", + "next_upgrade_versions": []string(nil), + "nodes_url": "", + "private_network_id": "", + "region": "", + "status": "", + "update_policy": "", + "url": "", + "version": "1.0", + kubeClusterProxyModeKey: "", + "customization_apiserver": []map[string]interface{}{ + { + "admissionplugins": []map[string]interface{}{ + { + "enabled": pointerArray([]string{"foo"}), + "disabled": pointerArray([]string{"bar"}), + }, + }, + }, + }, + }, + }, + { + name: "IPTables customization with one field", + fields: fields{ + ControlPlaneIsUpToDate: false, + Id: "", + IsUpToDate: false, + Name: "", + NextUpgradeVersions: nil, + NodesUrl: "", + PrivateNetworkId: "", + Region: "", + Status: "", + UpdatePolicy: "", + Url: "", + Version: "1.0.0", + Customization: Customization{ + APIServer: nil, + KubeProxy: &kubeProxyCustomization{ + IPTables: &kubeProxyCustomizationIPTables{ + MinSyncPeriod: strPtr("PT30S"), + }, + IPVS: nil, + }, + }, + KubeProxyMode: "iptables", + }, + args: args{}, + want: map[string]interface{}{ + "control_plane_is_up_to_date": false, + "id": "", + "is_up_to_date": false, + "name": "", + "next_upgrade_versions": []string(nil), + "nodes_url": "", + "private_network_id": "", + "region": "", + "status": "", + "update_policy": "", + "url": "", + "version": "1.0", + kubeClusterProxyModeKey: "iptables", + "customization_kube_proxy": []map[string]interface{}{ + { + "iptables": []map[string]interface{}{ + { + "min_sync_period": strPtr("PT30S"), + }, + }, + }, + }, + }, + }, + { + name: "IPTables customization", + fields: fields{ + ControlPlaneIsUpToDate: false, + Id: "", + IsUpToDate: false, + Name: "", + NextUpgradeVersions: nil, + NodesUrl: "", + PrivateNetworkId: "", + Region: "", + Status: "", + UpdatePolicy: "", + Url: "", + Version: "1.0.0", + Customization: Customization{ + APIServer: nil, + KubeProxy: &kubeProxyCustomization{ + IPTables: &kubeProxyCustomizationIPTables{ + MinSyncPeriod: strPtr("PT30S"), + SyncPeriod: strPtr("PT30S"), + }, + IPVS: nil, + }, + }, + KubeProxyMode: "iptables", + }, + args: args{}, + want: map[string]interface{}{ + "control_plane_is_up_to_date": false, + "id": "", + "is_up_to_date": false, + "name": "", + "next_upgrade_versions": []string(nil), + "nodes_url": "", + "private_network_id": "", + "region": "", + "status": "", + "update_policy": "", + "url": "", + "version": "1.0", + kubeClusterProxyModeKey: "iptables", + "customization_kube_proxy": []map[string]interface{}{ + { + "iptables": []map[string]interface{}{ + { + "min_sync_period": strPtr("PT30S"), + "sync_period": strPtr("PT30S"), + }, + }, + }, + }, + }, + }, + { + name: "IPVS customization with one field", + fields: fields{ + ControlPlaneIsUpToDate: false, + Id: "", + IsUpToDate: false, + Name: "", + NextUpgradeVersions: nil, + NodesUrl: "", + PrivateNetworkId: "", + Region: "", + Status: "", + UpdatePolicy: "", + Url: "", + Version: "1.0.0", + Customization: Customization{ + APIServer: nil, + KubeProxy: &kubeProxyCustomization{ + IPTables: nil, + IPVS: &kubeProxyCustomizationIPVS{ + MinSyncPeriod: strPtr("PT30S"), + }, + }, + }, + KubeProxyMode: "ipvs", + }, + args: args{}, + want: map[string]interface{}{ + "control_plane_is_up_to_date": false, + "id": "", + "is_up_to_date": false, + "name": "", + "next_upgrade_versions": []string(nil), + "nodes_url": "", + "private_network_id": "", + "region": "", + "status": "", + "update_policy": "", + "url": "", + "version": "1.0", + kubeClusterProxyModeKey: "ipvs", + "customization_kube_proxy": []map[string]interface{}{ + { + "ipvs": []map[string]interface{}{ + { + "min_sync_period": strPtr("PT30S"), + }, + }, + }, + }, + }, + }, + { + name: "IPVS customization", + fields: fields{ + ControlPlaneIsUpToDate: false, + Id: "", + IsUpToDate: false, + Name: "", + NextUpgradeVersions: nil, + NodesUrl: "", + PrivateNetworkId: "", + Region: "", + Status: "", + UpdatePolicy: "", + Url: "", + Version: "1.0.0", + Customization: Customization{ + APIServer: nil, + KubeProxy: &kubeProxyCustomization{ + IPTables: nil, + IPVS: &kubeProxyCustomizationIPVS{ + MinSyncPeriod: strPtr("PT30S"), + SyncPeriod: strPtr("PT30S"), + Scheduler: strPtr("rr"), + TCPFinTimeout: strPtr("PT30S"), + TCPTimeout: strPtr("PT30S"), + UDPTimeout: strPtr("PT30S"), + }, + }, + }, + KubeProxyMode: "ipvs", + }, + args: args{}, + want: map[string]interface{}{ + "control_plane_is_up_to_date": false, + "id": "", + "is_up_to_date": false, + "name": "", + "next_upgrade_versions": []string(nil), + "nodes_url": "", + "private_network_id": "", + "region": "", + "status": "", + "update_policy": "", + "url": "", + "version": "1.0", + kubeClusterProxyModeKey: "ipvs", + "customization_kube_proxy": []map[string]interface{}{ + { + "ipvs": []map[string]interface{}{ + { + "min_sync_period": strPtr("PT30S"), + "sync_period": strPtr("PT30S"), + "scheduler": strPtr("rr"), + "tcp_fin_timeout": strPtr("PT30S"), + "tcp_timeout": strPtr("PT30S"), + "udp_timeout": strPtr("PT30S"), + }, + }, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + v := CloudProjectKubeResponse{ + ControlPlaneIsUpToDate: tt.fields.ControlPlaneIsUpToDate, + Id: tt.fields.Id, + IsUpToDate: tt.fields.IsUpToDate, + Name: tt.fields.Name, + NextUpgradeVersions: tt.fields.NextUpgradeVersions, + NodesUrl: tt.fields.NodesUrl, + PrivateNetworkId: tt.fields.PrivateNetworkId, + Region: tt.fields.Region, + Status: tt.fields.Status, + UpdatePolicy: tt.fields.UpdatePolicy, + Url: tt.fields.Url, + Version: tt.fields.Version, + Customization: tt.fields.Customization, + KubeProxyMode: tt.fields.KubeProxyMode, + } + + got := v.ToMap(tt.args.d) + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("ToMap() mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func Test_readApiServerAdmissionPlugins(t *testing.T) { + type args struct { + admissionPlugins map[string]interface{} + apiServerOutput *APIServer + } + + pointerArray := func(s []string) *[]string { return &s } + + tests := []struct { + name string + args args + want *APIServer + }{ + { + name: "expected admission plugins", + args: args{ + admissionPlugins: map[string]interface{}{ + "enabled": []interface{}{"foo", "bar"}, + "disabled": []interface{}{"baz"}, + }, + apiServerOutput: &APIServer{ + AdmissionPlugins: &AdmissionPlugins{}, + }, + }, + want: &APIServer{ + AdmissionPlugins: &AdmissionPlugins{ + Enabled: pointerArray([]string{"foo", "bar"}), + Disabled: pointerArray([]string{"baz"}), + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + readApiServerAdmissionPlugins(tt.args.admissionPlugins, tt.args.apiServerOutput) + if diff := cmp.Diff(tt.want, tt.args.apiServerOutput); diff != "" { + t.Errorf("readApiServerAdmissionPlugins() mismatch (-want +got):\n%s", diff) + } + }) + } +} diff --git a/website/docs/d/cloud_project_kube.html.markdown b/website/docs/d/cloud_project_kube.html.markdown index 16baffc3a..51d6bf098 100644 --- a/website/docs/d/cloud_project_kube.html.markdown +++ b/website/docs/d/cloud_project_kube.html.markdown @@ -40,15 +40,29 @@ The following attributes are exported: * `region` - The OVHcloud public cloud region ID of the managed kubernetes cluster. * `version` - Kubernetes version of the managed kubernetes cluster. * `private_network_id` - OpenStack private network (or vrack) ID to use. -* `control_plane_is_up_to_date` - True if control-plane is up to date. -* `is_up_to_date` - True if all nodes and control-plane are up to date. +* `control_plane_is_up_to_date` - True if control-plane is up-to-date. +* `is_up_to_date` - True if all nodes and control-plane are up-to-date. * `next_upgrade_versions` - Kubernetes versions available for upgrade. * `nodes_url` - Cluster nodes URL. * `status` - Cluster status. Should be normally set to 'READY'. * `update_policy` - Cluster update policy. Choose between [ALWAYS_UPDATE,MINIMAL_DOWNTIME,NEVER_UPDATE]'. * `url` - Management URL of your cluster. -* `customization` - Customer customization object +* `kube_proxy_mode` - Selected mode for kube-proxy. +* `customization` - **Deprecated** (Optional) Use `customization_apiserver` and `customization_kube_proxy` instead. Kubernetes cluster customization * `apiserver` - Kubernetes API server customization - * `admissionplugins` - Kubernetes API server admission plugins customization - * `enabled` - Array of admission plugins enabled, default is ["NodeRestriction","AlwaysPulImages"] and only these admission plugins can be enabled at this time. - * `disabled` - Array of admission plugins disabled, default is [] and only AlwaysPulImages can be disabled at this time. + * `kube_proxy` - Kubernetes kube-proxy customization +* `customization_apiserver` - Kubernetes API server customization + * `admissionplugins` - Kubernetes API server admission plugins customization + * `enabled` - Array of admission plugins enabled, default is ["NodeRestriction","AlwaysPulImages"] and only these admission plugins can be enabled at this time. + * `disabled` - Array of admission plugins disabled, default is [] and only AlwaysPulImages can be disabled at this time. +* `customization_kube_proxy` - Kubernetes kube-proxy customization + * `iptables` - Kubernetes cluster kube-proxy customization of iptables specific config. + * `sync_period` - Minimum period that iptables rules are refreshed, in [RFC3339](https://www.rfc-editor.org/rfc/rfc3339) duration format. + * `min_sync_period` - Period that iptables rules are refreshed, in [RFC3339](https://www.rfc-editor.org/rfc/rfc3339) duration format. + * `ipvs` - Kubernetes cluster kube-proxy customization of IPVS specific config (durations format is [RFC3339](https://www.rfc-editor.org/rfc/rfc3339) duration. + * `sync_period` - Minimum period that IPVS rules are refreshed, in [RFC3339](https://www.rfc-editor.org/rfc/rfc3339) duration format. + * `min_sync_period` - Minimum period that IPVS rules are refreshed in [RFC3339](https://www.rfc-editor.org/rfc/rfc3339) duration. + * `scheduler` - IPVS scheduler. + * `tcp_timeout` - Timeout value used for idle IPVS TCP sessions in [RFC3339](https://www.rfc-editor.org/rfc/rfc3339) duration. + * `tcp_fin_timeout` - Timeout value used for IPVS TCP sessions after receiving a FIN in RFC3339 duration. + * `udp_timeout` - timeout value used for IPVS UDP packets in [RFC3339](https://www.rfc-editor.org/rfc/rfc3339) duration. diff --git a/website/docs/r/cloud_project_kube.html.markdown b/website/docs/r/cloud_project_kube.html.markdown index 2d388f426..be691dc98 100644 --- a/website/docs/r/cloud_project_kube.html.markdown +++ b/website/docs/r/cloud_project_kube.html.markdown @@ -33,7 +33,7 @@ resource "ovh_cloud_project_kube" "mycluster" { resource "local_file" "kubeconfig" { content = ovh_cloud_project_kube.mycluster.kubeconfig - filename = "mycluster.yml" + filename = "mycluster.yml" } ``` @@ -94,17 +94,42 @@ Create a Kubernetes cluster in `GRA5` region with API Server AdmissionPlugins co ```hcl resource "ovh_cloud_project_kube" "mycluster" { - service_name = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - name = "my_kube_cluster" - region = "GRA5" - customization { - apiserver { - admissionplugins { - enabled = ["NodeRestriction"] - disabled = ["AlwaysPullImages"] - } - } - } + service_name = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + name = "my_kube_cluster" + region = "GRA5" + customization_apiserver { + admissionplugins { + enabled = ["NodeRestriction"] + disabled = ["AlwaysPullImages"] + } + } +} +``` + +Create a Kubernetes cluster in `GRA5` region with Kube proxy configuration, by specifying iptables or ipvs configurations: + +```hcl +resource "ovh_cloud_project_kube" "mycluster" { + service_name = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + name = "my_kube_cluster" + region = "GRA5" + kube_proxy_mode = "ipvs" # or "iptables" + + customization_kube_proxy { + iptables { + min_sync_period = "PT0S" + sync_period = "PT0S" + } + + ipvs { + min_sync_period = "PT0S" + sync_period = "PT0S" + scheduler = "rr" + tcp_timeout = "PT0S" + tcp_fin_timeout = "PT0S" + udp_timeout = "PT0S" + } + } } ``` @@ -112,16 +137,16 @@ Kubernetes cluster creation attached to a VRack in `GRA5` region: ```hcl resource "ovh_vrack_cloudproject" "attach" { - service_name = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" # vrack ID - project_id = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" # Public Cloud service name + service_name = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" # vrack ID + project_id = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" # Public Cloud service name } resource "ovh_cloud_project_network_private" "network" { - service_name = ovh_vrack_cloudproject.attach.service_name - vlan_id = 0 - name = "terraform_testacc_private_net" - regions = ["GRA5"] - depends_on = [ovh_vrack_cloudproject.attach] + service_name = ovh_vrack_cloudproject.attach.service_name + vlan_id = 0 + name = "terraform_testacc_private_net" + regions = ["GRA5"] + depends_on = [ovh_vrack_cloudproject.attach] } resource "ovh_cloud_project_network_private_subnet" "networksubnet" { @@ -140,24 +165,22 @@ resource "ovh_cloud_project_network_private_subnet" "networksubnet" { } output "openstackID" { - value = one(ovh_cloud_project_network_private.network.regions_attributes[*].openstackid) + value = one(ovh_cloud_project_network_private.network.regions_attributes[*].openstackid) } resource "ovh_cloud_project_kube" "mycluster" { - service_name = var.service_name - name = "test-kube-attach" - region = "GRA5" + service_name = var.service_name + name = "test-kube-attach" + region = "GRA5" - private_network_id = tolist(ovh_cloud_project_network_private.network.regions_attributes[*].openstackid)[0] - - private_network_configuration { - default_vrack_gateway = "" - private_network_routing_as_default = false - } + private_network_id = tolist(ovh_cloud_project_network_private.network.regions_attributes[*].openstackid)[0] - depends_on = [ - ovh_cloud_project_network_private.network - ] + private_network_configuration { + default_vrack_gateway = "" + private_network_routing_as_default = false + } + + depends_on = [ovh_cloud_project_network_private.network] } ``` @@ -169,12 +192,30 @@ The following arguments are supported: * `name` - (Optional) The name of the kubernetes cluster. * `region` - a valid OVHcloud public cloud region ID in which the kubernetes cluster will be available. Ex.: "GRA1". Defaults to all public cloud regions. Changing this value recreates the resource. * `version` - (Optional) kubernetes version to use. Changing this value updates the resource. Defaults to the latest available. -* `customization` - (Optional) Customer customization object +* `kube_proxy_mode` - (Optional) Selected mode for kube-proxy. **Changing this value recreates the resource. This will result in the loss of all data stored in the etcd.** Defaults to `iptables`. +* `customization` - **Deprecated** (Optional) Use `customization_apiserver` and `customization_kube_proxy` instead. Kubernetes cluster customization * `apiserver` - Kubernetes API server customization - * `admissionplugins` - (Optional) Kubernetes API server admission plugins customization - * `enabled` - (Optional) Array of admission plugins enabled, default is ["NodeRestriction","AlwaysPulImages"] and only these admission plugins can be enabled at this time. - * `disabled` - (Optional) Array of admission plugins disabled, default is [] and only AlwaysPulImages can be disabled at this time. -* `private_network_id` - (Optional) OpenStack private network (or vrack) ID to use. Changing this value delete the resource (including ETCD user data). Defaults - not use private network. ~> __WARNING__ Updating the private network ID resets the cluster so that all user data is deleted. + * `kube_proxy` - Kubernetes kube-proxy customization +* `customization_apiserver` - Kubernetes API server customization + * `admissionplugins` - (Optional) Kubernetes API server admission plugins customization + * `enabled` - (Optional) Array of admission plugins enabled, default is ["NodeRestriction","AlwaysPulImages"] and only these admission plugins can be enabled at this time. + * `disabled` - (Optional) Array of admission plugins disabled, default is [] and only AlwaysPulImages can be disabled at this time. +* `customization_kube_proxy` - Kubernetes kube-proxy customization + * `iptables` - (Optional) Kubernetes cluster kube-proxy customization of iptables specific config (durations format is RFC3339 duration, e.g. `PT60S`) + * `sync_period` - (Optional) Minimum period that iptables rules are refreshed, in [RFC3339](https://www.rfc-editor.org/rfc/rfc3339) duration format (e.g. `PT60S`). + * `min_sync_period` - (Optional) Period that iptables rules are refreshed, in [RFC3339](https://www.rfc-editor.org/rfc/rfc3339) duration format (e.g. `PT60S`). Must be greater than 0. + * `ipvs` - (Optional) Kubernetes cluster kube-proxy customization of IPVS specific config (durations format is [RFC3339](https://www.rfc-editor.org/rfc/rfc3339) duration, e.g. `PT60S`) + * `sync_period` - (Optional) Minimum period that IPVS rules are refreshed, in [RFC3339](https://www.rfc-editor.org/rfc/rfc3339) duration format (e.g. `PT60S`). + * `min_sync_period` - (Optional) Minimum period that IPVS rules are refreshed in [RFC3339](https://www.rfc-editor.org/rfc/rfc3339) duration (e.g. `PT60S`). + * `scheduler` - (Optional) IPVS scheduler. + * `tcp_timeout` - (Optional) Timeout value used for idle IPVS TCP sessions in [RFC3339](https://www.rfc-editor.org/rfc/rfc3339) duration (e.g. `PT60S`). The default value is `PT0S`, which preserves the current timeout value on the system. + * `tcp_fin_timeout` - (Optional) Timeout value used for IPVS TCP sessions after receiving a FIN in RFC3339 duration (e.g. `PT60S`). The default value is `PT0S`, which preserves the current timeout value on the system. + * `udp_timeout` - (Optional) timeout value used for IPVS UDP packets in [RFC3339](https://www.rfc-editor.org/rfc/rfc3339) duration (e.g. `PT60S`). The default value is `PT0S`, which preserves the current timeout value on the system. +* `private_network_id` - (Optional) OpenStack private network (or vrack) ID to use. + Changing this value delete the resource(including ETCD user data). Defaults - not use private network. + +~> __WARNING__ Updating the private network ID resets the cluster so that all user data is deleted. + * `private_network_configuration` - (Optional) The private network configuration * `default_vrack_gateway` - If defined, all egress traffic will be routed towards this IP address, which should belong to the private network. Empty string means disabled. * `private_network_routing_as_default` - Defines whether routing should default to using the nodes' private interface, instead of their public interface. Default is false. @@ -204,7 +245,8 @@ The following attributes are exported: * `update_policy` - See Argument Reference above. * `url` - Management URL of your cluster. * `version` - See Argument Reference above. -* `customization` - See Argument Reference above. +* `customization_apiserver` - See Argument Reference above. +* `customization_kube_proxy` - See Argument Reference above. ## Timeouts