Introduce proper backend config merge (#61)

Author: lakkeger

README.md

@@ -50,6 +50,7 @@ please refer to the man pages of `terraform --help`.
 
 ## Change Log
 
+* v0.20.0: Fix S3 backend option merging
 * v0.19.0: Add `SKIP_ALIASES` configuration environment variable
 * v0.18.2: Fix warning on aliased custom endpoint names
 * v0.18.1: Fix issue with not proxied commands

bin/tflocal

@@ -55,17 +55,7 @@ provider "aws" {
 """
 TF_S3_BACKEND_CONFIG = """
 terraform {
-  backend "s3" {
-    region         = "<region>"
-    bucket         = "<bucket>"
-    key            = "<key>"
-    dynamodb_table = "<dynamodb_table>"
-
-    access_key        = "test"
-    secret_key        = "test"
-<endpoints>
-    skip_credentials_validation = true
-    skip_metadata_api_check     = true
+  backend "s3" {<configs>
   }
 }
 """
@@ -265,6 +255,10 @@ def generate_s3_backend_config() -> str:
         "key": "terraform.tfstate",
         "dynamodb_table": "tf-test-state",
         "region": get_region(),
+        "skip_credentials_validation": True,
+        "skip_metadata_api_check": True,
+        "secret_key": "test",
+
         "endpoints": {
             "s3": get_service_endpoint("s3"),
             "iam": get_service_endpoint("iam"),
@@ -278,40 +272,44 @@ def generate_s3_backend_config() -> str:
         print("Warning: Unsupported backend option(s) detected (`endpoints`). Please make sure you always use the corresponding options to your Terraform version.")
         exit(1)
     for legacy_endpoint, endpoint in legacy_endpoint_mappings.items():
+        if legacy_endpoint in backend_config and backend_config.get("endpoints") and endpoint in backend_config["endpoints"]:
+            del backend_config[legacy_endpoint]
+            continue
         if legacy_endpoint in backend_config and (not backend_config.get("endpoints") or endpoint not in backend_config["endpoints"]):
             if not backend_config.get("endpoints"):
                 backend_config["endpoints"] = {}
             backend_config["endpoints"].update({endpoint: backend_config[legacy_endpoint]})
+            del backend_config[legacy_endpoint]
     # Add any missing default endpoints
     if backend_config.get("endpoints"):
         backend_config["endpoints"] = {
             k: backend_config["endpoints"].get(k) or v
             for k, v in configs["endpoints"].items()}
+    backend_config["access_key"] = get_access_key(backend_config) if CUSTOMIZE_ACCESS_KEY else DEFAULT_ACCESS_KEY
     configs.update(backend_config)
     if not DRY_RUN:
         get_or_create_bucket(configs["bucket"])
         get_or_create_ddb_table(configs["dynamodb_table"], region=configs["region"])
     result = TF_S3_BACKEND_CONFIG
-    for key, value in configs.items():
+    config_options = ""
+    for key, value in sorted(configs.items()):
         if isinstance(value, bool):
             value = str(value).lower()
         elif isinstance(value, dict):
             if key == "endpoints" and is_tf_legacy:
-                value = textwrap.indent(
-                    text=textwrap.dedent(f"""\
-                        endpoint          = "{value["s3"]}"
-                        iam_endpoint      = "{value["iam"]}"
-                        sts_endpoint      = "{value["sts"]}"
-                        dynamodb_endpoint = "{value["dynamodb"]}"
-                    """),
-                    prefix=" " * 4)
+                for legacy_endpoint, endpoint in legacy_endpoint_mappings.items():
+                    config_options += f'\n    {legacy_endpoint} = "{configs[key][endpoint]}"'
+                continue
             else:
                 value = textwrap.indent(
                     text=f"{key} = {{\n" + "\n".join([f'  {k} = "{v}"' for k, v in value.items()]) + "\n}",
                     prefix=" " * 4)
+                config_options += f"\n{value}"
+                continue
         else:
-            value = str(value)
-        result = result.replace(f"<{key}>", value)
+            value = f'"{str(value)}"'
+        config_options += f'\n    {key} = {value}'
+    result = result.replace("<configs>", config_options)
     return result
 
 

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = terraform-local
-version = 0.19.0
+version = 0.20.0
 url = https://github.com/localstack/terraform-local
 author = LocalStack Team
 author_email = [email protected]

tests/test_apply.py

@@ -230,7 +230,7 @@ def test_dry_run(monkeypatch):
     override_file = os.path.join(temp_dir, "localstack_providers_override.tf")
     assert check_override_file_exists(override_file)
 
-    assert check_override_file_backend_content(override_file, is_legacy=is_legacy_tf)
+    assert check_override_file_backend_endpoints_content(override_file, is_legacy=is_legacy_tf)
 
     # assert that bucket with state file exists
     s3 = client("s3", region_name="us-east-2")
@@ -276,6 +276,50 @@ def check_override_file_content(override_file):
     return True
 
 
+def test_s3_backend_configs_merge(monkeypatch):
+    monkeypatch.setenv("DRY_RUN", "1")
+    state_bucket = "tf-state-conf-merge"
+    state_table = "tf-state-conf-merge"
+    # Temporarily change "." -> "-" as aws provider >5.55.0 fails with LocalStack
+    # by calling aws-global pseudo region at S3 bucket creation instead of us-east-1
+    bucket_name = "bucket-conf-merge"
+    config = """
+    terraform {
+      backend "s3" {
+        bucket = "%s"
+        key    = "terraform.tfstate"
+        dynamodb_table = "%s"
+        region = "us-east-2"
+        skip_credentials_validation = true
+        encryption = true
+        use_path_style = true
+        acl = "bucket-owner-full-control"
+      }
+    }
+    resource "aws_s3_bucket" "test-bucket" {
+      bucket = "%s"
+    }
+    """ % (state_bucket, state_table, bucket_name)
+    temp_dir = deploy_tf_script(config, cleanup=False, user_input="yes")
+    override_file = os.path.join(temp_dir, "localstack_providers_override.tf")
+    assert check_override_file_exists(override_file)
+    assert check_override_file_backend_extra_content(override_file)
+    rmtree(temp_dir)
+
+
+def check_override_file_backend_extra_content(override_file):
+    try:
+        with open(override_file, "r") as fp:
+            result = hcl2.load(fp)
+            result = result["terraform"][0]["backend"][0]["s3"]
+    except Exception as e:
+        raise Exception(f'Unable to parse "{override_file}" as HCL file: {e}')
+
+    return result.get("use_path_style") is True and \
+        result.get("encryption") is True and \
+        result.get("acl") == "bucket-owner-full-control"
+
+
 @pytest.mark.parametrize("endpoints", [
     '',
     'endpoint = "http://s3-localhost.localstack.cloud:4566"',
@@ -314,15 +358,15 @@ def test_s3_backend_endpoints_merge(monkeypatch, endpoints: str):
         temp_dir = deploy_tf_script(config, cleanup=False, user_input="yes")
         override_file = os.path.join(temp_dir, "localstack_providers_override.tf")
         assert check_override_file_exists(override_file)
-        assert check_override_file_backend_content(override_file, is_legacy=is_legacy_tf)
+        assert check_override_file_backend_endpoints_content(override_file, is_legacy=is_legacy_tf)
         rmtree(temp_dir)
 
 
 def check_override_file_exists(override_file):
     return os.path.isfile(override_file)
 
 
-def check_override_file_backend_content(override_file, is_legacy: bool = False):
+def check_override_file_backend_endpoints_content(override_file, is_legacy: bool = False):
     legacy_options = (
         "endpoint",
         "iam_endpoint",