diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl index e853b98..02e65c7 100644 --- a/.terraform.lock.hcl +++ b/.terraform.lock.hcl @@ -3,9 +3,10 @@ provider "registry.terraform.io/hashicorp/aws" { version = "5.8.0" - constraints = ">= 2.23.0, >= 3.72.0, >= 4.36.0, >= 4.47.0, >= 4.57.0, >= 4.59.0, >= 5.0.0, ~> 5.1" + constraints = ">= 2.23.0, >= 4.33.0, >= 4.36.0, >= 4.47.0, >= 4.57.0, >= 4.59.0, >= 5.0.0, ~> 5.1" hashes = [ "h1:CYWeH3ii7UQNc+rpNLixWilueA5sV9FF9kcBCz+D48U=", + "h1:vnjWfeuf4AflWsRq3ivVig8dR8PAg8BHTVyAtOzJ1yQ=", "zh:0974311d5e1becfdcbdae43d022d52689fdad32a4145659e56ac534bcb8cba02", "zh:100dc64a90fc0d36cf6e2882b4358fde17705edd8ab3c5f2c06d219c36b21565", "zh:467a86de8a7d77cde5c3386f9e82d7f1bf5972d1b3d177e797d1d9d2e87fd357", @@ -29,6 +30,7 @@ provider "registry.terraform.io/hashicorp/cloudinit" { constraints = ">= 2.0.0" hashes = [ "h1:Ar/DAbZQ9Nsj0BrqX6camrEE6U+Yq4E87DCNVqxqx8k=", + "h1:ocyv0lvfyvzW4krenxV5CL4Jq5DiA3EUfoy8DR6zFMw=", "zh:2487e498736ed90f53de8f66fe2b8c05665b9f8ff1506f751c5ee227c7f457d1", "zh:3d8627d142942336cf65eea6eb6403692f47e9072ff3fa11c3f774a3b93130b3", "zh:434b643054aeafb5df28d5529b72acc20c6f5ded24decad73b98657af2b53f4f", @@ -49,6 +51,7 @@ provider "registry.terraform.io/hashicorp/helm" { constraints = ">= 2.9.0, ~> 2.10" hashes = [ "h1:OFRsk+lMoRoNoJjJzRngH8hAq++Sb6LwrEKIjd7PeWA=", + "h1:rssAXPIBWhumMtToGhh63w1euKOgVOi7+9LK6qZtDUQ=", "zh:0717312baed39fb0a00576297241b69b419880cad8771bf72dec97ebdc96b200", "zh:0e0e287b4e8429a0700143c8159764502eba0b33b1d094bf0d4ef4d93c7802cb", "zh:4f74605377dab4065aaad35a2c5fa6186558c6e2e57b9058bdc8a62cf91857b9", @@ -68,6 +71,7 @@ provider "registry.terraform.io/hashicorp/kubernetes" { version = "2.22.0" constraints = ">= 2.10.0, >= 2.20.0, ~> 2.21" hashes = [ + "h1:DJr88+52tPK4Ft9xltF6YL+sRz8HWLP2ZOfFiKSB5Dc=", "h1:N2Nta6li+07oT02gcgLzAU4goGIWNXY2zqKUV/9rLLE=", "zh:1eac662b1f238042b2068401e510f0624efaf51fd6a4dd9c49d710a49d383b61", "zh:4c35651603493437b0b13e070148a330c034ac62c8967c2de9da6620b26adca4", @@ -89,6 +93,7 @@ provider "registry.terraform.io/hashicorp/time" { constraints = ">= 0.9.0" hashes = [ "h1:UHcDnIYFZ00uoou0TwPGMwOrE8gTkoRephIvdwDAK70=", + "h1:VxyoYYOCaJGDmLz4TruZQTSfQhvwEcMxvcKclWdnpbs=", "zh:00a1476ecf18c735cc08e27bfa835c33f8ac8fa6fa746b01cd3bcbad8ca84f7f", "zh:3007f8fc4a4f8614c43e8ef1d4b0c773a5de1dcac50e701d8abc9fdc8fcb6bf5", "zh:5f79d0730fdec8cb148b277de3f00485eff3e9cf1ff47fb715b1c969e5bbd9d4", @@ -108,6 +113,7 @@ provider "registry.terraform.io/hashicorp/tls" { version = "4.0.4" constraints = ">= 3.0.0" hashes = [ + "h1:GZcFizg5ZT2VrpwvxGBHQ/hO9r6g0vYdQqx3bFD3anY=", "h1:Wd3RqmQW60k2QWPN4sK5CtjGuO1d+CRNXgC+D4rKtXc=", "zh:23671ed83e1fcf79745534841e10291bbf34046b27d6e68a5d0aab77206f4a55", "zh:45292421211ffd9e8e3eb3655677700e3c5047f71d8f7650d2ce30242335f848", diff --git a/main.tf b/main.tf index 0086351..f0cc68c 100644 --- a/main.tf +++ b/main.tf @@ -5,10 +5,6 @@ data "aws_eks_cluster_auth" "this" { locals { resource_name = "comet-${var.environment}" - tags = { - Terraform = "true" - Environment = var.environment - } } module "comet_vpc" { diff --git a/modules/comet_ec2/main.tf b/modules/comet_ec2/main.tf index 9251e8f..7a871a9 100644 --- a/modules/comet_ec2/main.tf +++ b/modules/comet_ec2/main.tf @@ -4,11 +4,6 @@ locals { https_port = 443 any_port = 0 cidr_anywhere = "0.0.0.0/0" - - tags = { - Terraform = "true" - Environment = var.environment - } } data "aws_ami" "al2" { @@ -147,10 +142,13 @@ resource "aws_instance" "comet_ec2" { volume_size = var.comet_ec2_volume_size } - tags = merge(local.tags, { - Name = "${var.environment}-comet-ml-${count.index}" - }) - + tags = merge( + var.common_tags, + { + Name = "${var.environment}-comet-ml-${count.index}" + } + ) + lifecycle { create_before_destroy = true } diff --git a/modules/comet_ec2/variables.tf b/modules/comet_ec2/variables.tf index 2bdc738..4f21b07 100644 --- a/modules/comet_ec2/variables.tf +++ b/modules/comet_ec2/variables.tf @@ -68,4 +68,10 @@ variable "comet_ec2_s3_iam_policy" { variable "comet_ec2_alb_sg" { description = "ID of the security group attached to an associated application load balancer, for creating ingress EC2 SG rule" type = string -} \ No newline at end of file +} + +variable "common_tags" { + type = map(string) + description = "A map of common tags" + default = {} +} diff --git a/modules/comet_ec2_alb/main.tf b/modules/comet_ec2_alb/main.tf index 8af36ed..cf5c16b 100644 --- a/modules/comet_ec2_alb/main.tf +++ b/modules/comet_ec2_alb/main.tf @@ -3,11 +3,6 @@ locals { https_port = 443 any_port = 0 cidr_anywhere = "0.0.0.0/0" - - tags = { - Terraform = "true" - Environment = var.environment - } } resource "aws_security_group" "comet_alb_sg" { @@ -82,6 +77,4 @@ module "alb" { } } ] - - tags = local.tags } \ No newline at end of file diff --git a/modules/comet_ec2_alb/variables.tf b/modules/comet_ec2_alb/variables.tf index 9f429a7..62209ad 100644 --- a/modules/comet_ec2_alb/variables.tf +++ b/modules/comet_ec2_alb/variables.tf @@ -16,4 +16,5 @@ variable "public_subnets" { variable "ssl_certificate_arn" { description = "ARN of the ACM certificate to use for the ALB" type = string -} \ No newline at end of file +} + diff --git a/modules/comet_eks/main.tf b/modules/comet_eks/main.tf index 1592375..3b0c42b 100644 --- a/modules/comet_eks/main.tf +++ b/modules/comet_eks/main.tf @@ -1,8 +1,4 @@ locals { - tags = { - Terraform = "true" - Environment = var.environment - } volume_type = "gp3" volume_encrypted = false volume_delete_on_termination = true @@ -97,7 +93,6 @@ module "eks" { } } : {} ) - tags = local.tags } @@ -133,6 +128,4 @@ module "eks_blueprints_addons" { enable_aws_cloudwatch_metrics = var.eks_aws_cloudwatch_metrics enable_external_dns = var.eks_external_dns external_dns_route53_zone_arns = var.eks_external_dns_r53_zones - - tags = local.tags } \ No newline at end of file diff --git a/modules/comet_eks/variables.tf b/modules/comet_eks/variables.tf index d803ca3..5772ad5 100644 --- a/modules/comet_eks/variables.tf +++ b/modules/comet_eks/variables.tf @@ -113,4 +113,4 @@ variable "eks_airflow_instance_type" { variable "eks_airflow_node_count" { description = "Instance count for EKS Airflow nodes" type = number -} \ No newline at end of file +} diff --git a/modules/comet_elasticache/main.tf b/modules/comet_elasticache/main.tf index c75865c..44cf7d2 100644 --- a/modules/comet_elasticache/main.tf +++ b/modules/comet_elasticache/main.tf @@ -1,10 +1,5 @@ locals { redis_port = 6379 - - tags = { - Terraform = "true" - Environment = var.environment - } } resource "aws_elasticache_replication_group" "comet-ml-ec-redis" { diff --git a/modules/comet_elasticache/variables.tf b/modules/comet_elasticache/variables.tf index 33dd9ef..6281f3c 100644 --- a/modules/comet_elasticache/variables.tf +++ b/modules/comet_elasticache/variables.tf @@ -52,4 +52,4 @@ variable "elasticache_auth_token" { description = "Auth token for ElastiCache" type = string default = null -} \ No newline at end of file +} diff --git a/modules/comet_rds/main.tf b/modules/comet_rds/main.tf index a2d324c..1eb0106 100644 --- a/modules/comet_rds/main.tf +++ b/modules/comet_rds/main.tf @@ -1,18 +1,16 @@ locals { mysql_port = 3306 - - tags = { - Terraform = "true" - Environment = var.environment - } } resource "aws_db_subnet_group" "comet-ml-rds-subnet" { name = "cometml-rds-sgn-${var.environment}" subnet_ids = var.rds_private_subnets - tags = merge(local.tags, { - Name = "cometml-rds-sng-${var.environment}" - }) + tags = merge( + var.common_tags, + { + Name = "cometml-rds-sng-${var.environment}" + } + ) } resource "aws_rds_cluster_instance" "comet-ml-rds-mysql" { diff --git a/modules/comet_rds/variables.tf b/modules/comet_rds/variables.tf index 379f2ee..881019f 100644 --- a/modules/comet_rds/variables.tf +++ b/modules/comet_rds/variables.tf @@ -72,3 +72,9 @@ variable "rds_root_password" { description = "Root password for RDS database" type = string } + +variable "common_tags" { + type = map(string) + description = "A map of common tags" + default = {} +} diff --git a/modules/comet_s3/main.tf b/modules/comet_s3/main.tf index f05ee95..5403b99 100644 --- a/modules/comet_s3/main.tf +++ b/modules/comet_s3/main.tf @@ -1,8 +1,4 @@ locals { - tags = { - Terraform = "true" - Environment = var.environment - } suffix = substr(sha1("${var.environment}"), 0, 8) } @@ -11,9 +7,12 @@ resource "aws_s3_bucket" "comet_s3_bucket" { force_destroy = var.s3_force_destroy - tags = merge(local.tags, { - Name = var.comet_s3_bucket - }) + tags = merge( + var.common_tags, + { + Name = var.comet_s3_bucket + } + ) } resource "aws_s3_bucket" "comet_druid_bucket" { @@ -23,9 +22,12 @@ resource "aws_s3_bucket" "comet_druid_bucket" { force_destroy = var.s3_force_destroy - tags = merge(local.tags, { - Name = "comet-druid-${local.suffix}" - }) + tags = merge( + var.common_tags, + { + Name = "comet-druid-${local.suffix}" + } + ) } resource "aws_s3_bucket" "comet_airflow_bucket" { @@ -35,9 +37,12 @@ resource "aws_s3_bucket" "comet_airflow_bucket" { force_destroy = var.s3_force_destroy - tags = merge(local.tags, { - Name = "comet-airflow-${local.suffix}" - }) + tags = merge( + var.common_tags, + { + Name = "comet-airflow-${local.suffix}" + } + ) } resource "aws_iam_policy" "comet_s3_iam_policy" { diff --git a/modules/comet_s3/variables.tf b/modules/comet_s3/variables.tf index f2b5676..3db03d3 100644 --- a/modules/comet_s3/variables.tf +++ b/modules/comet_s3/variables.tf @@ -16,4 +16,10 @@ variable "s3_force_destroy" { variable "enable_mpm_infra" { description = "Sets buckets to be created for MPM Druid/Airflow" type = bool -} \ No newline at end of file +} + +variable "common_tags" { + type = map(string) + description = "A map of common tags" + default = {} +} diff --git a/modules/comet_vpc/main.tf b/modules/comet_vpc/main.tf index 379b2a7..2a0ab91 100644 --- a/modules/comet_vpc/main.tf +++ b/modules/comet_vpc/main.tf @@ -4,11 +4,6 @@ locals { resource_name = "comet-${var.environment}" vpc_cidr = "10.0.0.0/16" azs = slice(data.aws_availability_zones.available.names, 0, 3) - - tags = { - Terraform = "true" - Environment = var.environment - } } module "vpc" { @@ -37,6 +32,4 @@ module "vpc" { # if EKS deployment, set subnet tags for AWS Load Balancer Controller auto-discovery public_subnet_tags = var.eks_enabled ? { "kubernetes.io/role/elb" = 1 } : null private_subnet_tags = var.eks_enabled ? { "kubernetes.io/role/internal-elb" = 1 } : null - - tags = local.tags } \ No newline at end of file diff --git a/modules/comet_vpc/variables.tf b/modules/comet_vpc/variables.tf index 427953f..23a1c05 100644 --- a/modules/comet_vpc/variables.tf +++ b/modules/comet_vpc/variables.tf @@ -11,4 +11,4 @@ variable "eks_enabled" { variable "single_nat_gateway" { description = "Controls whether single NAT gateway used for all public subnets" type = bool -} \ No newline at end of file +} diff --git a/providers.tf b/providers.tf index 7e6e068..7ca0665 100644 --- a/providers.tf +++ b/providers.tf @@ -1,5 +1,15 @@ provider "aws" { region = var.region + + default_tags { + tags = merge( + { + Terraform = "true" + Environment = var.environment_tag + }, + var.common_tags + ) + } } provider "kubernetes" { diff --git a/terraform.tfvars b/terraform.tfvars index e142213..4a71312 100644 --- a/terraform.tfvars +++ b/terraform.tfvars @@ -1,29 +1,40 @@ +########################### +#### AWS Resource Tags #### +########################### +# common_tags = { +# # Place your dictionary of customized AWS resource tags here. eg. +# Owner = "firstName-lastName" +# DeployedBy = "Devops" +# TTL = "2025-01-01 12:00:00" +# Customer = "Model-Ops" +# } + ######################## #### Module toggles #### ######################## # Create a VPC to launch other resources in -enable_vpc = false +enable_vpc = true # Create an EC2 instance for running Comet -enable_ec2 = false +enable_ec2 = true # Create an ALB for the Comet EC2 instance -enable_ec2_alb = false +enable_ec2_alb = true # Create an EKS cluster for running Comet -enable_eks = false +enable_eks = true # Create ElastiCache resources for running Comet Redis -enable_elasticache = false +enable_elasticache = true # Create RDS resources for running Comet MySQL -enable_rds = false +enable_rds = true # Create S3 resources for storing Comet objects -enable_s3 = false +enable_s3 = true # Create EKS nodegroups for MPM compute -enable_mpm_infra = false +enable_mpm_infra = true ################ #### Global #### @@ -34,11 +45,14 @@ region = "us-east-1" # Name for Comet environment, for use in resource naming environment = "prod" -# If not setting enable_vpc to provision a VPC for the Comet resources, set the variables below to specify the existing VPC in which resources will be launched -comet_vpc_id = "vpc-012345abcdefghijkl" -availability_zones = ["us-east-1a", "us-east-1b", "us-east-1c"] -comet_public_subnets = ["subnet-012345abcdefghijkl", "subnet-012345abcdefghijkl", "subnet-012345abcdefghijkl"] -comet_private_subnets = ["subnet-012345abcdefghijkl", "subnet-012345abcdefghijkl", "subnet-012345abcdefghijkl"] +# Deployment identifier +environment_tag = "test" + +## If not setting enable_vpc to provision a VPC for the Comet resources, set the variables below to specify the existing VPC in which resources will be launched +# comet_vpc_id = "vpc-012345abcdefghijkl" +# availability_zones = ["us-east-1a", "us-east-1b", "us-east-1c"] +# comet_public_subnets = ["subnet-012345abcdefghijkl", "subnet-012345abcdefghijkl", "subnet-012345abcdefghijkl"] +# comet_private_subnets = ["subnet-012345abcdefghijkl", "subnet-012345abcdefghijkl", "subnet-012345abcdefghijkl"] ####################### #### Module inputs #### @@ -53,24 +67,24 @@ comet_private_subnets = ["subnet-012345abcdefghijkl", "subnet-012345abcdefghijkl ssl_certificate_arn = null #### comet_eks #### -# +# eks_aws_cloudwatch_metrics = false #### comet_elasticache #### # If setting enable_elasticache with existing compute, set the variable below to specify an SG that connections will be allowed from -elasticache_allow_from_sg = "sg-012345abcdefghijkl" +# elasticache_allow_from_sg = "sg-012345abcdefghijkl" ## Set the following to enable the auth token for Redis #elasticache_transit_encryption = true #elasticache_auth_token = "your-cometml-redis-token" #### comet_rds #### # If setting enable_rds, specify the root password for RDS below, or leave null and enter at the prompt during apply -rds_root_password = null +#rds_root_password = null # If setting enable_rds with existing compute, set the variable below to specify an SG that connections will be allowed from -rds_allow_from_sg = "sg-012345abcdefghijkl" +# rds_allow_from_sg = "sg-012345abcdefghijkl" #### comet_s3 #### # If setting enable_s3, specify the bucket name below -s3_bucket_name = null +#s3_bucket_name = null #### comet_vpc #### # diff --git a/variables.tf b/variables.tf index bf08efc..e4b6dfd 100644 --- a/variables.tf +++ b/variables.tf @@ -258,7 +258,7 @@ variable "elasticache_engine" { variable "elasticache_engine_version" { description = "Version number for ElastiCache engine" type = string - default = "7.1.0" + default = "7.1" } variable "elasticache_instance_type" { @@ -270,7 +270,7 @@ variable "elasticache_instance_type" { variable "elasticache_param_group_name" { description = "Name for the ElastiCache cluster parameter group" type = string - default = "default.redis5.0" + default = "default.redis7" } variable "elasticache_num_cache_nodes" { @@ -375,3 +375,13 @@ variable "single_nat_gateway" { type = bool default = true } + +variable "common_tags" { + description = "A map of tags to apply to resources" + type = map(string) +} + +variable "environment_tag" { + description = "Deployment identifier" + type = string +}