Merge pull request #22 from comet-ml/airflow-mng

Add airflow mng

Author: burmek

main.tf

@@ -80,6 +80,8 @@ module "comet_eks" {
 
   eks_druid_instance_type     = var.eks_druid_instance_type
   eks_druid_node_count        = var.eks_druid_node_count
+  eks_airflow_instance_type   = var.eks_airflow_instance_type
+  eks_airflow_node_count      = var.eks_airflow_node_count
 }
 
 module "comet_elasticache" {

modules/comet_eks/main.tf

@@ -72,6 +72,28 @@ module "eks" {
           nodegroup_name = "druid"
         }
         iam_role_additional_policies = var.s3_enabled ? { comet_s3_access = var.comet_ec2_s3_iam_policy } : {}
+      },
+      airflow = {
+        name           = "airflow"
+        instance_types = [var.eks_airflow_instance_type]
+        min_size       = var.eks_airflow_node_count
+        max_size       = var.eks_airflow_node_count
+        desired_size   = var.eks_airflow_node_count
+        block_device_mappings = {
+          xvda = {
+            device_name = "/dev/xvda"
+            ebs = {
+              volume_size           = var.eks_mng_disk_size
+              volume_type           = local.volume_type
+              encrypted             = local.volume_encrypted
+              delete_on_termination = local.volume_delete_on_termination
+            }
+          }
+        }
+        labels = {
+          nodegroup_name = "airflow"
+        }
+        iam_role_additional_policies = var.s3_enabled ? { comet_s3_access = var.comet_ec2_s3_iam_policy } : {}
       }
     } : {}
   )

modules/comet_eks/variables.tf

@@ -103,4 +103,14 @@ variable "eks_druid_instance_type" {
 variable "eks_druid_node_count" {
   description = "Instance count for EKS Druid nodes"
   type        = number
+}
+
+variable "eks_airflow_instance_type" {
+  description = "Instance type for EKS Airflow nodes"
+  type        = string
+}
+
+variable "eks_airflow_node_count" {
+  description = "Instance count for EKS Airflow nodes"
+  type        = number
 }

variables.tf

@@ -230,6 +230,18 @@ variable "eks_druid_node_count" {
   default     = 4
 }
 
+variable "eks_airflow_instance_type" {
+  description = "Instance type for EKS Airflow nodes"
+  type        = string
+  default     = "t3.medium"
+}
+
+variable "eks_airflow_node_count" {
+  description = "Instance count for EKS Airflow nodes"
+  type        = number
+  default     = 2
+}
+
 #### comet_elasticache ####
 variable "elasticache_allow_from_sg" {
   description = "Security group from which to allow connections to ElastiCache, to use when provisioning with existing compute"