kubecluster.yaml

heat_template_version: 2014-10-16

description: >
  This template will boot a Kubernetes cluster with a single master
  and one or more nodes (as specified by the initial_nodes
  parameter, which defaults to 1).

parameters:

  ssh_key_name:
    type: string
    description: name of ssh key to be provisioned on our server

  external_network:
    type: string
    description: uuid/name of a network to use for floating ip addresses
    default: public

  server_image:
    type: string
    description: glance image used to boot the server

  master_flavor:
    type: string
    default: m1.small
    description: flavor to use when booting the server

  node_flavor:
    type: string
    default: m1.small
    description: flavor to use when booting the server

  dns_nameserver:
    type: string
    description: address of a dns nameserver reachable in your environment
    default: 8.8.8.8

  initial_nodes:
    type: number
    description: how many kubernetes nodes to spawn initially
    default: 1

  max_nodes:
    type: number
    description: maximum number of kubernetes nodes to spawn
    default: 10

  fixed_network_cidr:
    type: string
    description: network range for fixed ip network
    default: 10.0.0.0/24

  portal_network_cidr:
    type: string
    description: >
      address range used by kubernetes for service portals
    default: 10.254.0.0/16

  flannel_network_cidr:
    type: string
    description: network range for flannel overlay network
    default: 10.100.0.0/16

  flannel_network_subnetlen:
    type: string
    description: size of subnet assigned to each node
    default: 24

  flannel_use_vxlan:
    type: string
    description: >
      if true use the vxlan backend, otherwise use the default
      udp backend
    default: "false"
    constraints:
      - allowed_values: ["true", "false"]

  kube_allow_priv:
    type: string
    description: >
      whether or not kubernetes should permit privileged containers.
    default: "true"
    constraints:
      - allowed_values: ["true", "false"]

  docker_volume_size:
    type: number
    description: >
      size of a cinder volume to allocate to docker for container/image
      storage
    default: 25

  wait_condition_timeout:
    type: number
    description : >
      timeout for the Wait Conditions
    default: 6000

  rhn_register_host:
    type: string
    description: >
      whether or not to try registering with RHN
    default: "false"
    constraints:
      - allowed_values: ["true", "false"]

  rhn_username:
    type: string
    description: >
      username to use for authenticating with RHN
    default: ''

  rhn_password:
    type: string
    description: >
      password to use for authenticating with RHN
    default: ''

  scale_up_period_length:
    type: number
    default: 60

  scale_up_period_count:
    type: number
    default: 1

  scale_up_threshold:
    type: number
    default: 70

  scale_down_period_length:
    type: number
    default: 600

  scale_down_period_count:
    type: number
    default: 1

  scale_down_threshold:
    type: number
    default: 15

  scale_up_cooldown:
    type: number
    default: 120

  scale_down_cooldown:
    type: number
    default: 60

resources:

  master_wait_handle:
    type: OS::Heat::WaitConditionHandle

  master_wait_condition:
    type: OS::Heat::WaitCondition
    depends_on: kube_master
    properties:
      handle: {get_resource: master_wait_handle}
      timeout: {get_param: wait_condition_timeout}

  ######################################################################
  #
  # network resources.  allocate a network and router for our server.
  #

  fixed_network:
    type: OS::Neutron::Net

  fixed_subnet:
    type: OS::Neutron::Subnet
    properties:
      cidr: {get_param: fixed_network_cidr}
      network: {get_resource: fixed_network}
      dns_nameservers:
        - {get_param: dns_nameserver}

  extrouter:
    type: OS::Neutron::Router
    properties:
      external_gateway_info:
        network: {get_param: external_network}

  extrouter_inside:
    type: OS::Neutron::RouterInterface
    properties:
      router_id: {get_resource: extrouter}
      subnet: {get_resource: fixed_subnet}

  ######################################################################
  #
  # security groups.  we need to permit network traffic of various
  # sorts.
  #

  secgroup_base:
    type: OS::Neutron::SecurityGroup
    properties:
      rules:
        - protocol: icmp
        - protocol: tcp
          port_range_min: 22
          port_range_max: 22

  secgroup_all_open:
    type: OS::Neutron::SecurityGroup
    properties:
      rules:
        - protocol: icmp
        - protocol: tcp
        - protocol: udp

  secgroup_kubernetes:
    type: OS::Neutron::SecurityGroup
    properties:
      rules:
        - protocol: tcp
          port_range_min: 7080
          port_range_max: 7080
        - protocol: tcp
          port_range_min: 8080
          port_range_max: 8080
        - protocol: tcp
          port_range_min: 4001
          port_range_max: 4001
        - protocol: tcp
          port_range_min: 7001
          port_range_max: 7001

  ######################################################################
  #
  # software configs.  these are components that are combined into
  # a multipart MIME user-data archive.
  #

  write_heat_params:
    type: OS::Heat::SoftwareConfig
    properties:
      group: ungrouped
      config:
        str_replace:
          template: {get_file: fragments/write-heat-params-master.yaml}
          params:
            "$KUBE_ALLOW_PRIV": {get_param: kube_allow_priv}
            "$FLANNEL_NETWORK_CIDR": {get_param: flannel_network_cidr}
            "$FLANNEL_NETWORK_SUBNETLEN": {get_param: flannel_network_subnetlen}
            "$FLANNEL_USE_VXLAN": {get_param: flannel_use_vxlan}
            "$PORTAL_NETWORK_CIDR": {get_param: portal_network_cidr}

  rhn_register:
    type: OS::Heat::SoftwareConfig
    properties:
      group: ungrouped
      config:
        str_replace:
          template: {get_file: fragments/rhn-register.yaml}
          params:
            "$RHN_REGISTER_HOST": {get_param: rhn_register_host}
            "$RHN_USERNAME": {get_param: rhn_username}
            "$RHN_PASSWORD": {get_param: rhn_password}

  atomic_upgrade:
    type: OS::Heat::SoftwareConfig
    properties:
      group: ungrouped
      config: {get_file: fragments/atomic-upgrade.yaml}

  configure_etcd:
    type: OS::Heat::SoftwareConfig
    properties:
      group: ungrouped
      config: {get_file: fragments/configure-etcd.sh}

  configure_kubernetes:
    type: OS::Heat::SoftwareConfig
    properties:
      group: ungrouped
      config: {get_file: fragments/configure-kubernetes-master.sh}

  write_flannel_config:
    type: OS::Heat::SoftwareConfig
    properties:
      group: ungrouped
      config: {get_file: fragments/write-flannel-config.sh}

  flannel_config_service:
    type: OS::Heat::SoftwareConfig
    properties:
      group: ungrouped
      config: {get_file: fragments/flannel-config.service.yaml}

  enable_services:
    type: OS::Heat::SoftwareConfig
    properties:
      group: ungrouped
      config: {get_file: fragments/enable-services-master.sh}

  kube_user:
    type: OS::Heat::SoftwareConfig
    properties:
      group: ungrouped
      config: {get_file: fragments/kube-user.yaml}

  kube_examples:
    type: OS::Heat::SoftwareConfig
    properties:
      group: ungrouped
      config: {get_file: fragments/kube-examples.yaml}

  master_wc_notify:
    type: OS::Heat::SoftwareConfig
    properties:
      group: ungrouped
      config:
        str_replace:
          template: |
            #!/bin/bash -v
            wc_notify --data-binary '{"status": "SUCCESS"}'
          params:
            wc_notify: {get_attr: [master_wait_handle, curl_cli]}

  disable_selinux:
    type: OS::Heat::SoftwareConfig
    properties:
      group: ungrouped
      config: {get_file: fragments/disable-selinux.sh}

  kube_master_init:
    type: OS::Heat::MultipartMime
    properties:
      parts:
        - config: {get_resource: disable_selinux}
        - config: {get_resource: rhn_register}
        - config: {get_resource: atomic_upgrade}
        - config: {get_resource: write_heat_params}
        - config: {get_resource: configure_etcd}
        - config: {get_resource: kube_user}
        - config: {get_resource: configure_kubernetes}
        - config: {get_resource: enable_services}
        - config: {get_resource: write_flannel_config}
        - config: {get_resource: flannel_config_service}
        - config: {get_resource: kube_examples}
        - config: {get_resource: master_wc_notify}

  ######################################################################
  #
  # kubernetes master server.
  #

  kube_master:
    type: OS::Nova::Server
    depends_on:
      - extrouter_inside
    properties:
      image: {get_param: server_image}
      flavor: {get_param: master_flavor}
      key_name: {get_param: ssh_key_name}
      user_data_format: RAW
      user_data: {get_resource: kube_master_init}
      networks:
        - port: {get_resource: kube_master_eth0}

  kube_master_eth0:
    type: OS::Neutron::Port
    properties:
      network: {get_resource: fixed_network}
      security_groups:
        - {get_resource: secgroup_base}
        - {get_resource: secgroup_kubernetes}
        - {get_resource: secgroup_all_open}
      fixed_ips:
        - subnet: {get_resource: fixed_subnet}
      replacement_policy: AUTO

  kube_master_floating:
    type: OS::Neutron::FloatingIP
    properties:
      floating_network: {get_param: external_network}
      port_id: {get_resource: kube_master_eth0}

  ######################################################################
  #
  # kubernetes nodes. This is an autoscaling group that will initially
  # create <initial_nodes> nodes, and will scale up to
  # <max_nodes> based on CPU utilization.
  #

  kube_nodes:
    type: OS::Heat::AutoScalingGroup
    depends_on:
      - extrouter_inside
      - master_wait_condition
    properties:
      resource:
        type: kubenode.yaml
        properties:
          rhn_register_host: {get_param: rhn_register_host}
          rhn_username: {get_param: rhn_username}
          rhn_password: {get_param: rhn_password}
          ssh_key_name: {get_param: ssh_key_name}
          server_image: {get_param: server_image}
          node_flavor: {get_param: node_flavor}
          fixed_network: {get_resource: fixed_network}
          fixed_subnet: {get_resource: fixed_subnet}
          kube_master_ip: {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]}
          external_network: {get_param: external_network}
          kube_allow_priv: {get_param: kube_allow_priv}
          docker_volume_size: {get_param: docker_volume_size}
          wait_condition_timeout: {get_param: wait_condition_timeout}
          metadata: {"metering.stack": {get_param: "OS::stack_id"}}
      min_size: {get_param: initial_nodes}
      desired_capacity: {get_param: initial_nodes}
      max_size: {get_param: max_nodes}

  scale_up_policy:
    type: OS::Heat::ScalingPolicy
    properties:
      adjustment_type: change_in_capacity
      auto_scaling_group_id: {get_resource: kube_nodes}
      cooldown: {get_param: scale_up_cooldown}
      scaling_adjustment: 1

  scale_down_policy:
    type: OS::Heat::ScalingPolicy
    properties:
      adjustment_type: change_in_capacity
      auto_scaling_group_id: {get_resource: kube_nodes}
      cooldown: {get_param: scale_down_cooldown}
      scaling_adjustment: -1

  # NB: Looking at this alarm you might expect it to trigger if cpu_util is
  # greater than 50% for one minute.  Unfortunately, the underlying
  # cpu metric in most default configurations only updates every
  # 10 minutes.  To modify the frequency of this metric, you will need
  # to edit /etc/ceilometer/pipeline.yaml and change the "interval"
  # key for "cpu" to "60".
  cpu_alarm_high:
    type: OS::Ceilometer::Alarm
    properties:
      description: Scale-up if the average CPU > <scale_up_threshold>% for 1 minute
      meter_name: cpu_util
      statistic: avg
      period: {get_param: scale_up_period_length}
      evaluation_periods: {get_param: scale_up_period_count}
      threshold: {get_param: scale_up_threshold}
      alarm_actions:
        - {get_attr: [scale_up_policy, alarm_url]}
      matching_metadata: {'metadata.user_metadata.stack': {get_param: "OS::stack_id"}}
      comparison_operator: gt
  cpu_alarm_low:
    type: OS::Ceilometer::Alarm
    properties:
      description: Scale-down if the average CPU < <scale_down_threshold>% for 10 minutes
      meter_name: cpu_util
      statistic: avg
      period: {get_param: scale_down_period_length}
      evaluation_periods: {get_param: scale_down_period_count}
      threshold: {get_param: scale_down_threshold}
      alarm_actions:
        - {get_attr: [scale_down_policy, alarm_url]}
      matching_metadata: {'metadata.user_metadata.stack': {get_param: "OS::stack_id"}}
      comparison_operator: lt

outputs:

  kube_master:
    value: {get_attr: [kube_master_floating, floating_ip_address]}
    description: >
      This is the "public" ip address of the Kubernetes master server.
      Use this address to log in to the Kubernetes master via ssh or
      to access the Kubernetes API from outside the cluster.

  kube_nodes:
    value: {get_attr: [kube_nodes, outputs_list, kube_node_ip]}
    description: >
      This is a list of of the "private" addresses of all the
      Kubernetes nodes.