Skip to content
This repository was archived by the owner on Dec 18, 2024. It is now read-only.

Commit a30032f

Browse files
rlaurikarlaurika
committed
Merge branch 'scaleup_enhancements' into 'master'
 support scaling masters and etcd See merge request !139
2 parents 47886b0 + 8b13bd2 commit a30032f

8 files changed

+169
-50
lines changed

‎README.md

+15
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,21 @@ is done with the official [installer playbook](https://github.com/openshift/open
4242
- customizes the default project template
4343
- optionally deploys default www page and Prometheus based monitoring
4444

45+
46+
### scaleup.yml
47+
48+
Running OpenShift Ansible scaleup playbooks for any host that does not
49+
have /var/lib/POC_INSTALLED flag on them. The playbooks are
50+
51+
playbooks/byo/openshift-master/scaleup.yml
52+
playbooks/byo/openshift-node/scaleup.yml
53+
playbooks/byo/openshift-etcd/scaleup.yml
54+
55+
NOTE: Set flag 'allow_first_master_scaleup' to true to allow
56+
replacing/recovering first master. This is a special case as the first
57+
master has the files for local CA, thus you must restore /etc/origin/master
58+
first from backup before configuring it with scaleup.
59+
4560
## Prerequisites
4661

4762
All that is needed is

‎playbooks/get_install_state.yml

+51
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
---
2+
- name: Check if hosts have been installed previously
3+
hosts: all
4+
become: yes
5+
tasks:
6+
- name: check for installation flag file
7+
stat:
8+
path: "/var/lib/POC_INSTALLED"
9+
register: check_poc_installed
10+
11+
- name: set poc_installed
12+
set_fact:
13+
poc_installed: True
14+
when: check_poc_installed.stat.exists
15+
16+
- name: Collect install flags and populate new_nodes
17+
hosts: localhost
18+
tasks:
19+
- name: populate new_masters -group for scaleup and detecting new installation
20+
add_host:
21+
name: "{{ item }}"
22+
groups: new_masters
23+
when:
24+
- hostvars[item].poc_installed|default(False)|bool == False
25+
with_items: "{{ groups['masters']|default([]) }}"
26+
27+
- name: safety for first master scaleup
28+
fail:
29+
msg: >
30+
Please set allow_first_master_scaleup to true to acknowledge that you have restored the
31+
certificates from backup before running scaleup based recovery on the first master.
32+
when:
33+
- cluster_name + '-master-1' in groups['new_masters']|default([])
34+
- groups['new_masters']|default([])|length < groups['masters']|default([])|length
35+
- allow_first_master_scaleup|default(False)|bool == False
36+
37+
- name: populate new_etcd -group for scaleup
38+
add_host:
39+
name: "{{ item }}"
40+
groups: new_etcd
41+
when:
42+
- hostvars[item].poc_installed|default(False)|bool == False
43+
with_items: "{{ groups['etcd']|default([]) }}"
44+
45+
- name: populate new_nodes -group for scaleup
46+
add_host:
47+
name: "{{ item }}"
48+
groups: new_nodes
49+
when:
50+
- hostvars[item].poc_installed|default(False)|bool == False
51+
with_items: "{{ groups['nodes']|default([]) }}"

‎playbooks/install.yml

+3-49
Original file line numberDiff line numberDiff line change
@@ -1,55 +1,9 @@
11
---
22

3-
- name: Check if hosts have been installed previously
4-
hosts: all
5-
become: yes
6-
tasks:
7-
- name: check for installation flag file
8-
stat:
9-
path: "/var/lib/POC_INSTALLED"
10-
register: check_poc_installed
11-
12-
- name: set poc_installed
13-
set_fact:
14-
poc_installed: True
15-
when: check_poc_installed.stat.exists
16-
17-
- name: Collect install flags and populate new_nodes
18-
hosts: localhost
19-
tasks:
20-
- set_fact:
21-
poc_installed_masters: []
22-
23-
- name: populate poc_installed_masters
24-
set_fact:
25-
poc_installed_masters: "{{ poc_installed_masters + [item] }}"
26-
when: hostvars[item].poc_installed|default(False)|bool
27-
with_items: "{{ groups['masters'] }}"
28-
29-
- name: populate new_nodes -group for scaleup
30-
add_host:
31-
name: "{{ item }}"
32-
groups: new_nodes
33-
when:
34-
- hostvars[item].poc_installed|default(False)|bool == False
35-
- allow_openshift_node_scaleup | default(False) | bool
36-
with_items: "{{ groups['nodes'] }}"
3+
- include: get_install_state.yml
374

385
- include: "{{ os_ansible_path|default('../../openshift-ansible') }}/playbooks/byo/config.yml"
39-
when: hostvars['localhost']['poc_installed_masters']|length == 0
40-
41-
- include: "{{ os_ansible_path|default('../../openshift-ansible') }}/playbooks/byo/openshift-node/scaleup.yml"
426
when:
43-
- allow_openshift_node_scaleup | default(False) | bool
44-
- groups['new_nodes']|default([])|length>0
7+
- groups['new_masters']|default([])|length == groups['masters']|default([])|length
458

46-
- name: Set install flags
47-
hosts: all
48-
become: yes
49-
tasks:
50-
- name: set installation flag
51-
copy:
52-
dest: "/var/lib/POC_INSTALLED"
53-
content: |
54-
POC installed this host at {{ ansible_date_time['iso8601'] }}
55-
when: poc_installed | default(False) | bool == False
9+
- include: set_install_state.yml

‎playbooks/post_install.yml

+5-1
Original file line numberDiff line numberDiff line change
@@ -61,10 +61,14 @@
6161
changed_when: false
6262
failed_when: false
6363

64-
- name: add a persistent volume to the registry
64+
- name: create and attach a persistent volume for registry
6565
shell: oc volume -n default dc/docker-registry --add --mount-path=/registry --overwrite --name=registry-storage -t pvc --claim-size=200Gi --claim-name=registry
6666
when: existing_registry_pv.stdout_lines | length == 0
6767

68+
- name: attach the existing persistent volume for registry
69+
shell: oc volume -n default dc/docker-registry --add --mount-path=/registry --overwrite --name=registry-storage --claim-name=registry
70+
when: existing_registry_pv.stdout_lines | length > 0
71+
6872
# block for conditionally deploying default-www-app
6973
- block:
7074
- name: check if project default-www exists

‎playbooks/scaleup.yml

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
---
2+
3+
- include: get_install_state.yml
4+
5+
- include: "{{ os_ansible_path|default('../../openshift-ansible') }}/playbooks/byo/openshift-master/scaleup.yml"
6+
when:
7+
- groups['new_masters']|default([])|length > 0
8+
9+
- include: "{{ os_ansible_path|default('../../openshift-ansible') }}/playbooks/byo/openshift-etcd/scaleup.yml"
10+
when:
11+
- groups['new_etcd']|default([])|length > 0
12+
13+
- include: "{{ os_ansible_path|default('../../openshift-ansible') }}/playbooks/byo/openshift-node/scaleup.yml"
14+
when:
15+
- groups['new_nodes']|default([])|length > 0
16+
17+
- include: set_install_state.yml

‎playbooks/set_install_state.yml

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
---
2+
- name: Set install flags
3+
hosts: all
4+
become: yes
5+
tasks:
6+
- name: set installation flag
7+
copy:
8+
dest: "/var/lib/POC_INSTALLED"
9+
content: |
10+
POC installed this host at {{ ansible_date_time['iso8601'] }}
11+
when: poc_installed | default(False) | bool == False

‎scripts/rolling_host_update.bash

+40
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
#!/usr/bin/env bash
2+
3+
# Proof of concept script to test rebuilding hosts one by one.
4+
#
5+
# NOTE:
6+
# - This code has been used for developing scaleup playbooks.
7+
# It has not been tested in other scenarios. Use with caution.
8+
# - updating first master is not supported, as that requires
9+
# restoring /etc/origin/master as part of the process
10+
#
11+
12+
set -e
13+
14+
for host in $*; do
15+
echo
16+
echo "updating $host"
17+
echo
18+
if [ ! -z $CONTROL_HOST ]; then
19+
echo "draining node"
20+
ssh $CONTROL_HOST oc adm drain $host --delete-local-data --force --ignore-daemonsets --grace-period=10
21+
else
22+
echo "no CONTROL_HOST set, proceeding without draining"
23+
fi
24+
echo "rebuilding"
25+
openstack server rebuild --image $IMAGE $host
26+
while ! openstack server list | grep $host | grep -q " ACTIVE "; do
27+
echo "waiting for server to be active"
28+
sleep 5
29+
done
30+
31+
echo "give it time to boot"
32+
sleep 30
33+
34+
echo "run pre-install just for $host"
35+
ansible-playbook -v -l $host pre_install.yml
36+
37+
echo "run scaleup"
38+
ansible-playbook -v scaleup.yml
39+
40+
done

‎scripts/select_lb.bash

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
#!/usr/bin/env bash
2+
3+
# Proof of concept script to check that OpenStack API responds and
4+
# assign floating ip to another load balancer in case it does not.
5+
# It assumes that there are two load balancers called
6+
# ${ENV_NAME}-lb-{1,2}
7+
#
8+
# NOTE: This code has been used for developing scaleup playbooks.
9+
# It has not been tested in other scenarios. Use with caution.
10+
#
11+
12+
13+
set -e
14+
15+
date
16+
17+
if ! curl -s -k -o /dev/null --connect-timeout 5 https://$PUBLIC_IP:8443; then
18+
19+
current_lb=$(openstack server list -c Name -c Networks | grep $PUBLIC_IP | sed -e "s/ //g" | cut -d "|" -f 2)
20+
echo "Currently active LB: '$current_lb'"
21+
current_idx=${current_lb:(-1)}
22+
new_idx=$(( current_idx%2 + 1 ))
23+
new_host=${ENV_NAME}-lb-$new_idx
24+
25+
echo "curl failed, re-assigning floating ip to $new_host"
26+
openstack server add floating ip $new_host $PUBLIC_IP
27+
fi

0 commit comments

Comments
 (0)