Skip to content

Commit 3e3690b

Browse files
imatza-rhrlobillo
imatza-rh
authored and
rlobillo
committed
Support Proxy installation in shiftstack-qa automation
https://issues.redhat.com/browse/OSPRH-4174 - Add a job definition for OCP 4.16 IPI proxy installation. - Prepare a restricted network for the installer-host. - Configure the installer VM for OpenShift proxy installation: - Configure squid on the installer vm. - Install and configure dnsmasq on the installer host. - Install NTP server on the installer host. - Add a template for generating an OpenShift install-config.yaml in a proxy installation. - Configure the NTP server of the workers and masters in the proxy installation post. Depends-on: https://review.gerrithub.io/c/shiftstack/shiftstack-qa/+/1196079 Change-Id: I64ddebfa53d3a41ec6193bb29fd31c6d6d9511fe Reviewed-on: https://review.gerrithub.io/c/shiftstack/shiftstack-qa/+/1194955 Tested-by: rhosqeauto <[email protected]> Reviewed-by: Ramón Lobillo <[email protected]> Tested-by: Itay Matza <[email protected]>
1 parent 96f4414 commit 3e3690b

25 files changed

+642
-24
lines changed

Diff for: ‎.ansible-lint

+1
Original file line numberDiff line numberDiff line change
@@ -20,3 +20,4 @@ skip_list:
2020
- metadata
2121
- var-naming[no-role-prefix]
2222
- name[template]
23+
- package-latest

Diff for: ‎collection/stages/roles/install/defaults/main.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,8 @@ upi_inventory_file_values:
1212
- {regexp: "os_subnet:.*", replace: "os_subnet: '{{ openshift_upi.subnets.ipv4.name }}'"}
1313
- {regexp: "os_subnet_range:.*", replace: "os_subnet_range: '{{ topology.machine_cidr }}'"}
1414
- {regexp: "os_external_network:.*", replace: "os_external_network: '{{ infra.external_network }}'"}
15-
- {regexp: "os_api_fip:.*", replace: "os_api_fip: '{{ resources.api_fip }}'"}
16-
- {regexp: "os_ingress_fip:.*", replace: "os_ingress_fip: '{{ resources.apps_fip }}'"}
15+
- {regexp: "os_api_fip:.*", replace: "os_api_fip: '{{ resources.api_accessible_ip }}'"}
16+
- {regexp: "os_ingress_fip:.*", replace: "os_ingress_fip: '{{ resources.apps_accessible_ip }}'"}
1717
- {regexp: "os_bootstrap_fip:.*", replace: "os_bootstrap_fip: '{{ resources.bootstrap_fip }}'"}
1818
- {regexp: "ansible_python_interpreter:", replace: "#ansible_python_interpreter:"}
1919

Diff for: ‎collection/stages/roles/install/tasks/generate_ocp_install-config.yml

+29-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
api_filter: "api.{{ ocp_cluster_name }}.{{ user_cloud }}.com"
1212
ingress_filter: "oauth-openshift.apps.{{ ocp_cluster_name }}.{{ user_cloud }}.com"
1313

14-
- name: Generate install-config.yaml from template
14+
- name: Generate an OpenShift install-config.yaml in a non-proxy installation
1515
ansible.builtin.template:
1616
src: install-config.yaml.j2
1717
dest: "{{ user_cloud_installation_dir }}/install-config.yaml"
@@ -29,6 +29,34 @@
2929
installcfg_dns_servers: "{{ infra.dns_servers }}"
3030
installcfg_api_floating_ip: "{{ precreated_api_fip }}"
3131
installcfg_ingress_floating_ip: "{{ precreated_ingress_fip }}"
32+
when: not openshift_proxy_installation
33+
34+
- name: Generate an OpenShift install-config.yaml in a proxy installation
35+
when: openshift_proxy_installation
36+
block:
37+
- name: Include vars from registered resources
38+
ansible.builtin.include_vars:
39+
file: "{{ resources_file }}"
40+
name: resources
41+
42+
- name: Generate install-config.yaml from install-config-proxy.yaml.j2 template
43+
ansible.builtin.template:
44+
src: install-config-proxy.yaml.j2
45+
dest: "{{ user_cloud_installation_dir }}/install-config.yaml"
46+
mode: u=rw,g=rw,o=r
47+
vars:
48+
installcfg_cluster_name: "{{ ocp_cluster_name }}"
49+
installcfg_worker_flavor: "{{ topology.flavors.worker.name }}"
50+
installcfg_worker_replicas: "{{ (installation_type == 'upi') | ternary('0', topology.replicas.worker) }}"
51+
installcfg_master_flavor: "{{ topology.flavors.master.name }}"
52+
installcfg_master_replicas: "{{ topology.replicas.master }}"
53+
installcfg_machine_cidr: "{{ resources.restricted_cidr }}"
54+
installcfg_network_type: "{{ openshift_network_type }}"
55+
installcfg_region: "{{ infra.region }}"
56+
installcfg_machines_subnet: "{{ resources.machines_subnet }}"
57+
installcfg_api_vip: "{{ precreated_api_fip }}"
58+
installcfg_ingress_vip: "{{ precreated_ingress_fip }}"
59+
installcfg_additional_trust_bundle: "{{ resources.additional_trust_bundle }}"
3260

3361
- name: Copy the install-config.yaml to installation dir
3462
ansible.builtin.copy:

Diff for: ‎collection/stages/roles/install/templates/install-config-proxy.yaml.j2

+45
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
# This file is autogenerated by shiftstack-qe automation
2+
apiVersion: v1
3+
baseDomain: "{{ user_cloud }}.com"
4+
compute:
5+
- name: worker
6+
platform:
7+
openstack:
8+
type: "{{ installcfg_worker_flavor }}"
9+
replicas: {{ installcfg_worker_replicas }}
10+
controlPlane:
11+
name: master
12+
platform:
13+
openstack:
14+
type: "{{ installcfg_master_flavor }}"
15+
replicas: {{ installcfg_master_replicas }}
16+
metadata:
17+
name: "{{ installcfg_cluster_name }}"
18+
networking:
19+
clusterNetworks:
20+
- cidr: 10.128.0.0/14
21+
hostSubnetLength: 9
22+
serviceNetwork:
23+
- 172.30.0.0/16
24+
machineNetwork:
25+
- cidr: "{{ installcfg_machine_cidr }}"
26+
networkType: "{{ installcfg_network_type }}"
27+
platform:
28+
openstack:
29+
cloud: "{{ user_cloud }}"
30+
externalNetwork: ""
31+
region: "{{ installcfg_region }}"
32+
machinesSubnet: {{ installcfg_machines_subnet }}
33+
apiVIP: "{{ installcfg_api_vip }}"
34+
ingressVIP: "{{ installcfg_ingress_vip }}"
35+
proxy:
36+
httpProxy: http://dummy:dummy@{{restricted_network.installer_ip}}:3128/
37+
httpsProxy: https://dummy:dummy@{{restricted_network.installer_ip}}:3130/
38+
pullSecret: |
39+
{{ ocp_pull_secret }}
40+
sshKey: |
41+
{{ ocp_public_key }}
42+
additionalTrustBundle: |
43+
{% for line in installcfg_additional_trust_bundle %}
44+
{{ line }}
45+
{% endfor %}

Diff for: ‎collection/stages/roles/post/tasks/main.yml

+4
Original file line numberDiff line numberDiff line change
@@ -11,3 +11,7 @@
1111
name: version
1212
spec:
1313
channel: ""
14+
15+
- name: Set the workers and masters NTP server
16+
ansible.builtin.include_tasks: set_ntp.yml
17+
when: openshift_proxy_installation

Diff for: ‎collection/stages/roles/post/tasks/set_ntp.yml

+82
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
---
2+
# We follow the procedure at:
3+
# https://docs.openshift.com/container-platform/4.10/post_installation_configuration/machine-configuration-tasks.html#installation-special-config-chrony_post-install-machine-configuration-tasks
4+
- name: Install Butane
5+
become: yes
6+
ansible.builtin.get_url:
7+
url: https://mirror.openshift.com/pub/openshift-v4/clients/butane/latest/butane
8+
dest: /usr/local/bin/butane
9+
mode: u=rwx,g=rwx,o=rwx
10+
11+
- name: Get OCP GA version from release.txt and Build the chronyc manifest
12+
block:
13+
- name: Download the OCP GA version from openshift mirror
14+
ansible.builtin.get_url:
15+
url: https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/stable/release.txt
16+
dest: /tmp/release.txt
17+
mode: u=rw,g=rw,o=r
18+
19+
- name: Extract OpenShift version
20+
ansible.builtin.shell: |
21+
grep -oP "Version:\s*\K4\.\d+" /tmp/release.txt
22+
changed_when: false
23+
register: openshift_ga_version
24+
25+
- name: Set ocp_latest_ga variable
26+
ansible.builtin.set_fact:
27+
ocp_latest_ga: "{{ openshift_ga_version.stdout }}"
28+
29+
- name: Build the chronyc manifest
30+
vars:
31+
ntp_server: "{{ openshift_mirror | default(False) | ternary(installer_vm.installer_fqdn, restricted_network.installer_ip) }}"
32+
ansible.builtin.template:
33+
src: 99-node-chronyc.j2
34+
dest: "{{ home_dir }}/99-{{ item }}-chrony"
35+
mode: u=rw,g=rw,o=r
36+
loop:
37+
- worker
38+
- master
39+
40+
# Use Butane to translate a human readable Butane config into a machine readable Ignition config
41+
- name: Build the chrony machine config
42+
ansible.builtin.shell: |
43+
butane {{ home_dir }}/99-{{ item }}-chrony -o {{ home_dir }}/99-{{ item }}-chrony.yaml
44+
changed_when: true
45+
loop:
46+
- worker
47+
- master
48+
49+
- name: Apply the chrony manifest
50+
kubernetes.core.k8s:
51+
state: present
52+
src: "{{ home_dir }}/99-{{ item }}-chrony.yaml"
53+
loop:
54+
- worker
55+
- master
56+
environment:
57+
KUBECONFIG: "{{ kubeconfig }}"
58+
59+
- name: Wait for the MCP to finish the cluster updates
60+
ansible.builtin.include_role:
61+
name: tools_cluster_checks
62+
tasks_from: wait_mcp_updated.yml
63+
64+
- name: Wait until cluster nodes are ready
65+
ansible.builtin.include_role:
66+
name: tools_cluster_checks
67+
tasks_from: wait_until_nodes_ready.yml
68+
69+
# Go over all the OCP nodes and check that NTP is configured correctly by checking that the Stratum from
70+
# The chronyc tracking command is not 0 in any of them
71+
- name: Check that the NTP server is reachable from all the OCP nodes
72+
ansible.builtin.shell: |
73+
set -o pipefail &&
74+
for i in $(oc get nodes -o name);
75+
do oc debug -q $i -- chroot /host sudo chronyc tracking|awk '/Stratum/{print $3}'; done | tr -d '\n' | awk '/0/{exit 1}'
76+
environment:
77+
KUBECONFIG: "{{ kubeconfig }}"
78+
changed_when: false
79+
register: ntp_output
80+
until: ntp_output is not failed
81+
retries: 5
82+
delay: 30

Diff for: ‎collection/stages/roles/post/templates/99-node-chronyc.j2

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
variant: openshift
2+
{% if discovered_openshift_release is version(ocp_latest_ga, '<=') %}
3+
version: {{ discovered_openshift_release }}.0
4+
{% else %}
5+
version: {{ ocp_latest_ga }}.0
6+
{% endif %}
7+
metadata:
8+
name: 99-{{ item }}-chrony
9+
labels:
10+
machineconfiguration.openshift.io/role: {{ item }}
11+
storage:
12+
files:
13+
- path: /etc/chrony.conf
14+
mode: 0664
15+
overwrite: true
16+
contents:
17+
inline: |
18+
server {{ ntp_server }} iburst
19+
driftfile /var/lib/chrony/drift
20+
makestep 1.0 3
21+
rtcsync
22+
logdir /var/log/chrony

Diff for: ‎collection/stages/roles/prepare/tasks/external_access.yml

+27-18
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,38 @@
11
---
2-
- name: Create (if needed) API FIP and get the FIP info
3-
ansible.builtin.include_tasks: create_fip.yml
4-
vars:
5-
description: "{{ ocp_api_description }}"
2+
- name: Create Floating IPs for API and APPS for a non-proxy installation
3+
when: not openshift_proxy_installation
4+
block:
5+
- name: Create (if needed) API FIP and get the FIP info
6+
ansible.builtin.include_tasks: create_fip.yml
7+
vars:
8+
description: "{{ ocp_api_description }}"
69

7-
- name: Store API FIP
8-
ansible.builtin.set_fact:
9-
api_fip: "{{ fip_address }}"
10+
- name: Store API FIP
11+
ansible.builtin.set_fact:
12+
api_accessible_ip: "{{ fip_address }}"
1013

11-
- name: Create (if needed) APPS FIP and get the FIP info
12-
ansible.builtin.include_tasks: create_fip.yml
13-
vars:
14-
description: "{{ ocp_apps_description }}"
14+
- name: Create (if needed) APPS FIP and get the FIP info
15+
ansible.builtin.include_tasks: create_fip.yml
16+
vars:
17+
description: "{{ ocp_apps_description }}"
18+
19+
- name: Store APPS FIP
20+
ansible.builtin.set_fact:
21+
apps_accessible_ip: "{{ fip_address }}"
1522

16-
- name: Store APPS FIP
23+
- name: Store restricted network IPs for API and APPS for a openshift-proxy installation
1724
ansible.builtin.set_fact:
18-
apps_fip: "{{ fip_address }}"
25+
api_accessible_ip: "{{ restricted_network.ocp_api_ip }}"
26+
apps_accessible_ip: "{{ restricted_network.ocp_apps_ip }}"
27+
when: openshift_proxy_installation
1928

20-
- name: Register Installer API and APP FIPS in resources.yml
29+
- name: Register Installer API and APP FIPS in resources.yml
2130
ansible.builtin.include_role:
2231
name: shiftstack.tools.tools_register_resources_file
2332
vars:
2433
input:
25-
api_fip: "{{ api_fip }}"
26-
apps_fip: "{{ apps_fip }}"
34+
api_accessible_ip: "{{ api_accessible_ip }}"
35+
apps_accessible_ip: "{{ apps_accessible_ip }}"
2736

2837
- name: Add resulting API and APPS fip/vip to /etc/hosts file
2938
become: true
@@ -33,8 +42,8 @@
3342
line: "{{ item.row }}"
3443
unsafe_writes: true
3544
vars:
36-
api_ip: "{{ api_fip }}"
37-
apps_ip: "{{ apps_fip }}"
45+
api_ip: "{{ api_accessible_ip }}"
46+
apps_ip: "{{ apps_accessible_ip }}"
3847
loop: "{{ etc_hosts_entries }}"
3948

4049
- name: Bootstrap fip pre-creation for UPI

Diff for: ‎collection/stages/roles/prepare/tasks/main.yml

+5
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,11 @@
55
- name: Update clouds.yml file with new Project
66
ansible.builtin.include_tasks: clouds.yml
77

8+
- name: Restricted Network Preparations
9+
ansible.builtin.include_tasks: restricted_network.yml
10+
when:
11+
- openshift_proxy_installation
12+
813
- name: Prepare setup for accessing the OCP Cluster
914
ansible.builtin.include_tasks: external_access.yml
1015

Diff for: ‎collection/stages/roles/prepare/tasks/restricted_network.yml

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
---
2+
- name: Create restricted network
3+
openstack.cloud.network:
4+
cloud: "{{ user_cloud }}"
5+
name: "{{ restricted_network.network_name }}"
6+
state: present
7+
8+
- name: Create restricted subnet
9+
openstack.cloud.subnet:
10+
cloud: "{{ user_cloud }}"
11+
name: "{{ restricted_network.subnet_name }}"
12+
network_name: "{{ restricted_network.network_name }}"
13+
cidr: "{{ restricted_network.cidr }}"
14+
dns_nameservers:
15+
- "{{ restricted_network.installer_ip }}"
16+
# OVNKubernetes requirement,further info in BZ1983951
17+
gateway_ip: "{{ (openshift_network_type == 'OVNKubernetes') |
18+
ternary(restricted_network.installer_ip, restricted_network.default_gw) }}"
19+
allocation_pool_start: "{{ restricted_network.pool_start }}"
20+
allocation_pool_end: "{{ restricted_network.pool_end }}"
21+
register: restricted_subnet
22+
23+
- name: Register restricted subnet cidr and subnet id for machinesSubnet var in install-config
24+
ansible.builtin.include_role:
25+
name: shiftstack.tools.tools_register_resources_file
26+
vars:
27+
input:
28+
restricted_cidr: "{{ restricted_subnet.subnet.cidr }}"
29+
machines_subnet: "{{ restricted_subnet.subnet.id }}"

Diff for: ‎collection/stages/roles/verification/tasks/main.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@
8686
vars:
8787
# Using internal-lb annotation when cluster is deployed on restricted or provider network (FIPless):
8888
internal_lb: |
89-
{{ (openshift_proxy | default(False) or bm_workers | default(False) or provider_network_primary | default(False)) | bool }}
89+
{{ (openshift_proxy_installation | default(False) or bm_workers | default(False) or provider_network_primary | default(False)) | bool }}
9090
ansible.builtin.include_tasks: check_lb_svc.yml
9191
when:
9292
- edge_nova_az is not defined # Skip due to Openstack Manila is not supported at the Edge

Diff for: ‎collection/tools/roles/tools_installer_host/defaults/main.yml

+8
Original file line numberDiff line numberDiff line change
@@ -93,3 +93,11 @@ installer_vm:
9393
protocol: udp
9494
port_range_min: '53'
9595
port_range_max: '53'
96+
ntp:
97+
sg_rules:
98+
- direction: 'ingress'
99+
ethertype: 'IPv4'
100+
remote_ip_prefix: '0.0.0.0/0'
101+
protocol: udp
102+
port_range_min: '123'
103+
port_range_max: '123'

Diff for: ‎collection/tools/roles/tools_installer_host/tasks/configuration_hosts.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,6 @@
1111
regexp: "{{ item.regex }}"
1212
line: "{{ item.row }}"
1313
vars:
14-
api_ip: "{{ resources.api_fip }}"
15-
apps_ip: "{{ resources.apps_fip }}"
14+
api_ip: "{{ resources.api_accessible_ip }}"
15+
apps_ip: "{{ resources.apps_accessible_ip }}"
1616
loop: "{{ etc_hosts_entries }}"

Diff for: ‎collection/tools/roles/tools_installer_host/tasks/install_cert.yml

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
---
2+
- name: Generate a self-signed SSL certificate using OpenSSL
3+
ansible.builtin.shell: |
4+
openssl req -newkey rsa:4096 -nodes -sha256 -keyout domain.key \
5+
-x509 -days 30 -addext "subjectAltName = IP:{{ restricted_network.installer_ip }},DNS:{{ installer_vm.installer_fqdn }}" \
6+
-subj "/C=US/ST=Denial/L=Springfield/O=RedHat/CN=shiftstack.com" \
7+
-out domain.crt
8+
args:
9+
chdir: "{{ certs_dir }}"
10+
changed_when: true
11+
become: "{{ openshift_proxy_installation | ternary('yes', omit) }}"
12+
13+
- name: Copy certificate to root chain dir
14+
ansible.builtin.copy:
15+
src: "{{ certs_dir }}/domain.crt"
16+
dest: /etc/pki/ca-trust/source/anchors/domain.crt
17+
remote_src: yes
18+
mode: u=rwx,g=rw,o=r
19+
become: yes
20+
21+
- name: Add the CA cert to the trusted root chain in the installer host VM
22+
ansible.builtin.command: update-ca-trust extract
23+
changed_when: true
24+
become: yes
25+
26+
# Store certificate on resources.yaml to be used on install-config.yaml on install role:
27+
- name: Register cacert content
28+
ansible.builtin.command: "cat {{ certs_dir }}/domain.crt"
29+
changed_when: false
30+
register: cacert_content
31+
become: "{{ openshift_proxy_installation | ternary('yes', omit) }}"
32+
33+
- name: Register cacert to resources.yml
34+
ansible.builtin.include_role:
35+
name: shiftstack.tools.tools_register_resources_file
36+
vars:
37+
input:
38+
additional_trust_bundle: "{{ cacert_content.stdout_lines }}"

0 commit comments

Comments
 (0)