Set the installation dir inside the artifacts dir

Change the installation dir path from the home dir to the artifacts
dir:

artifacts/
├── must-gather-verification
├── installation
│   └── shiftstack                       # `user_cloud_installation_dir`
│       ├── install-config.yaml
│       ├── openstack                    # `osp_config_dir`
│       │   ├── clouds.yaml
│       │   └── shiftstackrc
│       └── ostest                       # `ocp_installation_dir`

This way we can more easily synchronize the installation files from the
installer host to the Ansible Controller Node machine.

As a result of this change, we don't need the fetch_to_inventory task anymore.

Change-Id: I8a19afff1effb3f902671f69e15be963a5e823a2
Reviewed-on: https://review.gerrithub.io/c/shiftstack/shiftstack-qa/+/1196079
Reviewed-by: Yaakov Khodorkovski <[email protected]>
Tested-by: Itay Matza <[email protected]>
Tested-by: Yaakov Khodorkovski <[email protected]>
Tested-by: rhosqeauto <[email protected]>
Reviewed-by: Ramón Lobillo <[email protected]>

Author: imatza-rh

Diff for: collection/stages/roles/conformance_test/tasks/run_conformance_test.yml

@@ -109,4 +109,3 @@
         name: tools_must-gather
       vars:
         must_gather_dir_suffix: "must-gather-conformance"
-        fetch_to_inventory: true

Diff for: collection/stages/roles/csi_tests/tasks/run_csi_test.yml

@@ -38,4 +38,3 @@
         name: tools_must-gather
       vars:
         must_gather_dir_suffix: "must-gather-{{ csi_test_name }}"
-        fetch_to_inventory: true

Diff for: collection/stages/roles/install/tasks/generate_ocp_install-config.yml

@@ -14,7 +14,7 @@
 - name: Generate install-config.yaml from template
   ansible.builtin.template:
     src: install-config.yaml.j2
-    dest: "{{ home_dir }}/install-config.yaml"
+    dest: "{{ user_cloud_installation_dir }}/install-config.yaml"
     mode: u=rw,g=rw,o=r
   vars:
     installcfg_cluster_name: "{{ ocp_cluster_name }}"
@@ -32,7 +32,7 @@
 
 - name: Copy the install-config.yaml to installation dir
   ansible.builtin.copy:
-    src: "{{ home_dir }}/install-config.yaml"
-    dest: "{{ installation_dir }}/install-config.yaml"
+    src: "{{ user_cloud_installation_dir }}/install-config.yaml"
+    dest: "{{ ocp_installation_dir }}/install-config.yaml"
     remote_src: yes
     mode: u=rwx,g=rwx,o=rwx

Diff for: collection/stages/roles/install/tasks/ipi_tenant.yml

@@ -3,21 +3,21 @@
   block:
     - name: Install Openshift
       ansible.builtin.shell: |
-          openshift-install create cluster --log-level debug --dir {{ installation_dir }}
+          openshift-install create cluster --log-level debug --dir {{ ocp_installation_dir }}
       environment:
         OS_CLOUD: "{{ user_cloud }}"
       changed_when: true
   rescue:
     - name: Use an openshift-install flag to wait until the cluster is ready
       ansible.builtin.shell: |
-        openshift-install wait-for install-complete --log-level debug --dir {{ installation_dir }}
+        openshift-install wait-for install-complete --log-level debug --dir {{ ocp_installation_dir }}
       environment:
         OS_CLOUD: "{{ user_cloud }}"
       changed_when: true
 
     - name: Remove the bootstrap resources after the OCP installation succeeded
       ansible.builtin.shell: |
-        openshift-install destroy bootstrap --log-level debug --dir {{ installation_dir }}
+        openshift-install destroy bootstrap --log-level debug --dir {{ ocp_installation_dir }}
       environment:
         OS_CLOUD: "{{ user_cloud }}"
       changed_when: true
@@ -31,12 +31,11 @@
           The openshift installation passed but unexpectedly needed the wait-for flag.
           More info in Jira KURYRQE-1002.
   always:
-    - name: "Copy installation files to {{ artifacts_dir }}"
-      ansible.builtin.copy:
-        src: "{{ item.src }}"
-        dest: "{{ artifacts_dir }}/{{ user_cloud }}/{{ ocp_cluster_name }}"
-        mode: "{{ item.mode }}"
-      loop:
-        - {src: "{{ installation_dir }}/", mode: "0755"}
-        - {src: "{{ osp_config_dir }}", mode: "0755"}
-        - {src: "{{ home_dir }}/install-config.yaml", mode: "0644"}
+    - name: Synchronize artifacts from the Ansible Managed Node to Ansible Controller
+      ansible.posix.synchronize:
+        src: "{{ artifacts_dir }}"
+        dest: "{{ controller_home_dir }}"
+        mode: pull
+        recursive: yes
+        times: false
+        perms: false

Diff for: collection/stages/roles/install/tasks/main.yml

@@ -1,12 +1,12 @@
 ---
 - name: Remove the directory for the installation files (if exists)
   ansible.builtin.file:
-    path: "{{ installation_dir }}"
+    path: "{{ ocp_installation_dir }}"
     state: absent
 
 - name: Create a directory for the installation files
   ansible.builtin.file:
-    path: "{{ installation_dir }}"
+    path: "{{ ocp_installation_dir }}"
     state: directory
     mode: u=rwx,g=rw,o=r
 

Diff for: collection/stages/roles/install/tasks/upi/bootstrap.yml

@@ -45,7 +45,7 @@
     - name: Run the script that modifies the bootstrap ignition file
       ansible.builtin.script:
         cmd: "scripts/upi_bootstrap.py"
-        chdir: "{{ installation_dir }}"
+        chdir: "{{ ocp_installation_dir }}"
         executable: python3
       environment: "{{ env_vars }}"
 
@@ -62,7 +62,7 @@
     container_format: bare
     disk_format: raw
     state: present
-    filename: "{{ installation_dir }}/bootstrap.ign"
+    filename: "{{ ocp_installation_dir }}/bootstrap.ign"
 
 - name: Retrieve the image file string
   block:
@@ -123,5 +123,5 @@
 - name: Generate the secondary ignition config file
   ansible.builtin.template:
     src: "upi_bootstrap-ignition.json.j2"
-    dest: "{{ installation_dir }}/{{ ocp_infra_id }}-bootstrap-ignition.json"
+    dest: "{{ ocp_installation_dir }}/{{ ocp_infra_id }}-bootstrap-ignition.json"
     mode: u=rw,g=rw,o=r

Diff for: collection/stages/roles/install/tasks/upi/control_plane.yml

@@ -16,4 +16,4 @@
   ansible.builtin.shell: "{{ script }}" # noqa: command-instead-of-shell
   changed_when: true
   args:
-    chdir: "{{ installation_dir }}"
+    chdir: "{{ ocp_installation_dir }}"

Diff for: collection/stages/roles/install/tasks/upi/create_ocp_manifests.yml

@@ -1,14 +1,14 @@
 ---
 - name: Generate the Kubernetes manifests for the cluster
   ansible.builtin.shell: |
-    openshift-install create manifests --dir={{ installation_dir }}
+    openshift-install create manifests --dir={{ ocp_installation_dir }}
   changed_when: true
 
 - name: Remove the Kubernetes manifest files that define the control plane machines and compute machineSets
   block:
     - name: Find the Kubernetes manifest files that define the control plane machines and compute machineSets
       ansible.builtin.find:
-        paths: "{{ installation_dir }}/openshift"
+        paths: "{{ ocp_installation_dir }}/openshift"
         patterns:
           - "99_openshift-cluster-api_master-machines-*.yaml"
           - "99_openshift-cluster-api_worker-machineset-*.yaml"
@@ -23,7 +23,7 @@
 
 - name: Modify the Kubernetes manifest file to prevent Pods from being scheduled on the control plane machines
   ansible.builtin.replace:
-    path: "{{ installation_dir }}/manifests/cluster-scheduler-02-config.yml"
+    path: "{{ ocp_installation_dir }}/manifests/cluster-scheduler-02-config.yml"
     regexp: '^  mastersSchedulable: .*'
     replace: '  mastersSchedulable: false'
     after: 'spec'

Diff for: collection/stages/roles/install/tasks/upi/deploy_cluster.yml

@@ -37,7 +37,7 @@
 
 - name: Wait until the bootstrap is completed
   ansible.builtin.shell: |
-    openshift-install wait-for bootstrap-complete --log-level=debug --dir {{ installation_dir }}
+    openshift-install wait-for bootstrap-complete --log-level=debug --dir {{ ocp_installation_dir }}
   changed_when: false
 
 - name: Remove the bootstrap resources (down-bootstrap.yaml)
@@ -110,5 +110,5 @@
 
 - name: Wait until the installation is completed
   ansible.builtin.shell: |
-    openshift-install wait-for install-complete --log-level=debug --dir {{ installation_dir }}
+    openshift-install wait-for install-complete --log-level=debug --dir {{ ocp_installation_dir }}
   changed_when: false

Diff for: collection/stages/roles/install/tasks/upi/ignition_configs.yml

@@ -1,13 +1,13 @@
 ---
 - name: Create the ignition config files
   ansible.builtin.shell: |
-    openshift-install create ignition-configs --dir={{ installation_dir }}
+    openshift-install create ignition-configs --dir={{ ocp_installation_dir }}
   changed_when: true
 
 - name: Get the OCP cluster id
   ansible.builtin.shell:
     cmd: jq -r .infraID metadata.json
-    chdir: "{{ installation_dir }}"
+    chdir: "{{ ocp_installation_dir }}"
   changed_when: false # noqa: command-instead-of-shell
   register: jq_result
 

Diff for: collection/stages/roles/lb_tests/tasks/change_cloud_cm.yml

@@ -210,7 +210,6 @@
         name: tools_must-gather
       vars:
         must_gather_dir_suffix: "must-gather-change-cloud-cm"
-        fetch_to_inventory: true
 
     - name: Fail inside rescue block
       ansible.builtin.fail:

Diff for: collection/stages/roles/openstack_test/tasks/run_openstack_test.yml

@@ -95,4 +95,3 @@
         name: tools_must-gather
       vars:
         must_gather_dir_suffix: "must-gather-openstack-tests"
-        fetch_to_inventory: true

Diff for: collection/stages/roles/prepare/defaults/main.yml

@@ -1,6 +1,4 @@
 ---
-clouds_yaml_file_path: "{{ home_dir }}/.config/openstack/clouds.yaml"
-secure_yaml_file_path: "{{ home_dir }}/.config/openstack/secure.yaml"
 project:
   name: "{{ user_cloud }}"
   user: user

Diff for: collection/stages/roles/prepare/tasks/clouds.yml

@@ -42,7 +42,7 @@
 - name: Generate RC file for {{ user_cloud }}
   ansible.builtin.template:
     src: user_env_file.j2
-    dest: "{{ osp_config_dir }}/{{ user_cloud }}rc"
+    dest: "{{ user_env_file }}"
     mode: u=rw,g=rw,o=r
 
 - name: "Copy clouds.yaml to {{ osp_config_dir }}"

Diff for: collection/stages/roles/verification/tasks/main.yml

@@ -163,7 +163,6 @@
       ansible.builtin.include_role:
         name: tools_must-gather
       vars:
-        fetch_to_inventory: true
         must_gather_dir_suffix: "must-gather-verification"
 
 - name: Fail the playbook in a case of a cluster verification test failure

Diff for: collection/tools/roles/tools_cluster_checks/tasks/print_cluster_status.yml

@@ -62,23 +62,17 @@
       block:
         - name: Gather bootstrap logs
           ansible.builtin.shell: |
-            openshift-install gather bootstrap --dir {{ installation_dir }} --log-level=debug
+            openshift-install gather bootstrap --dir {{ ocp_installation_dir }} --log-level=debug
           environment:
             SSH_AUTH_SOCK: ""
           failed_when: false
           changed_when: true
 
-        - name: Compress {{ installation_dir }} directory
+        - name: Compress {{ ocp_installation_dir }} directory
           community.general.archive:
-            path: "{{ installation_dir }}"
-            dest: "{{ installation_dir }}.tar.gz"
-            mode: u=rw,g=rw,o=r
-
-        - name: Fetch {{ installation_dir }}.tar.gz to {{ artifacts_dir }}
-          ansible.builtin.fetch:
-            src: "{{ installation_dir }}.tar.gz"
+            path: "{{ ocp_installation_dir }}"
             dest: "{{ artifacts_dir }}/failing_installation_logs.tar.gz"
-            flat: yes
+            mode: u=rw,g=rw,o=r
 
     - name: Get ocp cluster version
       ansible.builtin.command: oc get clusterversion

Diff for: collection/tools/roles/tools_installer_host/defaults/main.yml

@@ -8,8 +8,14 @@ osp_rhel_matrix:
 installer_image: "{{ osp_rhel_matrix[openstack_version] }}"
 
 files_to_copy:
-  - "{{ user_env_file | basename }}"
-  - "clouds.yaml"
+  - "{{ user_env_file }}"
+  - "{{ osp_config_dir }}/clouds.yaml"
+
+installer_vm_home_dir: "/home/{{ installer_vm_ssh_user }}"
+installer_vm_artifacts_dir: "{{ installer_vm_home_dir }}/artifacts"
+installer_vm_installation_dir: "{{ installer_vm_artifacts_dir }}/installation"
+installer_vm_user_cloud_installation_dir: "{{ installer_vm_installation_dir }}/{{ user_cloud }}"
+installer_vm_osp_config_dir: "{{ installer_vm_user_cloud_installation_dir }}/openstack"
 
 installer_required_packages:
   - 'python3-openstackclient'

Diff for: collection/tools/roles/tools_installer_host/tasks/configuration_copy_workspace.yml

@@ -1,8 +1,8 @@
 ---
 - name: Copy files from the OSP config directory to the installer host VM
   ansible.builtin.copy:
-    src: "{{ osp_config_dir }}/{{ item }}"
-    dest: "~/{{ item }}"
+    src: "{{ item }}"
+    dest: "{{ installer_vm_osp_config_dir }}/"
     mode: u=rw,g=rw,o=r
   loop: "{{ files_to_copy }}"
 
@@ -18,7 +18,7 @@
 - name: Copy the secure.yaml file from the OSP config directory to the installer host VM
   ansible.builtin.copy:
     src: "{{ osp_config_dir }}/secure.yaml"
-    dest: "~/secure.yaml"
+    dest: "{{ installer_vm_osp_config_dir }}/secure.yaml"
     mode: u=rw,g=rw,o=r
   when: not (legacy_installation | default(False))
 
@@ -32,14 +32,14 @@
 
     - name: Create a symlink to ~/clouds.yaml in ~/.config/openstack/clouds.yaml
       ansible.builtin.file:
-        src: ~/clouds.yaml
+        src: "{{ installer_vm_osp_config_dir }}/clouds.yaml"
         dest: ~/.config/openstack/clouds.yaml
         force: true
         state: link
 
     - name: Create a symlink to ~/secure.yaml in ~/.config/openstack/secure.yaml
       ansible.builtin.file:
-        src: ~/secure.yaml
+        src: "{{ installer_vm_osp_config_dir }}/secure.yaml"
         dest: ~/.config/openstack/secure.yaml
         force: true
         state: link

Diff for: collection/tools/roles/tools_must-gather/defaults/main.yml

@@ -1,6 +1,5 @@
 ---
 # defaults file for tools_must-gather
-must_gather_dir_base: "{{ home_dir }}"
-fetch_to_inventory: false # must be defined when calling the role
+must_gather_dir_base: "{{ artifacts_dir }}"
 must_gather_dir_suffix: None # must be defined when calling the role
 must_gather_dir: "{{ must_gather_dir_base }}/{{ must_gather_dir_suffix }}"

Diff for: collection/tools/roles/tools_must-gather/tasks/main.yml

@@ -54,11 +54,3 @@
         dest: "{{ must_gather_dir_base }}/{{ must_gather_dir_suffix }}.tar.gz"
         mode: u=rw,g=rw,o=r
       no_log: true
-
-    - name: Fetch {{ must_gather_dir_base }}/{{ must_gather_dir_suffix }}.tar.gz to {{ artifacts_dir }}
-      ansible.builtin.fetch:
-        src: "{{ must_gather_dir_base }}/{{ must_gather_dir_suffix }}.tar.gz"
-        dest: "{{ artifacts_dir }}/{{ must_gather_dir_suffix }}.tar.gz"
-        flat: yes
-      no_log: true
-      when: fetch_to_inventory

Diff for: configs/global.yml

@@ -1,13 +1,28 @@
 ---
 # Sets the Ansible Target (a.k.a Managed Node) HOME environment variable to its actual value
 # This ensures scripts and applications running on the target machine know the user's home directory
+# artifacts/
+# ├── installation
+# │   └── shiftstack                   # Dedicated dir for OSP user_cloud - `user_cloud_installation_dir`
+# │       ├── install-config.yaml
+# │       ├── openstack                # OSP config dir - `osp_config_dir`
+# │       │   ├── clouds.yaml
+# │       │   └── shiftstackrc
+# │       └── ostest                   # OCP cluster installation dir - `ocp_installation_dir`
+#
+# Note: The ci-framework job overwriting the 'user_cloud' and 'ocp_cluster_name'.
+#       Reference: https://github.com/openstack-k8s-operators/ci-framework/pull/1882
 home_dir: "{{ ansible_facts.env['HOME'] }}"
 artifacts_dir: "{{ home_dir }}/artifacts"
-installation_dir: "{{ home_dir }}/installation/{{ user_cloud }}/{{ ocp_cluster_name }}"
-osp_config_dir: "{{ home_dir }}/installation/{{ user_cloud }}/openstack"
+installation_dir: "{{ artifacts_dir }}/installation"
+user_cloud: "shiftstack"
+user_cloud_installation_dir: "{{ installation_dir }}/{{ user_cloud }}"
+osp_config_dir: "{{ user_cloud_installation_dir }}/openstack"
+ocp_cluster_name: "ostest"
+ocp_installation_dir: "{{ user_cloud_installation_dir }}/{{ ocp_cluster_name }}"
+kubeconfig: "{{ ocp_installation_dir }}/auth/kubeconfig"
+upi_playbooks_dir: "{{ ocp_installation_dir }}"
 polarion_dir: "{{ artifacts_dir }}/test_results"
-kubeconfig: "{{ installation_dir }}/auth/kubeconfig"
-upi_playbooks_dir: "{{ installation_dir }}"
 
 # Sets the Ansible Controller Node machine's HOME environment variable
 controller_home_dir: "{{ lookup('ansible.builtin.env', 'HOME') }}"
@@ -16,11 +31,11 @@ resources_file: "{{ controller_home_dir }}/artifacts/resources.yml"
 stage_unstable_flag: "{{ controller_home_dir }}/artifacts/stage_unstable_flag"
 
 admin_cloud: "overcloud"
-admin_env_file: "{{ home_dir }}/{{ admin_cloud }}rc"
+admin_env_file: "{{ osp_config_dir }}/{{ admin_cloud }}rc"
 admin_verify_cacert: false
-user_cloud: "shiftstack"
-user_env_file: "{{ home_dir }}/{{ user_cloud }}rc"
-ocp_cluster_name: "ostest"
+user_env_file: "{{ osp_config_dir }}/{{ user_cloud }}rc"
+clouds_yaml_file_path: "{{ home_dir }}/.config/openstack/clouds.yaml"
+secure_yaml_file_path: "{{ home_dir }}/.config/openstack/secure.yaml"
 
 topology:
   flavors: