Fix registry and registry console certs

The registry is by default deployed using a certificate signed by the
OpenShift CA. As this is not generally recognized, create a new
re-encrypting edge route to the registry that uses a proper certificate.
This way it is possible to login to the registry normally.

Using the re-encrypting route triggers a bug in the Docker registry that
is fixed in a newer version. To work around this bug, manually update
the Docker image used for the registry to a newer one. See:
openshift/origin#14249 and also
openshift/origin#11391.

The registry console also needs to be reconfigured with the new route to
the registry. Make this configuration change using the oc_env module
from openshift-ansible. For this to work, add modules from the
lib_openshift role into the library path.

Replace the certificate of the registry console with a proper recognized
certificate so that warnings are not shown when accessing the console
from a browser.

Write tests for checking correct connectivity to the routes used for the
registry and the registry console. These should verify that there are no
certificate issues with the endpoints.

Split the registry config changes into their own playbook from the
post-install playbook to keep things tidy.

Author: rlaurika

container-src/poc-deployer/init_env.bash

@@ -25,7 +25,8 @@ echo "ANSIBLE_INVENTORY set to $ANSIBLE_INVENTORY"
 echo
 
 export ANSIBLE_LIBRARY="/usr/share/ansible:\
-$HOME/openshift-ansible/roles/lib_utils/library"
+$HOME/openshift-ansible/roles/lib_utils/library:\
+$HOME/openshift-ansible/roles/lib_openshift/library"
 echo "ANSIBLE_LIBRARY set to $ANSIBLE_LIBRARY"
 
 pushd /opt/deployment/poc/playbooks > /dev/null

playbooks/post_install.yml

@@ -55,20 +55,12 @@
       when: existing_storageclass.stdout_lines | length > 0
       changed_when: storageclass_template.changed
 
-    - name: check if registry PVC exists
-      shell: oc get pvc -n default registry
-      register: existing_registry_pv
-      changed_when: false
-      failed_when: false
-
-    - name: create and attach a persistent volume for registry
-      shell: oc volume -n default dc/docker-registry --add --mount-path=/registry --overwrite --name=registry-storage -t pvc --claim-size=200Gi --claim-name=registry
-      when: existing_registry_pv.stdout_lines | length == 0
-
-    - name: attach the existing persistent volume for registry
-      shell: oc volume -n default dc/docker-registry --add --mount-path=/registry --overwrite --name=registry-storage --claim-name=registry
-      when: existing_registry_pv.stdout_lines | length > 0
+- name: Additional Docker registry configuration
+  include: registry_config.yml
 
+- name: Setup default www app
+  hosts: masters[0]
+  tasks:
     # block for conditionally deploying default-www-app
     - block:
       - name: check if project default-www exists

playbooks/registry_config.yml

@@ -0,0 +1,107 @@
+---
+- name: Do additional Docker registry configuration
+  hosts: masters[0]
+  tasks:
+    - name: update Docker image used for the registry
+      oc_edit:
+        name: docker-registry
+        namespace: default
+        kind: dc
+        content:
+          spec.template.spec.containers[0].image: "{{ registry_docker_image }}"
+
+    - name: get certificate for registry
+      slurp:
+        src: '/etc/origin/master/registry.crt'
+      register: registry_cert_file
+
+    - name: put registry certificate content into variable
+      set_fact:
+        registry_dest_cert: "{{ registry_cert_file['content'] | b64decode }}"
+
+    - name: create a re-encrypt route with a proper cert for the registry
+      oc_route:
+        name: docker-registry-reencrypt
+        namespace: default
+        cert_path: "/etc/origin/master/named_certificates/{{ openshift_public_hostname }}.crt"
+        key_path: "/etc/origin/master/named_certificates/{{ openshift_public_hostname }}.key"
+        cacert_path: "/etc/origin/master/named_certificates/ext_ca.crt"
+        dest_cacert_content: "{{ registry_dest_cert }}"
+        service_name: "docker-registry"
+        port: "5000"
+        host: "docker-registry.{{ openshift_public_hostname }}"
+        tls_termination: "reencrypt"
+      run_once: true
+
+    - name: update registry URL in registry console
+      oc_env:
+        state: present
+        name: registry-console
+        namespace: default
+        kind: dc
+        env_vars:
+          REGISTRY_HOST: docker-registry.{{ openshift_public_hostname }}
+      run_once: true
+
+    - name: check if we already have registry-console.cert created
+      stat:
+        path: /etc/origin/master/registry-console.cert
+      register: registry_console_cert
+
+    - name: create registry console cert file with a proper cert+key
+      shell: >
+        cat /etc/origin/master/{{ openshift_public_hostname }}.crt
+        /etc/origin/master/{{ openshift_public_hostname }}.key
+        > /etc/origin/master/registry-console.cert
+      when: registry_console_cert.stat.exists == False
+
+    - name: set access rights for registry-console.cert
+      file:
+        path: /etc/origin/master/registry-console.cert
+        owner: root
+        group: root
+        mode: 0640
+
+    - name: create cert secret for registry console
+      oc_secret:
+        state: present
+        namespace: default
+        name: console-secret
+        files:
+          - name: registry-console.cert
+            path: '/etc/origin/master/registry-console.cert'
+      run_once: true
+
+    - name: add cert secret as a volume to dc/registry-console
+      oc_volume:
+        state: present
+        name: registry-console
+        namespace: default
+        kind: dc
+        mount_type: secret
+        secret_name: console-secret
+        vol_name: console-secret-vol
+        mount_path: /etc/cockpit/ws-certs.d
+      run_once: true
+
+    - name: check if registry PVC exists
+      shell: oc get pvc -n default registry
+      register: existing_registry_pv
+      changed_when: false
+      failed_when: false
+
+    - name: create and attach a persistent volume for registry
+      shell: >
+        oc volume -n default dc/docker-registry --add
+        --mount-path=/registry --overwrite
+        --name=registry-storage
+        --claim-name=registry
+        -t pvc --claim-size=200Gi
+      when: existing_registry_pv.stdout_lines | length == 0
+
+    - name: attach the existing persistent volume for registry
+      shell: >
+        oc volume -n default dc/docker-registry --add
+        --mount-path=/registry --overwrite
+        --name=registry-storage --claim-name=registry
+      when: existing_registry_pv.stdout_lines | length > 0

tests/files/bats_test_master_openshift_health/run.bats

@@ -1,8 +1,37 @@
 #!/usr/bin/env bats
 
+check_route_url() {
+  # OpenShift adds its own CA cert to the CA bundle. For testing, we don't
+  # want to have that cert available, so create a copy of the CA bundle
+  # without the OpenShift cert added.
+  ca_bundle=$(mktemp)
+  sed '/openshift-signer/,/^$/d' /etc/ssl/certs/ca-bundle.crt > $ca_bundle
+
+  url=$(oc get route $1 -o json -o jsonpath='{.spec.host}')
+
+  curl --cacert $ca_bundle https://$url >&2
+  curl_status=$?
+
+  rm $ca_bundle
+
+  return $curl_status
+}
+
 @test "test default namespace pod health" {
     all_pods_count=$(oc get pods -n default -o json | jq '[.items[].status.phase]|length')
     running_pods_count=$(oc get pods -n default -o json | jq '[.items[].status.phase|select(. == "Running")]|length')
 
     [ $all_pods_count -eq $running_pods_count ]
 }
+
+@test "test connectivity to registry URL" {
+    run check_route_url docker-registry-reencrypt
+
+    [ $status -eq 0 ]
+}
+
+@test "test connectivity to registry console URL" {
+    run check_route_url registry-console
+
+    [ $status -eq 0 ]
+}