Distgen - add ability to generate files

Author: dhodovsk

Dockerfile.template

@@ -1,4 +1,4 @@
-FROM centos:centos7
+FROM {{ config.docker.from }}
 
 # PostgreSQL image for OpenShift.
 # Volumes:
@@ -10,8 +10,8 @@ FROM centos:centos7
 #  * $POSTGRESQL_ADMIN_PASSWORD (Optional) - Password for the 'postgres'
 #                           PostgreSQL administrative account
 
-ENV POSTGRESQL_VERSION=9.6 \
-    POSTGRESQL_PREV_VERSION=9.5 \
+ENV POSTGRESQL_VERSION={{ spec.version }} \
+    POSTGRESQL_PREV_VERSION={{ spec.prev_version }} \
     HOME=/var/lib/pgsql \
     PGUSER=postgres
 
@@ -23,14 +23,20 @@ create, run, maintain and access a PostgreSQL DBMS server."
 LABEL summary=$SUMMARY \
       description="$DESCRIPTION" \
       io.k8s.description="$DESCRIPTION" \
-      io.k8s.display-name="PostgreSQL 9.6" \
+      io.k8s.display-name="PostgreSQL {{ spec.version }}" \
       io.openshift.expose-services="5432:postgresql" \
-      io.openshift.tags="database,postgresql,postgresql96,rh-postgresql96" \
-      name="centos/postgresql-96-centos7" \
-      com.redhat.component="rh-postgresql96-docker" \
-      version="9.6" \
-      release="1" \
-      maintainer="SoftwareCollections.org <[email protected]>"
+      io.openshift.tags="{{ spec.openshift_tags }}" \
+      name="{{ spec.common_image_name }}" \
+      com.redhat.component="{{ spec.redhat_component }}" \
+{%- if spec.version in spec.rhscl3_introduced %}
+      version="1"
+{%- else %}
+      version="{{ spec.version }}" \
+      release="1"
+{%- endif %}
+{%- if spec.maintainer %} \
+      maintainer="{{ spec.maintainer }}"
+{%- endif %}
 
 EXPOSE 5432
 
@@ -39,10 +45,14 @@ COPY root/usr/libexec/fix-permissions /usr/libexec/fix-permissions
 # This image must forever use UID 26 for postgres user so our volumes are
 # safe in the future. This should *never* change, the last test is there
 # to make sure of that.
-RUN yum install -y centos-release-scl-rh && \
-    yum-config-manager --add-repo https://cbs.centos.org/repos/sclo7-rh-postgresql96-rh-candidate/x86_64/os/ && \
-    echo gpgcheck=0 >> /etc/yum.repos.d/cbs.centos.org_repos_sclo7-rh-postgresql96-rh-candidate_x86_64_os_.repo && \
-    INSTALL_PKGS="rsync tar gettext bind-utils nss_wrapper rh-postgresql96 rh-postgresql96-postgresql-contrib rh-postgresql95-postgresql-server" && \
+{%- if spec.repo_enable_reason %}
+{{ spec.repo_enable_reason }}
+{%- endif %}
+RUN {{ spec.environment_setup }}
+{%- if not spec.stable %}
+{{ spec.staging_repo_setup }}
+{%- endif %}
+    INSTALL_PKGS="rsync tar gettext bind-utils nss_wrapper {{ spec.pkgs }}" && \
     yum -y --setopt=tsflags=nodocs install $INSTALL_PKGS && \
     rpm -V $INSTALL_PKGS && \
     yum clean all && \
@@ -54,7 +64,7 @@ RUN yum install -y centos-release-scl-rh && \
 
 # Get prefix path and path to scripts rather than hard-code them in scripts
 ENV CONTAINER_SCRIPTS_PATH=/usr/share/container-scripts/postgresql \
-    ENABLED_COLLECTIONS=rh-postgresql96
+    ENABLED_COLLECTIONS={{ spec.enabled_collection }}
 
 COPY root /
 

Makefile

@@ -2,6 +2,7 @@
 BASE_IMAGE_NAME = postgresql
 VERSIONS = 9.4 9.5 9.6
 OPENSHIFT_NAMESPACES = 9.2
+export VERSIONS
 
 # HACK:  Ensure that 'git pull' for old clones doesn't cause confusion.
 # New clones should use '--recursive'.
@@ -13,11 +14,5 @@ include common/common.mk
 upgrade-tests: $(VERSIONS)
 	test/run_upgrade_test 9.2:remote 9.4:local 9.5:local 9.6:local
 
-DG = /usr/bin/dg
-DG_EXEC = ${DG} --max-passes 25 --template Dockerfile.template --multispec specs/multispec.yml
-
-generate_dockerfiles:
-	for version in ${VERSIONS}; do \
-		${DG_EXEC} --distro centos-7-x86_64.yaml --multispec-selector version=$${version} --output $${version}/Dockerfile; \
-		${DG_EXEC} --distro rhel-7-x86_64.yaml --multispec-selector version=$${version} --output $${version}/Dockerfile.rhel7; \
-	done
+distgen:
+	./gen-dirs.sh

gen-dirs.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+set -e
+cd templates
+FILES=$(find -follow)
+
+pre_gen_cleanup() {
+    rm -rf ../latest
+    for version in ${VERSIONS}; do
+        rm -rf ../${version}
+    done
+}
+
+# postgresql specific
+pg_generate_symlinks() {
+    echo "Creating symlinks:"
+    for version in ${VERSIONS}; do
+        cd ../${version}
+        echo "    ${version}/README.md -> ${version}/root/usr/share/container-scripts/postgresql/README.md"
+        ln -s root/usr/share/container-scripts/postgresql/README.md README.md
+        echo "    ${version}/test test"
+        ln -s ../test test
+    done
+    cd ..
+    echo "Moving 9.6 version to latest"
+    if [ -d "./latest" ]; then
+        echo "Failed to copy directory: ./latest already exists"
+        exit 1
+    fi
+    mv 9.6 ./latest
+    echo "Creating symlink 9.6->latest"
+    ln -s latest 9.6
+}
+
+generate_file() {
+    if [ -L $1 ]; then
+        echo "Skipping symbolic link: $1"
+        return
+    elif [[ -d $1 && ! -d ../${version}/$1 ]]; then
+        mkdir ../${version}/$1
+    else
+        echo "Generating: ${version}/$1"
+        /usr/bin/dg --max-passes 1 --multispec ../specs/multispec.yml \
+        --template $1 --distro centos-7-x86_64.yaml \
+        --multispec-selector version=${version} --output ../${version}/$1
+    fi
+    if [ -x $1 ]; then
+        chmod +x ../${version}/$1
+    fi
+}
+
+generate_dockerfiles() {
+    mkdir ../$1
+    echo "Generating Dockerfile: $1/Dockerfile"
+    /usr/bin/dg --max-passes 25 --template ${PWD}/../Dockerfile.template \
+        --multispec ../specs/multispec.yml --distro centos-7-x86_64.yaml \
+        --multispec-selector version=$1 --output ../$1/Dockerfile
+    echo "Generating Dockerfile: $1/Dockerfile.rhel7"
+    /usr/bin/dg --max-passes 25 --template ${PWD}/../Dockerfile.template \
+        --multispec ../specs/multispec.yml --distro rhel-7-x86_64.yaml \
+        --multispec-selector version=$1 --output ../$1/Dockerfile.rhel7
+
+}
+
+pre_gen_cleanup
+
+for version in ${VERSIONS}; do
+    generate_dockerfiles $version
+
+    # iterate all directories and files in templates except of "."
+    for file in ${FILES[@]:1}; do
+        generate_file $file
+    done
+done
+
+pg_generate_symlinks

specs/multispec.yml

@@ -42,6 +42,9 @@ specs:
       enabled_collection: "rh-postgresql94"
       openshift_tags: "database,postgresql,postgresql94,rh-postgresql94"
       redhat_component: "rh-postgresql94-docker"
+      common_image_name: "{{ spec.org }}/postgresql-94-{{ spec.prod }}"
+      rhel_image_name: "rhscl/postgresql-94-rhel7"
+      centos_image_name: "centos/postgresql-94-centos7"
       stable: True
 
     "9.5":
@@ -52,6 +55,9 @@ specs:
       enabled_collection: "rh-postgresql95"
       openshift_tags: "database,postgresql,postgresql95,rh-postgresql95"
       redhat_component: "rh-postgresql95-docker"
+      common_image_name: "{{ spec.org }}/postgresql-95-{{ spec.prod }}"
+      rhel_image_name: "rhscl/postgresql-95-rhel7"
+      centos_image_name: "centos/postgresql-95-centos7"
       stable: True
 
     "9.6":
@@ -62,4 +68,7 @@ specs:
       enabled_collection: "rh-postgresql96"
       openshift_tags: "database,postgresql,postgresql96,rh-postgresql96"
       redhat_component: "rh-postgresql96-docker"
+      common_image_name: "{{ spec.org }}/postgresql-96-{{ spec.prod }}"
+      rhel_image_name: "rhscl/postgresql-96-rhel7"
+      centos_image_name: "centos/postgresql-96-centos7"
       stable: False

templates/Dockerfile.rhel7

@@ -1 +1 @@
-job-id: postgresql-96-centos7
+job-id: postgresql-{{ spec.short }}-{{ spec.prod }}

templates/cccp.yml

@@ -1,12 +1,12 @@
-PostgreSQL 9.6 SQL Database Server Docker image
+PostgreSQL {{ spec.version }} SQL Database Server Docker image
 ===============================================
 
-This container image includes PostgreSQL 9.6 SQL database server for OpenShift and general usage.
+This container image includes PostgreSQL {{ spec.version }} SQL database server for OpenShift and general usage.
 Users can choose between RHEL and CentOS based images.
-The RHEL image is available in the [Red Hat Container Catalog](https://access.redhat.com/containers/#/registry.access.redhat.com/rhscl/postgresql-96-rhel7)
-as registry.access.redhat.com/rhscl/postgresql-96-rhel7.
-The CentOS image is then available on [Docker Hub](https://hub.docker.com/r/centos/postgresql-96-centos7/)
-as centos/postgresql-96-centos7.
+The RHEL image is available in the [Red Hat Container Catalog](https://access.redhat.com/containers/#/registry.access.redhat.com/{{ spec.rhel_image_name }})
+as registry.access.redhat.com/{{ spec.rhel_image_name }}.
+The CentOS image is then available on [Docker Hub](https://hub.docker.com/r/{{ spec.centos_image_name }}/)
+as {{ spec.centos_image_name }}.
 
 
 Description
@@ -22,12 +22,12 @@ You can find more information on the PostgreSQL project from the project Web sit
 Usage
 -----
 
-For this, we will assume that you are using the `rhscl/postgresql-96-rhel7` image.
+For this, we will assume that you are using the `{{ spec.rhel_image_name }}` image.
 If you want to set only the mandatory environment variables and not store the database
 in a host directory, execute the following command:
 
 ```
-$ docker run -d --name postgresql_database -e POSTGRESQL_USER=user -e POSTGRESQL_PASSWORD=pass -e POSTGRESQL_DATABASE=db -p 5432:5432 rhscl/postgresql-96-rhel7
+$ docker run -d --name postgresql_database -e POSTGRESQL_USER=user -e POSTGRESQL_PASSWORD=pass -e POSTGRESQL_DATABASE=db -p 5432:5432 {{ spec.rhel_image_name }}
 ```
 
 This will create a container named `postgresql_database` running PostgreSQL with
@@ -37,9 +37,9 @@ executions, also add a `-v /host/db/path:/var/lib/pgsql/data` argument (see
 below). This will be the PostgreSQL database cluster directory.
 
 If the database cluster directory is not initialized, the entrypoint script will
-first run [`initdb`](http://www.postgresql.org/docs/9.6/static/app-initdb.html)
+first run [`initdb`](http://www.postgresql.org/docs/{{ spec.version }}/static/app-initdb.html)
 and setup necessary database users and passwords. After the database is initialized,
-or if it was already present, [`postgres`](http://www.postgresql.org/docs/9.6/static/app-postgres.html)
+or if it was already present, [`postgres`](http://www.postgresql.org/docs/{{ spec.version }}/static/app-postgres.html)
 is executed and will run as PID 1. You can stop the detached container by running
 `docker stop postgresql_database`.
 
@@ -205,7 +205,7 @@ ensure that you've carefully backed up all your data and that you are OK with
 potential manual rollback! **
 
 This image supports automatic upgrade of data directory created by
-the PostgreSQL server version 9.5 (and _only_ this version) - provided by sclorg
+the PostgreSQL server version {{ spec.prev_version }} (and _only_ this version) - provided by sclorg
 image.  The upgrade process is designed so that you should be able to just
 switch from *image A* to *image B*, and set the `$POSTGRESQL_UPGRADE` variable
 appropriately to explicitly request the database data transformation.

templates/root/usr/share/container-scripts/postgresql/README.md

@@ -70,8 +70,10 @@ function check_env_vars() {
     [[ "$POSTGRESQL_USER"     =~ $psql_identifier_regex ]] || usage
     [[ "$POSTGRESQL_PASSWORD" =~ $psql_password_regex   ]] || usage
     [[ "$POSTGRESQL_DATABASE" =~ $psql_identifier_regex ]] || usage
+{%- raw %}
     [ ${#POSTGRESQL_USER}     -le 63 ] || usage "PostgreSQL username too long (maximum 63 characters)"
     [ ${#POSTGRESQL_DATABASE} -le 63 ] || usage "Database name too long (maximum 63 characters)"
+{%- endraw %}
     postinitdb_actions+=",simple_db"
   fi
 
@@ -148,7 +150,7 @@ initdb_wrapper ()
 {
   # Initialize the database cluster with utf8 support enabled by default.
   # This might affect performance, see:
-  # http://www.postgresql.org/docs/9.6/static/locale.html
+  # http://www.postgresql.org/docs/{{ spec.version }}/static/locale.html
   LANG=${LANG:-en_US.utf8} "$@"
 }