Minimize the number of docker jobs (#10202)

Co-authored-by: Jirka Borovec <[email protected]>

Author: carmocca

.github/workflows/ci_dockers.yml

@@ -23,8 +23,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python_version: ["3.8"]
-        pytorch_version: ["1.6", "1.8"]
+        # should be the config used in '.github/workflows/release-docker.yml', but we just keep one to check.
+        python_version: ["3.9"]
+        pytorch_version: ["1.9"]
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -45,8 +46,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
+        # the config used in '.circleci/config.yml`'
         python_version: ["3.7"]
-        xla_version: ["1.6", "1.8", "1.10"]
+        xla_version: ["1.8"]
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -67,24 +69,19 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        include:
-          # todo: see notes in Dockerfile
-          - python_version: "3.7"
-            pytorch_version: "1.6"
-          - python_version: "3.9"
-            pytorch_version: "1.9"
+        # the config used in '.azure-pipelines/gpu-tests.yml'
+        python_version: ["3.7"]
+        pytorch_version: ["1.8"]
     steps:
       - name: Checkout
         uses: actions/checkout@v2
-
       - name: Build CUDA Docker
         # publish master/release
         uses: docker/build-push-action@v2
         with:
           build-args: |
             PYTHON_VERSION=${{ matrix.python_version }}
             PYTORCH_VERSION=${{ matrix.pytorch_version }}
-            CUDA_VERSION=10.2
           file: dockers/base-cuda/Dockerfile
           push: false
         timeout-minutes: 75
@@ -94,24 +91,17 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - python_version: "3.7"
-            pytorch_version: "1.6"
-          - python_version: "3.8"
-            pytorch_version: "1.9"
-          - python_version: "3.9"
-            pytorch_version: "1.10"
+        # the config used in '.github/workflows/ci_test-conda.yml'
+        python_version: ["3.7"]
+        pytorch_version: ["1.6", "1.7", "1.8", "1.9", "1.10"]
     steps:
       - name: Checkout
         uses: actions/checkout@v2
-
-      # see: https://pytorch.org/get-started/previous-versions/
       - run: |
           cuda=$(python -c "from distutils.version import LooseVersion as LVer ; print(11.1 if LVer('${{matrix.pytorch_version}}') > LVer('1.7') else 10.2)" 2>&1)
           echo "::set-output name=CUDA::$cuda"
         id: extend
-
-      - name: Build CUDA Docker
+      - name: Build Conda Docker
         # publish master/release
         uses: docker/build-push-action@v2
         with:
@@ -128,9 +118,11 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - python_version: "3.8"
-            pytorch_version: "1.7"
+        # the config used in 'dockers/ipu-ci-runner/Dockerfile'
+        python_version: ["3.8"]  # latest
+        # TODO: upgrade - PopTorch 2.2 uses torch 1.9, see:
+        # https://docs.graphcore.ai/projects/poptorch-user-guide/en/latest/installation.html#version-compatibility
+        pytorch_version: ["1.7"]
     steps:
       - name: Checkout
         uses: actions/checkout@v2

.github/workflows/ci_test-conda.yml

@@ -14,7 +14,6 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # os: [ubuntu-20.04]
         python-version: ["3.7"]
         pytorch-version: ["1.6", "1.7", "1.8", "1.9", "1.10"]
 

.github/workflows/events-nightly.yml

@@ -53,8 +53,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
+        # the config used in '.circleci/config.yml`'
         python_version: ["3.7"]
-        xla_version: ["1.6", "1.7", "1.8", "1.9"]
+        xla_version: ["1.8"]
 
     steps:
       - name: Checkout
@@ -87,11 +88,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python_version: ["3.6", "3.7", "3.8", "3.9"]
-        pytorch_version: ["1.6", "1.7", "1.8", "1.9"]
-        exclude:
-          - python_version: "3.9"
-            pytorch_version: "1.6"
+        # the config used in '.azure-pipelines/gpu-tests.yml'
+        python_version: ["3.7"]
+        pytorch_version: ["1.8"]
 
     steps:
       - name: Checkout
@@ -122,11 +121,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python_version: ["3.6", "3.7", "3.8", "3.9"]
+        # the config used in '.github/workflows/ci_test-conda.yml'
+        python_version: ["3.7"]
         pytorch_version: ["1.6", "1.7", "1.8", "1.9", "1.10"]
-        exclude:
-          - python_version: "3.9"
-            pytorch_version: "1.6"
 
     steps:
       - name: Checkout
@@ -164,6 +161,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
+        # the config used in 'dockers/ipu-ci-runner/Dockerfile'
         include:
           - python_version: "3.8"
             pytorch_version: "1.7"

dockers/base-conda/Dockerfile

@@ -129,4 +129,4 @@ RUN \
     conda info && \
     pip list && \
     python -c "import sys; ver = sys.version_info ; assert f'{ver.major}.{ver.minor}' == '$PYTHON_VERSION', ver" && \
-    python -c "from torch import __version__ as ver; assert '.'.join(ver.split('.')[:2]) == '$PYTORCH_VERSION', ver"
+    python -c "import torch; assert torch.__version__.startswith('$PYTORCH_VERSION'), torch.__version__"

dockers/base-cuda/Dockerfile

@@ -119,4 +119,4 @@ RUN \
     pip --version && \
     pip list && \
     python -c "import sys; ver = sys.version_info ; assert f'{ver.major}.{ver.minor}' == '$PYTHON_VERSION', ver" && \
-    python -c "import torch; assert torch.__version__[:3] == '$PYTORCH_VERSION', torch.__version__"
+    python -c "import torch; assert torch.__version__.startswith('$PYTORCH_VERSION'), torch.__version__"

dockers/base-ipu/Dockerfile

@@ -95,4 +95,4 @@ RUN \
     conda info && \
     pip list && \
     python -c "import sys; assert sys.version[:3] == '$PYTHON_VERSION', sys.version" && \
-    python -c "import torch; assert torch.__version__[:3] == '$PYTORCH_VERSION', torch.__version__"
+    python -c "import torch; assert torch.__version__.startswith('$PYTORCH_VERSION'), torch.__version__"

dockers/base-xla/Dockerfile

@@ -109,4 +109,4 @@ RUN \
     conda info && \
     pip list && \
     python -c "import sys; assert sys.version[:3] == '$PYTHON_VERSION', sys.version" && \
-    python -c "import torch; ver = '$XLA_VERSION' ; assert torch.__version__.startswith(ver), torch.__version__"
+    python -c "import torch; assert torch.__version__.startswith('$XLA_VERSION'), torch.__version__"

dockers/nvidia/Dockerfile

@@ -61,7 +61,7 @@ RUN \
     python --version && \
     pip --version && \
     pip list | grep torch && \
-    python -c "from torch import __version__ as ver ; assert ver == '$TORCH_VERSION', ver" && \
+    python -c "import torch; assert torch.__version__.startswith('$TORCH_VERSION'), torch.__version__" && \
     python -c "import pytorch_lightning as pl; print(pl.__version__)"
 
 CMD ["jupyter", "notebook", "--port=8888", "--no-browser", "--ip=0.0.0.0", "--allow-root"]