Sync CI configuration into 1.5.x branch

.github/workflows/ci_dockers.yml

@@ -1,4 +1,4 @@
-name: CI build Docker
+name: Docker
 # https://www.docker.com/blog/first-docker-github-action-is-here
 # https://github.com/docker/build-push-action
 # see: https://help.github.com/en/actions/reference/events-that-trigger-workflows
@@ -23,9 +23,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # should be the config used in '.github/workflows/release-docker.yml', but we just keep one to check.
-        python_version: ["3.9"]
-        pytorch_version: ["1.9"]
+        # the config used in '.azure-pipelines/gpu-tests.yml' since the Dockerfile uses the cuda image
+        python_version: ["3.7"]
+        pytorch_version: ["1.8"]
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -93,7 +93,7 @@ jobs:
       matrix:
         # the config used in '.github/workflows/ci_test-conda.yml'
         python_version: ["3.8"]
-        pytorch_version: ["1.6", "1.7", "1.8", "1.9", "1.10"]
+        pytorch_version: ["1.7", "1.8", "1.9", "1.10"]
     steps:
       - name: Checkout
         uses: actions/checkout@v2

.github/workflows/ci_pkg-install.yml

@@ -1,4 +1,4 @@
-name: Install pkg
+name: Package
 
 # see: https://help.github.com/en/actions/reference/events-that-trigger-workflows
 on:  # Trigger the workflow on push or pull request, but only for the master branch
@@ -9,7 +9,7 @@ on:  # Trigger the workflow on push or pull request, but only for the master bra
 
 jobs:
 
-  pkg-install:
+  install:
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false

.github/workflows/ci_schema.yml

@@ -1,11 +1,11 @@
-name: CI action schema
+name: Schema
 on: # Trigger the workflow on push or pull request, but only for the master branch
   push: {}
   pull_request:
     branches: [master, "release/*"]
 
 jobs:
-  validate-schema:
+  check:
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout

.github/workflows/ci_test-base.yml

@@ -1,6 +1,6 @@
 # this jobs runs `pytest` over the source directory. It does not install any extra dependencies.
 # this is useful to catch errors where an import has been added which is not part of the basic dependencies.
-name: CI basic testing
+name: Test
 
 # see: https://help.github.com/en/actions/reference/events-that-trigger-workflows
 on:  # Trigger the workflow on push or pull request, but only for the master branch
@@ -10,8 +10,7 @@ on:  # Trigger the workflow on push or pull request, but only for the master bra
     branches: [master, "release/*"]
 
 jobs:
-  doctest:
-
+  source:
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false

.github/workflows/ci_test-conda.yml

@@ -1,4 +1,4 @@
-name: PyTorch & Conda
+name: Test
 
 # see: https://help.github.com/en/actions/reference/events-that-trigger-workflows
 on:  # Trigger the workflow on push or pull request, but only for the master branch
@@ -15,7 +15,7 @@ jobs:
       fail-fast: false
       matrix:
         python-version: ["3.8"]  # previous to last Python version as that one is already used in test-full
-        pytorch-version: ["1.6", "1.7", "1.8", "1.9", "1.10"]
+        pytorch-version: ["1.7", "1.8", "1.9", "1.10"]  # nightly: add when there's a release candidate
 
     timeout-minutes: 35
     steps:

.github/workflows/ci_test-full.yml

@@ -1,4 +1,4 @@
-name: CI complete testing
+name: Test
 
 # see: https://help.github.com/en/actions/reference/events-that-trigger-workflows
 on:  # Trigger the workflow on push or pull request, but only for the master branch
@@ -10,7 +10,7 @@ on:  # Trigger the workflow on push or pull request, but only for the master bra
 
 jobs:
 
-  pytest:
+  cpu:
 
     runs-on: ${{ matrix.os }}
     if: github.event.pull_request.draft == false
@@ -28,11 +28,6 @@ jobs:
           - {os: macOS-10.15, python-version: "3.6", requires: "oldest", release: "stable"}
           # nightly: add when there's a release candidate
           #- {os: ubuntu-20.04, python-version: "3.10", requires: "latest", release: "pre"}
-        exclude:
-          # PyTorch 1.6 is not available with Python 3.9: https://github.com/pytorch/pytorch/issues/46205
-          - {os: ubuntu-18.04, python-version: "3.9", requires: "oldest", release: "stable"}
-          - {os: windows-2019, python-version: "3.9", requires: "oldest", release: "stable"}
-          - {os: macOS-10.15, python-version: "3.9", requires: "oldest", release: "stable"}
 
     timeout-minutes: 40
 

.github/workflows/code-checks.yml

@@ -1,4 +1,4 @@
-name: "Check code"
+name: Test
 
 on:  # Trigger the workflow on push or pull request, but only for the master branch
   push:
@@ -7,8 +7,7 @@ on:  # Trigger the workflow on push or pull request, but only for the master bra
     branches: [master, "release/*"]
 
 jobs:
-  python-typing-mypy:
-    name: Python typing Mypy
+  mypy:
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@master

.github/workflows/docs-checks.yml

@@ -1,4 +1,4 @@
-name: "Docs check"
+name: Test
 # https://github.com/marketplace/actions/sphinx-build
 
 on:  # Trigger the workflow on push or pull request, but only for the master branch
@@ -8,7 +8,7 @@ on:  # Trigger the workflow on push or pull request, but only for the master bra
     branches: [master, "release/*"]
 
 jobs:
-  test-docs:
+  doctest:
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v2

.github/workflows/events-nightly.yml

@@ -1,4 +1,4 @@
-name: Nightly events
+name: Nightly
 
 # https://jasonet.co/posts/scheduled-actions/
 # https://github.community/t/distinct-job-for-each-schedule/17811/2
@@ -123,7 +123,7 @@ jobs:
       matrix:
         # the config used in '.github/workflows/ci_test-conda.yml'
         python_version: ["3.8"]
-        pytorch_version: ["1.6", "1.7", "1.8", "1.9", "1.10"]
+        pytorch_version: ["1.7", "1.8", "1.9", "1.10"]
 
     steps:
       - name: Checkout

.github/workflows/events-recurrent.yml

@@ -1,4 +1,4 @@
-name: Recurrent events
+name: Recurrent
 
 # https://jasonet.co/posts/scheduled-actions/
 # https://github.community/t/distinct-job-for-each-schedule/17811/2

.github/workflows/probot-auto-cc.yml

@@ -0,0 +1,16 @@
+name: Probot
+
+on:
+  issues:
+    types: [labeled]
+  pull_request:
+    types: [labeled, ready_for_review]
+
+jobs:
+  auto-cc:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'issue' || github.event.pull_request.draft == false
+    steps:
+      - uses: carmocca/probot@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/release-docker.yml

@@ -1,4 +1,4 @@
-name: Publish Docker Releases
+name: Docker
 # https://www.docker.com/blog/first-docker-github-action-is-here
 # https://github.com/docker/build-push-action
 on:
@@ -8,15 +8,15 @@ on:
     types: [published]
 
 jobs:
-  cuda-PL:
+  publish:
     runs-on: ubuntu-20.04
     # only on releases
     if: startsWith(github.ref, 'refs/tags/') || github.event_name == 'release'
     strategy:
       fail-fast: false
       matrix:
         python_version: ["3.6", "3.7", "3.8", "3.9"]
-        pytorch_version: ["1.6", "1.7", "1.8", "1.9"]
+        pytorch_version: ["1.7", "1.8", "1.9", "1.10"]
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -39,7 +39,7 @@ jobs:
       - name: Publish Latest to Docker
         uses: docker/[email protected]
         # only on releases and latest Python and PyTorch
-        if: matrix.python_version == 3.9 && matrix.pytorch_version == 1.9
+        if: matrix.python_version == "3.9" && matrix.pytorch_version == "1.10"
         with:
           repository: pytorchlightning/pytorch_lightning
           username: ${{ secrets.DOCKER_USERNAME }}

.github/workflows/release-pypi.yml

@@ -1,4 +1,4 @@
-name: PyPI Release
+name: PyPI
 
 # https://help.github.com/en/actions/reference/events-that-trigger-workflows
 on:  # Trigger the workflow on push or pull request, but only for the master branch

dockers/base-cuda/Dockerfile

@@ -17,7 +17,7 @@ ARG CUDA_VERSION=10.2
 FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu18.04
 
 ARG PYTHON_VERSION=3.9
-ARG PYTORCH_VERSION=1.6
+ARG PYTORCH_VERSION=1.8
 
 SHELL ["/bin/bash", "-c"]
 # https://techoverflow.net/2019/05/18/how-to-fix-configuring-tzdata-interactive-input-when-building-docker-images/

dockers/base-xla/Dockerfile

@@ -19,7 +19,7 @@ LABEL maintainer="PyTorchLightning <https://github.com/PyTorchLightning>"
 # CALL: docker image build -t pytorch-lightning:XLA-extras-py3.6 -f dockers/base-xla/Dockerfile . --build-arg PYTHON_VERSION=3.8
 ARG PYTHON_VERSION=3.9
 ARG CONDA_VERSION=4.9.2
-ARG XLA_VERSION=1.6
+ARG XLA_VERSION=1.8
 
 SHELL ["/bin/bash", "-c"]
 # for skipping configurations

dockers/release/Dockerfile

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 ARG PYTHON_VERSION=3.9
-ARG PYTORCH_VERSION=1.6
+ARG PYTORCH_VERSION=1.8
 
 FROM pytorchlightning/pytorch_lightning:base-cuda-py${PYTHON_VERSION}-torch${PYTORCH_VERSION}
 

dockers/tpu-tests/Dockerfile

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 ARG PYTHON_VERSION=3.9
-ARG PYTORCH_VERSION=1.6
+ARG PYTORCH_VERSION=1.8
 
 FROM pytorchlightning/pytorch_lightning:base-xla-py${PYTHON_VERSION}-torch${PYTORCH_VERSION}
 

environment.yml

@@ -29,7 +29,7 @@ dependencies:
     - python>=3.6
     - pip>20.1
     - numpy>=1.17.2
-    - pytorch>=1.6
+    - pytorch>=1.7.*
     - future>=0.17.1
     - PyYAML>=5.1
     - tqdm>=4.41.0
@@ -41,9 +41,10 @@ dependencies:
     - scikit-learn>=0.20.0
     - matplotlib>=3.1.1
     - omegaconf>=2.0.5
+    - torchtext>=0.8.*
 
     # Examples
-    - torchvision>=0.6
+    - torchvision>=0.8.*
 
     - pip:
         - test-tube>=0.7.5

requirements.txt

@@ -1,7 +1,7 @@
 # the default package dependencies
 
 numpy>=1.17.2
-torch>=1.6
+torch>=1.7.*
 future>=0.17.1  # required for builtins in setup.py
 tqdm>=4.41.0
 PyYAML>=5.1

requirements/adjust_versions.py

@@ -14,7 +14,6 @@
     dict(torch="1.8.0", torchvision="0.9.0", torchtext="0.9.0"),
     dict(torch="1.7.1", torchvision="0.8.2", torchtext="0.8.1"),
     dict(torch="1.7.0", torchvision="0.8.1", torchtext="0.8.0"),
-    dict(torch="1.6.0", torchvision="0.7.0", torchtext="0.7"),
 ]
 
 
@@ -33,28 +32,59 @@ def find_latest(ver: str) -> Dict[str, str]:
     raise ValueError(f"Missing {ver} in {VERSIONS}")
 
 
-def main(path_req: str, torch_version: Optional[str] = None) -> None:
+def main(req: str, torch_version: Optional[str] = None) -> str:
     if not torch_version:
         import torch
 
         torch_version = torch.__version__
     assert torch_version, f"invalid torch: {torch_version}"
 
-    with open(path_req) as fp:
-        req = fp.read()
-    # remove comments
-    req = re.sub(rf"\s*#.*{os.linesep}", os.linesep, req)
+    # remove comments and strip whitespace
+    req = re.sub(rf"\s*#.*{os.linesep}", os.linesep, req).strip()
 
     latest = find_latest(torch_version)
     for lib, version in latest.items():
-        replace = f"{lib}=={version}" if version else lib
-        replace += os.linesep
-        req = re.sub(rf"{lib}[>=]*[\d\.]*{os.linesep}", replace, req)
+        replace = f"{lib}=={version}" if version else ""
+        req = re.sub(rf"\b{lib}(?!\w).*", replace, req)
 
-    print(req)  # on purpose - to debug
-    with open(path_req, "w") as fp:
-        fp.write(req)
+    return req
+
+
+def test():
+    requirements = """
+    torch>=1.2.*
+    torch==1.2.3
+    torch==1.4
+    torch
+    future>=0.17.1
+    pytorch==1.5.6+123dev0
+    torchvision
+    torchmetrics>=0.4.1
+    """
+    expected = """
+    torch==1.9.1
+    torch==1.9.1
+    torch==1.9.1
+    torch==1.9.1
+    future>=0.17.1
+    pytorch==1.5.6+123dev0
+    torchvision==0.10.1
+    torchmetrics>=0.4.1
+    """.strip()
+    actual = main(requirements, "1.9")
+    assert actual == expected, (actual, expected)
 
 
 if __name__ == "__main__":
-    main(*sys.argv[1:])
+    test()  # sanity check
+
+    if len(sys.argv) == 3:
+        requirements_path, torch_version = sys.argv[1:]
+    else:
+        requirements_path, torch_version = sys.argv[1], None
+
+    with open(requirements_path, "r+") as fp:
+        requirements = fp.read()
+        requirements = main(requirements, torch_version)
+        print(requirements)  # on purpose - to debug
+        fp.write(requirements)

requirements/examples.txt

@@ -1,3 +1,3 @@
-torchvision>=0.7
+torchvision>=0.8.*
 gym>=0.17.0
 ipython[all]

requirements/extra.txt

@@ -2,9 +2,9 @@
 
 matplotlib>3.1
 horovod>=0.21.2  # no need to install with [pytorch] as pytorch is already installed
-torchtext>=0.7
+torchtext>=0.8.*
 omegaconf>=2.0.5
 hydra-core>=1.0.5
-jsonargparse[signatures]>=3.19.3
+jsonargparse[signatures]>=4.0.0
 gcsfs>=2021.5.0
 rich>=10.2.2