From 886f1b7009c493279960deef04cf18d6e853bc6d Mon Sep 17 00:00:00 2001
From: Akihiro Nitta <nitta@akihironitta.com>
Date: Wed, 22 Dec 2021 11:42:37 +0900
Subject: [PATCH 01/65] [skip azpipelines]

---
 .github/workflows/ci_test-full.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index 2a17c07b05666..e79ac71364edf 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -133,7 +133,7 @@ jobs:
     - name: Tests
       run: |
         # NOTE: do not include coverage report here, see: https://github.com/nedbat/coveragepy/issues/1003
-        coverage run --source pytorch_lightning -m pytest pytorch_lightning tests -v --durations=50 --junitxml=junit/test-results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml
+        pytest pytorch_lightning tests -v --durations=50 --junitxml=junit/test-results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml
 
     - name: Examples
       run: |

From a3764c3981b294faf400f46fb57d24a25a52811f Mon Sep 17 00:00:00 2001
From: Akihiro Nitta <nitta@akihironitta.com>
Date: Thu, 23 Dec 2021 01:34:37 +0900
Subject: [PATCH 02/65] Disable coverage

---
 .github/workflows/ci_test-full.yml | 34 +++++++++++++++---------------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index e79ac71364edf..2e9fd3e1a3a53 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -146,20 +146,20 @@ jobs:
         path: junit/test-results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml
       if: failure()
 
-    - name: Statistics
-      if: success()
-      run: |
-        coverage report
-        coverage xml
-
-    - name: Upload coverage to Codecov
-      uses: codecov/codecov-action@v1
-      if: always()
-      # see: https://github.com/actions/toolkit/issues/399
-      continue-on-error: true
-      with:
-        token: ${{ secrets.CODECOV_TOKEN }}
-        file: coverage.xml
-        flags: cpu,pytest,python${{ matrix.python-version }}
-        name: CPU-coverage
-        fail_ci_if_error: false
+    # - name: Statistics
+    #   if: success()
+    #   run: |
+    #     coverage report
+    #     coverage xml
+
+    # - name: Upload coverage to Codecov
+    #   uses: codecov/codecov-action@v1
+    #   if: always()
+    #   # see: https://github.com/actions/toolkit/issues/399
+    #   continue-on-error: true
+    #   with:
+    #     token: ${{ secrets.CODECOV_TOKEN }}
+    #     file: coverage.xml
+    #     flags: cpu,pytest,python${{ matrix.python-version }}
+    #     name: CPU-coverage
+    #     fail_ci_if_error: false

From 99ab83ca2d9da7a966cbf768ac9466a08e57e1ea Mon Sep 17 00:00:00 2001
From: Akihiro Nitta <nitta@akihironitta.com>
Date: Sun, 26 Dec 2021 11:33:25 +0900
Subject: [PATCH 03/65] python -X faulthander

---
 .github/workflows/ci_test-full.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index 2e9fd3e1a3a53..645a06b8d4c23 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -133,7 +133,7 @@ jobs:
     - name: Tests
       run: |
         # NOTE: do not include coverage report here, see: https://github.com/nedbat/coveragepy/issues/1003
-        pytest pytorch_lightning tests -v --durations=50 --junitxml=junit/test-results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml
+        python -X faulthander -m pytest pytorch_lightning tests -v --durations=50 --junitxml=junit/test-results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml
 
     - name: Examples
       run: |

From 1c99eeba85c193977c2df037b198f3221561ead8 Mon Sep 17 00:00:00 2001
From: Akihiro Nitta <nitta@akihironitta.com>
Date: Sun, 26 Dec 2021 11:50:57 +0900
Subject: [PATCH 04/65] pytest -p no:faulthandler

---
 .github/workflows/ci_test-full.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index 645a06b8d4c23..71c4950ec90ea 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -133,7 +133,7 @@ jobs:
     - name: Tests
       run: |
         # NOTE: do not include coverage report here, see: https://github.com/nedbat/coveragepy/issues/1003
-        python -X faulthander -m pytest pytorch_lightning tests -v --durations=50 --junitxml=junit/test-results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml
+        python -X faulthander -m pytest pytorch_lightning tests -v --durations=50 --junitxml=junit/test-results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml -p no:faulthandler
 
     - name: Examples
       run: |

From 65fd952df84cf8dd510ba880892234e9cf6227b8 Mon Sep 17 00:00:00 2001
From: Akihiro Nitta <nitta@akihironitta.com>
Date: Tue, 28 Dec 2021 05:51:25 +0900
Subject: [PATCH 05/65] revert python's faulthandler

---
 .github/workflows/ci_test-full.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index 71c4950ec90ea..2e9fd3e1a3a53 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -133,7 +133,7 @@ jobs:
     - name: Tests
       run: |
         # NOTE: do not include coverage report here, see: https://github.com/nedbat/coveragepy/issues/1003
-        python -X faulthander -m pytest pytorch_lightning tests -v --durations=50 --junitxml=junit/test-results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml -p no:faulthandler
+        pytest pytorch_lightning tests -v --durations=50 --junitxml=junit/test-results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml
 
     - name: Examples
       run: |

From 7fe8c9694ba96f3b48fa18b7d2b5efe2bda02d25 Mon Sep 17 00:00:00 2001
From: Akihiro Nitta <nitta@akihironitta.com>
Date: Tue, 28 Dec 2021 06:23:31 +0900
Subject: [PATCH 06/65] gdb

---
 .github/workflows/ci_test-full.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index 2e9fd3e1a3a53..15d92fe6594cd 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -133,7 +133,8 @@ jobs:
     - name: Tests
       run: |
         # NOTE: do not include coverage report here, see: https://github.com/nedbat/coveragepy/issues/1003
-        pytest pytorch_lightning tests -v --durations=50 --junitxml=junit/test-results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml
+        # pytest pytorch_lightning tests -v --durations=50 --junitxml=junit/test-results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml
+        gdb --batch -q -ex r -ex "thread apply all bt" --args "$(readlink -f $(which python))" -m pytest pytorch_lightning tests -v --durations=50 --junitxml=junit/test-results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml -p no:faulthandler
 
     - name: Examples
       run: |

From 46f9ae0c5e6349417392b9be5e4c6afde53cce58 Mon Sep 17 00:00:00 2001
From: Akihiro Nitta <nitta@akihironitta.com>
Date: Tue, 28 Dec 2021 06:39:06 +0900
Subject: [PATCH 07/65] install gdb

---
 .github/workflows/ci_test-full.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index 15d92fe6594cd..0aa7d39f03eb9 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -50,12 +50,18 @@ jobs:
       if: runner.os == 'macOS'
       run: |
         brew install openmpi libuv  # Horovod on macOS requires OpenMPI, Gloo not currently supported
+        brew install gdb
 
     - name: Setup Windows
       if: runner.os == 'windows'
       run: |
         python .github/prune-packages.py requirements/extra.txt "horovod"
 
+    - name: Setup Ubuntu
+      if: runner.os == 'Linux'
+      run: |
+        sudo apt install gdb
+
     - name: Set min. dependencies
       if: matrix.requires == 'oldest'
       run: |

From 871ae6bfb52f62f3e64d96a365aa5da43208f9d7 Mon Sep 17 00:00:00 2001
From: Akihiro Nitta <nitta@akihironitta.com>
Date: Tue, 28 Dec 2021 07:17:10 +0900
Subject: [PATCH 08/65] pytest --color=no for visibility in raw log

---
 setup.cfg | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.cfg b/setup.cfg
index ca6c0dc17b7e6..77d7fcbd18924 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -22,7 +22,7 @@ norecursedirs =
 addopts =
     --strict-markers
     --doctest-modules
-    --color=yes
+    --color=no
     --disable-pytest-warnings
 filterwarnings =
     # error out on our deprecation warnings - ensures the code and tests are kept up-to-date

From 24fb0841666e50150a0d6d6aae200c8728c217ec Mon Sep 17 00:00:00 2001
From: Akihiro Nitta <nitta@akihironitta.com>
Date: Tue, 28 Dec 2021 07:56:37 +0900
Subject: [PATCH 09/65] suppress pytest output and return its status

---
 .github/workflows/ci_test-full.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index 0aa7d39f03eb9..7debede2ea783 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -140,7 +140,7 @@ jobs:
       run: |
         # NOTE: do not include coverage report here, see: https://github.com/nedbat/coveragepy/issues/1003
         # pytest pytorch_lightning tests -v --durations=50 --junitxml=junit/test-results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml
-        gdb --batch -q -ex r -ex "thread apply all bt" --args "$(readlink -f $(which python))" -m pytest pytorch_lightning tests -v --durations=50 --junitxml=junit/test-results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml -p no:faulthandler
+        gdb --return-child-result --batch -q -ex r -ex "thread apply all bt" --args "$(readlink -f $(which python))" -m pytest pytorch_lightning tests -v --durations=50 --junitxml=junit/test-results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml -p no:faulthandler -q
 
     - name: Examples
       run: |

From a476ef2a82bb57534afdbea806f98eb1a4f14961 Mon Sep 17 00:00:00 2001
From: Akihiro Nitta <nitta@akihironitta.com>
Date: Tue, 28 Dec 2021 08:05:24 +0900
Subject: [PATCH 10/65] Simplify pytest options under gdb

---
 .github/workflows/ci_test-full.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index 7debede2ea783..25b16d3e21ce1 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -140,7 +140,7 @@ jobs:
       run: |
         # NOTE: do not include coverage report here, see: https://github.com/nedbat/coveragepy/issues/1003
         # pytest pytorch_lightning tests -v --durations=50 --junitxml=junit/test-results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml
-        gdb --return-child-result --batch -q -ex r -ex "thread apply all bt" --args "$(readlink -f $(which python))" -m pytest pytorch_lightning tests -v --durations=50 --junitxml=junit/test-results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml -p no:faulthandler -q
+        gdb --return-child-result --batch -q -ex r -ex "thread apply all bt" --args "$(readlink -f $(which python))" -m pytest pytorch_lightning tests -p no:faulthandler -q
 
     - name: Examples
       run: |

From b88c25779d5042daa9e031b7057308f91be4215b Mon Sep 17 00:00:00 2001
From: Akihiro Nitta <nitta@akihironitta.com>
Date: Tue, 28 Dec 2021 08:05:46 +0900
Subject: [PATCH 11/65] [azp skip]


From 1317ba0c647bad3bf76b4991d557842ed8f3a2ad Mon Sep 17 00:00:00 2001
From: Akihiro Nitta <nitta@akihironitta.com>
Date: Tue, 28 Dec 2021 08:07:54 +0900
Subject: [PATCH 12/65] Temporarily disable example tests and uploading results

---
 .github/workflows/ci_test-full.yml | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index 25b16d3e21ce1..9e56baf86fc9f 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -142,16 +142,16 @@ jobs:
         # pytest pytorch_lightning tests -v --durations=50 --junitxml=junit/test-results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml
         gdb --return-child-result --batch -q -ex r -ex "thread apply all bt" --args "$(readlink -f $(which python))" -m pytest pytorch_lightning tests -p no:faulthandler -q
 
-    - name: Examples
-      run: |
-        python -m pytest pl_examples -v --durations=10
+    # - name: Examples
+    #   run: |
+    #     python -m pytest pl_examples -v --durations=10
 
-    - name: Upload pytest results
-      uses: actions/upload-artifact@v2
-      with:
-        name: pytest-results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}
-        path: junit/test-results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml
-      if: failure()
+    # - name: Upload pytest results
+    #   uses: actions/upload-artifact@v2
+    #   with:
+    #     name: pytest-results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}
+    #     path: junit/test-results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml
+    #   if: failure()
 
     # - name: Statistics
     #   if: success()

From 10ec84445dea65ce51c64cea3b1767999e84ac52 Mon Sep 17 00:00:00 2001
From: Akihiro Nitta <nitta@akihironitta.com>
Date: Tue, 28 Dec 2021 08:08:18 +0900
Subject: [PATCH 13/65] echo after tests

---
 .github/workflows/ci_test-full.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index 9e56baf86fc9f..8a857b83a5c09 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -141,6 +141,7 @@ jobs:
         # NOTE: do not include coverage report here, see: https://github.com/nedbat/coveragepy/issues/1003
         # pytest pytorch_lightning tests -v --durations=50 --junitxml=junit/test-results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml
         gdb --return-child-result --batch -q -ex r -ex "thread apply all bt" --args "$(readlink -f $(which python))" -m pytest pytorch_lightning tests -p no:faulthandler -q
+        echo TESTINGISFINISHEDHERE
 
     # - name: Examples
     #   run: |

From baf2bf8340be70a857078296d5af2189aeaffab6 Mon Sep 17 00:00:00 2001
From: Akihiro Nitta <nitta@akihironitta.com>
Date: Tue, 28 Dec 2021 08:19:30 +0900
Subject: [PATCH 14/65] fix illegal option -f on macos

---
 .github/workflows/ci_test-full.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index 8a857b83a5c09..b7febf6a49802 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -140,7 +140,7 @@ jobs:
       run: |
         # NOTE: do not include coverage report here, see: https://github.com/nedbat/coveragepy/issues/1003
         # pytest pytorch_lightning tests -v --durations=50 --junitxml=junit/test-results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml
-        gdb --return-child-result --batch -q -ex r -ex "thread apply all bt" --args "$(readlink -f $(which python))" -m pytest pytorch_lightning tests -p no:faulthandler -q
+        gdb --return-child-result --batch -q -ex r -ex "thread apply all bt" --args $(which python) -m pytest pytorch_lightning tests -p no:faulthandler -q
         echo TESTINGISFINISHEDHERE
 
     # - name: Examples

From ca1c639db354dd530e421022a99f90e2a44ec6d5 Mon Sep 17 00:00:00 2001
From: Akihiro Nitta <nitta@akihironitta.com>
Date: Tue, 28 Dec 2021 08:25:51 +0900
Subject: [PATCH 15/65] execute command running under gdb for checking if
 working

---
 .github/workflows/ci_test-full.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index b7febf6a49802..0f107555031b1 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -140,7 +140,8 @@ jobs:
       run: |
         # NOTE: do not include coverage report here, see: https://github.com/nedbat/coveragepy/issues/1003
         # pytest pytorch_lightning tests -v --durations=50 --junitxml=junit/test-results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml
-        gdb --return-child-result --batch -q -ex r -ex "thread apply all bt" --args $(which python) -m pytest pytorch_lightning tests -p no:faulthandler -q
+        # gdb --return-child-result --batch -q -ex r -ex "thread apply all bt" --args $(which python) -m pytest pytorch_lightning tests -p no:faulthandler -q
+        $(which python) -m pytest pytorch_lightning tests -p no:faulthandler -q
         echo TESTINGISFINISHEDHERE
 
     # - name: Examples

From cc37fd06ffc37de2c40e3deb7d4ce9f587eeaec5 Mon Sep 17 00:00:00 2001
From: Akihiro Nitta <nitta@akihironitta.com>
Date: Tue, 28 Dec 2021 15:12:45 +0900
Subject: [PATCH 16/65] enable gdb again

---
 .github/workflows/ci_test-full.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index 0f107555031b1..617b27f3a0f77 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -140,8 +140,8 @@ jobs:
       run: |
         # NOTE: do not include coverage report here, see: https://github.com/nedbat/coveragepy/issues/1003
         # pytest pytorch_lightning tests -v --durations=50 --junitxml=junit/test-results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml
-        # gdb --return-child-result --batch -q -ex r -ex "thread apply all bt" --args $(which python) -m pytest pytorch_lightning tests -p no:faulthandler -q
-        $(which python) -m pytest pytorch_lightning tests -p no:faulthandler -q
+        gdb --return-child-result --batch -q -ex r -ex "thread apply all bt" --args $(which python) -m pytest pytorch_lightning tests -p no:faulthandler -q
+        # $(which python) -m pytest pytorch_lightning tests -p no:faulthandler -q
         echo TESTINGISFINISHEDHERE
 
     # - name: Examples

From 7995994d841009e8847c7d61d8aa729a9243f066 Mon Sep 17 00:00:00 2001
From: Akihiro Nitta <nitta@akihironitta.com>
Date: Tue, 28 Dec 2021 18:19:33 +0900
Subject: [PATCH 17/65] Temporarily disable win and mac

---
 .github/workflows/ci_test-full.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index 617b27f3a0f77..eb5b183ffcb37 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -17,7 +17,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-18.04, windows-2019, macOS-10.15]
+        os: [ubuntu-18.04]
         python-version: ["3.7", "3.9"]  # minimum, maximum
         requires: ["oldest", "latest"]
         release: ["stable"]

From 999d01d7221dd925d47561c90f3960df97d04181 Mon Sep 17 00:00:00 2001
From: Akihiro Nitta <nitta@akihironitta.com>
Date: Thu, 6 Jan 2022 15:13:00 +0900
Subject: [PATCH 18/65] Temporarily rm horovod installation

---
 .github/workflows/ci_test-full.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index eb5b183ffcb37..7627b4c18fad9 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -61,6 +61,7 @@ jobs:
       if: runner.os == 'Linux'
       run: |
         sudo apt install gdb
+        python .github/prune-packages.py requirements/extra.txt "horovod"
 
     - name: Set min. dependencies
       if: matrix.requires == 'oldest'

From 3c554056ecc690a57b6a5bc7a1e34245b2cc5ee2 Mon Sep 17 00:00:00 2001
From: Akihiro Nitta <nitta@akihironitta.com>
Date: Thu, 6 Jan 2022 15:20:16 +0900
Subject: [PATCH 19/65] tmp: no reinstallation of horovod

---
 .github/workflows/ci_test-full.yml | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index 7627b4c18fad9..aab332379a0ca 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -115,21 +115,21 @@ jobs:
         pip list
       shell: bash
 
-    - name: Reinstall Horovod if necessary
-      if: runner.os != 'windows'
-      env:
-        HOROVOD_BUILD_ARCH_FLAGS: "-mfma"
-        HOROVOD_WITHOUT_MXNET: 1
-        HOROVOD_WITHOUT_TENSORFLOW: 1
-      run: |
-        HOROVOD_BUILT=$(python -c "import horovod.torch; horovod.torch.nccl_built(); print('SUCCESS')" || true)
-        if [[ $HOROVOD_BUILT != "SUCCESS" ]]; then
-          pip uninstall -y horovod
-          echo $(grep "horovod" requirements/extra.txt) > requirements/horovod.txt
-          pip install --no-cache-dir -r requirements/horovod.txt
-        fi
-        horovodrun --check-build
-      shell: bash
+    # - name: Reinstall Horovod if necessary
+    #   if: runner.os != 'windows'
+    #   env:
+    #     HOROVOD_BUILD_ARCH_FLAGS: "-mfma"
+    #     HOROVOD_WITHOUT_MXNET: 1
+    #     HOROVOD_WITHOUT_TENSORFLOW: 1
+    #   run: |
+    #     HOROVOD_BUILT=$(python -c "import horovod.torch; horovod.torch.nccl_built(); print('SUCCESS')" || true)
+    #     if [[ $HOROVOD_BUILT != "SUCCESS" ]]; then
+    #       pip uninstall -y horovod
+    #       echo $(grep "horovod" requirements/extra.txt) > requirements/horovod.txt
+    #       pip install --no-cache-dir -r requirements/horovod.txt
+    #     fi
+    #     horovodrun --check-build
+    #   shell: bash
 
     - name: Cache datasets
       uses: actions/cache@v2

From 3494f1020b9122ddb78f2009bade37deb35dbc5a Mon Sep 17 00:00:00 2001
From: Akihiro Nitta <nitta@akihironitta.com>
Date: Thu, 6 Jan 2022 15:32:38 +0900
Subject: [PATCH 20/65] disable gdb for clean output

---
 .github/workflows/ci_test-full.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index aab332379a0ca..517e9cc95ef05 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -140,8 +140,8 @@ jobs:
     - name: Tests
       run: |
         # NOTE: do not include coverage report here, see: https://github.com/nedbat/coveragepy/issues/1003
-        # pytest pytorch_lightning tests -v --durations=50 --junitxml=junit/test-results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml
-        gdb --return-child-result --batch -q -ex r -ex "thread apply all bt" --args $(which python) -m pytest pytorch_lightning tests -p no:faulthandler -q
+        pytest pytorch_lightning tests -v --durations=50 --junitxml=junit/test-results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml
+        # gdb --return-child-result --batch -q -ex r -ex "thread apply all bt" --args $(which python) -m pytest pytorch_lightning tests -p no:faulthandler -q
         # $(which python) -m pytest pytorch_lightning tests -p no:faulthandler -q
         echo TESTINGISFINISHEDHERE
 

From ebe3af1946f892faa3248dcb64ca0f4546853130 Mon Sep 17 00:00:00 2001
From: Akihiro Nitta <nitta@akihironitta.com>
Date: Thu, 6 Jan 2022 15:49:14 +0900
Subject: [PATCH 21/65] Revert "disable gdb for clean output"

This reverts commit 3494f1020b9122ddb78f2009bade37deb35dbc5a.
---
 .github/workflows/ci_test-full.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index 517e9cc95ef05..aab332379a0ca 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -140,8 +140,8 @@ jobs:
     - name: Tests
       run: |
         # NOTE: do not include coverage report here, see: https://github.com/nedbat/coveragepy/issues/1003
-        pytest pytorch_lightning tests -v --durations=50 --junitxml=junit/test-results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml
-        # gdb --return-child-result --batch -q -ex r -ex "thread apply all bt" --args $(which python) -m pytest pytorch_lightning tests -p no:faulthandler -q
+        # pytest pytorch_lightning tests -v --durations=50 --junitxml=junit/test-results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml
+        gdb --return-child-result --batch -q -ex r -ex "thread apply all bt" --args $(which python) -m pytest pytorch_lightning tests -p no:faulthandler -q
         # $(which python) -m pytest pytorch_lightning tests -p no:faulthandler -q
         echo TESTINGISFINISHEDHERE
 

From e766a293c46269270827c39a6533339e7c07cb3d Mon Sep 17 00:00:00 2001
From: Akihiro Nitta <nitta@akihironitta.com>
Date: Thu, 6 Jan 2022 16:14:22 +0900
Subject: [PATCH 22/65] [skip azp] wish the latest pytest somehow works

---
 .github/workflows/ci_test-full.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index aab332379a0ca..7bab3a39154ed 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -103,6 +103,12 @@ jobs:
         pip list
       shell: bash
 
+    - name: Use latest pytest
+      run: |
+        pip install pytest -U
+        pip list
+      shell: bash
+
     - name: Install extra dependencies
       env:
         HOROVOD_BUILD_ARCH_FLAGS: "-mfma"

From d8cabe7082bfc58bf0dcc83669dd005c47559410 Mon Sep 17 00:00:00 2001
From: Akihiro Nitta <nitta@akihironitta.com>
Date: Thu, 6 Jan 2022 16:41:58 +0900
Subject: [PATCH 23/65] [skip azp] Don't use old versions of packages for
 testing

---
 .github/set-min-requirements.py    | 2 +-
 .github/workflows/ci_test-full.yml | 6 ------
 2 files changed, 1 insertion(+), 7 deletions(-)

diff --git a/.github/set-min-requirements.py b/.github/set-min-requirements.py
index b67ba224662ab..5562b66c4a50e 100644
--- a/.github/set-min-requirements.py
+++ b/.github/set-min-requirements.py
@@ -2,7 +2,7 @@
     "requirements.txt",
     "requirements/extra.txt",
     "requirements/loggers.txt",
-    "requirements/test.txt",
+    # "requirements/test.txt",  # don't use old versions of testing packages
     "requirements/examples.txt",
 )
 
diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index 7bab3a39154ed..aab332379a0ca 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -103,12 +103,6 @@ jobs:
         pip list
       shell: bash
 
-    - name: Use latest pytest
-      run: |
-        pip install pytest -U
-        pip list
-      shell: bash
-
     - name: Install extra dependencies
       env:
         HOROVOD_BUILD_ARCH_FLAGS: "-mfma"

From 7ca993cdb95cb3a18658d5209de4e57f3f05242f Mon Sep 17 00:00:00 2001
From: Akihiro Nitta <nitta@akihironitta.com>
Date: Thu, 6 Jan 2022 17:08:54 +0900
Subject: [PATCH 24/65] [skip azp] disable gdb for clean output

---
 .github/workflows/ci_test-full.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index aab332379a0ca..da6399e4c87ed 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -141,8 +141,8 @@ jobs:
       run: |
         # NOTE: do not include coverage report here, see: https://github.com/nedbat/coveragepy/issues/1003
         # pytest pytorch_lightning tests -v --durations=50 --junitxml=junit/test-results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml
-        gdb --return-child-result --batch -q -ex r -ex "thread apply all bt" --args $(which python) -m pytest pytorch_lightning tests -p no:faulthandler -q
-        # $(which python) -m pytest pytorch_lightning tests -p no:faulthandler -q
+        # gdb --return-child-result --batch -q -ex r -ex "thread apply all bt" --args $(which python) -m pytest pytorch_lightning tests -p no:faulthandler -q
+        $(which python) -m pytest pytorch_lightning tests
         echo TESTINGISFINISHEDHERE
 
     # - name: Examples

From 73f78f069857400d764e21a0c67b1a6517c2c5ea Mon Sep 17 00:00:00 2001
From: Akihiro Nitta <nitta@akihironitta.com>
Date: Thu, 6 Jan 2022 17:32:57 +0900
Subject: [PATCH 25/65] [skip azp] python -X faulthandler with typo fixed (redo
 of 99ab83c)

---
 .github/workflows/ci_test-full.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index da6399e4c87ed..518761669bc7d 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -142,7 +142,7 @@ jobs:
         # NOTE: do not include coverage report here, see: https://github.com/nedbat/coveragepy/issues/1003
         # pytest pytorch_lightning tests -v --durations=50 --junitxml=junit/test-results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml
         # gdb --return-child-result --batch -q -ex r -ex "thread apply all bt" --args $(which python) -m pytest pytorch_lightning tests -p no:faulthandler -q
-        $(which python) -m pytest pytorch_lightning tests
+        $(which python) -X faulthandler -m pytest pytorch_lightning tests -p no:faulthandler -q
         echo TESTINGISFINISHEDHERE
 
     # - name: Examples

From aaaf7d17f85bf128882e263860df60e1bed5b3c5 Mon Sep 17 00:00:00 2001
From: Akihiro Nitta <nitta@akihironitta.com>
Date: Thu, 6 Jan 2022 18:02:24 +0900
Subject: [PATCH 26/65] [skip azp] enable gdb

---
 .github/workflows/ci_test-full.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index 518761669bc7d..2d55835bc1792 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -141,8 +141,8 @@ jobs:
       run: |
         # NOTE: do not include coverage report here, see: https://github.com/nedbat/coveragepy/issues/1003
         # pytest pytorch_lightning tests -v --durations=50 --junitxml=junit/test-results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml
-        # gdb --return-child-result --batch -q -ex r -ex "thread apply all bt" --args $(which python) -m pytest pytorch_lightning tests -p no:faulthandler -q
-        $(which python) -X faulthandler -m pytest pytorch_lightning tests -p no:faulthandler -q
+        gdb --return-child-result --batch -q -ex r -ex "thread apply all bt" --args $(which python) -m pytest pytorch_lightning tests -p no:faulthandler -q
+        # $(which python) -X faulthandler -m pytest pytorch_lightning tests -p no:faulthandler -q
         echo TESTINGISFINISHEDHERE
 
     # - name: Examples

From e61abc1036e7452842c672b3a515a4c83204cb1d Mon Sep 17 00:00:00 2001
From: Akihiro Nitta <nitta@akihironitta.com>
Date: Thu, 6 Jan 2022 18:15:02 +0900
Subject: [PATCH 27/65] Temporarily disable all other CI jobs

---
 .azure-pipelines/gpu-tests.yml       | 1 -
 .azure-pipelines/ipu-tests.yml       | 1 -
 .circleci/config.yml                 | 1 -
 .github/workflows/ci_pkg-install.yml | 2 +-
 .github/workflows/ci_schema.yml      | 2 +-
 .github/workflows/ci_test-base.yml   | 2 +-
 .github/workflows/ci_test-conda.yml  | 2 +-
 .github/workflows/ci_test-slow.yml   | 2 +-
 .github/workflows/code-checks.yml    | 2 +-
 .github/workflows/docs-checks.yml    | 2 +-
 10 files changed, 7 insertions(+), 10 deletions(-)

diff --git a/.azure-pipelines/gpu-tests.yml b/.azure-pipelines/gpu-tests.yml
index ca8c54a61479e..4eacce5d41251 100644
--- a/.azure-pipelines/gpu-tests.yml
+++ b/.azure-pipelines/gpu-tests.yml
@@ -13,7 +13,6 @@ trigger:
       - "release/*"
       - "refs/tags/*"
 pr:
-  - "master"
   - "release/*"
 
 jobs:
diff --git a/.azure-pipelines/ipu-tests.yml b/.azure-pipelines/ipu-tests.yml
index 1b68a05ce012f..2dde23f665f44 100644
--- a/.azure-pipelines/ipu-tests.yml
+++ b/.azure-pipelines/ipu-tests.yml
@@ -8,7 +8,6 @@ trigger:
       - release/*
       - refs/tags/*
 pr:
-  - master
   - release/*
 
 variables:
diff --git a/.circleci/config.yml b/.circleci/config.yml
index 48797fab85034..9f15314354968 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -15,7 +15,6 @@ trigger:
       - "release/*"
       - "refs/tags/*"
 pr:
-  - "master"
   - "release/*"
 
 # Workflow Steps:
diff --git a/.github/workflows/ci_pkg-install.yml b/.github/workflows/ci_pkg-install.yml
index 0be8b37188787..0afc1e265ac01 100644
--- a/.github/workflows/ci_pkg-install.yml
+++ b/.github/workflows/ci_pkg-install.yml
@@ -5,7 +5,7 @@ on:  # Trigger the workflow on push or pull request, but only for the master bra
   push:
     branches: [master, "release/*"]
   pull_request:
-    branches: [master, "release/*"]
+    branches: ["release/*"]
 
 jobs:
 
diff --git a/.github/workflows/ci_schema.yml b/.github/workflows/ci_schema.yml
index d635285fae39a..a5fb5952a2eb0 100644
--- a/.github/workflows/ci_schema.yml
+++ b/.github/workflows/ci_schema.yml
@@ -2,7 +2,7 @@ name: Schema
 on: # Trigger the workflow on push or pull request, but only for the master branch
   push: {}
   pull_request:
-    branches: [master, "release/*"]
+    branches: ["release/*"]
 
 jobs:
   check:
diff --git a/.github/workflows/ci_test-base.yml b/.github/workflows/ci_test-base.yml
index c2f1d370e2d1a..bd7be230a288a 100644
--- a/.github/workflows/ci_test-base.yml
+++ b/.github/workflows/ci_test-base.yml
@@ -7,7 +7,7 @@ on:  # Trigger the workflow on push or pull request, but only for the master bra
   push:
     branches: [master, "release/*"]
   pull_request:
-    branches: [master, "release/*"]
+    branches: ["release/*"]
 
 jobs:
   source:
diff --git a/.github/workflows/ci_test-conda.yml b/.github/workflows/ci_test-conda.yml
index fa366e645f1d9..18033702478b8 100644
--- a/.github/workflows/ci_test-conda.yml
+++ b/.github/workflows/ci_test-conda.yml
@@ -5,7 +5,7 @@ on:  # Trigger the workflow on push or pull request, but only for the master bra
   push:
     branches: [master, "release/*"]
   pull_request:
-    branches: [master, "release/*"]
+    branches: ["release/*"]
 
 jobs:
   conda:
diff --git a/.github/workflows/ci_test-slow.yml b/.github/workflows/ci_test-slow.yml
index b9a07c4b6a26f..8d37f313c3021 100644
--- a/.github/workflows/ci_test-slow.yml
+++ b/.github/workflows/ci_test-slow.yml
@@ -5,7 +5,7 @@ on:  # Trigger the workflow on push or pull request, but only for the master bra
   push:
     branches: [master, "release/*"]
   pull_request:
-    branches: [master, "release/*"]
+    branches: ["release/*"]
 
 jobs:
   slow:
diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml
index 8cd4206ab61b7..5b8f88490c7b2 100644
--- a/.github/workflows/code-checks.yml
+++ b/.github/workflows/code-checks.yml
@@ -4,7 +4,7 @@ on:  # Trigger the workflow on push or pull request, but only for the master bra
   push:
     branches: [master, "release/*"]
   pull_request:
-    branches: [master, "release/*"]
+    branches: ["release/*"]
 
 jobs:
   mypy:
diff --git a/.github/workflows/docs-checks.yml b/.github/workflows/docs-checks.yml
index 841f9128da8b1..e7a9579966ba5 100644
--- a/.github/workflows/docs-checks.yml
+++ b/.github/workflows/docs-checks.yml
@@ -5,7 +5,7 @@ on:  # Trigger the workflow on push or pull request, but only for the master bra
   push:
     branches: [master, "release/*"]
   pull_request:
-    branches: [master, "release/*"]
+    branches: ["release/*"]
 
 jobs:
   doctest:

From 4dde7e8a7c86a32d7af0e8fd652f273f5e3b677f Mon Sep 17 00:00:00 2001
From: Akihiro Nitta <nitta@akihironitta.com>
Date: Thu, 6 Jan 2022 18:38:21 +0900
Subject: [PATCH 28/65] split into doctests and tests

---
 .github/workflows/ci_test-full.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index 2d55835bc1792..aaaea9cbcedca 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -137,13 +137,13 @@ jobs:
         path: Datasets
         key: pl-dataset
 
-    - name: Tests
+    - name: Tests - ./pytorch_lightning/
       run: |
-        # NOTE: do not include coverage report here, see: https://github.com/nedbat/coveragepy/issues/1003
-        # pytest pytorch_lightning tests -v --durations=50 --junitxml=junit/test-results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml
-        gdb --return-child-result --batch -q -ex r -ex "thread apply all bt" --args $(which python) -m pytest pytorch_lightning tests -p no:faulthandler -q
-        # $(which python) -X faulthandler -m pytest pytorch_lightning tests -p no:faulthandler -q
-        echo TESTINGISFINISHEDHERE
+        python -m pytest pytorch_lightning -v
+
+    - name: Tests - ./tests/
+      run: |
+        python -m pytest tests -v
 
     # - name: Examples
     #   run: |

From fd8ee636641a8f02954b5203b33a387829ba7757 Mon Sep 17 00:00:00 2001
From: Akihiro Nitta <nitta@akihironitta.com>
Date: Thu, 6 Jan 2022 18:56:37 +0900
Subject: [PATCH 29/65] split tests into subdirectories

---
 .github/workflows/ci_test-full.yml | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index aaaea9cbcedca..c297c8a274355 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -143,7 +143,21 @@ jobs:
 
     - name: Tests - ./tests/
       run: |
-        python -m pytest tests -v
+        python -m pytest tests/accelerators -v
+        python -m pytest tests/callbacks -v
+        python -m pytest tests/core -v
+        python -m pytest tests/distributed -v
+        python -m pytest tests/lite -v
+        python -m pytest tests/loggers -v
+        python -m pytest tests/loops -v
+        python -m pytest tests/overrides -v
+        python -m pytest tests/plugins -v
+        python -m pytest tests/profiler -v
+        python -m pytest tests/strategies -v
+        python -m pytest tests/trainer -v
+        python -m pytest tests/tuner -v
+        python -m pytest tests/utilities -v
+        python -m pytest tests -v  # all
 
     # - name: Examples
     #   run: |

From eb4fe8933e7e0d44c287325f5db509712e7df713 Mon Sep 17 00:00:00 2001
From: Akihiro Nitta <nitta@akihironitta.com>
Date: Thu, 6 Jan 2022 19:21:55 +0900
Subject: [PATCH 30/65] Fix 'split tests into subdirectories'

---
 .github/workflows/ci_test-full.yml | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index c297c8a274355..7901022b01f34 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -144,12 +144,16 @@ jobs:
     - name: Tests - ./tests/
       run: |
         python -m pytest tests/accelerators -v
+        python -m pytest tests/base -v
         python -m pytest tests/callbacks -v
+        python -m pytest tests/checkpointing -v
         python -m pytest tests/core -v
-        python -m pytest tests/distributed -v
+        python -m pytest tests/deprecated_api -v
+        python -m pytest tests/helpers -v
         python -m pytest tests/lite -v
         python -m pytest tests/loggers -v
         python -m pytest tests/loops -v
+        python -m pytest tests/models -v
         python -m pytest tests/overrides -v
         python -m pytest tests/plugins -v
         python -m pytest tests/profiler -v
@@ -157,7 +161,6 @@ jobs:
         python -m pytest tests/trainer -v
         python -m pytest tests/tuner -v
         python -m pytest tests/utilities -v
-        python -m pytest tests -v  # all
 
     # - name: Examples
     #   run: |

From dcd67ed123e9edc4d335ece924c1c8dc38246655 Mon Sep 17 00:00:00 2001
From: Akihiro Nitta <nitta@akihironitta.com>
Date: Thu, 6 Jan 2022 19:26:01 +0900
Subject: [PATCH 31/65] Fix 'split tests into subdirectories'

---
 .github/workflows/ci_test-full.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index 7901022b01f34..59502d337d253 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -144,7 +144,6 @@ jobs:
     - name: Tests - ./tests/
       run: |
         python -m pytest tests/accelerators -v
-        python -m pytest tests/base -v
         python -m pytest tests/callbacks -v
         python -m pytest tests/checkpointing -v
         python -m pytest tests/core -v

From 8213659b27a58be9512c5e26924f7006dd90c8e2 Mon Sep 17 00:00:00 2001
From: Akihiro Nitta <nitta@akihironitta.com>
Date: Thu, 6 Jan 2022 19:37:22 +0900
Subject: [PATCH 32/65] Disable tests unrelated to segfault

---
 .github/workflows/ci_test-full.yml | 32 +++++++++++++++---------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index 59502d337d253..1ced7b8e992cb 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -144,22 +144,22 @@ jobs:
     - name: Tests - ./tests/
       run: |
         python -m pytest tests/accelerators -v
-        python -m pytest tests/callbacks -v
-        python -m pytest tests/checkpointing -v
-        python -m pytest tests/core -v
-        python -m pytest tests/deprecated_api -v
-        python -m pytest tests/helpers -v
-        python -m pytest tests/lite -v
-        python -m pytest tests/loggers -v
-        python -m pytest tests/loops -v
-        python -m pytest tests/models -v
-        python -m pytest tests/overrides -v
-        python -m pytest tests/plugins -v
-        python -m pytest tests/profiler -v
-        python -m pytest tests/strategies -v
-        python -m pytest tests/trainer -v
-        python -m pytest tests/tuner -v
-        python -m pytest tests/utilities -v
+        # python -m pytest tests/callbacks -v
+        # python -m pytest tests/checkpointing -v
+        # python -m pytest tests/core -v
+        # python -m pytest tests/deprecated_api -v
+        # python -m pytest tests/helpers -v
+        # python -m pytest tests/lite -v
+        # python -m pytest tests/loggers -v
+        # python -m pytest tests/loops -v
+        # python -m pytest tests/models -v
+        # python -m pytest tests/overrides -v
+        # python -m pytest tests/plugins -v
+        # python -m pytest tests/profiler -v
+        # python -m pytest tests/strategies -v
+        # python -m pytest tests/trainer -v
+        # python -m pytest tests/tuner -v
+        # python -m pytest tests/utilities -v
 
     # - name: Examples
     #   run: |

From cdc1bb1a6874ce0106766f8a862715b8b6b90564 Mon Sep 17 00:00:00 2001
From: Akihiro Nitta <nitta@akihironitta.com>
Date: Thu, 6 Jan 2022 20:24:24 +0900
Subject: [PATCH 33/65] use ubuntu20.04

---
 .github/workflows/ci_test-full.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index 1ced7b8e992cb..ed6b7e6a1ed59 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -17,7 +17,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-18.04]
+        os: [ubuntu-20.04]
         python-version: ["3.7", "3.9"]  # minimum, maximum
         requires: ["oldest", "latest"]
         release: ["stable"]

From f1a79968fb2b7d9a327d3c622265efd1fb7489fc Mon Sep 17 00:00:00 2001
From: Akihiro Nitta <nitta@akihironitta.com>
Date: Thu, 6 Jan 2022 22:04:45 +0900
Subject: [PATCH 34/65] uninstall horovod

---
 .github/workflows/ci_test-full.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index ed6b7e6a1ed59..9fc5ba93412c6 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -113,6 +113,8 @@ jobs:
         python ./requirements/adjust_versions.py requirements/extra.txt
         pip install --requirement ./requirements/extra.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --upgrade
         pip list
+        pip uninstall horovod
+        pip list
       shell: bash
 
     # - name: Reinstall Horovod if necessary

From ea4d419a3698f21c5dbf6bbc79b4ca83e68f6f2d Mon Sep 17 00:00:00 2001
From: Akihiro Nitta <nitta@akihironitta.com>
Date: Thu, 6 Jan 2022 22:06:32 +0900
Subject: [PATCH 35/65] _HOROVOD_AVAILABLE = False

---
 pytorch_lightning/utilities/imports.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pytorch_lightning/utilities/imports.py b/pytorch_lightning/utilities/imports.py
index d4f615b2e7cc0..4345549eb70b7 100644
--- a/pytorch_lightning/utilities/imports.py
+++ b/pytorch_lightning/utilities/imports.py
@@ -77,7 +77,7 @@ def _compare_version(package: str, op: Callable, version: str, use_base_version:
 _FAIRSCALE_OSS_FP16_BROADCAST_AVAILABLE = _FAIRSCALE_AVAILABLE and _compare_version("fairscale", operator.ge, "0.3.3")
 _FAIRSCALE_FULLY_SHARDED_AVAILABLE = _FAIRSCALE_AVAILABLE and _compare_version("fairscale", operator.ge, "0.3.4")
 _GROUP_AVAILABLE = not _IS_WINDOWS and _module_available("torch.distributed.group")
-_HOROVOD_AVAILABLE = _module_available("horovod.torch")
+_HOROVOD_AVAILABLE = False
 _HYDRA_AVAILABLE = _module_available("hydra")
 _HYDRA_EXPERIMENTAL_AVAILABLE = _module_available("hydra.experimental")
 _JSONARGPARSE_AVAILABLE = _module_available("jsonargparse") and _compare_version("jsonargparse", operator.ge, "4.0.0")

From b048c9b4bb11faed3a87a2110f0c6c138d9356ad Mon Sep 17 00:00:00 2001
From: Akihiro Nitta <nitta@akihironitta.com>
Date: Thu, 6 Jan 2022 22:13:22 +0900
Subject: [PATCH 36/65] run tests/callbacks and then tests/accelerators

---
 .github/workflows/ci_test-full.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index 9fc5ba93412c6..400467898cc78 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -145,8 +145,8 @@ jobs:
 
     - name: Tests - ./tests/
       run: |
+        python -m pytest tests/callbacks -v
         python -m pytest tests/accelerators -v
-        # python -m pytest tests/callbacks -v
         # python -m pytest tests/checkpointing -v
         # python -m pytest tests/core -v
         # python -m pytest tests/deprecated_api -v

From 3d52b34359600736fd85fae84dd8e27b35e509cb Mon Sep 17 00:00:00 2001
From: Akihiro Nitta <nitta@akihironitta.com>
Date: Thu, 6 Jan 2022 22:26:09 +0900
Subject: [PATCH 37/65] Add a fake test

---
 .github/workflows/ci_test-full.yml |  1 +
 tests/foo/test_foo.py              | 16 ++++++++++++++++
 2 files changed, 17 insertions(+)
 create mode 100644 tests/foo/test_foo.py

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index 400467898cc78..abc1744c96837 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -145,6 +145,7 @@ jobs:
 
     - name: Tests - ./tests/
       run: |
+        python -m pytest tests/foo -v
         python -m pytest tests/callbacks -v
         python -m pytest tests/accelerators -v
         # python -m pytest tests/checkpointing -v
diff --git a/tests/foo/test_foo.py b/tests/foo/test_foo.py
new file mode 100644
index 0000000000000..67693a5c51680
--- /dev/null
+++ b/tests/foo/test_foo.py
@@ -0,0 +1,16 @@
+# Copyright The PyTorch Lightning team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+def test_foo(tmpdir):
+    assert None is None

From a5721a41fb99e90777ccab81b62e9252b55318ac Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Thu, 6 Jan 2022 13:28:58 +0000
Subject: [PATCH 38/65] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 tests/foo/test_foo.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/foo/test_foo.py b/tests/foo/test_foo.py
index 67693a5c51680..6eaad56bc330c 100644
--- a/tests/foo/test_foo.py
+++ b/tests/foo/test_foo.py
@@ -12,5 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
 def test_foo(tmpdir):
     assert None is None

From e05effded366da5d7e2f9d1b93aa2e7bdf40ecf6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Carlos=20Mochol=C3=AD?= <carlossmocholi@gmail.com>
Date: Thu, 6 Jan 2022 14:40:23 +0100
Subject: [PATCH 39/65] Update .github/workflows/ci_test-full.yml

---
 .github/workflows/ci_test-full.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index abc1744c96837..67b9cabccaae8 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -146,8 +146,8 @@ jobs:
     - name: Tests - ./tests/
       run: |
         python -m pytest tests/foo -v
-        python -m pytest tests/callbacks -v
-        python -m pytest tests/accelerators -v
+        # python -m pytest tests/callbacks -v
+        # python -m pytest tests/accelerators -v
         # python -m pytest tests/checkpointing -v
         # python -m pytest tests/core -v
         # python -m pytest tests/deprecated_api -v

From 61fbd2b76ae046b058e315ae693386914aa1562d Mon Sep 17 00:00:00 2001
From: Carlos Mocholi <carlossmocholi@gmail.com>
Date: Thu, 6 Jan 2022 14:55:27 +0100
Subject: [PATCH 40/65] Move foo test with callbacks tests

---
 .github/workflows/ci_test-full.yml |  3 +--
 tests/callbacks/test_callbacks.py  |  4 ++++
 tests/foo/test_foo.py              | 17 -----------------
 3 files changed, 5 insertions(+), 19 deletions(-)
 delete mode 100644 tests/foo/test_foo.py

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index 67b9cabccaae8..764eb40bd5e27 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -145,8 +145,7 @@ jobs:
 
     - name: Tests - ./tests/
       run: |
-        python -m pytest tests/foo -v
-        # python -m pytest tests/callbacks -v
+        python -m pytest tests/callbacks/test_callbacks.py::test_foo -v
         # python -m pytest tests/accelerators -v
         # python -m pytest tests/checkpointing -v
         # python -m pytest tests/core -v
diff --git a/tests/callbacks/test_callbacks.py b/tests/callbacks/test_callbacks.py
index d97af8e211249..17822612c5302 100644
--- a/tests/callbacks/test_callbacks.py
+++ b/tests/callbacks/test_callbacks.py
@@ -164,3 +164,7 @@ def test_resume_incomplete_callbacks_list_warning(tmpdir):
     )
     with no_warning_call(UserWarning, match="Please add the following callbacks:"):
         trainer.fit(model, ckpt_path=ckpt_path)
+
+
+def test_foo():
+    assert True
diff --git a/tests/foo/test_foo.py b/tests/foo/test_foo.py
deleted file mode 100644
index 6eaad56bc330c..0000000000000
--- a/tests/foo/test_foo.py
+++ /dev/null
@@ -1,17 +0,0 @@
-# Copyright The PyTorch Lightning team.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-def test_foo(tmpdir):
-    assert None is None

From b73261e0f76cbc1262de42ca89a19ec1364b6db9 Mon Sep 17 00:00:00 2001
From: Carlos Mocholi <carlossmocholi@gmail.com>
Date: Thu, 6 Jan 2022 14:58:24 +0100
Subject: [PATCH 41/65] Revert Horovod changes

---
 .github/workflows/ci_test-full.yml     | 33 ++++++++++++--------------
 pytorch_lightning/utilities/imports.py |  2 +-
 2 files changed, 16 insertions(+), 19 deletions(-)

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index 764eb40bd5e27..7815fb06e542d 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -61,7 +61,6 @@ jobs:
       if: runner.os == 'Linux'
       run: |
         sudo apt install gdb
-        python .github/prune-packages.py requirements/extra.txt "horovod"
 
     - name: Set min. dependencies
       if: matrix.requires == 'oldest'
@@ -113,25 +112,23 @@ jobs:
         python ./requirements/adjust_versions.py requirements/extra.txt
         pip install --requirement ./requirements/extra.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --upgrade
         pip list
-        pip uninstall horovod
-        pip list
       shell: bash
 
-    # - name: Reinstall Horovod if necessary
-    #   if: runner.os != 'windows'
-    #   env:
-    #     HOROVOD_BUILD_ARCH_FLAGS: "-mfma"
-    #     HOROVOD_WITHOUT_MXNET: 1
-    #     HOROVOD_WITHOUT_TENSORFLOW: 1
-    #   run: |
-    #     HOROVOD_BUILT=$(python -c "import horovod.torch; horovod.torch.nccl_built(); print('SUCCESS')" || true)
-    #     if [[ $HOROVOD_BUILT != "SUCCESS" ]]; then
-    #       pip uninstall -y horovod
-    #       echo $(grep "horovod" requirements/extra.txt) > requirements/horovod.txt
-    #       pip install --no-cache-dir -r requirements/horovod.txt
-    #     fi
-    #     horovodrun --check-build
-    #   shell: bash
+    - name: Reinstall Horovod if necessary
+      if: runner.os != 'windows'
+      env:
+        HOROVOD_BUILD_ARCH_FLAGS: "-mfma"
+        HOROVOD_WITHOUT_MXNET: 1
+        HOROVOD_WITHOUT_TENSORFLOW: 1
+      run: |
+        HOROVOD_BUILT=$(python -c "import horovod.torch; horovod.torch.nccl_built(); print('SUCCESS')" || true)
+        if [[ $HOROVOD_BUILT != "SUCCESS" ]]; then
+          pip uninstall -y horovod
+          echo $(grep "horovod" requirements/extra.txt) > requirements/horovod.txt
+          pip install --no-cache-dir -r requirements/horovod.txt
+        fi
+        horovodrun --check-build
+      shell: bash
 
     - name: Cache datasets
       uses: actions/cache@v2
diff --git a/pytorch_lightning/utilities/imports.py b/pytorch_lightning/utilities/imports.py
index 4345549eb70b7..d4f615b2e7cc0 100644
--- a/pytorch_lightning/utilities/imports.py
+++ b/pytorch_lightning/utilities/imports.py
@@ -77,7 +77,7 @@ def _compare_version(package: str, op: Callable, version: str, use_base_version:
 _FAIRSCALE_OSS_FP16_BROADCAST_AVAILABLE = _FAIRSCALE_AVAILABLE and _compare_version("fairscale", operator.ge, "0.3.3")
 _FAIRSCALE_FULLY_SHARDED_AVAILABLE = _FAIRSCALE_AVAILABLE and _compare_version("fairscale", operator.ge, "0.3.4")
 _GROUP_AVAILABLE = not _IS_WINDOWS and _module_available("torch.distributed.group")
-_HOROVOD_AVAILABLE = False
+_HOROVOD_AVAILABLE = _module_available("horovod.torch")
 _HYDRA_AVAILABLE = _module_available("hydra")
 _HYDRA_EXPERIMENTAL_AVAILABLE = _module_available("hydra.experimental")
 _JSONARGPARSE_AVAILABLE = _module_available("jsonargparse") and _compare_version("jsonargparse", operator.ge, "4.0.0")

From b70319b2ca739b3775ddbdfd4cdb2a59c46a8dcc Mon Sep 17 00:00:00 2001
From: Carlos Mocholi <carlossmocholi@gmail.com>
Date: Thu, 6 Jan 2022 15:27:45 +0100
Subject: [PATCH 42/65] Just foo worked

---
 .github/workflows/ci_test-full.yml | 2 +-
 tests/callbacks/test_callbacks.py  | 4 ----
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index 7815fb06e542d..08a243015f55a 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -142,7 +142,7 @@ jobs:
 
     - name: Tests - ./tests/
       run: |
-        python -m pytest tests/callbacks/test_callbacks.py::test_foo -v
+        python -m pytest tests/callbacks/test_callbacks.py -v
         # python -m pytest tests/accelerators -v
         # python -m pytest tests/checkpointing -v
         # python -m pytest tests/core -v
diff --git a/tests/callbacks/test_callbacks.py b/tests/callbacks/test_callbacks.py
index 17822612c5302..d97af8e211249 100644
--- a/tests/callbacks/test_callbacks.py
+++ b/tests/callbacks/test_callbacks.py
@@ -164,7 +164,3 @@ def test_resume_incomplete_callbacks_list_warning(tmpdir):
     )
     with no_warning_call(UserWarning, match="Please add the following callbacks:"):
         trainer.fit(model, ckpt_path=ckpt_path)
-
-
-def test_foo():
-    assert True

From 306676eb91cf936052be639388d7e7c839e45dae Mon Sep 17 00:00:00 2001
From: Carlos Mocholi <carlossmocholi@gmail.com>
Date: Thu, 6 Jan 2022 15:35:04 +0100
Subject: [PATCH 43/65] Trim down steps

---
 .github/workflows/ci_test-full.yml | 44 ------------------------------
 1 file changed, 44 deletions(-)

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index 08a243015f55a..a5ab885850e44 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -82,64 +82,20 @@ jobs:
         restore-keys: |
           ${{ runner.os }}-pip-td${{ steps.times.outputs.period }}-py${{ matrix.python-version }}-${{ matrix.release }}-${{ matrix.requires }}-
 
-    - name: Pull checkpoints from S3
-      working-directory: ./legacy
-      run: |
-        # wget is simpler but does not work on Windows
-        python -c "from urllib.request import urlretrieve ; urlretrieve('https://pl-public-data.s3.amazonaws.com/legacy/checkpoints.zip', 'checkpoints.zip')"
-        ls -l .
-        unzip -o checkpoints.zip
-        ls -l checkpoints/
-
     - name: Install dependencies
       run: |
         flag=$(python -c "print('--pre' if '${{matrix.release}}' == 'pre' else '')" 2>&1)
         pip install --requirement requirements.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --upgrade $flag
-        # adjust versions according installed Torch version
-        python ./requirements/adjust_versions.py requirements/examples.txt
-        pip install --requirement requirements/examples.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --upgrade
         pip install --requirement requirements/test.txt --upgrade
         pip list
       shell: bash
 
-    - name: Install extra dependencies
-      env:
-        HOROVOD_BUILD_ARCH_FLAGS: "-mfma"
-        HOROVOD_WITHOUT_MXNET: 1
-        HOROVOD_WITHOUT_TENSORFLOW: 1
-      run: |
-        # adjust versions according installed Torch version
-        python ./requirements/adjust_versions.py requirements/extra.txt
-        pip install --requirement ./requirements/extra.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --upgrade
-        pip list
-      shell: bash
-
-    - name: Reinstall Horovod if necessary
-      if: runner.os != 'windows'
-      env:
-        HOROVOD_BUILD_ARCH_FLAGS: "-mfma"
-        HOROVOD_WITHOUT_MXNET: 1
-        HOROVOD_WITHOUT_TENSORFLOW: 1
-      run: |
-        HOROVOD_BUILT=$(python -c "import horovod.torch; horovod.torch.nccl_built(); print('SUCCESS')" || true)
-        if [[ $HOROVOD_BUILT != "SUCCESS" ]]; then
-          pip uninstall -y horovod
-          echo $(grep "horovod" requirements/extra.txt) > requirements/horovod.txt
-          pip install --no-cache-dir -r requirements/horovod.txt
-        fi
-        horovodrun --check-build
-      shell: bash
-
     - name: Cache datasets
       uses: actions/cache@v2
       with:
         path: Datasets
         key: pl-dataset
 
-    - name: Tests - ./pytorch_lightning/
-      run: |
-        python -m pytest pytorch_lightning -v
-
     - name: Tests - ./tests/
       run: |
         python -m pytest tests/callbacks/test_callbacks.py -v

From 3eba49ea1ebce126c23701647e19a8b4356b70f0 Mon Sep 17 00:00:00 2001
From: Carlos Mocholi <carlossmocholi@gmail.com>
Date: Thu, 6 Jan 2022 15:35:29 +0100
Subject: [PATCH 44/65] Just test_callbaks.py worked

---
 .github/workflows/ci_test-full.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index a5ab885850e44..1bf77f5fd7ea4 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -98,7 +98,7 @@ jobs:
 
     - name: Tests - ./tests/
       run: |
-        python -m pytest tests/callbacks/test_callbacks.py -v
+        python -m pytest tests/callbacks -v
         # python -m pytest tests/accelerators -v
         # python -m pytest tests/checkpointing -v
         # python -m pytest tests/core -v

From adc4a597ee3758059423efcf2de685557470a1ff Mon Sep 17 00:00:00 2001
From: Carlos Mocholi <carlossmocholi@gmail.com>
Date: Thu, 6 Jan 2022 15:52:08 +0100
Subject: [PATCH 45/65] Fix rich bug

---
 pytorch_lightning/callbacks/progress/rich_progress.py | 3 +--
 pytorch_lightning/callbacks/rich_model_summary.py     | 2 +-
 tests/callbacks/test_rich_model_summary.py            | 2 +-
 tests/callbacks/test_rich_progress_bar.py             | 2 +-
 4 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/pytorch_lightning/callbacks/progress/rich_progress.py b/pytorch_lightning/callbacks/progress/rich_progress.py
index f983ccdaab12e..7306c36aec7b4 100644
--- a/pytorch_lightning/callbacks/progress/rich_progress.py
+++ b/pytorch_lightning/callbacks/progress/rich_progress.py
@@ -17,7 +17,6 @@
 from typing import Any, Dict, Optional, Union
 
 from pytorch_lightning.callbacks.progress.base import ProgressBarBase
-from pytorch_lightning.utilities.exceptions import MisconfigurationException
 from pytorch_lightning.utilities.imports import _RICH_AVAILABLE
 
 Task, Style = None, None
@@ -231,7 +230,7 @@ def __init__(
         console_kwargs: Optional[Dict[str, Any]] = None,
     ) -> None:
         if not _RICH_AVAILABLE:
-            raise MisconfigurationException(
+            raise ModuleNotFoundError(
                 "`RichProgressBar` requires `rich` >= 10.2.2. Install it by running `pip install -U rich`."
             )
 
diff --git a/pytorch_lightning/callbacks/rich_model_summary.py b/pytorch_lightning/callbacks/rich_model_summary.py
index cce4eb316a3b0..14c078a273ece 100644
--- a/pytorch_lightning/callbacks/rich_model_summary.py
+++ b/pytorch_lightning/callbacks/rich_model_summary.py
@@ -61,7 +61,7 @@ class RichModelSummary(ModelSummary):
     def __init__(self, max_depth: int = 1) -> None:
         if not _RICH_AVAILABLE:
             raise ModuleNotFoundError(
-                "`RichProgressBar` requires `rich` to be installed. Install it by running `pip install -U rich`."
+                "`RichModelSummary` requires `rich` to be installed. Install it by running `pip install -U rich`."
             )
         super().__init__(max_depth)
 
diff --git a/tests/callbacks/test_rich_model_summary.py b/tests/callbacks/test_rich_model_summary.py
index 5ab091bd01445..88c5f9ab531f0 100644
--- a/tests/callbacks/test_rich_model_summary.py
+++ b/tests/callbacks/test_rich_model_summary.py
@@ -35,7 +35,7 @@ def test_rich_model_summary_callback():
 
 def test_rich_progress_bar_import_error():
     if not _RICH_AVAILABLE:
-        with pytest.raises(ImportError, match="`RichModelSummary` requires `rich` to be installed."):
+        with pytest.raises(ModuleNotFoundError, match="`RichModelSummary` requires `rich` to be installed."):
             Trainer(callbacks=RichModelSummary())
 
 
diff --git a/tests/callbacks/test_rich_progress_bar.py b/tests/callbacks/test_rich_progress_bar.py
index f2e75006f7ecb..d9a9ad8c3c726 100644
--- a/tests/callbacks/test_rich_progress_bar.py
+++ b/tests/callbacks/test_rich_progress_bar.py
@@ -85,7 +85,7 @@ def predict_dataloader(self):
 
 def test_rich_progress_bar_import_error():
     if not _RICH_AVAILABLE:
-        with pytest.raises(ImportError, match="`RichProgressBar` requires `rich` >= 10.2.2."):
+        with pytest.raises(ModuleNotFoundError, match="`RichProgressBar` requires `rich` >= 10.2.2."):
             Trainer(callbacks=RichProgressBar())
 
 

From 814f2fe996bf7113ce0fc4671c0ef822df1a0271 Mon Sep 17 00:00:00 2001
From: Carlos Mocholi <carlossmocholi@gmail.com>
Date: Thu, 6 Jan 2022 16:02:42 +0100
Subject: [PATCH 46/65] tests/callbacks failed. try now each file separately

---
 .github/workflows/ci_test-full.yml | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index 1bf77f5fd7ea4..9c7c5ed622868 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -98,7 +98,27 @@ jobs:
 
     - name: Tests - ./tests/
       run: |
-        python -m pytest tests/callbacks -v
+        python -m pytest tests/callbacks/test_callback_hook_outputs.py -v
+        python -m pytest tests/callbacks/test_callbacks.py -v
+        python -m pytest tests/callbacks/test_device_stats_monitor.py -v
+        python -m pytest tests/callbacks/test_early_stopping.py -v
+        python -m pytest tests/callbacks/test_finetuning_callback.py -v
+        python -m pytest tests/callbacks/test_gpu_stats_monitor.py -v
+        python -m pytest tests/callbacks/test_gradient_accumulation_scheduler.py -v
+        python -m pytest tests/callbacks/test_lambda_function.py -v
+        python -m pytest tests/callbacks/test_lr_monitor.py -v
+        python -m pytest tests/callbacks/test_model_summary.py -v
+        python -m pytest tests/callbacks/test_prediction_writer.py -v
+        python -m pytest tests/callbacks/test_pruning.py -v
+        python -m pytest tests/callbacks/test_quantization.py -v
+        python -m pytest tests/callbacks/test_rich_model_summary.py -v
+        python -m pytest tests/callbacks/test_rich_progress_bar.py -v
+        python -m pytest tests/callbacks/test_stochastic_weight_avg.py -v
+        python -m pytest tests/callbacks/test_timer.py -v
+        python -m pytest tests/callbacks/test_tqdm_progress_bar.py -v
+        python -m pytest tests/callbacks/test_xla_stats_monitor.py -v
+
+        # python -m pytest tests/callbacks -v
         # python -m pytest tests/accelerators -v
         # python -m pytest tests/checkpointing -v
         # python -m pytest tests/core -v

From d1b595dc7ea7e2dc40eceedffa55a726ffb51460 Mon Sep 17 00:00:00 2001
From: Carlos Mocholi <carlossmocholi@gmail.com>
Date: Thu, 6 Jan 2022 16:10:01 +0100
Subject: [PATCH 47/65] Narrow down

---
 .github/workflows/ci_test-full.yml | 40 +++---------------------------
 1 file changed, 4 insertions(+), 36 deletions(-)

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index 9c7c5ed622868..e1c8366e37310 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -98,43 +98,11 @@ jobs:
 
     - name: Tests - ./tests/
       run: |
+        python -m pytest tests/callbacks/test_callback_hook_outputs.py::test_free_memory_on_eval_outputs -v
+        python -m pytest tests/callbacks/test_callback_hook_outputs.py::test_train_step_no_return[False] -v
+        python -m pytest tests/callbacks/test_callback_hook_outputs.py::test_train_step_no_return[True] -v
+        # in case the previous did not fail
         python -m pytest tests/callbacks/test_callback_hook_outputs.py -v
-        python -m pytest tests/callbacks/test_callbacks.py -v
-        python -m pytest tests/callbacks/test_device_stats_monitor.py -v
-        python -m pytest tests/callbacks/test_early_stopping.py -v
-        python -m pytest tests/callbacks/test_finetuning_callback.py -v
-        python -m pytest tests/callbacks/test_gpu_stats_monitor.py -v
-        python -m pytest tests/callbacks/test_gradient_accumulation_scheduler.py -v
-        python -m pytest tests/callbacks/test_lambda_function.py -v
-        python -m pytest tests/callbacks/test_lr_monitor.py -v
-        python -m pytest tests/callbacks/test_model_summary.py -v
-        python -m pytest tests/callbacks/test_prediction_writer.py -v
-        python -m pytest tests/callbacks/test_pruning.py -v
-        python -m pytest tests/callbacks/test_quantization.py -v
-        python -m pytest tests/callbacks/test_rich_model_summary.py -v
-        python -m pytest tests/callbacks/test_rich_progress_bar.py -v
-        python -m pytest tests/callbacks/test_stochastic_weight_avg.py -v
-        python -m pytest tests/callbacks/test_timer.py -v
-        python -m pytest tests/callbacks/test_tqdm_progress_bar.py -v
-        python -m pytest tests/callbacks/test_xla_stats_monitor.py -v
-
-        # python -m pytest tests/callbacks -v
-        # python -m pytest tests/accelerators -v
-        # python -m pytest tests/checkpointing -v
-        # python -m pytest tests/core -v
-        # python -m pytest tests/deprecated_api -v
-        # python -m pytest tests/helpers -v
-        # python -m pytest tests/lite -v
-        # python -m pytest tests/loggers -v
-        # python -m pytest tests/loops -v
-        # python -m pytest tests/models -v
-        # python -m pytest tests/overrides -v
-        # python -m pytest tests/plugins -v
-        # python -m pytest tests/profiler -v
-        # python -m pytest tests/strategies -v
-        # python -m pytest tests/trainer -v
-        # python -m pytest tests/tuner -v
-        # python -m pytest tests/utilities -v
 
     # - name: Examples
     #   run: |

From 3f34ca5ea2af51cf7feae185cf84d0a3488a8bf9 Mon Sep 17 00:00:00 2001
From: Carlos Mocholi <carlossmocholi@gmail.com>
Date: Thu, 6 Jan 2022 16:21:08 +0100
Subject: [PATCH 48/65] Narrow down

---
 .github/workflows/ci_test-full.yml            | 11 +++++++----
 tests/callbacks/test_callback_hook_outputs.py |  4 ++++
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index e1c8366e37310..5e87674b33ec6 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -98,11 +98,14 @@ jobs:
 
     - name: Tests - ./tests/
       run: |
+        # sanity check
+        python -m pytest tests/callbacks/test_callback_hook_outputs.py::test_foo -v
+
+        # test if it's unrelated to pytest
+        python pl_examples/bug_report/bug_report_model.py
+
+        # we know this fails
         python -m pytest tests/callbacks/test_callback_hook_outputs.py::test_free_memory_on_eval_outputs -v
-        python -m pytest tests/callbacks/test_callback_hook_outputs.py::test_train_step_no_return[False] -v
-        python -m pytest tests/callbacks/test_callback_hook_outputs.py::test_train_step_no_return[True] -v
-        # in case the previous did not fail
-        python -m pytest tests/callbacks/test_callback_hook_outputs.py -v
 
     # - name: Examples
     #   run: |
diff --git a/tests/callbacks/test_callback_hook_outputs.py b/tests/callbacks/test_callback_hook_outputs.py
index f7c9321cd0e2e..7f2ec5c2e01a4 100644
--- a/tests/callbacks/test_callback_hook_outputs.py
+++ b/tests/callbacks/test_callback_hook_outputs.py
@@ -78,3 +78,7 @@ def on_epoch_end(self, trainer, pl_module):
     )
 
     trainer.fit(model)
+
+
+def test_foo():
+    assert True

From a741f6399faefa0ab4b3dab13f323b7aabe14b46 Mon Sep 17 00:00:00 2001
From: Carlos Mocholi <carlossmocholi@gmail.com>
Date: Thu, 6 Jan 2022 16:23:47 +0100
Subject: [PATCH 49/65] Need the local import

---
 .github/workflows/ci_test-full.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index 5e87674b33ec6..4427ecd5fd07c 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -102,7 +102,8 @@ jobs:
         python -m pytest tests/callbacks/test_callback_hook_outputs.py::test_foo -v
 
         # test if it's unrelated to pytest
-        python pl_examples/bug_report/bug_report_model.py
+        cp pl_examples/bug_report/bug_report_model.py .
+        python bug_report_model.py
 
         # we know this fails
         python -m pytest tests/callbacks/test_callback_hook_outputs.py::test_free_memory_on_eval_outputs -v

From 24e3645e9e9e293edd97302482e53f1b63d3ed13 Mon Sep 17 00:00:00 2001
From: Carlos Mocholi <carlossmocholi@gmail.com>
Date: Thu, 6 Jan 2022 16:31:49 +0100
Subject: [PATCH 50/65] Try bug_report_model with pytest

---
 .github/workflows/ci_test-full.yml         |  5 +---
 pl_examples/bug_report/bug_report_model.py | 27 +++++++---------------
 2 files changed, 9 insertions(+), 23 deletions(-)

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index 4427ecd5fd07c..159ac7dc74530 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -98,12 +98,9 @@ jobs:
 
     - name: Tests - ./tests/
       run: |
-        # sanity check
-        python -m pytest tests/callbacks/test_callback_hook_outputs.py::test_foo -v
-
         # test if it's unrelated to pytest
         cp pl_examples/bug_report/bug_report_model.py .
-        python bug_report_model.py
+        python -m pytest bug_report_model.py
 
         # we know this fails
         python -m pytest tests/callbacks/test_callback_hook_outputs.py::test_free_memory_on_eval_outputs -v
diff --git a/pl_examples/bug_report/bug_report_model.py b/pl_examples/bug_report/bug_report_model.py
index 7739630237d32..b3d5a8a6e225c 100644
--- a/pl_examples/bug_report/bug_report_model.py
+++ b/pl_examples/bug_report/bug_report_model.py
@@ -1,5 +1,3 @@
-import os
-
 import torch
 from torch.utils.data import DataLoader, Dataset
 
@@ -35,32 +33,23 @@ def validation_step(self, batch, batch_idx):
         loss = self(batch).sum()
         self.log("valid_loss", loss)
 
-    def test_step(self, batch, batch_idx):
-        loss = self(batch).sum()
-        self.log("test_loss", loss)
-
     def configure_optimizers(self):
         return torch.optim.SGD(self.layer.parameters(), lr=0.1)
 
+    def train_dataloader(self):
+        return DataLoader(RandomDataset(32, 64))
 
-def run():
-    train_data = DataLoader(RandomDataset(32, 64), batch_size=2)
-    val_data = DataLoader(RandomDataset(32, 64), batch_size=2)
-    test_data = DataLoader(RandomDataset(32, 64), batch_size=2)
+    def val_dataloader(self):
+        return DataLoader(RandomDataset(32, 64))
 
+
+def test_something(tmpdir):
     model = BoringModel()
     trainer = Trainer(
-        default_root_dir=os.getcwd(),
+        default_root_dir=tmpdir,
         limit_train_batches=1,
         limit_val_batches=1,
-        limit_test_batches=1,
-        num_sanity_val_steps=0,
         max_epochs=1,
         enable_model_summary=False,
     )
-    trainer.fit(model, train_dataloaders=train_data, val_dataloaders=val_data)
-    trainer.test(model, dataloaders=test_data)
-
-
-if __name__ == "__main__":
-    run()
+    trainer.fit(model)

From 30e1d880447be2bd895e5c1ba0bfe829adc7929f Mon Sep 17 00:00:00 2001
From: Carlos Mocholi <carlossmocholi@gmail.com>
Date: Thu, 6 Jan 2022 16:35:32 +0100
Subject: [PATCH 51/65] Clean unused code

---
 .github/workflows/ci_test-full.yml            | 1 -
 tests/callbacks/test_callback_hook_outputs.py | 4 ----
 2 files changed, 5 deletions(-)

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index 159ac7dc74530..1ceb73690c91c 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -98,7 +98,6 @@ jobs:
 
     - name: Tests - ./tests/
       run: |
-        # test if it's unrelated to pytest
         cp pl_examples/bug_report/bug_report_model.py .
         python -m pytest bug_report_model.py
 
diff --git a/tests/callbacks/test_callback_hook_outputs.py b/tests/callbacks/test_callback_hook_outputs.py
index 7f2ec5c2e01a4..f7c9321cd0e2e 100644
--- a/tests/callbacks/test_callback_hook_outputs.py
+++ b/tests/callbacks/test_callback_hook_outputs.py
@@ -78,7 +78,3 @@ def on_epoch_end(self, trainer, pl_module):
     )
 
     trainer.fit(model)
-
-
-def test_foo():
-    assert True

From 2cb5f914ff50a812003d71d52dd3d48ded3223c3 Mon Sep 17 00:00:00 2001
From: Carlos Mocholi <carlossmocholi@gmail.com>
Date: Thu, 6 Jan 2022 16:38:17 +0100
Subject: [PATCH 52/65] Try SSH

---
 .github/workflows/ci_test-full.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index 1ceb73690c91c..734ede96f2847 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -25,8 +25,6 @@ jobs:
           # nightly: add when there's a release candidate
           #- {os: ubuntu-20.04, python-version: "3.10", requires: "latest", release: "pre"}
 
-    timeout-minutes: 40
-
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}
@@ -104,6 +102,10 @@ jobs:
         # we know this fails
         python -m pytest tests/callbacks/test_callback_hook_outputs.py::test_free_memory_on_eval_outputs -v
 
+    - name: Setup tmate session
+      if: ${{ failure() }}
+      uses: mxschmitt/action-tmate@v3
+
     # - name: Examples
     #   run: |
     #     python -m pytest pl_examples -v --durations=10

From a85861afe857a706dd182d2a6b08bf11ffa0874f Mon Sep 17 00:00:00 2001
From: Carlos Mocholi <carlossmocholi@gmail.com>
Date: Thu, 6 Jan 2022 16:39:57 +0100
Subject: [PATCH 53/65] Disable checkpointing

---
 pl_examples/bug_report/bug_report_model.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pl_examples/bug_report/bug_report_model.py b/pl_examples/bug_report/bug_report_model.py
index b3d5a8a6e225c..b9ac254749ced 100644
--- a/pl_examples/bug_report/bug_report_model.py
+++ b/pl_examples/bug_report/bug_report_model.py
@@ -51,5 +51,6 @@ def test_something(tmpdir):
         limit_val_batches=1,
         max_epochs=1,
         enable_model_summary=False,
+        enable_checkpointing=False,
     )
     trainer.fit(model)

From 34a16f2ecbebf5ba801048e12678a85e2531c349 Mon Sep 17 00:00:00 2001
From: Carlos Mocholi <carlossmocholi@gmail.com>
Date: Thu, 6 Jan 2022 17:26:56 +0100
Subject: [PATCH 54/65] Narrow down

---
 .github/workflows/ci_test-full.yml            | 12 +----------
 tests/callbacks/test_callback_hook_outputs.py | 20 +++++++++++++++++++
 2 files changed, 21 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index 734ede96f2847..061b23f591e76 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -88,19 +88,9 @@ jobs:
         pip list
       shell: bash
 
-    - name: Cache datasets
-      uses: actions/cache@v2
-      with:
-        path: Datasets
-        key: pl-dataset
-
     - name: Tests - ./tests/
       run: |
-        cp pl_examples/bug_report/bug_report_model.py .
-        python -m pytest bug_report_model.py
-
-        # we know this fails
-        python -m pytest tests/callbacks/test_callback_hook_outputs.py::test_free_memory_on_eval_outputs -v
+        python -m pytest tests/callbacks/test_callback_hook_outputs.py::test_foo -v
 
     - name: Setup tmate session
       if: ${{ failure() }}
diff --git a/tests/callbacks/test_callback_hook_outputs.py b/tests/callbacks/test_callback_hook_outputs.py
index f7c9321cd0e2e..60f699bd5a5ac 100644
--- a/tests/callbacks/test_callback_hook_outputs.py
+++ b/tests/callbacks/test_callback_hook_outputs.py
@@ -78,3 +78,23 @@ def on_epoch_end(self, trainer, pl_module):
     )
 
     trainer.fit(model)
+
+
+class SegfaultModel(BoringModel):
+    def backward(self, loss, optimizer, optimizer_idx, *args, **kwargs):
+        # this is causing the segfault, no segfault if commented
+        loss.backward(*args, **kwargs)
+
+
+def test_foo():
+    model = SegfaultModel()
+    trainer = Trainer(
+        limit_train_batches=1,
+        limit_val_batches=1,
+        max_epochs=1,
+        enable_model_summary=False,
+        enable_progress_bar=False,
+        enable_checkpointing=False,
+        logger=False,
+    )
+    trainer.fit(model)

From fe84e8dc931319e93e2d0b9747d962b9daa79fbc Mon Sep 17 00:00:00 2001
From: Akihiro Nitta <nitta@akihironitta.com>
Date: Fri, 7 Jan 2022 17:03:56 +0900
Subject: [PATCH 55/65] empty commit for rerun ci


From 0560e7760a93b1ccb8c15a7f5223813a569003b1 Mon Sep 17 00:00:00 2001
From: Akihiro Nitta <nitta@akihironitta.com>
Date: Fri, 7 Jan 2022 17:42:35 +0900
Subject: [PATCH 56/65] Revert "use ubuntu20.04"

This reverts commit cdc1bb1a6874ce0106766f8a862715b8b6b90564.
---
 .github/workflows/ci_test-full.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index 061b23f591e76..cc9feba5eed04 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -17,7 +17,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-20.04]
+        os: [ubuntu-18.04]
         python-version: ["3.7", "3.9"]  # minimum, maximum
         requires: ["oldest", "latest"]
         release: ["stable"]

From 9ad12b857ec99abc9a2042a060dd7bd27164469a Mon Sep 17 00:00:00 2001
From: Akihiro Nitta <nitta@akihironitta.com>
Date: Fri, 7 Jan 2022 17:49:33 +0900
Subject: [PATCH 57/65] Undo unrelated changes

---
 .github/workflows/ci_test-full.yml | 121 +++++++++++++++++++----------
 1 file changed, 80 insertions(+), 41 deletions(-)

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index cc9feba5eed04..343aa35cba52d 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -25,6 +25,8 @@ jobs:
           # nightly: add when there's a release candidate
           #- {os: ubuntu-20.04, python-version: "3.10", requires: "latest", release: "pre"}
 
+    timeout-minutes: 40
+
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}
@@ -48,18 +50,12 @@ jobs:
       if: runner.os == 'macOS'
       run: |
         brew install openmpi libuv  # Horovod on macOS requires OpenMPI, Gloo not currently supported
-        brew install gdb
 
     - name: Setup Windows
       if: runner.os == 'windows'
       run: |
         python .github/prune-packages.py requirements/extra.txt "horovod"
 
-    - name: Setup Ubuntu
-      if: runner.os == 'Linux'
-      run: |
-        sudo apt install gdb
-
     - name: Set min. dependencies
       if: matrix.requires == 'oldest'
       run: |
@@ -80,47 +76,90 @@ jobs:
         restore-keys: |
           ${{ runner.os }}-pip-td${{ steps.times.outputs.period }}-py${{ matrix.python-version }}-${{ matrix.release }}-${{ matrix.requires }}-
 
+    - name: Pull checkpoints from S3
+      working-directory: ./legacy
+      run: |
+        # wget is simpler but does not work on Windows
+        python -c "from urllib.request import urlretrieve ; urlretrieve('https://pl-public-data.s3.amazonaws.com/legacy/checkpoints.zip', 'checkpoints.zip')"
+        ls -l .
+        unzip -o checkpoints.zip
+        ls -l checkpoints/
+
     - name: Install dependencies
       run: |
         flag=$(python -c "print('--pre' if '${{matrix.release}}' == 'pre' else '')" 2>&1)
         pip install --requirement requirements.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --upgrade $flag
+        # adjust versions according installed Torch version
+        python ./requirements/adjust_versions.py requirements/examples.txt
+        pip install --requirement requirements/examples.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --upgrade
         pip install --requirement requirements/test.txt --upgrade
         pip list
       shell: bash
 
-    - name: Tests - ./tests/
+    - name: Install extra dependencies
+      env:
+        HOROVOD_BUILD_ARCH_FLAGS: "-mfma"
+        HOROVOD_WITHOUT_MXNET: 1
+        HOROVOD_WITHOUT_TENSORFLOW: 1
+      run: |
+        # adjust versions according installed Torch version
+        python ./requirements/adjust_versions.py requirements/extra.txt
+        pip install --requirement ./requirements/extra.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --upgrade
+        pip list
+      shell: bash
+
+    - name: Reinstall Horovod if necessary
+      if: runner.os != 'windows'
+      env:
+        HOROVOD_BUILD_ARCH_FLAGS: "-mfma"
+        HOROVOD_WITHOUT_MXNET: 1
+        HOROVOD_WITHOUT_TENSORFLOW: 1
+      run: |
+        HOROVOD_BUILT=$(python -c "import horovod.torch; horovod.torch.nccl_built(); print('SUCCESS')" || true)
+        if [[ $HOROVOD_BUILT != "SUCCESS" ]]; then
+          pip uninstall -y horovod
+          echo $(grep "horovod" requirements/extra.txt) > requirements/horovod.txt
+          pip install --no-cache-dir -r requirements/horovod.txt
+        fi
+        horovodrun --check-build
+      shell: bash
+
+    - name: Cache datasets
+      uses: actions/cache@v2
+      with:
+        path: Datasets
+        key: pl-dataset
+
+    - name: Tests
+      run: |
+        # NOTE: do not include coverage report here, see: https://github.com/nedbat/coveragepy/issues/1003
+        coverage run --source pytorch_lightning -m pytest pytorch_lightning tests -v --durations=50 --junitxml=junit/test-results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml
+
+    - name: Examples
       run: |
-        python -m pytest tests/callbacks/test_callback_hook_outputs.py::test_foo -v
-
-    - name: Setup tmate session
-      if: ${{ failure() }}
-      uses: mxschmitt/action-tmate@v3
-
-    # - name: Examples
-    #   run: |
-    #     python -m pytest pl_examples -v --durations=10
-
-    # - name: Upload pytest results
-    #   uses: actions/upload-artifact@v2
-    #   with:
-    #     name: pytest-results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}
-    #     path: junit/test-results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml
-    #   if: failure()
-
-    # - name: Statistics
-    #   if: success()
-    #   run: |
-    #     coverage report
-    #     coverage xml
-
-    # - name: Upload coverage to Codecov
-    #   uses: codecov/codecov-action@v1
-    #   if: always()
-    #   # see: https://github.com/actions/toolkit/issues/399
-    #   continue-on-error: true
-    #   with:
-    #     token: ${{ secrets.CODECOV_TOKEN }}
-    #     file: coverage.xml
-    #     flags: cpu,pytest,python${{ matrix.python-version }}
-    #     name: CPU-coverage
-    #     fail_ci_if_error: false
+        python -m pytest pl_examples -v --durations=10
+
+    - name: Upload pytest results
+      uses: actions/upload-artifact@v2
+      with:
+        name: pytest-results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}
+        path: junit/test-results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml
+      if: failure()
+
+    - name: Statistics
+      if: success()
+      run: |
+        coverage report
+        coverage xml
+
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v1
+      if: always()
+      # see: https://github.com/actions/toolkit/issues/399
+      continue-on-error: true
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        file: coverage.xml
+        flags: cpu,pytest,python${{ matrix.python-version }}
+        name: CPU-coverage
+        fail_ci_if_error: false

From 42aa3285098d512133a606e55c588e2064df455b Mon Sep 17 00:00:00 2001
From: Akihiro Nitta <nitta@akihironitta.com>
Date: Fri, 7 Jan 2022 17:51:30 +0900
Subject: [PATCH 58/65] Revert "pytest --color=no for visibility in raw log"

This reverts commit 871ae6bfb52f62f3e64d96a365aa5da43208f9d7.
---
 setup.cfg | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.cfg b/setup.cfg
index 77d7fcbd18924..ca6c0dc17b7e6 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -22,7 +22,7 @@ norecursedirs =
 addopts =
     --strict-markers
     --doctest-modules
-    --color=no
+    --color=yes
     --disable-pytest-warnings
 filterwarnings =
     # error out on our deprecation warnings - ensures the code and tests are kept up-to-date

From a71ee3244838c572e457876399e1a64e84b8a69d Mon Sep 17 00:00:00 2001
From: Akihiro Nitta <nitta@akihironitta.com>
Date: Fri, 7 Jan 2022 17:53:23 +0900
Subject: [PATCH 59/65] Undo unrelated change - set-min-requirements.py

---
 .github/set-min-requirements.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/set-min-requirements.py b/.github/set-min-requirements.py
index 5562b66c4a50e..b67ba224662ab 100644
--- a/.github/set-min-requirements.py
+++ b/.github/set-min-requirements.py
@@ -2,7 +2,7 @@
     "requirements.txt",
     "requirements/extra.txt",
     "requirements/loggers.txt",
-    # "requirements/test.txt",  # don't use old versions of testing packages
+    "requirements/test.txt",
     "requirements/examples.txt",
 )
 

From beafe533385a0186ab722507b40ef57b4bb4c868 Mon Sep 17 00:00:00 2001
From: Akihiro Nitta <nitta@akihironitta.com>
Date: Fri, 7 Jan 2022 18:17:23 +0900
Subject: [PATCH 60/65] Revert "Fix rich bug"

This reverts commit adc4a597ee3758059423efcf2de685557470a1ff.
---
 pytorch_lightning/callbacks/progress/rich_progress.py | 3 ++-
 pytorch_lightning/callbacks/rich_model_summary.py     | 2 +-
 tests/callbacks/test_rich_model_summary.py            | 2 +-
 tests/callbacks/test_rich_progress_bar.py             | 2 +-
 4 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/pytorch_lightning/callbacks/progress/rich_progress.py b/pytorch_lightning/callbacks/progress/rich_progress.py
index 7306c36aec7b4..f983ccdaab12e 100644
--- a/pytorch_lightning/callbacks/progress/rich_progress.py
+++ b/pytorch_lightning/callbacks/progress/rich_progress.py
@@ -17,6 +17,7 @@
 from typing import Any, Dict, Optional, Union
 
 from pytorch_lightning.callbacks.progress.base import ProgressBarBase
+from pytorch_lightning.utilities.exceptions import MisconfigurationException
 from pytorch_lightning.utilities.imports import _RICH_AVAILABLE
 
 Task, Style = None, None
@@ -230,7 +231,7 @@ def __init__(
         console_kwargs: Optional[Dict[str, Any]] = None,
     ) -> None:
         if not _RICH_AVAILABLE:
-            raise ModuleNotFoundError(
+            raise MisconfigurationException(
                 "`RichProgressBar` requires `rich` >= 10.2.2. Install it by running `pip install -U rich`."
             )
 
diff --git a/pytorch_lightning/callbacks/rich_model_summary.py b/pytorch_lightning/callbacks/rich_model_summary.py
index 14c078a273ece..cce4eb316a3b0 100644
--- a/pytorch_lightning/callbacks/rich_model_summary.py
+++ b/pytorch_lightning/callbacks/rich_model_summary.py
@@ -61,7 +61,7 @@ class RichModelSummary(ModelSummary):
     def __init__(self, max_depth: int = 1) -> None:
         if not _RICH_AVAILABLE:
             raise ModuleNotFoundError(
-                "`RichModelSummary` requires `rich` to be installed. Install it by running `pip install -U rich`."
+                "`RichProgressBar` requires `rich` to be installed. Install it by running `pip install -U rich`."
             )
         super().__init__(max_depth)
 
diff --git a/tests/callbacks/test_rich_model_summary.py b/tests/callbacks/test_rich_model_summary.py
index 88c5f9ab531f0..5ab091bd01445 100644
--- a/tests/callbacks/test_rich_model_summary.py
+++ b/tests/callbacks/test_rich_model_summary.py
@@ -35,7 +35,7 @@ def test_rich_model_summary_callback():
 
 def test_rich_progress_bar_import_error():
     if not _RICH_AVAILABLE:
-        with pytest.raises(ModuleNotFoundError, match="`RichModelSummary` requires `rich` to be installed."):
+        with pytest.raises(ImportError, match="`RichModelSummary` requires `rich` to be installed."):
             Trainer(callbacks=RichModelSummary())
 
 
diff --git a/tests/callbacks/test_rich_progress_bar.py b/tests/callbacks/test_rich_progress_bar.py
index d9a9ad8c3c726..f2e75006f7ecb 100644
--- a/tests/callbacks/test_rich_progress_bar.py
+++ b/tests/callbacks/test_rich_progress_bar.py
@@ -85,7 +85,7 @@ def predict_dataloader(self):
 
 def test_rich_progress_bar_import_error():
     if not _RICH_AVAILABLE:
-        with pytest.raises(ModuleNotFoundError, match="`RichProgressBar` requires `rich` >= 10.2.2."):
+        with pytest.raises(ImportError, match="`RichProgressBar` requires `rich` >= 10.2.2."):
             Trainer(callbacks=RichProgressBar())
 
 

From bc569bcdbd8265bba6ebec250f159df1f9b58270 Mon Sep 17 00:00:00 2001
From: Akihiro Nitta <nitta@akihironitta.com>
Date: Fri, 7 Jan 2022 18:23:50 +0900
Subject: [PATCH 61/65] revert bug_report_model

---
 pl_examples/bug_report/bug_report_model.py | 28 +++++++++++++++-------
 1 file changed, 19 insertions(+), 9 deletions(-)

diff --git a/pl_examples/bug_report/bug_report_model.py b/pl_examples/bug_report/bug_report_model.py
index b9ac254749ced..7739630237d32 100644
--- a/pl_examples/bug_report/bug_report_model.py
+++ b/pl_examples/bug_report/bug_report_model.py
@@ -1,3 +1,5 @@
+import os
+
 import torch
 from torch.utils.data import DataLoader, Dataset
 
@@ -33,24 +35,32 @@ def validation_step(self, batch, batch_idx):
         loss = self(batch).sum()
         self.log("valid_loss", loss)
 
+    def test_step(self, batch, batch_idx):
+        loss = self(batch).sum()
+        self.log("test_loss", loss)
+
     def configure_optimizers(self):
         return torch.optim.SGD(self.layer.parameters(), lr=0.1)
 
-    def train_dataloader(self):
-        return DataLoader(RandomDataset(32, 64))
 
-    def val_dataloader(self):
-        return DataLoader(RandomDataset(32, 64))
+def run():
+    train_data = DataLoader(RandomDataset(32, 64), batch_size=2)
+    val_data = DataLoader(RandomDataset(32, 64), batch_size=2)
+    test_data = DataLoader(RandomDataset(32, 64), batch_size=2)
 
-
-def test_something(tmpdir):
     model = BoringModel()
     trainer = Trainer(
-        default_root_dir=tmpdir,
+        default_root_dir=os.getcwd(),
         limit_train_batches=1,
         limit_val_batches=1,
+        limit_test_batches=1,
+        num_sanity_val_steps=0,
         max_epochs=1,
         enable_model_summary=False,
-        enable_checkpointing=False,
     )
-    trainer.fit(model)
+    trainer.fit(model, train_dataloaders=train_data, val_dataloaders=val_data)
+    trainer.test(model, dataloaders=test_data)
+
+
+if __name__ == "__main__":
+    run()

From 5e2508ce7c863c57fbf03ebe73db111e02112131 Mon Sep 17 00:00:00 2001
From: Akihiro Nitta <nitta@akihironitta.com>
Date: Fri, 7 Jan 2022 18:24:34 +0900
Subject: [PATCH 62/65] revert foo test

---
 tests/callbacks/test_callback_hook_outputs.py | 20 -------------------
 1 file changed, 20 deletions(-)

diff --git a/tests/callbacks/test_callback_hook_outputs.py b/tests/callbacks/test_callback_hook_outputs.py
index 60f699bd5a5ac..f7c9321cd0e2e 100644
--- a/tests/callbacks/test_callback_hook_outputs.py
+++ b/tests/callbacks/test_callback_hook_outputs.py
@@ -78,23 +78,3 @@ def on_epoch_end(self, trainer, pl_module):
     )
 
     trainer.fit(model)
-
-
-class SegfaultModel(BoringModel):
-    def backward(self, loss, optimizer, optimizer_idx, *args, **kwargs):
-        # this is causing the segfault, no segfault if commented
-        loss.backward(*args, **kwargs)
-
-
-def test_foo():
-    model = SegfaultModel()
-    trainer = Trainer(
-        limit_train_batches=1,
-        limit_val_batches=1,
-        max_epochs=1,
-        enable_model_summary=False,
-        enable_progress_bar=False,
-        enable_checkpointing=False,
-        logger=False,
-    )
-    trainer.fit(model)

From b8c0bcdc1831b12944472bd62df01cc04c64079d Mon Sep 17 00:00:00 2001
From: Akihiro Nitta <nitta@akihironitta.com>
Date: Fri, 7 Jan 2022 18:35:23 +0900
Subject: [PATCH 63/65] Skip (ubuntu, py3.9, oldest)

---
 .github/workflows/ci_test-full.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index 343aa35cba52d..330613655508a 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -21,6 +21,9 @@ jobs:
         python-version: ["3.7", "3.9"]  # minimum, maximum
         requires: ["oldest", "latest"]
         release: ["stable"]
+        exclude:
+          # Skip if torch<1.8 and py3.9 on Linux: https://github.com/pytorch/pytorch/issues/50014
+          - {os: ubuntu-18.04, python-version: "3.9", requires: "oldest", release: "stable"}
         #include:
           # nightly: add when there's a release candidate
           #- {os: ubuntu-20.04, python-version: "3.10", requires: "latest", release: "pre"}

From e48738df26fbc322edb17f8a02058703e8a83fdd Mon Sep 17 00:00:00 2001
From: Akihiro Nitta <nitta@akihironitta.com>
Date: Fri, 7 Jan 2022 19:21:45 +0900
Subject: [PATCH 64/65] Revert "Temporarily disable all other CI jobs"

This reverts commit e61abc1036e7452842c672b3a515a4c83204cb1d.
---
 .azure-pipelines/gpu-tests.yml       | 1 +
 .azure-pipelines/ipu-tests.yml       | 1 +
 .circleci/config.yml                 | 1 +
 .github/workflows/ci_pkg-install.yml | 2 +-
 .github/workflows/ci_schema.yml      | 2 +-
 .github/workflows/ci_test-base.yml   | 2 +-
 .github/workflows/ci_test-conda.yml  | 2 +-
 .github/workflows/ci_test-slow.yml   | 2 +-
 .github/workflows/code-checks.yml    | 2 +-
 .github/workflows/docs-checks.yml    | 2 +-
 10 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/.azure-pipelines/gpu-tests.yml b/.azure-pipelines/gpu-tests.yml
index 4eacce5d41251..ca8c54a61479e 100644
--- a/.azure-pipelines/gpu-tests.yml
+++ b/.azure-pipelines/gpu-tests.yml
@@ -13,6 +13,7 @@ trigger:
       - "release/*"
       - "refs/tags/*"
 pr:
+  - "master"
   - "release/*"
 
 jobs:
diff --git a/.azure-pipelines/ipu-tests.yml b/.azure-pipelines/ipu-tests.yml
index 2dde23f665f44..1b68a05ce012f 100644
--- a/.azure-pipelines/ipu-tests.yml
+++ b/.azure-pipelines/ipu-tests.yml
@@ -8,6 +8,7 @@ trigger:
       - release/*
       - refs/tags/*
 pr:
+  - master
   - release/*
 
 variables:
diff --git a/.circleci/config.yml b/.circleci/config.yml
index 9f15314354968..48797fab85034 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -15,6 +15,7 @@ trigger:
       - "release/*"
       - "refs/tags/*"
 pr:
+  - "master"
   - "release/*"
 
 # Workflow Steps:
diff --git a/.github/workflows/ci_pkg-install.yml b/.github/workflows/ci_pkg-install.yml
index 0afc1e265ac01..0be8b37188787 100644
--- a/.github/workflows/ci_pkg-install.yml
+++ b/.github/workflows/ci_pkg-install.yml
@@ -5,7 +5,7 @@ on:  # Trigger the workflow on push or pull request, but only for the master bra
   push:
     branches: [master, "release/*"]
   pull_request:
-    branches: ["release/*"]
+    branches: [master, "release/*"]
 
 jobs:
 
diff --git a/.github/workflows/ci_schema.yml b/.github/workflows/ci_schema.yml
index a5fb5952a2eb0..d635285fae39a 100644
--- a/.github/workflows/ci_schema.yml
+++ b/.github/workflows/ci_schema.yml
@@ -2,7 +2,7 @@ name: Schema
 on: # Trigger the workflow on push or pull request, but only for the master branch
   push: {}
   pull_request:
-    branches: ["release/*"]
+    branches: [master, "release/*"]
 
 jobs:
   check:
diff --git a/.github/workflows/ci_test-base.yml b/.github/workflows/ci_test-base.yml
index bd7be230a288a..c2f1d370e2d1a 100644
--- a/.github/workflows/ci_test-base.yml
+++ b/.github/workflows/ci_test-base.yml
@@ -7,7 +7,7 @@ on:  # Trigger the workflow on push or pull request, but only for the master bra
   push:
     branches: [master, "release/*"]
   pull_request:
-    branches: ["release/*"]
+    branches: [master, "release/*"]
 
 jobs:
   source:
diff --git a/.github/workflows/ci_test-conda.yml b/.github/workflows/ci_test-conda.yml
index 18033702478b8..fa366e645f1d9 100644
--- a/.github/workflows/ci_test-conda.yml
+++ b/.github/workflows/ci_test-conda.yml
@@ -5,7 +5,7 @@ on:  # Trigger the workflow on push or pull request, but only for the master bra
   push:
     branches: [master, "release/*"]
   pull_request:
-    branches: ["release/*"]
+    branches: [master, "release/*"]
 
 jobs:
   conda:
diff --git a/.github/workflows/ci_test-slow.yml b/.github/workflows/ci_test-slow.yml
index 8d37f313c3021..b9a07c4b6a26f 100644
--- a/.github/workflows/ci_test-slow.yml
+++ b/.github/workflows/ci_test-slow.yml
@@ -5,7 +5,7 @@ on:  # Trigger the workflow on push or pull request, but only for the master bra
   push:
     branches: [master, "release/*"]
   pull_request:
-    branches: ["release/*"]
+    branches: [master, "release/*"]
 
 jobs:
   slow:
diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml
index 5b8f88490c7b2..8cd4206ab61b7 100644
--- a/.github/workflows/code-checks.yml
+++ b/.github/workflows/code-checks.yml
@@ -4,7 +4,7 @@ on:  # Trigger the workflow on push or pull request, but only for the master bra
   push:
     branches: [master, "release/*"]
   pull_request:
-    branches: ["release/*"]
+    branches: [master, "release/*"]
 
 jobs:
   mypy:
diff --git a/.github/workflows/docs-checks.yml b/.github/workflows/docs-checks.yml
index e7a9579966ba5..841f9128da8b1 100644
--- a/.github/workflows/docs-checks.yml
+++ b/.github/workflows/docs-checks.yml
@@ -5,7 +5,7 @@ on:  # Trigger the workflow on push or pull request, but only for the master bra
   push:
     branches: [master, "release/*"]
   pull_request:
-    branches: ["release/*"]
+    branches: [master, "release/*"]
 
 jobs:
   doctest:

From fc8f3c3f60b44f435b141e3baf38102bca98383c Mon Sep 17 00:00:00 2001
From: Akihiro Nitta <nitta@akihironitta.com>
Date: Fri, 7 Jan 2022 19:22:29 +0900
Subject: [PATCH 65/65] Enable testing on mac and win

---
 .github/workflows/ci_test-full.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
index 330613655508a..6cd4108c7fc36 100644
--- a/.github/workflows/ci_test-full.yml
+++ b/.github/workflows/ci_test-full.yml
@@ -17,7 +17,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-18.04]
+        os: [ubuntu-18.04, windows-2019, macOS-10.15]
         python-version: ["3.7", "3.9"]  # minimum, maximum
         requires: ["oldest", "latest"]
         release: ["stable"]