Relax app dependencies for lightning | breaking imports from L (#18386)

* make App conditional import

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* ci: min req.

* data + store

* typo

* module_available

* extra

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* typo

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* envs

* doctest

* more

* docs

* **

* rm

* -f

* Apply suggestions from code review

* docs

* Revert "docs"

This reverts commit 20e8e6b.

* Apply suggestions from code review

* warnings

* breaking

* ...

* fixing source

* fixing examples

* fixing tests

* fixing docs

* fixing source

* fix mock

* fix trainer

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: Borda

.actions/assistant.py

@@ -36,9 +36,9 @@
         "requirements/pytorch/examples.txt",
     ),
     "app": (
-        "requirements/app/base.txt",
-        "requirements/app/ui.txt",
+        "requirements/app/app.txt",
         "requirements/app/cloud.txt",
+        "requirements/app/ui.txt",
     ),
     "fabric": (
         "requirements/fabric/base.txt",

.github/actions/pkg-install/action.yml

@@ -8,6 +8,10 @@ inputs:
   pkg-name:
     description: Package name to import
     required: true
+  pkg-extra:
+    description: argument for install extra
+    required: false
+    default: ""
   pip-flags:
     description: Additional pip install flags
     required: false
@@ -19,24 +23,32 @@ runs:
   - name: Choose package import
     working-directory: ${{ inputs.pkg-folder }}
     run: |
-      ls -l
-      python -c "print('PKG_IMPORT=' + {'app': 'lightning_app', 'fabric': 'lightning_fabric', 'pytorch': 'pytorch_lightning'}.get('${{matrix.pkg-name}}', 'lightning'))" >> $GITHUB_ENV
-      python -c "import glob ; ls = glob.glob('*.tar.gz') ; print('PKG_SOURCE=' + ls[0])" >> $GITHUB_ENV
-      python -c "import glob ; ls = glob.glob('*.whl') ; print('PKG_WHEEL=' + ls[0])" >> $GITHUB_ENV
-    shell: bash
+      import os, glob
+
+      lut = {'app': 'lightning_app', 'fabric': 'lightning_fabric', 'pytorch': 'pytorch_lightning'}
+      act_pkg = lut.get('${{inputs.pkg-name}}', 'lightning')
+      pkg_sdist = glob.glob('*.tar.gz')[0]
+      pkg_wheel = glob.glob('*.whl')[0]
+      extra = '${{inputs.pkg-extra}}'
+      extra = f'[{extra}]' if extra else ''
+
+      envs = [f'PKG_IMPORT={act_pkg}', f'PKG_SOURCE={pkg_sdist}', f'PKG_WHEEL={pkg_wheel}', f'PKG_EXTRA={extra}']
+      with open(os.getenv('GITHUB_ENV'), "a") as gh_env:
+          gh_env.write(os.linesep.join(envs))
+    shell: python
 
   - name: Install package - wheel
     working-directory: ${{ inputs.pkg-folder }}
     run: |
-      pip install ${PKG_WHEEL} ${{ inputs.pip-flags }}
+      pip install "${PKG_WHEEL}${PKG_EXTRA}" ${{ inputs.pip-flags }}
       pip list | grep lightning
       python -c "import ${{ env.PKG_IMPORT }}; print(${{ env.PKG_IMPORT }}.__version__)"
     shell: bash
 
   - name: Install package - archive
     working-directory: ${{ inputs.pkg-folder }}
     run: |
-      pip install ${PKG_SOURCE} ${{ inputs.pip-flags }}
+      pip install "${PKG_SOURCE}${PKG_EXTRA}" ${{ inputs.pip-flags }}
       pip list | grep lightning
       python -c "import ${{ env.PKG_IMPORT }}; print(${{ env.PKG_IMPORT }}.__version__)"
     shell: bash

.github/workflows/ci-pkg-install.yml

@@ -42,10 +42,15 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-22.04, macOS-12, windows-2022]
+        os: ["ubuntu-22.04", "macOS-12", "windows-2022"]
         pkg-name: ["app", "fabric", "pytorch", "lightning", "notset"]
         python-version: ["3.8" , "3.11"]
         # TODO: add also install from source
+        include:
+          - {os: "macOS-12", pkg-name: "lightning", python-version: "3.9", pkg-extra: "app"}
+          - {os: "macOS-12", pkg-name: "notset", python-version: "3.9", pkg-extra: "app"}
+          - {os: "ubuntu-22.04", pkg-name: "lightning", python-version: "3.9", pkg-extra: "app"}
+          - {os: "ubuntu-22.04", pkg-name: "notset", python-version: "3.9", pkg-extra: "app"}
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
@@ -56,19 +61,23 @@ jobs:
           name: dist-packages-${{ github.sha }}
           path: dist
 
-      - run: |
+      - name: Set package dir
+        run: |
           python -c "print('PKG_DIR=' + {'notset': 'lightning'}.get('${{matrix.pkg-name}}', '${{matrix.pkg-name}}'))" >> $GITHUB_ENV
       - name: Install package - wheel & archive
         uses: ./.github/actions/pkg-install
         with:
           pkg-folder: dist/${{ env.PKG_DIR }}
           pkg-name: ${{ matrix.pkg-name }}
+          pkg-extra: ${{ matrix.pkg-extra }}
 
       - name: Run CLI (via python)
-        if: ${{ matrix.pkg-name == 'lightning' || matrix.pkg-name == 'notset' }}
+        if: ${{ (matrix.pkg-name == 'lightning' || matrix.pkg-name == 'notset') && matrix.pkg-extra == 'app' }}
         run: python -m lightning --version
       - name: Run CLI (direct bash)
-        if: ${{ matrix.pkg-name == 'lightning' || matrix.pkg-name == 'notset' || matrix.pkg-name == 'app' }}
+        if: |
+          ((matrix.pkg-name == 'lightning' || matrix.pkg-name == 'notset') && matrix.pkg-extra == 'app') ||
+          matrix.pkg-name == 'app'
         run: lightning --version
 
       - name: DocTests actions
@@ -77,25 +86,33 @@ jobs:
           pip install -q pytest -r requirements.txt
           python -m pytest assistant.py
 
-      - name: build source
-        run: python setup.py build
       - name: Adjust code for standalone
         if: contains(fromJSON('["app", "fabric", "pytorch"]'), matrix.pkg-name)
         run: |
           python .actions/assistant.py copy_replace_imports --source_dir="./src" \
             --source_import="lightning.pytorch,lightning.fabric,lightning.app" \
             --target_import="pytorch_lightning,lightning_fabric,lightning_app"
-          rm src/lightning/__*.py
       - name: Rename src folders
         working-directory: src/
-        run: mv lightning lit
+        run: |
+          python -c "n = '${{matrix.pkg-name}}' ; n = n if n in ('app', 'fabric', 'pytorch') else '' ; print('PKG_NAME=' + n)" >> $GITHUB_ENV
+          rm -f ./*/__*.py
+          rm -f ./**/__*.py
+          mv lightning lit  # rename lightning folder to prevent accidental local imports
+      - name: drop App doctest
+        if: ${{ (matrix.pkg-name == 'lightning' || matrix.pkg-name == 'notset') && matrix.pkg-extra != 'app' }}
+        working-directory: src/lit
+        run: |
+          rm -rf app
+          rm -rf data
+          rm -rf store
+      - name: Install pytest doctest extension
+        run: |
+          pip install -q "pytest-doctestplus>=0.9.0"
+          pip list
+
       - name: DocTest package
         env:
           LIGHTING_TESTING: 1  # path for require wrapper
           PY_IGNORE_IMPORTMISMATCH: 1
-        run: |
-          pip install -q "pytest-doctestplus>=0.9.0"
-          pip list
-          PKG_NAME=$(python -c "n = '${{matrix.pkg-name}}'; print(n if n in ('app', 'fabric', 'pytorch') else '')")
-          echo $PKG_NAME
-          python -m pytest src/lit/${PKG_NAME} --ignore-glob="**/cli/*-template/**" --doctest-plus
+        run: python -m pytest src/lit/${PKG_NAME} --ignore-glob="**/cli/*-template/**" --doctest-plus

.github/workflows/ci-tests-store.yml

@@ -68,7 +68,7 @@ jobs:
     - name: Install package & dependencies
       run: |
         python -m pip install -q pip -U
-        pip install -e ".[store-test]" "pytest-timeout" -U -f ${TORCH_URL} --prefer-binary
+        pip install -e ".[store,store-test]" "pytest-timeout" -U -f ${TORCH_URL} --prefer-binary
         pip list
 
     - name: Testing Store

.github/workflows/docs-build.yml

@@ -33,6 +33,7 @@ env:
   FREEZE_REQUIREMENTS: "1"
   TORCH_URL: "https://download.pytorch.org/whl/cpu/torch_stable.html"
   PYPI_CACHE_DIR: "_pip-wheels"
+  PYPI_LOCAL_DIR: "pypi_pkgs/"
 
 jobs:
   docs-checks:
@@ -63,12 +64,12 @@ jobs:
           unzip -qq awscli.zip
           bash ./aws/install
           aws --version
-      - run: aws s3 sync s3://sphinx-packages/ pypi_pkgs/
+      - run: aws s3 sync s3://sphinx-packages/ ${PYPI_LOCAL_DIR}
+        if: ${{ matrix.pkg-name != 'pytorch' }}
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY }}
           AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
-        if: ${{ matrix.pkg-name != 'pytorch' }}
 
       - name: pip wheels cache
         uses: actions/cache/restore@v3
@@ -83,11 +84,11 @@ jobs:
           sudo apt-get install -y pandoc texlive-latex-extra dvipng texlive-pictures
       - name: Install package & dependencies
         run: |
-          mkdir -p $PYPI_CACHE_DIR  # in case cache was not hit
-          ls -lh $PYPI_CACHE_DIR
-          mkdir -p pypi_pkgs  # in case template is not pulled
-          pip install .[extra,cloud,ui] -U -r requirements/${{ matrix.pkg-name }}/docs.txt \
-            -f pypi_pkgs/ -f $PYPI_CACHE_DIR -f ${TORCH_URL}
+          mkdir -p ${PYPI_CACHE_DIR}  # in case cache was not hit
+          ls -lh ${PYPI_CACHE_DIR}
+          mkdir -p ${PYPI_LOCAL_DIR}  # in case template is not pulled
+          pip install .[app,extra,cloud,ui] -U -r requirements/${{ matrix.pkg-name }}/docs.txt \
+            -f ${PYPI_LOCAL_DIR} -f ${PYPI_CACHE_DIR} -f ${TORCH_URL}
           pip list
 
       - name: Test Documentation
@@ -128,12 +129,12 @@ jobs:
           unzip -qq awscli.zip
           bash ./aws/install
           aws --version
-      - run: aws s3 sync s3://sphinx-packages/ pypi_pkgs/
+      - run: aws s3 sync s3://sphinx-packages/ ${PYPI_LOCAL_DIR}
+        if: ${{ matrix.pkg-name != 'pytorch' }}
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY }}
           AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
-        if: ${{ matrix.pkg-name != 'pytorch' }}
 
       - name: pip wheels cache
         uses: actions/cache/restore@v3
@@ -148,12 +149,12 @@ jobs:
           sudo apt-get install -y pandoc texlive-latex-extra dvipng texlive-pictures
       - name: Install package & dependencies
         run: |
-          mkdir -p $PYPI_CACHE_DIR  # in case cache was not hit
-          ls -lh $PYPI_CACHE_DIR
-          mkdir -p pypi_pkgs  # in case template is not pulled
+          mkdir -p ${PYPI_CACHE_DIR}  # in case cache was not hit
+          ls -lh ${PYPI_CACHE_DIR}
+          mkdir -p ${PYPI_LOCAL_DIR}  # in case template is not pulled
           pip --version
-          pip install . -U -r requirements/${{ matrix.pkg-name }}/docs.txt \
-            -f pypi_pkgs/ -f $PYPI_CACHE_DIR -f ${TORCH_URL}
+          pip install .[app] -U -r requirements/${{ matrix.pkg-name }}/docs.txt \
+            -f ${PYPI_LOCAL_DIR} -f ${PYPI_CACHE_DIR} -f ${TORCH_URL}
           pip list
         shell: bash
 

docs/source-app/code_samples/basics/0.py

@@ -1,7 +1,7 @@
-import lightning as L
+from lightning.app import LightningWork
 
 
-class ExampleWork(L.LightningWork):
+class ExampleWork(LightningWork):
     def run(self, *args, **kwargs):
         print(f"I received the following props: args: {args} kwargs: {kwargs}")
 

docs/source-app/code_samples/basics/1.py

@@ -1,7 +1,7 @@
-import lightning as L
+from lightning.app import LightningWork
 
 
-class ExampleWork(L.LightningWork):
+class ExampleWork(LightningWork):
     def __init__(self):
         super().__init__(cache_calls=False)
 

docs/source-app/code_samples/convert_pl_to_app/app.py

@@ -1,17 +1,17 @@
-import lightning as L
+from lightning.app import LightningFlow, LightningApp, CloudCompute
 from lightning.app.components import TracerPythonScript
 
 
-class RootFlow(L.LightningFlow):
+class RootFlow(LightningFlow):
     def __init__(self):
         super().__init__()
         self.runner = TracerPythonScript(
             "train.py",
-            cloud_compute=L.CloudCompute("gpu"),
+            cloud_compute=CloudCompute("gpu"),
         )
 
     def run(self):
         self.runner.run()
 
 
-app = L.LightningApp(RootFlow())
+app = LightningApp(RootFlow())

docs/source-app/code_samples/quickstart/app/app_0.py

@@ -1,9 +1,9 @@
-import lightning as L
+from lightning.app import LightningWork, LightningFlow, LightningApp
 from docs.quickstart.app_02 import HourLongWork
 
 
-class RootFlow(L.LightningFlow):
-    def __init__(self, child_work_1: L.LightningWork, child_work_2: L.LightningWork):
+class RootFlow(LightningFlow):
+    def __init__(self, child_work_1: LightningWork, child_work_2: LightningWork):
         super().__init__()
         self.child_work_1 = child_work_1
         self.child_work_2 = child_work_2
@@ -18,4 +18,4 @@ def run(self):
             print("1 hour later `child_work_2` started!")
 
 
-app = L.LightningApp(RootFlow(HourLongWork(parallel=True), HourLongWork(parallel=True)))
+app = LightningApp(RootFlow(HourLongWork(parallel=True), HourLongWork(parallel=True)))

docs/source-app/code_samples/quickstart/app/app_1.py

@@ -2,14 +2,14 @@
 from flash.core.data.utils import download_data
 from flash.image import ImageClassificationData, ImageClassifier
 
-import lightning as L
-from pytorch_lightning.callbacks import ModelCheckpoint
+from lightning.app import LightningWork, LightningFlow, LightningApp, CloudCompute
+from lightning.pytorch.callbacks import ModelCheckpoint
 
 
 # Step 1: Create a training LightningWork component that gets a backbone as input
 # and saves the best model and its score
-class ImageClassifierTrainWork(L.LightningWork):
-    def __init__(self, max_epochs: int, backbone: str, cloud_compute: L.CloudCompute):
+class ImageClassifierTrainWork(LightningWork):
+    def __init__(self, max_epochs: int, backbone: str, cloud_compute: CloudCompute):
         # parallel is set to True to run asynchronously
         super().__init__(parallel=True, cloud_compute=cloud_compute)
         # Number of epochs to run
@@ -44,7 +44,7 @@ def run(self, train_folder):
 
 
 # Step 2: Create a serving LightningWork component that gets a model input and serves it
-class ImageClassifierServeWork(L.LightningWork):
+class ImageClassifierServeWork(LightningWork):
     def run(self, best_model_path: str):
         # Load the model from the model path
         model = ImageClassifier.load_from_checkpoint(best_model_path)
@@ -53,7 +53,7 @@ def run(self, best_model_path: str):
 
 # Step 3: Create a root LightningFlow component that gets number of epochs and a path to
 # a dataset as inputs, initialize 2 training components and serves the best model
-class RootFlow(L.LightningFlow):
+class RootFlow(LightningFlow):
     def __init__(self, max_epochs: int, data_dir: str):
         super().__init__()
         self.data_dir = data_dir
@@ -89,4 +89,4 @@ def run(self):
 download_data("https://pl-flash-data.s3.amazonaws.com/hymenoptera_data.zip", "./data")
 
 # Initialize your Lightning app with 5 epochs
-app = L.LightningApp(RootFlow(5, "./data/hymenoptera_data"))
+app = LightningApp(RootFlow(5, "./data/hymenoptera_data"))

docs/source-app/code_samples/quickstart/app_01.py

@@ -1,8 +1,8 @@
-import lightning as L
+from lightning.app import LightningWork, LightningFlow, LightningApp
 from lightning.app.utilities.app_helpers import pretty_state
 
 
-class Work(L.LightningWork):
+class Work(LightningWork):
     def __init__(self):
         super().__init__(cache_calls=False)
         # Attributes are registered automatically in the state.
@@ -13,7 +13,7 @@ def run(self):
         self.counter += 1
 
 
-class Flow(L.LightningFlow):
+class Flow(LightningFlow):
     def __init__(self):
         super().__init__()
         self.w = Work()
@@ -24,4 +24,4 @@ def run(self):
         self.w.run()
 
 
-app = L.LightningApp(Flow())
+app = LightningApp(Flow())

docs/source-app/code_samples/quickstart/app_02.py

@@ -1,10 +1,10 @@
 from time import sleep
 
-import lightning as L
+from lightning.app import LightningWork, LightningFlow, LightningApp
 
 
 # This work takes an hour to run
-class HourLongWork(L.LightningWork):
+class HourLongWork(LightningWork):
     def __init__(self, parallel: bool = False):
         super().__init__(parallel=parallel)
         self.progress = 0.0
@@ -16,8 +16,8 @@ def run(self):
             sleep(1)
 
 
-class RootFlow(L.LightningFlow):
-    def __init__(self, child_work: L.LightningWork):
+class RootFlow(LightningFlow):
+    def __init__(self, child_work: LightningWork):
         super().__init__()
         self.child_work = child_work
 
@@ -29,4 +29,4 @@ def run(self):
             print("1 hour later!")
 
 
-app = L.LightningApp(RootFlow(HourLongWork()))
+app = LightningApp(RootFlow(HourLongWork()))