Merge branch 'master' into refactor/spawn/setup-environment

Author: awaelchli

.actions/assistant.py

@@ -4,7 +4,7 @@
 import os
 import re
 import shutil
-from distutils.version import LooseVersion, StrictVersion
+from distutils.version import LooseVersion
 from importlib.util import module_from_spec, spec_from_file_location
 from itertools import chain
 from pathlib import Path
@@ -16,6 +16,7 @@
 
 import fire
 import pkg_resources
+from packaging.version import parse as version_parse
 
 REQUIREMENT_FILES = {
     "pytorch": (
@@ -30,15 +31,20 @@
 PACKAGE_MAPPING = {"app": "lightning-app", "pytorch": "pytorch-lightning"}
 
 
-def pypi_versions(package_name: str) -> List[str]:
-    """Return a list of released versions of a provided pypi name."""
+def pypi_versions(package_name: str, drop_pre: bool = True) -> List[str]:
+    """Return a list of released versions of a provided pypi name.
+
+    >>> _ = pypi_versions("lightning_app", drop_pre=False)
+    """
     # https://stackoverflow.com/a/27239645/4521646
     url = f"https://pypi.org/pypi/{package_name}/json"
     data = json.load(urlopen(Request(url)))
     versions = list(data["releases"].keys())
     # todo: drop this line after cleaning Pypi history from invalid versions
-    versions = list(filter(lambda v: v.count(".") == 2 and "rc" not in v, versions))
-    versions.sort(key=StrictVersion)
+    versions = list(filter(lambda v: v.count(".") == 2, versions))
+    if drop_pre:
+        versions = list(filter(lambda v: all(c not in v for c in ["rc", "dev"]), versions))
+    versions.sort(key=version_parse)
     return versions
 
 
@@ -122,15 +128,16 @@ def download_package(package: str, folder: str = ".", version: Optional[str] = N
         url = f"https://pypi.org/pypi/{PACKAGE_MAPPING[package]}/json"
         data = json.load(urlopen(Request(url)))
         if not version:
-            versions = list(data["releases"].keys())
-            version = sorted(versions, key=LooseVersion)[-1]
+            pypi_vers = pypi_versions(PACKAGE_MAPPING[package], drop_pre=False)
+            version = pypi_vers[-1]
         releases = list(filter(lambda r: r["packagetype"] == "sdist", data["releases"][version]))
         assert releases, f"Missing 'sdist' for this package/version aka {package}/{version}"
         release = releases[0]
         pkg_url = release["url"]
         pkg_file = os.path.basename(pkg_url)
         pkg_path = os.path.join(folder, pkg_file)
         os.makedirs(folder, exist_ok=True)
+        print(f"downloading: {pkg_url}")
         request.urlretrieve(pkg_url, pkg_path)
 
     @staticmethod

.actions/requirements.txt

@@ -0,0 +1,3 @@
+fire
+packaging
+requests

.actions/setup_tools.py

@@ -21,11 +21,14 @@
 import tempfile
 import urllib.request
 from datetime import datetime
+from distutils.version import LooseVersion
 from importlib.util import module_from_spec, spec_from_file_location
 from itertools import chain, groupby
 from types import ModuleType
 from typing import List
 
+from pkg_resources import parse_requirements
+
 _PROJECT_ROOT = os.path.dirname(os.path.dirname(__file__))
 _PACKAGE_MAPPING = {"pytorch": "pytorch_lightning", "app": "lightning_app"}
 
@@ -42,45 +45,92 @@ def _load_py_module(name: str, location: str) -> ModuleType:
     return py
 
 
+def _augment_requirement(ln: str, comment_char: str = "#", unfreeze: str = "all") -> str:
+    """Adjust the upper version contrains.
+
+    Args:
+        ln: raw line from requirement
+        comment_char: charter marking comment
+        unfreeze: Enum or "all"|"major"|""
+
+    Returns:
+        adjusted requirement
+
+    >>> _augment_requirement("arrow>=1.2.0, <=1.2.2  # anything", unfreeze="")
+    'arrow>=1.2.0, <=1.2.2'
+    >>> _augment_requirement("arrow>=1.2.0, <=1.2.2  # strict", unfreeze="")
+    'arrow>=1.2.0, <=1.2.2  # strict'
+    >>> _augment_requirement("arrow>=1.2.0, <=1.2.2  # my name", unfreeze="all")
+    'arrow>=1.2.0'
+    >>> _augment_requirement("arrow>=1.2.0, <=1.2.2  # strict", unfreeze="all")
+    'arrow>=1.2.0, <=1.2.2  # strict'
+    >>> _augment_requirement("arrow", unfreeze="all")
+    'arrow'
+    >>> _augment_requirement("arrow>=1.2.0, <=1.2.2  # cool", unfreeze="major")
+    'arrow>=1.2.0, <2.0  # strict'
+    >>> _augment_requirement("arrow>=1.2.0, <=1.2.2  # strict", unfreeze="major")
+    'arrow>=1.2.0, <=1.2.2  # strict'
+    >>> _augment_requirement("arrow>=1.2.0", unfreeze="major")
+    'arrow>=1.2.0, <2.0  # strict'
+    >>> _augment_requirement("arrow", unfreeze="major")
+    'arrow'
+    """
+    # filer all comments
+    if comment_char in ln:
+        comment = ln[ln.index(comment_char) :]
+        ln = ln[: ln.index(comment_char)]
+        is_strict = "strict" in comment
+    else:
+        is_strict = False
+    req = ln.strip()
+    # skip directly installed dependencies
+    if not req or req.startswith("http") or "@http" in req:
+        return ""
+    # extract the major version from all listed versions
+    if unfreeze == "major":
+        req_ = list(parse_requirements([req]))[0]
+        vers = [LooseVersion(v) for s, v in req_.specs if s not in ("==", "~=")]
+        ver_major = sorted(vers)[-1].version[0] if vers else None
+    else:
+        ver_major = None
+
+    # remove version restrictions unless they are strict
+    if unfreeze and "<" in req and not is_strict:
+        req = re.sub(r",? *<=? *[\d\.\*]+", "", req).strip()
+    if ver_major is not None and not is_strict:
+        # add , only if there are already some versions
+        req += f"{',' if any(c in req for c in '<=>') else ''} <{int(ver_major) + 1}.0"
+
+    # adding strict back to the comment
+    if is_strict or ver_major is not None:
+        req += "  # strict"
+
+    return req
+
+
 def load_requirements(
-    path_dir: str, file_name: str = "base.txt", comment_char: str = "#", unfreeze: bool = True
+    path_dir: str, file_name: str = "base.txt", comment_char: str = "#", unfreeze: str = "all"
 ) -> List[str]:
     """Loading requirements from a file.
 
     >>> path_req = os.path.join(_PROJECT_ROOT, "requirements")
-    >>> load_requirements(path_req)  # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
-    ['numpy...', 'torch...', ...]
+    >>> load_requirements(path_req, unfreeze="major")  # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
+    ['pytorch_lightning...', 'lightning_app...']
     """
     with open(os.path.join(path_dir, file_name)) as file:
         lines = [ln.strip() for ln in file.readlines()]
     reqs = []
     for ln in lines:
-        # filer all comments
-        comment = ""
-        if comment_char in ln:
-            comment = ln[ln.index(comment_char) :]
-            ln = ln[: ln.index(comment_char)]
-        req = ln.strip()
-        # skip directly installed dependencies
-        if not req or req.startswith("http") or "@http" in req:
-            continue
-        # remove version restrictions unless they are strict
-        if unfreeze and "<" in req and "strict" not in comment:
-            req = re.sub(r",? *<=? *[\d\.\*]+", "", req).strip()
-
-        # adding strict back to the comment
-        if "strict" in comment:
-            req += "  # strict"
-
-        reqs.append(req)
-    return reqs
+        reqs.append(_augment_requirement(ln, comment_char=comment_char, unfreeze=unfreeze))
+    # filter empty lines
+    return [str(req) for req in reqs if req]
 
 
 def load_readme_description(path_dir: str, homepage: str, version: str) -> str:
     """Load readme as decribtion.
 
     >>> load_readme_description(_PROJECT_ROOT, "", "")  # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
-    '<div align="center">...'
+    '...'
     """
     path_readme = os.path.join(path_dir, "README.md")
     text = open(path_readme, encoding="utf-8").read()
@@ -127,7 +177,18 @@ def replace_block_with_imports(lines: List[str], import_path: str, kword: str =
     >>> lines = replace_block_with_imports(lines, import_path, "def")
     """
     body, tracking, skip_offset = [], False, 0
-    for ln in lines:
+    for i, ln in enumerate(lines):
+        # support for defining a class with this condition
+        conditional_class_definitions = ("if TYPE_CHECKING", "if typing.TYPE_CHECKING", "if torch.", "if _TORCH_")
+        if (
+            any(ln.startswith(pattern) for pattern in conditional_class_definitions)
+            # avoid bug in CI for the <1.7 meta code
+            and "pytorch_lightning.utilities.meta" not in import_path
+        ):
+            # dedent the next line
+            lines[i + 1] = lines[i + 1].lstrip()
+            continue
+
         offset = len(ln) - len(ln.lstrip())
         # in case of mating the class args are multi-line
         if tracking and ln and offset <= skip_offset and not any(ln.lstrip().startswith(c) for c in ")]"):
@@ -439,12 +500,14 @@ def _download_frontend(root: str = _PROJECT_ROOT):
         print("The Lightning UI downloading has failed!")
 
 
-def _adjust_require_versions(source_dir: str = "src", req_dir: str = "requirements") -> None:
-    """Parse the base requirements and append  as version adjustments if needed `pkg>=X1.Y1.Z1,==X2.Y2.*`."""
+def _relax_require_versions(source_dir: str = "src", req_dir: str = "requirements") -> None:
+    """Parse the base requirements and append  as version adjustments if needed `pkg>=X1.Y1.Z1,==X2.Y2.*`.
+
+    >>> _relax_require_versions("../src", "../requirements")
+    """
     reqs = load_requirements(req_dir, file_name="base.txt")
-    for i, req in enumerate(reqs):
-        pkg_name = req[: min(req.index(c) for c in ">=" if c in req)]
-        ver_ = parse_version_from_file(os.path.join(source_dir, pkg_name))
+    for i, req in enumerate(parse_requirements(reqs)):
+        ver_ = parse_version_from_file(os.path.join(source_dir, req.name))
         if not ver_:
             continue
         ver2 = ".".join(ver_.split(".")[:2] + ["*"])

.azure/app-cloud-e2e.yml

@@ -12,24 +12,38 @@ trigger:
       - "master"
       - "release/*"
       - "refs/tags/*"
+  paths:
+    include:
+    - ".azure/app-cloud-e2e.yml"
+    - "requirements/app/**"
+    - "src/lightning_app/**"
+    - "examples/app_*"
 
 pr:
-  - "master"
-  - "release/*"
+  branches:
+    include:
+      - "master"
+      - "release/*"
+  paths:
+    include:
+    - ".azure/app-cloud-e2e.yml"
+    - "requirements/app/**"
+    - "src/lightning_app/**"
+    - "examples/app_*"
 
+# variables are automatically exported as environment variables so this will override pip's default cache dir
 variables:
-  # variables are automatically exported as environment variables so this will override pip's default cache dir
   - name: pip_cache_dir
     value: $(Pipeline.Workspace)/.pip
+  - name: local_id
+    value: $(Build.BuildId)
 
 jobs:
   - job: App_cloud_e2e_testing
     pool: azure-cpus
     container:
       image: mcr.microsoft.com/playwright/python:v1.25.2-focal
       options: "--shm-size=2g"
-    timeoutInMinutes: "30"
-    cancelTimeoutInMinutes: "2"
     strategy:
       matrix:
         'App: v0_app':
@@ -54,20 +68,24 @@ jobs:
           name: "payload"
         'App: commands_and_api':
           name: "commands_and_api"
+    timeoutInMinutes: "30"
+    cancelTimeoutInMinutes: "2"
+    # values: https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#workspace
     workspace:
       clean: all
     steps:
+
+    - script: echo '##vso[task.setvariable variable=local_id]$(System.PullRequest.PullRequestNumber)'
+      displayName: "Set id for this PR"
+      condition: eq(variables['Build.Reason'], 'PullRequest')
+
     - bash: |
         whoami
+        printf "local id: $(local_id)\n"
         python --version
         pip --version
       displayName: 'Info'
 
-    # TODO: parse the PR number
-    - bash: |
-        ID=$(date +%s)
-        echo "##vso[task.setvariable variable=local_id]$ID"
-
     - task: Cache@2
       inputs:
         key: 'pip | "$(name)" | requirements/app/base.txt'
@@ -90,12 +108,15 @@ jobs:
       displayName: 'Install lightning'
 
     - bash: |
+        rm -rf examples/app_template_jupyterlab || true
         git clone https://github.com/Lightning-AI/LAI-lightning-template-jupyterlab-App examples/app_template_jupyterlab
         cp examples/app_template_jupyterlab/tests/test_template_jupyterlab.py tests/tests_app_examples/test_template_jupyterlab.py
       condition: eq(variables['name'], 'template_jupyterlab')
       displayName: 'Clone Template Jupyter Lab Repo'
 
-    - bash: git clone https://github.com/Lightning-AI/lightning-template-react examples/app_template_react_ui
+    - bash:  |
+        rm -rf examples/app_template_react_ui || true
+        git clone https://github.com/Lightning-AI/lightning-template-react examples/app_template_react_ui
       condition: eq(variables['name'], 'template_react_ui')
       displayName: 'Clone Template React UI Repo'
 
@@ -119,14 +140,17 @@ jobs:
         LIGHTNING_API_KEY: $(LIGHTNING_API_KEY_PROD)
         LIGHTNING_USERNAME: $(LIGHTNING_USERNAME)
         LIGHTNING_CLOUD_URL: $(LIGHTNING_CLOUD_URL_PROD)
+        LIGHTNING_DEBUG: '1'
       displayName: 'Run the tests'
 
     - publish: '$(Build.ArtifactStagingDirectory)/videos'
+      condition: failed()
       displayName: 'Publish videos'
       artifact: $(name)
 
     - bash: |
         time python -c "from lightning.app import testing; testing.delete_cloud_lightning_apps()"
+      condition: always()
       env:
         # LAI_USER: $(LAI_USER)
         # LAI_PASS: $(LAI_PASS)