From 0e6bc859912ebaf35cc2410d2db09278013c02cc Mon Sep 17 00:00:00 2001 From: BanzaiTokyo Date: Thu, 1 May 2025 11:17:17 +0000 Subject: [PATCH 01/10] uv in pytorch-version-tests.yml --- .github/workflows/pytorch-version-tests.yml | 73 ++++++++------------- 1 file changed, 28 insertions(+), 45 deletions(-) diff --git a/.github/workflows/pytorch-version-tests.yml b/.github/workflows/pytorch-version-tests.yml index 1f8f41e4650f..230ab31355a5 100644 --- a/.github/workflows/pytorch-version-tests.yml +++ b/.github/workflows/pytorch-version-tests.yml @@ -16,75 +16,58 @@ jobs: fail-fast: false matrix: python-version: [3.9, "3.10", "3.11"] - pytorch-version: - [2.5.1, 2.4.1, 2.3.1, 2.2.2, 1.13.1, 1.12.1, 1.10.0] + pytorch-version: [2.5.1, 2.4.1, 2.3.1, 2.2.2, 1.13.1, 1.12.1, 1.10.0] exclude: - pytorch-version: 1.10.0 python-version: "3.10" - pytorch-version: 1.10.0 python-version: "3.11" - - pytorch-version: 1.11.0 python-version: "3.10" - pytorch-version: 1.11.0 python-version: "3.11" - pytorch-version: 1.12.1 python-version: "3.11" - # Conda fails to install cpuonly version and few cpu distributed tests are - # failing with unrelated errors - pytorch-version: 1.13.1 python-version: "3.11" steps: - uses: actions/checkout@v4 - - name: Get year & week number - id: get-date - run: echo "date=$(/bin/date "+%Y-%U")" >> $GITHUB_OUTPUT - shell: bash -l {0} - - - name: Get pip cache dir - id: pip-cache - run: | - python3 -m pip install -U pip - echo "pip_cache=$(python3 -m pip cache dir)" >> $GITHUB_OUTPUT - shell: bash -l {0} - - - uses: actions/cache@v3 - with: - path: | - ~/conda_pkgs_dir - ${{ steps.pip-cache.outputs.pip_cache }} - key: ${{ steps.get-date.outputs.date }}-pytorch-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.pytorch-version }}-${{ hashFiles('requirements-dev.txt') }} - restore-keys: | - ${{ steps.get-date.outputs.date }}-pytorch-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.pytorch-version }}- - - - name: Setup Miniconda - uses: conda-incubator/setup-miniconda@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 with: - miniconda-version: "latest" python-version: ${{ matrix.python-version }} - use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly! + cache: pip + + - name: Install uv + uses: astral-sh/setup-uv@v5 - - name: Install dependencies - shell: bash -l {0} + - name: Install PyTorch ${{ matrix.pytorch-version }} run: | - conda install pytorch=${{ matrix.pytorch-version }} torchvision cpuonly python=${{ matrix.python-version }} -c pytorch + # Use appropriate install index + if [[ "${{ matrix.pytorch-version }}" == 2.* || "${{ matrix.pytorch-version }}" == 1.13* || "${{ matrix.pytorch-version }}" == 1.12* || "${{ matrix.pytorch-version }}" == 1.10* ]]; then + pip install torch==${{ matrix.pytorch-version }} torchvision --extra-index-url https://download.pytorch.org/whl/cpu + else + pip install torch==${{ matrix.pytorch-version }} torchvision + fi - # We should install numpy<2.0 for pytorch<2.3 - numpy_one_pth_version=$(python -c "import torch; print(float('.'.join(torch.__version__.split('.')[:2])) < 2.3)") - if [ "${numpy_one_pth_version}" == "True" ]; then - pip install -U "numpy<2.0" + - name: Install numpy conditionally for PyTorch < 2.3 + run: | + version=$(python -c "import torch; print(float('.'.join(torch.__version__.split('.')[:2])))") + if (( $(echo "$version < 2.3" | bc -l) )); then + pip install "numpy<2.0" fi - pip install -r requirements-dev.txt - python setup.py install + - name: Install Ignite dependencies + run: | + uv pip install -r requirements-dev.txt + uv pip install -e . - # pytorch>=1.9.0,<1.11.0 is using "from setuptools import distutils; distutils.version.LooseVersion" anti-pattern - # which raises the error: AttributeError: module 'distutils' has no attribute 'version' for setuptools>59 - bad_pth_version=$(python -c "import torch; print('.'.join(torch.__version__.split('.')[:2]) in ['1.9', '1.10'])") - if [ "${bad_pth_version}" == "True" ]; then - pip install --upgrade "setuptools<59" + # Patch setuptools for old PyTorch versions + version=$(python -c "import torch; print('.'.join(torch.__version__.split('.')[:2]))") + if [[ "$version" == "1.9" || "$version" == "1.10" ]]; then + pip install "setuptools<59" python -c "from setuptools import distutils; distutils.version.LooseVersion" fi @@ -99,7 +82,7 @@ jobs: max_attempts: 5 timeout_minutes: 15 shell: bash - command: bash -l tests/run_cpu_tests.sh "not test_time_profilers" + command: bash tests/run_cpu_tests.sh "not test_time_profilers" new_command_on_retry: USE_LAST_FAILED=1 bash -l tests/run_cpu_tests.sh "not test_time_profilers" create-issue: From fd51a835e06766c450a2f94328fe15aac32221cc Mon Sep 17 00:00:00 2001 From: BanzaiTokyo Date: Thu, 1 May 2025 12:47:47 +0000 Subject: [PATCH 02/10] run action on pr --- .github/workflows/pytorch-version-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/pytorch-version-tests.yml b/.github/workflows/pytorch-version-tests.yml index 230ab31355a5..8646d024fbf8 100644 --- a/.github/workflows/pytorch-version-tests.yml +++ b/.github/workflows/pytorch-version-tests.yml @@ -6,6 +6,9 @@ on: # Run at 00:00 UTC Every Day - cron: "0 0 * * *" workflow_dispatch: + pull_request: + paths: + - ".github/workflows/pytorch-version-tests.yml" jobs: build: From e4c5e9986e1b96d41e3cf5c016966dbb1ffc9ff0 Mon Sep 17 00:00:00 2001 From: BanzaiTokyo Date: Thu, 1 May 2025 12:59:21 +0000 Subject: [PATCH 03/10] remove cache, install python with uv --- .github/workflows/pytorch-version-tests.yml | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/.github/workflows/pytorch-version-tests.yml b/.github/workflows/pytorch-version-tests.yml index 8646d024fbf8..d7efc2942487 100644 --- a/.github/workflows/pytorch-version-tests.yml +++ b/.github/workflows/pytorch-version-tests.yml @@ -37,29 +37,25 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + - name: Install uv and Python + uses: astral-sh/setup-uv@v5 with: python-version: ${{ matrix.python-version }} - cache: pip - - - name: Install uv - uses: astral-sh/setup-uv@v5 - + cache: true - name: Install PyTorch ${{ matrix.pytorch-version }} run: | # Use appropriate install index if [[ "${{ matrix.pytorch-version }}" == 2.* || "${{ matrix.pytorch-version }}" == 1.13* || "${{ matrix.pytorch-version }}" == 1.12* || "${{ matrix.pytorch-version }}" == 1.10* ]]; then - pip install torch==${{ matrix.pytorch-version }} torchvision --extra-index-url https://download.pytorch.org/whl/cpu + uv pip install --system torch==${{ matrix.pytorch-version }} torchvision --extra-index-url https://download.pytorch.org/whl/cpu else - pip install torch==${{ matrix.pytorch-version }} torchvision + uv pip install --system torch==${{ matrix.pytorch-version }} torchvision fi - name: Install numpy conditionally for PyTorch < 2.3 run: | version=$(python -c "import torch; print(float('.'.join(torch.__version__.split('.')[:2])))") if (( $(echo "$version < 2.3" | bc -l) )); then - pip install "numpy<2.0" + uv pip install --system "numpy<2.0" fi - name: Install Ignite dependencies @@ -86,7 +82,7 @@ jobs: timeout_minutes: 15 shell: bash command: bash tests/run_cpu_tests.sh "not test_time_profilers" - new_command_on_retry: USE_LAST_FAILED=1 bash -l tests/run_cpu_tests.sh "not test_time_profilers" + new_command_on_retry: USE_LAST_FAILED=1 bash tests/run_cpu_tests.sh "not test_time_profilers" create-issue: runs-on: ubuntu-latest From 68526684985fefcca0e770249b4b9743ddff2c20 Mon Sep 17 00:00:00 2001 From: BanzaiTokyo Date: Thu, 1 May 2025 13:02:13 +0000 Subject: [PATCH 04/10] set environment variable for systemwide installs through uv --- .github/workflows/pytorch-version-tests.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pytorch-version-tests.yml b/.github/workflows/pytorch-version-tests.yml index d7efc2942487..930835daacc9 100644 --- a/.github/workflows/pytorch-version-tests.yml +++ b/.github/workflows/pytorch-version-tests.yml @@ -13,6 +13,8 @@ on: jobs: build: runs-on: ubuntu-latest + env: + UV_SYSTEM_PIP: 1 timeout-minutes: 85 strategy: max-parallel: 5 @@ -46,16 +48,16 @@ jobs: run: | # Use appropriate install index if [[ "${{ matrix.pytorch-version }}" == 2.* || "${{ matrix.pytorch-version }}" == 1.13* || "${{ matrix.pytorch-version }}" == 1.12* || "${{ matrix.pytorch-version }}" == 1.10* ]]; then - uv pip install --system torch==${{ matrix.pytorch-version }} torchvision --extra-index-url https://download.pytorch.org/whl/cpu + uv pip install torch==${{ matrix.pytorch-version }} torchvision --extra-index-url https://download.pytorch.org/whl/cpu else - uv pip install --system torch==${{ matrix.pytorch-version }} torchvision + uv pip install torch==${{ matrix.pytorch-version }} torchvision fi - name: Install numpy conditionally for PyTorch < 2.3 run: | version=$(python -c "import torch; print(float('.'.join(torch.__version__.split('.')[:2])))") if (( $(echo "$version < 2.3" | bc -l) )); then - uv pip install --system "numpy<2.0" + uv pip install "numpy<2.0" fi - name: Install Ignite dependencies From c346cb8eaf6b1854db63b177c7f19635599b9816 Mon Sep 17 00:00:00 2001 From: BanzaiTokyo Date: Thu, 1 May 2025 13:05:07 +0000 Subject: [PATCH 05/10] set up cache --- .github/workflows/pytorch-version-tests.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pytorch-version-tests.yml b/.github/workflows/pytorch-version-tests.yml index 930835daacc9..b6cb9e9ff070 100644 --- a/.github/workflows/pytorch-version-tests.yml +++ b/.github/workflows/pytorch-version-tests.yml @@ -38,12 +38,14 @@ jobs: steps: - uses: actions/checkout@v4 - + - uses: actions/cache@v3 + with: + path: ~/.cache/uv + key: ${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('requirements-dev.txt') }} - name: Install uv and Python uses: astral-sh/setup-uv@v5 with: python-version: ${{ matrix.python-version }} - cache: true - name: Install PyTorch ${{ matrix.pytorch-version }} run: | # Use appropriate install index From 4dbce0399b90e9a32dfe15ea318b08ac26df1661 Mon Sep 17 00:00:00 2001 From: BanzaiTokyo Date: Thu, 1 May 2025 15:04:42 +0000 Subject: [PATCH 06/10] clearml defaults --- .github/workflows/pytorch-version-tests.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/pytorch-version-tests.yml b/.github/workflows/pytorch-version-tests.yml index b6cb9e9ff070..ea0b5ba73fc5 100644 --- a/.github/workflows/pytorch-version-tests.yml +++ b/.github/workflows/pytorch-version-tests.yml @@ -15,6 +15,10 @@ jobs: runs-on: ubuntu-latest env: UV_SYSTEM_PIP: 1 + CLEARML_NO_DEFAULT_SERVER: 1 + CLEARML_API_HOST: "http://localhost" + CLEARML_WEB_HOST: "http://localhost" + CLEARML_FILES_HOST: "http://localhost" timeout-minutes: 85 strategy: max-parallel: 5 From 6dc434bc2b8e127fd0c201c2101d7a52fe579d5c Mon Sep 17 00:00:00 2001 From: BanzaiTokyo Date: Fri, 2 May 2025 07:28:03 +0000 Subject: [PATCH 07/10] setuptools --- .github/workflows/pytorch-version-tests.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pytorch-version-tests.yml b/.github/workflows/pytorch-version-tests.yml index ea0b5ba73fc5..46593cba5cce 100644 --- a/.github/workflows/pytorch-version-tests.yml +++ b/.github/workflows/pytorch-version-tests.yml @@ -15,10 +15,7 @@ jobs: runs-on: ubuntu-latest env: UV_SYSTEM_PIP: 1 - CLEARML_NO_DEFAULT_SERVER: 1 - CLEARML_API_HOST: "http://localhost" - CLEARML_WEB_HOST: "http://localhost" - CLEARML_FILES_HOST: "http://localhost" + CLEARML_OFFLINE_MODE: 1 timeout-minutes: 85 strategy: max-parallel: 5 @@ -52,12 +49,12 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install PyTorch ${{ matrix.pytorch-version }} run: | - # Use appropriate install index if [[ "${{ matrix.pytorch-version }}" == 2.* || "${{ matrix.pytorch-version }}" == 1.13* || "${{ matrix.pytorch-version }}" == 1.12* || "${{ matrix.pytorch-version }}" == 1.10* ]]; then uv pip install torch==${{ matrix.pytorch-version }} torchvision --extra-index-url https://download.pytorch.org/whl/cpu else uv pip install torch==${{ matrix.pytorch-version }} torchvision fi + uv pip install setuptools - name: Install numpy conditionally for PyTorch < 2.3 run: | From 663f127a19fe9d0abcc00e0f3d1a81ca436c71b8 Mon Sep 17 00:00:00 2001 From: BanzaiTokyo Date: Fri, 2 May 2025 07:35:52 +0000 Subject: [PATCH 08/10] clearml --- .github/workflows/pytorch-version-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/pytorch-version-tests.yml b/.github/workflows/pytorch-version-tests.yml index 46593cba5cce..dacce358b9a5 100644 --- a/.github/workflows/pytorch-version-tests.yml +++ b/.github/workflows/pytorch-version-tests.yml @@ -16,6 +16,9 @@ jobs: env: UV_SYSTEM_PIP: 1 CLEARML_OFFLINE_MODE: 1 + CLEARML_API_HOST: http://localhost + CLEARML_WEB_HOST: http://localhost + CLEARML_FILES_HOST: http://localhost timeout-minutes: 85 strategy: max-parallel: 5 From da2fbbc9bcc593a5a7a97df293ca785c8289181f Mon Sep 17 00:00:00 2001 From: BanzaiTokyo Date: Fri, 2 May 2025 07:58:02 +0000 Subject: [PATCH 09/10] distutils --- .github/workflows/pytorch-version-tests.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/pytorch-version-tests.yml b/.github/workflows/pytorch-version-tests.yml index dacce358b9a5..3b69d0fb8583 100644 --- a/.github/workflows/pytorch-version-tests.yml +++ b/.github/workflows/pytorch-version-tests.yml @@ -74,8 +74,7 @@ jobs: # Patch setuptools for old PyTorch versions version=$(python -c "import torch; print('.'.join(torch.__version__.split('.')[:2]))") if [[ "$version" == "1.9" || "$version" == "1.10" ]]; then - pip install "setuptools<59" - python -c "from setuptools import distutils; distutils.version.LooseVersion" + pip install "setuptools<59" "distutils==0.3" fi - name: Download MNIST From 7586d562517f723f6a11c0b0f764a9489c4411a9 Mon Sep 17 00:00:00 2001 From: BanzaiTokyo Date: Fri, 2 May 2025 13:51:57 +0000 Subject: [PATCH 10/10] remove distutils --- .github/workflows/pytorch-version-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytorch-version-tests.yml b/.github/workflows/pytorch-version-tests.yml index 3b69d0fb8583..249c5638f408 100644 --- a/.github/workflows/pytorch-version-tests.yml +++ b/.github/workflows/pytorch-version-tests.yml @@ -74,7 +74,7 @@ jobs: # Patch setuptools for old PyTorch versions version=$(python -c "import torch; print('.'.join(torch.__version__.split('.')[:2]))") if [[ "$version" == "1.9" || "$version" == "1.10" ]]; then - pip install "setuptools<59" "distutils==0.3" + uv pip install "setuptools<59" fi - name: Download MNIST