Feat/use uv python management (#994)

* Remove requirements files and add dependencies into pyproject.toml instead

* Remove old conda files since we will use uv as our primary method for developers to set up environments

* Working through CI changes to use uv instead of pip and conda

* Add uv lock to exclude files

* Revert "Remove old conda files since we will use uv as our primary method for developers to set up environments"

This reverts commit 88aff7e.

* Windows workflows don't use source command

* Add in extra include for ignoreing rat

* Use uv commands in CI

* Remove conda recipes and CI stages

* Working on CI using uv

* Install doc requirements

* Remove caching uv

* Set uv venv

* Add requirements for building

* Revert github action to allowed one

* Call uv sync with verbose mode so users can see the build occuring in CI

* Test setting specific hash on action

* Test setting rust-toolchain github action with pinned version

* Testing night rust toolchain against apache rejection criteria

* Github action is fickle with the pattern matching

* Switch all Ci to use nightly rust toolchain until infra team whitelists the stable toolchain

* Speed up CI by preventing build during uv sync

* Additional uv commands missing no-project option

* Setting python versions of dependencies to match lowest supported python version, 3.8

* Update maturin and move to deps for dev

* CI ordering was wrong and maturin needed uv option

* Switch to stable toolchain

* uv requires two dashes

* Submodule init

* change directories for unit tests

* Add deps for build

* Maturin build doesn't take uv as parameter

* Update documentation for setting up with uv

* Enable cache in CI

* Update documentation to use uv

* Small adjustment to CI config

Author: timsaucer

.github/workflows/build.yml

@@ -31,28 +31,33 @@ jobs:
       - name: Install Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: "3.12"
+
+      - uses: astral-sh/setup-uv@v5
+        with:
+            enable-cache: true
+
+      # Use the --no-install-package to only install the dependencies
+      # but do not yet build the rust library
       - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install ruff
+        run: uv sync --dev --no-install-package datafusion
+
       # Update output format to enable automatic inline annotations.
       - name: Run Ruff
         run: |
-          ruff check --output-format=github python/
-          ruff format --check python/
+          uv run --no-project ruff check --output-format=github python/
+          uv run --no-project ruff format --check python/
 
   generate-license:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: actions-rs/toolchain@v1
+      - uses: astral-sh/setup-uv@v5
         with:
-          profile: minimal
-          toolchain: stable
-          override: true
+            enable-cache: true
+
       - name: Generate license file
-        run: python ./dev/create_license.py
+        run: uv run --no-project python ./dev/create_license.py
       - uses: actions/upload-artifact@v4
         with:
           name: python-wheel-license
@@ -74,15 +79,7 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-
-      - name: Upgrade pip
-        run: python -m pip install --upgrade pip
-
-      - name: Install maturin
-        run: pip install maturin==1.5.1
+      - uses: dtolnay/rust-toolchain@stable
 
       - run: rm LICENSE.txt
       - name: Download LICENSE.txt
@@ -97,8 +94,14 @@ jobs:
           version: "27.4"
           repo-token: ${{ secrets.GITHUB_TOKEN }}
 
+      - uses: astral-sh/setup-uv@v5
+        with:
+            enable-cache: true
+
       - name: Build Python package
-        run: maturin build --release --strip --features substrait
+        run: |
+          uv sync --dev --no-install-package datafusion
+          uv run --no-project maturin build --release --strip --features substrait
 
       - name: List Windows wheels
         if: matrix.os == 'windows-latest'
@@ -132,15 +135,7 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-
-      - name: Upgrade pip
-        run: python -m pip install --upgrade pip
-
-      - name: Install maturin
-        run: pip install maturin==1.5.1
+      - uses: dtolnay/rust-toolchain@stable
 
       - run: rm LICENSE.txt
       - name: Download LICENSE.txt
@@ -155,9 +150,14 @@ jobs:
           version: "27.4"
           repo-token: ${{ secrets.GITHUB_TOKEN }}
 
+      - uses: astral-sh/setup-uv@v5
+        with:
+            enable-cache: true
+
       - name: Build Python package
         run: |
-          maturin build --release --strip --features substrait
+          uv sync --dev --no-install-package datafusion
+          uv run --no-project maturin build --release --strip --features substrait
 
       - name: List Mac wheels
         run: find target/wheels/

.github/workflows/conda.yml

@@ -57,27 +57,24 @@ jobs:
           version: '27.4'
           repo-token: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Install dependencies
-        run: |
-          set -x
-          python3 -m venv venv
-          source venv/bin/activate
-          pip install -r requirements-311.txt
-          pip install -r docs/requirements.txt
-      - name: Build Datafusion
+      - name: Install dependencies and build
+        uses: astral-sh/setup-uv@v5
+        with:
+            enable-cache: true
+
+      - name: Build repo
         run: |
-          set -x
-          source venv/bin/activate
-          maturin develop
+          uv venv
+          uv sync --dev --no-install-package datafusion --group docs
+          uv run --no-project maturin develop --uv
 
       - name: Build docs
         run: |
           set -x
-          source venv/bin/activate
           cd docs
           curl -O https://gist.githubusercontent.com/ritchie46/cac6b337ea52281aa23c049250a4ff03/raw/89a957ff3919d90e6ef2d34235e6bf22304f3366/pokemon.csv
           curl -O https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2021-01.parquet
-          make html
+          uv run --no-project make html
 
       - name: Copy & push the generated HTML
         if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref_type == 'tag')

.github/workflows/docs.yaml

@@ -43,11 +43,10 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Setup Rust Toolchain
-        uses: actions-rs/toolchain@v1
+        uses: dtolnay/rust-toolchain@stable
         id: rust-toolchain
         with:
-          toolchain: ${{ matrix.toolchain }}
-          override: true
+          components: clippy,rustfmt
 
       - name: Install Protoc
         uses: arduino/setup-protoc@v3
@@ -64,60 +63,35 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ~/.cargo
-          key: cargo-cache-${{ steps.rust-toolchain.outputs.rustc_hash }}-${{ hashFiles('Cargo.lock') }}
+          key: cargo-cache-${{ steps.rust-toolchain.outputs.cachekey }}-${{ hashFiles('Cargo.lock') }}
 
       - name: Check Formatting
-        uses: actions-rs/cargo@v1
         if: ${{ matrix.python-version == '3.10' && matrix.toolchain == 'stable' }}
-        with:
-          command: fmt
-          args: -- --check
+        run: cargo fmt -- --check
 
       - name: Run Clippy
-        uses: actions-rs/cargo@v1
         if: ${{ matrix.python-version == '3.10' && matrix.toolchain == 'stable' }}
-        with:
-          command: clippy
-          args: --all-targets --all-features -- -D clippy::all -A clippy::redundant_closure
-
-      - name: Create Virtualenv (3.12)
-        if: ${{ matrix.python-version == '3.12' }}
-        run: |
-          python -m venv venv
-          source venv/bin/activate
-          pip install -r requirements-312.txt
+        run: cargo clippy --all-targets --all-features -- -D clippy::all -A clippy::redundant_closure
 
-      - name: Create Virtualenv (3.10)
-        if: ${{ matrix.python-version == '3.10' }}
-        run: |
-          python -m venv venv
-          source venv/bin/activate
-          pip install -r requirements-310.txt
-
-      - name: Create Virtualenv (3.11)
-        if: ${{ matrix.python-version == '3.11' }}
-        run: |
-          python -m venv venv
-          source venv/bin/activate
-          pip install -r requirements-311.txt
+      - name: Install dependencies and build
+        uses: astral-sh/setup-uv@v5
+        with:
+            enable-cache: true
 
       - name: Run tests
         env:
           RUST_BACKTRACE: 1
         run: |
           git submodule update --init
-          source venv/bin/activate
-          pip install -e . -vv
-          pytest -v .
+          uv sync --dev --no-install-package datafusion
+          uv run --no-project maturin develop --uv
+          uv run --no-project pytest -v .
 
       - name: FFI unit tests
         run: |
-          source venv/bin/activate
-          pip install -e . -vv
-          pip install maturin==1.5.1
           cd examples/ffi-table-provider
-          maturin develop --release --strip
-          pytest python/tests/_test_table_provider.py
+          uv run --no-project maturin develop --uv
+          uv run --no-project pytest python/tests/_test_table_provider.py
 
       - name: Cache the generated dataset
         id: cache-tpch-dataset
@@ -134,7 +108,6 @@ jobs:
 
       - name: Run TPC-H examples
         run: |
-          source venv/bin/activate
           cd examples/tpch
-          python convert_data_to_parquet.py
-          pytest _tests.py
+          uv run --no-project python convert_data_to_parquet.py
+          uv run --no-project pytest _tests.py

.github/workflows/test.yaml

@@ -17,9 +17,9 @@
 
 repos:
       - repo: https://github.com/rhysd/actionlint
-        rev: v1.6.23
+        rev: v1.7.6
         hooks:
-              - id: actionlint-docker
+          - id: actionlint-docker
       - repo: https://github.com/astral-sh/ruff-pre-commit
         # Ruff version.
         rev: v0.3.0