Add conda.yaml baseline workflow file (#281)

* point to custom upstream arrow-datafusion with updated build script

* Add conda.yaml baseline workflow file

* Change conda recipe path

* Add some debug output

* fix syntax issues

* update rustfmt path

* test invoking rustfmt

* Allow for RUSTFMT environment to pass through to the conda build

* conda build updates

* github workflow syntax issue

* workflow syntax

* more syntax

* updates

* use minimal rustup profile

* use minimal rustup profile

* use minimal rustup profile

* use minimal rustup profile

* use minimal rustup profile

* changes

* changes

* trigger workflow rebuild

* Add container

* Ouput Rust env varialbes

* full rustup command path

* Point to typify fork for testing

* Adjust conda CI workflow

* Adjust conda build instead of mambabuild

* Adjust command

* Install conda-build

* Some initial changes to get noarch builds working

* Remove beta rust toolchain from CI

* Add libprotobuf to host/build deps

* Bump libprotobuf version

* Revert back to 3 for libprotobuf

---------

Co-authored-by: Charles Blackmon-Luca <[email protected]>

Author: jdye64

.github/workflows/conda.yml

@@ -0,0 +1,61 @@
+name: Build conda nightly
+on: [push, pull_request]
+
+# Cancel any already running instances of this build
+concurrency:
+  group: conda-${{ github.head_ref }}
+  cancel-in-progress: true
+
+# Required shell entrypoint to have properly activated conda environments
+defaults:
+  run:
+    shell: bash -l {0}
+
+jobs:
+  conda:
+    name: Build (and optionally upload) the conda nightly
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Set up Python
+        uses: conda-incubator/[email protected]
+        with:
+          miniforge-variant: Mambaforge
+          use-mamba: true
+          python-version: "3.10"
+          channel-priority: strict
+      - name: Install dependencies
+        run: |
+          mamba install boa conda-verify
+
+          which python
+          pip list
+          mamba list
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+          components: clippy, rustfmt
+      - name: Build conda package
+        run: |
+          # suffix for nightly package versions
+          export VERSION_SUFFIX=a`date +%y%m%d`
+
+          conda mambabuild conda/recipes \
+                           --no-anaconda-upload \
+                           --output-folder .
+      # - name: Upload conda package
+      #   if: |
+      #     github.event_name == 'push'
+      #     && github.repository == 'apache/arrow-datafusion-python'
+      #   env:
+      #     ANACONDA_API_TOKEN: ${{ secrets.DASK_CONDA_TOKEN }}
+      #     LABEL: ${{ github.ref == 'refs/heads/datafusion-sql-planner' && 'dev_datafusion' || 'dev' }}
+      #   run: |
+      #     # install anaconda for upload
+      #     mamba install anaconda-client
+
+      #     anaconda upload --label $LABEL linux-64/*.tar.bz2

.github/workflows/test.yaml

@@ -36,7 +36,6 @@ jobs:
           - "3.10"
         toolchain:
           - "stable"
-          - "beta"
           # we are not that much eager in walking on the edge yet
           # - nightly
         # build stable for only 3.7

Cargo.lock

@@ -15,32 +15,41 @@
 # specific language governing permissions and limitations
 # under the License.
 
-{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %}
-{% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %}
+{% set name = "datafusion" %}
+{% set major_minor_patch = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').split('.') %}
+{% set new_patch = major_minor_patch[2] | int + 1 %}
+{% set version = (major_minor_patch[:2] + [new_patch]) | join('.') + environ.get('VERSION_SUFFIX', '') %}
+
 
 package:
-  name: datafusion
+  name: {{ name|lower }}
   version: {{ version }}
 
 source:
   git_url: ../..
 
 build:
   noarch: python
+  script: {{ PYTHON }} -m pip install . -vv
   number: 0
 
 requirements:
   host:
-    - {{ compiler('rust') }}
-    - python
-    - setuptools
-    - maturin
+    - python >=3.6
+    - maturin >=0.14,<0.15
+    - libprotobuf =3
+    - pip
   run:
-    - python
+    - python >=3.6
+    - pyarrow >=11.0.0
 
 test:
   imports:
     - datafusion
+  commands:
+    - pip check
+  requires:
+    - pip
 
 about:
   home: https://arrow.apache.org/datafusion

conda/recipes/build.sh

@@ -43,7 +43,7 @@ classifier = [
     "Programming Language :: Rust",
 ]
 dependencies = [
-    "pyarrow>=6.0.1",
+    "pyarrow>=11.0.0",
 ]
 
 [project.urls]