add a CI that tests xarray with all optional dependencies but dask (#3919)

* add a new ci that does not install dask

* remove iris since it depends on dask

* rename the new CI to py38-all-but-dask

* decorate the some of the failing tests with requires_dask

* decorate the zarr tests using dask or obj.chunk with requires_dask

* don't chunk if dask is not available

* pass an indexer instead of a tuple to _arrayize_vectorized_indexer

* xfail the remaining tests if dask is not available

* update whats-new.rst

Author: keewis

azure-pipelines.yml

@@ -20,6 +20,8 @@ jobs:
         conda_env: py37
       py38:
         conda_env: py38
+      py38-all-but-dask:
+        conda_env: py38-all-but-dask
       py38-upstream-dev:
         conda_env: py38
         upstream_dev: true

ci/requirements/py38-all-but-dask.yml

@@ -0,0 +1,44 @@
+name: xarray-tests
+channels:
+  - conda-forge
+dependencies:
+  - python=3.8
+  - black
+  - boto3
+  - bottleneck
+  - cartopy
+  - cdms2
+  - cfgrib
+  - cftime
+  - coveralls
+  - flake8
+  - h5netcdf
+  - h5py
+  - hdf5
+  - hypothesis
+  - isort
+  - lxml    # Optional dep of pydap
+  - matplotlib
+  - mypy=0.761  # Must match .pre-commit-config.yaml
+  - nc-time-axis
+  - netcdf4
+  - numba
+  - numpy
+  - pandas
+  - pint
+  - pip
+  - pseudonetcdf
+  - pydap
+  - pynio
+  - pytest
+  - pytest-cov
+  - pytest-env
+  - rasterio
+  - scipy
+  - seaborn
+  - setuptools
+  - sparse
+  - toolz
+  - zarr
+  - pip:
+    - numbagg

doc/whats-new.rst

@@ -71,6 +71,9 @@ Internal Changes
 - Run the ``isort`` pre-commit hook only on python source files
   and update the ``flake8`` version. (:issue:`3750`, :pull:`3711`)
   By `Justus Magin <https://github.com/keewis>`_.
+- Add a CI job that runs the tests with every optional dependency
+  except ``dask``. (:issue:`3794`, :pull:`3919`)
+  By `Justus Magin <https://github.com/keewis>`_.
 
 
 .. _whats-new.0.15.1:

xarray/backends/zarr.py

@@ -56,7 +56,7 @@ def __getitem__(self, key):
             return array[key.tuple]
         elif isinstance(key, indexing.VectorizedIndexer):
             return array.vindex[
-                indexing._arrayize_vectorized_indexer(key.tuple, self.shape).tuple
+                indexing._arrayize_vectorized_indexer(key, self.shape).tuple
             ]
         else:
             assert isinstance(key, indexing.OuterIndexer)
@@ -586,6 +586,12 @@ def open_zarr(
                 "Instead found %s. " % chunks
             )
 
+    if chunks == "auto":
+        try:
+            import dask.array  # noqa
+        except ImportError:
+            chunks = None
+
     if not decode_cf:
         mask_and_scale = False
         decode_times = False

xarray/tests/test_backends.py

@@ -581,6 +581,10 @@ def test_orthogonal_indexing(self):
             actual = on_disk.isel(**indexers)
             assert_identical(expected, actual)
 
+    @pytest.mark.xfail(
+        not has_dask,
+        reason="the code for indexing without dask handles negative steps in slices incorrectly",
+    )
     def test_vectorized_indexing(self):
         in_memory = create_test_data()
         with self.roundtrip(in_memory) as on_disk:
@@ -1539,6 +1543,7 @@ def test_roundtrip_consolidated(self):
             self.check_dtypes_roundtripped(expected, actual)
             assert_identical(expected, actual)
 
+    @requires_dask
     def test_auto_chunk(self):
         original = create_test_data().chunk()
 
@@ -1556,6 +1561,7 @@ def test_auto_chunk(self):
                 # chunk size should be the same as original
                 assert v.chunks == original[k].chunks
 
+    @requires_dask
     @pytest.mark.filterwarnings("ignore:Specified Dask chunks")
     def test_manual_chunk(self):
         original = create_test_data().chunk({"dim1": 3, "dim2": 4, "dim3": 3})
@@ -1598,6 +1604,7 @@ def test_manual_chunk(self):
                 assert_identical(actual, auto)
                 assert_identical(actual.load(), auto.load())
 
+    @requires_dask
     def test_warning_on_bad_chunks(self):
         original = create_test_data().chunk({"dim1": 4, "dim2": 3, "dim3": 5})
 
@@ -1620,6 +1627,7 @@ def test_warning_on_bad_chunks(self):
                         assert v._in_memory == (k in actual.dims)
             assert len(record) == 0
 
+    @requires_dask
     def test_deprecate_auto_chunk(self):
         original = create_test_data().chunk()
         with pytest.warns(FutureWarning):
@@ -1638,6 +1646,7 @@ def test_deprecate_auto_chunk(self):
                     # there should be no chunks
                     assert v.chunks is None
 
+    @requires_dask
     def test_write_uneven_dask_chunks(self):
         # regression for GH#2225
         original = create_test_data().chunk({"dim1": 3, "dim2": 4, "dim3": 3})
@@ -1662,6 +1671,7 @@ def test_chunk_encoding(self):
             with self.roundtrip(data) as actual:
                 pass
 
+    @requires_dask
     def test_chunk_encoding_with_dask(self):
         # These datasets DO have dask chunks. Need to check for various
         # interactions between dask and zarr chunks
@@ -1896,6 +1906,7 @@ def test_append_with_new_variable(self):
             combined["new_var"] = ds_with_new_var["new_var"]
             assert_identical(combined, xr.open_zarr(store_target))
 
+    @requires_dask
     def test_to_zarr_compute_false_roundtrip(self):
         from dask.delayed import Delayed
 
@@ -1915,6 +1926,7 @@ def test_to_zarr_compute_false_roundtrip(self):
             with self.open(store) as actual:
                 assert_identical(original, actual)
 
+    @requires_dask
     def test_to_zarr_append_compute_false_roundtrip(self):
         from dask.delayed import Delayed
 
@@ -1951,6 +1963,7 @@ def test_to_zarr_append_compute_false_roundtrip(self):
                 with self.open(store) as actual:
                     assert_identical(xr.concat([ds, ds_to_append], dim="time"), actual)
 
+    @requires_dask
     def test_encoding_chunksizes(self):
         # regression test for GH2278
         # see also test_encoding_chunksizes_unlimited
@@ -3513,6 +3526,7 @@ def test_uamiv_format_read(self):
         assert_allclose(expected, actual)
         camxfile.close()
 
+    @requires_dask
     def test_uamiv_format_mfread(self):
         """
         Open a CAMx file and test data variables
@@ -3939,6 +3953,9 @@ def test_chunks(self):
                 ex = expected.sel(band=1).mean(dim="x")
                 assert_allclose(ac, ex)
 
+    @pytest.mark.xfail(
+        not has_dask, reason="without dask, a non-serializable lock is used"
+    )
     def test_pickle_rasterio(self):
         # regression test for https://github.com/pydata/xarray/issues/2121
         with create_tmp_geotiff() as (tmp_file, expected):
@@ -4012,6 +4029,7 @@ def test_geotiff_tags(self):
             with xr.open_rasterio(tmp_file) as rioda:
                 assert isinstance(rioda.attrs["AREA_OR_POINT"], str)
 
+    @requires_dask
     def test_no_mftime(self):
         # rasterio can accept "filename" urguments that are actually urls,
         # including paths to remote files.

xarray/tests/test_sparse.py

@@ -837,6 +837,7 @@ def test_sparse_coords(self):
         )
 
 
+@requires_dask
 def test_chunk():
     s = sparse.COO.from_numpy(np.array([0, 0, 1, 2]))
     a = DataArray(s)