Fix gen_cluster failures; dask_version tweaks (#5610)

* gen_cluster no longer accepts timeout=None

* dask_version tweaks

* pep8

* [test-upstream]

Author: crusaderky

doc/getting-started-guide/installing.rst

@@ -96,7 +96,7 @@ dependencies:
 - **setuptools:** 42 months (but no older than 40.4)
 - **numpy:** 18 months
   (`NEP-29 <https://numpy.org/neps/nep-0029-deprecation_policy.html>`_)
-- **dask and dask.distributed:** 12 months (but no older than 2.9)
+- **dask and dask.distributed:** 12 months
 - **sparse, pint** and other libraries that rely on
   `NEP-18 <https://numpy.org/neps/nep-0018-array-function-protocol.html>`_
   for integration: very latest available versions only, until the technology will have

xarray/tests/test_backends.py

@@ -87,12 +87,8 @@
 try:
     import dask
     import dask.array as da
-
-    dask_version = dask.__version__
 except ImportError:
-    # needed for xfailed tests when dask < 2.4.0
-    # remove when min dask > 2.4.0
-    dask_version = "10.0"
+    pass
 
 ON_WINDOWS = sys.platform == "win32"
 default_value = object()
@@ -1961,7 +1957,6 @@ def test_hidden_zarr_keys(self):
                 with xr.decode_cf(store):
                     pass
 
-    @pytest.mark.skipif(LooseVersion(dask_version) < "2.4", reason="dask GH5334")
     @pytest.mark.parametrize("group", [None, "group1"])
     def test_write_persistence_modes(self, group):
         original = create_test_data()
@@ -2039,7 +2034,6 @@ def test_encoding_kwarg_fixed_width_string(self):
     def test_dataset_caching(self):
         super().test_dataset_caching()
 
-    @pytest.mark.skipif(LooseVersion(dask_version) < "2.4", reason="dask GH5334")
     def test_append_write(self):
         super().test_append_write()
 
@@ -2122,7 +2116,6 @@ def test_check_encoding_is_consistent_after_append(self):
                 xr.concat([ds, ds_to_append], dim="time"),
             )
 
-    @pytest.mark.skipif(LooseVersion(dask_version) < "2.4", reason="dask GH5334")
     def test_append_with_new_variable(self):
 
         ds, ds_to_append, ds_with_new_var = create_append_test_data()

xarray/tests/test_computation.py

@@ -1,7 +1,6 @@
 import functools
 import operator
 import pickle
-from distutils.version import LooseVersion
 
 import numpy as np
 import pandas as pd
@@ -21,6 +20,7 @@
     result_name,
     unified_dim_sizes,
 )
+from xarray.core.pycompat import dask_version
 
 from . import has_dask, raise_if_dask_computes, requires_dask
 
@@ -1307,7 +1307,7 @@ def test_vectorize_dask_dtype_without_output_dtypes(data_array):
 
 
 @pytest.mark.skipif(
-    LooseVersion(dask.__version__) > "2021.06",
+    dask_version > "2021.06",
     reason="dask/dask#7669: can no longer pass output_dtypes and meta",
 )
 @requires_dask

xarray/tests/test_dask.py

@@ -2,7 +2,6 @@
 import pickle
 import sys
 from contextlib import suppress
-from distutils.version import LooseVersion
 from textwrap import dedent
 
 import numpy as np
@@ -13,6 +12,7 @@
 import xarray.ufuncs as xu
 from xarray import DataArray, Dataset, Variable
 from xarray.core import duck_array_ops
+from xarray.core.pycompat import dask_version
 from xarray.testing import assert_chunks_equal
 from xarray.tests import mock
 
@@ -111,10 +111,7 @@ def test_indexing(self):
         self.assertLazyAndIdentical(u[:1], v[:1])
         self.assertLazyAndIdentical(u[[0, 1], [0, 1, 2]], v[[0, 1], [0, 1, 2]])
 
-    @pytest.mark.skipif(
-        LooseVersion(dask.__version__) < LooseVersion("2021.04.1"),
-        reason="Requires dask v2021.04.1 or later",
-    )
+    @pytest.mark.skipif(dask_version < "2021.04.1", reason="Requires dask >= 2021.04.1")
     @pytest.mark.parametrize(
         "expected_data, index",
         [
@@ -133,10 +130,7 @@ def test_setitem_dask_array(self, expected_data, index):
         arr[index] = 99
         assert_identical(arr, expected)
 
-    @pytest.mark.skipif(
-        LooseVersion(dask.__version__) >= LooseVersion("2021.04.1"),
-        reason="Requires dask v2021.04.0 or earlier",
-    )
+    @pytest.mark.skipif(dask_version >= "2021.04.1", reason="Requires dask < 2021.04.1")
     def test_setitem_dask_array_error(self):
         with pytest.raises(TypeError, match=r"stored in a dask array"):
             v = self.lazy_var
@@ -612,25 +606,6 @@ def test_dot(self):
         lazy = self.lazy_array.dot(self.lazy_array[0])
         self.assertLazyAndAllClose(eager, lazy)
 
-    @pytest.mark.skipif(LooseVersion(dask.__version__) >= "2.0", reason="no meta")
-    def test_dataarray_repr_legacy(self):
-        data = build_dask_array("data")
-        nonindex_coord = build_dask_array("coord")
-        a = DataArray(data, dims=["x"], coords={"y": ("x", nonindex_coord)})
-        expected = dedent(
-            """\
-            <xarray.DataArray 'data' (x: 1)>
-            {!r}
-            Coordinates:
-                y        (x) int64 dask.array<chunksize=(1,), meta=np.ndarray>
-            Dimensions without coordinates: x""".format(
-                data
-            )
-        )
-        assert expected == repr(a)
-        assert kernel_call_count == 0  # should not evaluate dask array
-
-    @pytest.mark.skipif(LooseVersion(dask.__version__) < "2.0", reason="needs meta")
     def test_dataarray_repr(self):
         data = build_dask_array("data")
         nonindex_coord = build_dask_array("coord")
@@ -648,7 +623,6 @@ def test_dataarray_repr(self):
         assert expected == repr(a)
         assert kernel_call_count == 0  # should not evaluate dask array
 
-    @pytest.mark.skipif(LooseVersion(dask.__version__) < "2.0", reason="needs meta")
     def test_dataset_repr(self):
         data = build_dask_array("data")
         nonindex_coord = build_dask_array("coord")
@@ -1645,7 +1619,7 @@ def test_optimize():
 
 # The graph_manipulation module is in dask since 2021.2 but it became usable with
 # xarray only since 2021.3
-@pytest.mark.skipif(LooseVersion(dask.__version__) <= "2021.02.0", reason="new module")
+@pytest.mark.skipif(dask_version <= "2021.02.0", reason="new module")
 def test_graph_manipulation():
     """dask.graph_manipulation passes an optional parameter, "rename", to the rebuilder
     function returned by __dask_postperist__; also, the dsk passed to the rebuilder is

xarray/tests/test_distributed.py

@@ -184,11 +184,7 @@ def test_dask_distributed_cfgrib_integration_test(loop):
                     assert_allclose(actual, expected)
 
 
-@pytest.mark.skipif(
-    distributed.__version__ <= "1.19.3",
-    reason="Need recent distributed version to clean up get",
-)
-@gen_cluster(client=True, timeout=None)
+@gen_cluster(client=True)
 async def test_async(c, s, a, b):
     x = create_test_data()
     assert not dask.is_dask_collection(x)

xarray/tests/test_formatting_html.py

@@ -1,5 +1,3 @@
-from distutils.version import LooseVersion
-
 import numpy as np
 import pandas as pd
 import pytest
@@ -57,19 +55,9 @@ def test_short_data_repr_html_non_str_keys(dataset):
 
 
 def test_short_data_repr_html_dask(dask_dataarray):
-    import dask
-
-    if LooseVersion(dask.__version__) < "2.0.0":
-        assert not hasattr(dask_dataarray.data, "_repr_html_")
-        data_repr = fh.short_data_repr_html(dask_dataarray)
-        assert (
-            data_repr
-            == "dask.array&lt;xarray-&lt;this-array&gt;, shape=(4, 6), dtype=float64, chunksize=(4, 6)&gt;"
-        )
-    else:
-        assert hasattr(dask_dataarray.data, "_repr_html_")
-        data_repr = fh.short_data_repr_html(dask_dataarray)
-        assert data_repr == dask_dataarray.data._repr_html_()
+    assert hasattr(dask_dataarray.data, "_repr_html_")
+    data_repr = fh.short_data_repr_html(dask_dataarray)
+    assert data_repr == dask_dataarray.data._repr_html_()
 
 
 def test_format_dims_no_dims():