Fix sparse ops that were calling bottleneck (#3254)

min and max are now working.

notnull was already fixed by one of my earlier PRs.

std/var/median are still broken, but only because sparse hasn't implemented
the corresponding NumPy functions yet (nanstd, nanvar and nanmedian).

rank needs pure NumPy implementation (not via bottleneck) if we want it to
work on sparse or dask arrays.

Author: shoyer

xarray/core/nputils.py

@@ -209,6 +209,7 @@ def f(values, axis=None, **kwargs):
 
         if (
             _USE_BOTTLENECK
+            and isinstance(values, np.ndarray)
             and bn_func is not None
             and not isinstance(axis, tuple)
             and values.dtype.kind in "uifc"

xarray/core/variable.py

@@ -1697,18 +1697,24 @@ def rank(self, dim, pct=False):
         """
         import bottleneck as bn
 
-        if isinstance(self.data, dask_array_type):
+        data = self.data
+
+        if isinstance(data, dask_array_type):
             raise TypeError(
                 "rank does not work for arrays stored as dask "
                 "arrays. Load the data via .compute() or .load() "
                 "prior to calling this method."
             )
+        elif not isinstance(data, np.ndarray):
+            raise TypeError(
+                "rank is not implemented for {} objects.".format(type(data))
+            )
 
         axis = self.get_axis_num(dim)
         func = bn.nanrankdata if self.dtype.kind == "f" else bn.rankdata
-        ranked = func(self.data, axis=axis)
+        ranked = func(data, axis=axis)
         if pct:
-            count = np.sum(~np.isnan(self.data), axis=axis, keepdims=True)
+            count = np.sum(~np.isnan(data), axis=axis, keepdims=True)
             ranked /= count
         return Variable(self.dims, ranked)
 

xarray/tests/test_sparse.py

@@ -171,11 +171,13 @@ def test_variable_property(prop):
             False,
             marks=xfail(reason="'COO' object has no attribute 'item'"),
         ),
-        param(do("max"), False, marks=xfail(reason="Coercion to dense via bottleneck")),
         param(
-            do("median"), False, marks=xfail(reason="Coercion to dense via bottleneck")
+            do("median"),
+            False,
+            marks=xfail(reason="Missing implementation for np.nanmedian"),
         ),
-        param(do("min"), False, marks=xfail(reason="Coercion to dense via bottleneck")),
+        param(do("max"), False),
+        param(do("min"), False),
         param(
             do("no_conflicts", other=make_xrvar({"x": 10, "y": 5})),
             True,
@@ -199,7 +201,7 @@ def test_variable_property(prop):
         param(
             do("rank", dim="x"),
             False,
-            marks=xfail(reason="Coercion to dense via bottleneck"),
+            marks=xfail(reason="Only implemented for NumPy arrays (via bottleneck)"),
         ),
         param(
             do("reduce", func=np.sum, dim="x"),
@@ -214,13 +216,17 @@ def test_variable_property(prop):
         param(
             do("shift", x=2), True, marks=xfail(reason="mixed sparse-dense operation")
         ),
-        param(do("std"), False, marks=xfail(reason="Coercion to dense via bottleneck")),
+        param(
+            do("std"), False, marks=xfail(reason="Missing implementation for np.nanstd")
+        ),
         param(
             do("sum"),
             False,
             marks=xfail(reason="Missing implementation for np.result_type"),
         ),
-        param(do("var"), False, marks=xfail(reason="Coercion to dense via bottleneck")),
+        param(
+            do("var"), False, marks=xfail(reason="Missing implementation for np.nanvar")
+        ),
         param(do("to_dict"), False, marks=xfail(reason="Coercion to dense")),
         param(
             do("where", cond=make_xrvar({"x": 10, "y": 5}) > 0.5),
@@ -476,16 +482,14 @@ def test_dataarray_property(prop):
             False,
             marks=xfail(reason="'COO' object has no attribute 'item'"),
         ),
-        param(do("max"), False, marks=xfail(reason="Coercion to dense via bottleneck")),
+        param(do("max"), False),
+        param(do("min"), False),
         param(
-            do("median"), False, marks=xfail(reason="Coercion to dense via bottleneck")
-        ),
-        param(do("min"), False, marks=xfail(reason="Coercion to dense via bottleneck")),
-        param(
-            do("notnull"),
+            do("median"),
             False,
-            marks=xfail(reason="'COO' object has no attribute 'notnull'"),
+            marks=xfail(reason="Missing implementation for np.nanmedian"),
         ),
+        param(do("notnull"), True),
         param(
             do("pipe", np.sum, axis=1),
             True,
@@ -504,7 +508,7 @@ def test_dataarray_property(prop):
         param(
             do("rank", "x"),
             False,
-            marks=xfail(reason="Coercion to dense via bottleneck"),
+            marks=xfail(reason="Only implemented for NumPy arrays (via bottleneck)"),
         ),
         param(
             do("reduce", np.sum, dim="x"),
@@ -532,13 +536,17 @@ def test_dataarray_property(prop):
             True,
             marks=xfail(reason="Indexing COO with more than one iterable index"),
         ),  # noqa
-        param(do("std"), False, marks=xfail(reason="Coercion to dense via bottleneck")),
+        param(
+            do("std"), False, marks=xfail(reason="Missing implementation for np.nanstd")
+        ),
         param(
             do("sum"),
             False,
             marks=xfail(reason="Missing implementation for np.result_type"),
         ),
-        param(do("var"), False, marks=xfail(reason="Coercion to dense via bottleneck")),
+        param(
+            do("var"), False, marks=xfail(reason="Missing implementation for np.nanvar")
+        ),
         param(
             do("where", make_xrarray({"x": 10, "y": 5}) > 0.5),
             False,