Avoid computing dask variables on __repr__ and __getattr__ (#1532)

* stop repr(Dataset) from resolving dask variables

Also stop resolving non-index coords in Datasets

* stop DataArray.__getattr__('NOTEXIST') from computing dask variables

* PEP8 fixes

* PEP8 fixes

* Tests for __repr__, __getattr__ and __getstate__

* int is coerced to int64 or int32 on different systems

* What's New

* flake8 tweaks

* print summary of dask arrays

* More compact printing for dask arrays

* Cleared changelog and moved to breaking changes

* Deduplicate code

Author: crusaderky

doc/whats-new.rst

@@ -27,6 +27,11 @@ Breaking changes
   (:issue:`727`).
   By `Joe Hamman <https://github.com/jhamman>`_.
 
+- ``repr`` and the Jupyter Notebook won't automatically compute dask variables.
+  Datasets loaded with ``open_dataset`` won't automatically read coords from
+  disk when calling ``repr`` (:issue:`1522`).
+  By `Guido Imperiale <https://github.com/crusaderky>`_.
+
 Backward Incompatible Changes
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -153,7 +158,7 @@ Bug fixes
   ``rtol`` arguments when called on ``DataArray`` objects.
   By `Stephan Hoyer <https://github.com/shoyer>`_.
 
-- Xarray ``quantile`` methods now properly raise a ``TypeError`` when applied to
+- xarray ``quantile`` methods now properly raise a ``TypeError`` when applied to
   objects with data stored as ``dask`` arrays (:issue:`1529`).
   By `Joe Hamman <https://github.com/jhamman>`_.
 

xarray/core/dataarray.py

@@ -447,8 +447,8 @@ def _level_coords(self):
         """
         level_coords = OrderedDict()
         for cname, var in self._coords.items():
-            if var.ndim == 1:
-                level_names = var.to_index_variable().level_names
+            if var.ndim == 1 and isinstance(var, IndexVariable):
+                level_names = var.level_names
                 if level_names is not None:
                     dim, = var.dims
                     level_coords.update({lname: dim for lname in level_names})

xarray/core/dataset.py

@@ -654,8 +654,8 @@ def _level_coords(self):
         level_coords = OrderedDict()
         for cname in self._coord_names:
             var = self.variables[cname]
-            if var.ndim == 1:
-                level_names = var.to_index_variable().level_names
+            if var.ndim == 1 and isinstance(var, IndexVariable):
+                level_names = var.level_names
                 if level_names is not None:
                     dim, = var.dims
                     level_coords.update({lname: dim for lname in level_names})
@@ -1669,12 +1669,12 @@ def expand_dims(self, dim, axis=None):
         for d in dim:
             if d in self.dims:
                 raise ValueError(
-                            'Dimension {dim} already exists.'.format(dim=d))
+                    'Dimension {dim} already exists.'.format(dim=d))
             if (d in self._variables and
                     not utils.is_scalar(self._variables[d])):
                 raise ValueError(
-                            '{dim} already exists as coordinate or'
-                            ' variable name.'.format(dim=d))
+                    '{dim} already exists as coordinate or'
+                    ' variable name.'.format(dim=d))
 
         if len(dim) != len(set(dim)):
             raise ValueError('dims should not contain duplicate values.')
@@ -1691,7 +1691,7 @@ def expand_dims(self, dim, axis=None):
                             raise IndexError(
                                 'Axis {a} is out of bounds of the expanded'
                                 ' dimension size {dim}.'.format(
-                                               a=a, v=k, dim=result_ndim))
+                                    a=a, v=k, dim=result_ndim))
 
                     axis_pos = [a if a >= 0 else result_ndim + a
                                 for a in axis]
@@ -3008,8 +3008,8 @@ def filter_by_attrs(self, **kwargs):
         for var_name, variable in self.data_vars.items():
             for attr_name, pattern in kwargs.items():
                 attr_value = variable.attrs.get(attr_name)
-                if ((callable(pattern) and pattern(attr_value))
-                        or attr_value == pattern):
+                if ((callable(pattern) and pattern(attr_value)) or
+                        attr_value == pattern):
                     selection.append(var_name)
         return self[selection]
 

xarray/core/formatting.py

@@ -196,8 +196,8 @@ def format_array_flat(items_ndarray, max_width):
     return pprint_str
 
 
-def _summarize_var_or_coord(name, var, col_width, show_values=True,
-                            marker=' ', max_width=None):
+def summarize_variable(name, var, col_width, show_values=True,
+                       marker=' ', max_width=None):
     if max_width is None:
         max_width = OPTIONS['display_width']
     first_col = pretty_print(u'  %s %s ' % (marker, name), col_width)
@@ -208,6 +208,8 @@ def _summarize_var_or_coord(name, var, col_width, show_values=True,
     front_str = u'%s%s%s ' % (first_col, dims_str, var.dtype)
     if show_values:
         values_str = format_array_flat(var, max_width - len(front_str))
+    elif isinstance(var.data, dask_array_type):
+        values_str = short_dask_repr(var, show_dtype=False)
     else:
         values_str = u'...'
 
@@ -222,38 +224,29 @@ def _summarize_coord_multiindex(coord, col_width, marker):
 def _summarize_coord_levels(coord, col_width, marker=u'-'):
     relevant_coord = coord[:30]
     return u'\n'.join(
-        [_summarize_var_or_coord(lname,
-                                 relevant_coord.get_level_variable(lname),
-                                 col_width, marker=marker)
+        [summarize_variable(lname,
+                            relevant_coord.get_level_variable(lname),
+                            col_width, marker=marker)
          for lname in coord.level_names])
 
 
-def _not_remote(var):
-    """Helper function to identify if array is positively identifiable as
-    coming from a remote source.
-    """
-    source = var.encoding.get('source')
-    if source and source.startswith('http') and not var._in_memory:
-        return False
-    return True
-
-
-def summarize_var(name, var, col_width):
-    show_values = _not_remote(var)
-    return _summarize_var_or_coord(name, var, col_width, show_values)
+def summarize_datavar(name, var, col_width):
+    show_values = var._in_memory
+    return summarize_variable(name, var.variable, col_width, show_values)
 
 
 def summarize_coord(name, var, col_width):
     is_index = name in var.dims
-    show_values = is_index or _not_remote(var)
+    show_values = var._in_memory
     marker = u'*' if is_index else u' '
     if is_index:
         coord = var.variable.to_index_variable()
         if coord.level_names is not None:
             return u'\n'.join(
                 [_summarize_coord_multiindex(coord, col_width, marker),
                  _summarize_coord_levels(coord, col_width)])
-    return _summarize_var_or_coord(name, var, col_width, show_values, marker)
+    return summarize_variable(
+        name, var.variable, col_width, show_values, marker)
 
 
 def summarize_attr(key, value, col_width=None):
@@ -307,7 +300,7 @@ def _mapping_repr(mapping, title, summarizer, col_width=None):
 
 
 data_vars_repr = functools.partial(_mapping_repr, title=u'Data variables',
-                                   summarizer=summarize_var)
+                                   summarizer=summarize_datavar)
 
 
 attrs_repr = functools.partial(_mapping_repr, title=u'Attributes',
@@ -370,6 +363,19 @@ def short_array_repr(array):
         return repr(array)
 
 
+def short_dask_repr(array, show_dtype=True):
+    """Similar to dask.array.DataArray.__repr__, but without
+    redundant information that's already printed by the repr
+    function of the xarray wrapper.
+    """
+    chunksize = tuple(c[0] for c in array.chunks)
+    if show_dtype:
+        return 'dask.array<shape=%s, dtype=%s, chunksize=%s>' % (
+            array.shape, array.dtype, chunksize)
+    else:
+        return 'dask.array<shape=%s, chunksize=%s>' % (array.shape, chunksize)
+
+
 def array_repr(arr):
     # used for DataArray, Variable and IndexVariable
     if hasattr(arr, 'name') and arr.name is not None:
@@ -381,7 +387,7 @@ def array_repr(arr):
                % (type(arr).__name__, name_str, dim_summary(arr))]
 
     if isinstance(getattr(arr, 'variable', arr)._data, dask_array_type):
-        summary.append(repr(arr.data))
+        summary.append(short_dask_repr(arr))
     elif arr._in_memory or arr.size < 1e5:
         summary.append(short_array_repr(arr.values))
     else:

xarray/core/merge.py

@@ -113,7 +113,7 @@ def merge_variables(
         list_of_variables_dicts,  # type: List[Mapping[Any, Variable]]
         priority_vars=None,       # type: Optional[Mapping[Any, Variable]]
         compat='minimal',         # type: str
-        ):
+):
     # type: (...) -> OrderedDict[Any, Variable]
     """Merge dicts of variables, while resolving conflicts appropriately.
 
@@ -180,7 +180,7 @@ def expand_variable_dicts(list_of_variable_dicts):
     Parameters
     ----------
     list_of_variable_dicts : list of dict or Dataset objects
-        The each value for the mappings must be of the following types:
+        Each value for the mappings must be of the following types:
         - an xarray.Variable
         - a tuple `(dims, data[, attrs[, encoding]])` that can be converted in
           an xarray.Variable

xarray/core/variable.py

@@ -284,7 +284,7 @@ def nbytes(self):
 
     @property
     def _in_memory(self):
-        return (isinstance(self._data, (np.ndarray, PandasIndexAdapter)) or
+        return (isinstance(self._data, (np.ndarray, np.number, PandasIndexAdapter)) or
                 (isinstance(self._data, indexing.MemoryCachedArray) and
                  isinstance(self._data.array, np.ndarray)))
 
@@ -1210,6 +1210,7 @@ def func(self, other):
             return self
         return func
 
+
 ops.inject_all_ops_and_reduce_methods(Variable)
 
 
@@ -1374,6 +1375,7 @@ def name(self):
     def name(self, value):
         raise AttributeError('cannot modify name of IndexVariable in-place')
 
+
 # for backwards compatibility
 Coordinate = utils.alias(IndexVariable, 'Coordinate')