Merge pull request #3293 from plotly/gh_3292

Fix: customdata is sometimes a tuple

Author: nicolaskruchten

Diff for: CHANGELOG.md

@@ -4,6 +4,8 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 ## UNRELEASED
 
+### Fixed
+  - Fixed regression introduced in version 5.0.0 where pandas/numpy arrays with `dtype` of Object were being converted to `list` values when added to a Figure ([#3292](https://github.com/plotly/plotly.py/issues/3292), [#3293](https://github.com/plotly/plotly.py/pull/3293))
 
 ## [5.1.0] - 2021-06-28
 

Diff for: packages/python/plotly/_plotly_utils/basevalidators.py

@@ -53,7 +53,7 @@ def to_scalar_or_list(v):
         return v
 
 
-def copy_to_readonly_numpy_array_or_list(v, kind=None, force_numeric=False):
+def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):
     """
     Convert an array-like value into a read-only numpy array
 
@@ -94,7 +94,6 @@ def copy_to_readonly_numpy_array_or_list(v, kind=None, force_numeric=False):
         "i": "int32",
         "f": "float64",
         "O": "object",
-        "U": "U",
     }
 
     # Handle pandas Series and Index objects
@@ -119,12 +118,18 @@ def copy_to_readonly_numpy_array_or_list(v, kind=None, force_numeric=False):
     if not isinstance(v, np.ndarray):
         # v has its own logic on how to convert itself into a numpy array
         if is_numpy_convertable(v):
-            return copy_to_readonly_numpy_array_or_list(
+            return copy_to_readonly_numpy_array(
                 np.array(v), kind=kind, force_numeric=force_numeric
             )
         else:
             # v is not homogenous array
-            return [to_scalar_or_list(e) for e in v]
+            v_list = [to_scalar_or_list(e) for e in v]
+
+            # Lookup dtype for requested kind, if any
+            dtype = kind_default_dtypes.get(first_kind, None)
+
+            # construct new array from list
+            new_v = np.array(v_list, order="C", dtype=dtype)
     elif v.dtype.kind in numeric_kinds:
         # v is a homogenous numeric array
         if kind and v.dtype.kind not in kind:
@@ -135,12 +140,6 @@ def copy_to_readonly_numpy_array_or_list(v, kind=None, force_numeric=False):
         else:
             # Either no kind was requested or requested kind is satisfied
             new_v = np.ascontiguousarray(v.copy())
-    elif v.dtype.kind == "O":
-        if kind:
-            dtype = kind_default_dtypes.get(first_kind, None)
-            return np.array(v, dtype=dtype)
-        else:
-            return v.tolist()
     else:
         # v is a non-numeric homogenous array
         new_v = v.copy()
@@ -155,12 +154,12 @@ def copy_to_readonly_numpy_array_or_list(v, kind=None, force_numeric=False):
     if "U" not in kind:
         # Force non-numeric arrays to have object type
         # --------------------------------------------
-        # Here we make sure that non-numeric arrays become lists
-        # This works around cases like np.array([1, 2, '3']) where
+        # Here we make sure that non-numeric arrays have the object
+        # datatype. This works around cases like np.array([1, 2, '3']) where
         # numpy converts the integers to strings and returns array of dtype
         # '<U21'
         if new_v.dtype.kind not in ["u", "i", "f", "O", "M"]:
-            return v.tolist()
+            new_v = np.array(v, dtype="object")
 
     # Set new array to be read-only
     # -----------------------------
@@ -399,7 +398,7 @@ def validate_coerce(self, v):
             # Pass None through
             pass
         elif is_homogeneous_array(v):
-            v = copy_to_readonly_numpy_array_or_list(v)
+            v = copy_to_readonly_numpy_array(v)
         elif is_simple_array(v):
             v = to_scalar_or_list(v)
         else:
@@ -610,7 +609,7 @@ def validate_coerce(self, v):
                 self.raise_invalid_elements(invalid_els[:10])
 
             if is_homogeneous_array(v):
-                v = copy_to_readonly_numpy_array_or_list(v)
+                v = copy_to_readonly_numpy_array(v)
             else:
                 v = to_scalar_or_list(v)
         else:
@@ -766,7 +765,7 @@ def validate_coerce(self, v):
         elif self.array_ok and is_homogeneous_array(v):
             np = get_module("numpy")
             try:
-                v_array = copy_to_readonly_numpy_array_or_list(v, force_numeric=True)
+                v_array = copy_to_readonly_numpy_array(v, force_numeric=True)
             except (ValueError, TypeError, OverflowError):
                 self.raise_invalid_val(v)
 
@@ -893,7 +892,7 @@ def validate_coerce(self, v):
             pass
         elif self.array_ok and is_homogeneous_array(v):
             np = get_module("numpy")
-            v_array = copy_to_readonly_numpy_array_or_list(
+            v_array = copy_to_readonly_numpy_array(
                 v, kind=("i", "u"), force_numeric=True
             )
 
@@ -1054,7 +1053,25 @@ def validate_coerce(self, v):
                 if invalid_els:
                     self.raise_invalid_elements(invalid_els)
 
-            if is_simple_array(v) or is_homogeneous_array(v):
+            if is_homogeneous_array(v):
+                np = get_module("numpy")
+
+                # If not strict, let numpy cast elements to strings
+                v = copy_to_readonly_numpy_array(v, kind="U")
+
+                # Check no_blank
+                if self.no_blank:
+                    invalid_els = v[v == ""][:10].tolist()
+                    if invalid_els:
+                        self.raise_invalid_elements(invalid_els)
+
+                # Check values
+                if self.values:
+                    invalid_inds = np.logical_not(np.isin(v, self.values))
+                    invalid_els = v[invalid_inds][:10].tolist()
+                    if invalid_els:
+                        self.raise_invalid_elements(invalid_els)
+            elif is_simple_array(v):
                 if not self.strict:
                     v = [StringValidator.to_str_or_unicode_or_none(e) for e in v]
 
@@ -1331,12 +1348,8 @@ def validate_coerce(self, v, should_raise=True):
             # Pass None through
             pass
         elif self.array_ok and is_homogeneous_array(v):
-            v = copy_to_readonly_numpy_array_or_list(v)
-            if (
-                not isinstance(v, list)
-                and self.numbers_allowed()
-                and v.dtype.kind in ["u", "i", "f"]
-            ):
+            v = copy_to_readonly_numpy_array(v)
+            if self.numbers_allowed() and v.dtype.kind in ["u", "i", "f"]:
                 # Numbers are allowed and we have an array of numbers.
                 # All good
                 pass
@@ -1350,9 +1363,9 @@ def validate_coerce(self, v, should_raise=True):
 
                 # ### Check that elements have valid colors types ###
                 elif self.numbers_allowed() or invalid_els:
-                    v = copy_to_readonly_numpy_array_or_list(validated_v, kind="O")
+                    v = copy_to_readonly_numpy_array(validated_v, kind="O")
                 else:
-                    v = copy_to_readonly_numpy_array_or_list(validated_v, kind="U")
+                    v = copy_to_readonly_numpy_array(validated_v, kind="U")
         elif self.array_ok and is_simple_array(v):
             validated_v = [self.validate_coerce(e, should_raise=False) for e in v]
 
@@ -1867,7 +1880,7 @@ def validate_coerce(self, v):
                 self.raise_invalid_elements(invalid_els)
 
             if is_homogeneous_array(v):
-                v = copy_to_readonly_numpy_array_or_list(validated_v, kind="U")
+                v = copy_to_readonly_numpy_array(validated_v, kind="U")
             else:
                 v = to_scalar_or_list(v)
         else:
@@ -1915,7 +1928,7 @@ def validate_coerce(self, v):
             # Pass None through
             pass
         elif self.array_ok and is_homogeneous_array(v):
-            v = copy_to_readonly_numpy_array_or_list(v, kind="O")
+            v = copy_to_readonly_numpy_array(v, kind="O")
         elif self.array_ok and is_simple_array(v):
             v = to_scalar_or_list(v)
         return v

Diff for: packages/python/plotly/_plotly_utils/tests/validators/test_dataarray_validator.py

@@ -32,29 +32,15 @@ def test_validator_acceptance_simple(val, validator):
 
 
 @pytest.mark.parametrize(
-    "val", [np.array([2, 3, 4]), np.array([[1, 2, 3], [4, 5, 6]])],
+    "val",
+    [np.array([2, 3, 4]), pd.Series(["a", "b", "c"]), np.array([[1, 2, 3], [4, 5, 6]])],
 )
 def test_validator_acceptance_homogeneous(val, validator):
     coerce_val = validator.validate_coerce(val)
     assert isinstance(coerce_val, np.ndarray)
     assert np.array_equal(validator.present(coerce_val), val)
 
 
-# Accept object array as list
-@pytest.mark.parametrize(
-    "val",
-    [
-        ["A", "B", "C"],
-        np.array(["A", "B", "C"], dtype="object"),
-        pd.Series(["a", "b", "c"]),
-    ],
-)
-def test_validator_accept_object_array_as_list(val, validator):
-    coerce_val = validator.validate_coerce(val)
-    assert isinstance(coerce_val, list)
-    assert coerce_val == list(val)
-
-
 # ### Rejection ###
 @pytest.mark.parametrize("val", ["Hello", 23, set(), {}])
 def test_rejection(val, validator):

Diff for: packages/python/plotly/_plotly_utils/tests/validators/test_enumerated_validator.py

@@ -135,7 +135,7 @@ def test_acceptance_aok(val, validator_aok_re):
     # Values should be accepted and returned unchanged
     coerce_val = validator_aok_re.validate_coerce(val)
     if isinstance(val, (np.ndarray, pd.Series)):
-        assert coerce_val == list(np.array(val))
+        assert np.array_equal(coerce_val, np.array(val, dtype=coerce_val.dtype))
     elif isinstance(val, (list, tuple)):
         assert validator_aok_re.present(coerce_val) == tuple(val)
     else:

Diff for: packages/python/plotly/_plotly_utils/tests/validators/test_pandas_series_input.py

@@ -149,10 +149,13 @@ def test_color_validator_object(color_validator, color_object_pandas):
     res = color_validator.validate_coerce(color_object_pandas)
 
     # Check type
-    assert isinstance(res, list)
+    assert isinstance(res, np.ndarray)
+
+    # Check dtype
+    assert res.dtype == "object"
 
     # Check values
-    assert res == color_object_pandas.tolist()
+    np.testing.assert_array_equal(res, color_object_pandas)
 
 
 def test_color_validator_categorical(color_validator, color_categorical_pandas):
@@ -161,10 +164,13 @@ def test_color_validator_categorical(color_validator, color_categorical_pandas):
 
     # Check type
     assert color_categorical_pandas.dtype == "category"
-    assert isinstance(res, list)
+    assert isinstance(res, np.ndarray)
+
+    # Check dtype
+    assert res.dtype == "object"
 
     # Check values
-    assert res == color_categorical_pandas.tolist()
+    np.testing.assert_array_equal(res, np.array(color_categorical_pandas))
 
 
 def test_data_array_validator_dates_series(
@@ -174,10 +180,13 @@ def test_data_array_validator_dates_series(
     res = data_array_validator.validate_coerce(datetime_pandas)
 
     # Check type
-    assert isinstance(res, list)
+    assert isinstance(res, np.ndarray)
+
+    # Check dtype
+    assert res.dtype == "object"
 
     # Check values
-    assert res == dates_array.tolist()
+    np.testing.assert_array_equal(res, dates_array)
 
 
 def test_data_array_validator_dates_dataframe(
@@ -188,7 +197,10 @@ def test_data_array_validator_dates_dataframe(
     res = data_array_validator.validate_coerce(df)
 
     # Check type
-    assert isinstance(res, list)
+    assert isinstance(res, np.ndarray)
+
+    # Check dtype
+    assert res.dtype == "object"
 
     # Check values
-    assert res == dates_array.reshape(len(dates_array), 1).tolist()
+    np.testing.assert_array_equal(res, dates_array.reshape(len(dates_array), 1))

Diff for: packages/python/plotly/_plotly_utils/tests/validators/test_string_validator.py

@@ -138,7 +138,8 @@ def test_acceptance_aok_scalars(val, validator_aok):
 def test_acceptance_aok_list(val, validator_aok):
     coerce_val = validator_aok.validate_coerce(val)
     if isinstance(val, np.ndarray):
-        assert coerce_val == val.tolist()
+        assert isinstance(coerce_val, np.ndarray)
+        assert np.array_equal(coerce_val, np.array(val, dtype=coerce_val.dtype))
     elif isinstance(val, list):
         assert validator_aok.present(val) == tuple(val)
     else:
@@ -177,7 +178,9 @@ def test_rejection_aok_values(val, validator_aok_values):
 )
 def test_acceptance_no_blanks_aok(val, validator_no_blanks_aok):
     coerce_val = validator_no_blanks_aok.validate_coerce(val)
-    if isinstance(val, (list, np.ndarray)):
+    if isinstance(val, np.ndarray):
+        assert np.array_equal(coerce_val, np.array(val, dtype=coerce_val.dtype))
+    elif isinstance(val, list):
         assert validator_no_blanks_aok.present(coerce_val) == tuple(val)
     else:
         assert coerce_val == val

Diff for: packages/python/plotly/_plotly_utils/tests/validators/test_xarray_input.py

@@ -126,7 +126,10 @@ def test_color_validator_object(color_validator, color_object_xarray):
     res = color_validator.validate_coerce(color_object_xarray)
 
     # Check type
-    assert isinstance(res, list)
+    assert isinstance(res, np.ndarray)
+
+    # Check dtype
+    assert res.dtype == "object"
 
     # Check values
-    assert res == list(color_object_xarray)
+    np.testing.assert_array_equal(res, color_object_xarray)

Diff for: packages/python/plotly/plotly/tests/test_io/test_to_from_plotly_json.py

@@ -1,6 +1,7 @@
 import pytest
 import plotly.io.json as pio
 import plotly.graph_objects as go
+import plotly.express as px
 import numpy as np
 import pandas as pd
 import json
@@ -202,3 +203,9 @@ def to_str(v):
     expected = build_test_dict_string(array_str)
     assert result == expected
     check_roundtrip(result, engine=engine, pretty=pretty)
+
+
+def test_object_array(engine, pretty):
+    fig = px.scatter(px.data.tips(), x="total_bill", y="tip", custom_data=["sex"])
+    result = fig.to_plotly_json()
+    check_roundtrip(result, engine=engine, pretty=pretty)

Diff for: packages/python/plotly/plotly/tests/test_optional/test_px/test_px.py

@@ -38,8 +38,8 @@ def test_custom_data_scatter():
         hover_data=["petal_length", "petal_width"],
         custom_data=["species_id", "species"],
     )
-    assert [e[0] for e in fig.data[0].customdata] == iris.species_id.to_list()
-    assert len(fig.data[0].customdata[0]) == 4
+    assert np.all(fig.data[0].customdata[:, 0] == iris.species_id)
+    assert fig.data[0].customdata.shape[1] == 4
     # Hover and custom data, with repeated arguments
     fig = px.scatter(
         iris,
@@ -48,8 +48,8 @@ def test_custom_data_scatter():
         hover_data=["petal_length", "petal_width", "species_id"],
         custom_data=["species_id", "species"],
     )
-    assert [e[0] for e in fig.data[0].customdata] == iris.species_id.tolist()
-    assert len(fig.data[0].customdata[0]) == 4
+    assert np.all(fig.data[0].customdata[:, 0] == iris.species_id)
+    assert fig.data[0].customdata.shape[1] == 4
     assert (
         fig.data[0].hovertemplate
         == "sepal_width=%{x}<br>sepal_length=%{y}<br>petal_length=%{customdata[2]}<br>petal_width=%{customdata[3]}<br>species_id=%{customdata[0]}<extra></extra>"

Diff for: packages/python/plotly/plotly/tests/test_optional/test_px/test_px_functions.py

@@ -229,9 +229,9 @@ def test_sunburst_treemap_with_path_color():
     df["hover"] = [el.lower() for el in vendors]
     fig = px.sunburst(df, path=path, color="calls", hover_data=["hover"])
     custom = fig.data[0].customdata
-    assert [el[0] for el in custom[:8]] == df["hover"].tolist()
-    assert [el[0] for el in custom[8:]] == ["(?)"] * 7
-    assert [el[1] for el in custom[:8]] == df["calls"].tolist()
+    assert np.all(custom[:8, 0] == df["hover"])
+    assert np.all(custom[8:, 0] == "(?)")
+    assert np.all(custom[:8, 1] == df["calls"])
 
     # Discrete color
     fig = px.sunburst(df, path=path, color="vendors")

Diff for: packages/python/plotly/plotly/tests/test_optional/test_px/test_px_input.py

@@ -126,7 +126,7 @@ def test_repeated_name():
         hover_data=["petal_length", "petal_width", "species_id"],
         custom_data=["species_id", "species"],
     )
-    assert len(fig.data[0].customdata[0]) == 4
+    assert fig.data[0].customdata.shape[1] == 4
 
 
 def test_arrayattrable_numpy():