Use plotly.js base64 API to store and pass typed arrays declared by numpy, pandas, etc.

.circleci/config.yml

@@ -81,7 +81,7 @@ commands:
             pytest plotly/tests/test_io
           no_output_timeout: 20m
       - run:
-          name: Test dependencdies not imported
+          name: Test dependencies not imported
           command: |
             cd packages/python/plotly
             . venv/bin/activate
@@ -92,46 +92,6 @@ commands:
             cd packages/python/plotly
             . venv/bin/activate
             pytest -x test_init/test_lazy_imports.py
-  test_orca:
-    parameters:
-      py:
-        default: "310"
-        type: string
-    steps:
-      - checkout
-      - browser-tools/install-chrome
-      - browser-tools/install-chromedriver
-      - run:
-          name: Install dependencies
-          command: |
-            cd packages/python/plotly
-            python -m venv venv
-            . venv/bin/activate
-            pip install --upgrade pip wheel
-            pip install -r ./test_requirements/requirements_<<parameters.py>>_optional.txt
-      - run:
-          name: Install plotly-geo
-          command: |
-            cd packages/python/plotly-geo
-            . ../plotly/venv/bin/activate
-            pip install -e .
-      - run:
-          name: Install orca
-          command: |
-            npm install [email protected]
-            npm install orca
-            sudo apt-get update
-            sudo apt-get install -y poppler-utils libxtst6 xvfb libgtk2.0-0 libgconf-2-4 libnss3 libasound2 rename
-            echo 'export PATH="/home/circleci/project/node_modules/.bin:$PATH"' >> $BASH_ENV
-      - run:
-          name: Test orca
-          command: |
-            cd packages/python/plotly
-            . venv/bin/activate
-            pytest plotly/tests/test_orca
-          no_output_timeout: 20m
-      - store_artifacts:
-          path: packages/python/plotly/plotly/tests/test_orca/images/linux/failed
 
 jobs:
   check-code-formatting:

CHANGELOG.md

@@ -2,6 +2,10 @@
 All notable changes to this project will be documented in this file.
 This project adheres to [Semantic Versioning](http://semver.org/).
 
+### Updated
+
+- Updated plotly.py to use base64 encoding of arrays in plotly JSON to improve performance. 
+
 ## [5.24.1] - 2024-09-12
 
 ### Updated

packages/python/plotly/_plotly_utils/basevalidators.py

@@ -1,6 +1,7 @@
 import base64
 import numbers
 import textwrap
+import traceback
 import uuid
 from importlib import import_module
 import copy
@@ -207,6 +208,17 @@ def is_homogeneous_array(v):
     return False
 
 
+def is_typed_array_spec(v):
+    """
+    Return whether a value is considered to be a typed array spec for plotly.js
+    """
+    return isinstance(v, dict) and "bdata" in v and "dtype" in v
+
+
+def is_none_or_typed_array_spec(v):
+    return v is None or is_typed_array_spec(v)
+
+
 def is_simple_array(v):
     """
     Return whether a value is considered to be an simple array
@@ -400,9 +412,7 @@ def description(self):
         )
 
     def validate_coerce(self, v):
-
-        if v is None:
-            # Pass None through
+        if is_none_or_typed_array_spec(v):
             pass
         elif is_homogeneous_array(v):
             v = copy_to_readonly_numpy_array(v)
@@ -599,8 +609,7 @@ def in_values(self, e):
         return False
 
     def validate_coerce(self, v):
-        if v is None:
-            # Pass None through
+        if is_none_or_typed_array_spec(v):
             pass
         elif self.array_ok and is_array(v):
             v_replaced = [self.perform_replacemenet(v_el) for v_el in v]
@@ -760,8 +769,7 @@ def description(self):
         return desc
 
     def validate_coerce(self, v):
-        if v is None:
-            # Pass None through
+        if is_none_or_typed_array_spec(v):
             pass
         elif self.array_ok and is_homogeneous_array(v):
             np = get_module("numpy")
@@ -907,8 +915,7 @@ def description(self):
         return desc
 
     def validate_coerce(self, v):
-        if v is None:
-            # Pass None through
+        if is_none_or_typed_array_spec(v):
             pass
         elif v in self.extras:
             return v
@@ -1071,8 +1078,7 @@ def description(self):
         return desc
 
     def validate_coerce(self, v):
-        if v is None:
-            # Pass None through
+        if is_none_or_typed_array_spec(v):
             pass
         elif self.array_ok and is_array(v):
 
@@ -1373,8 +1379,7 @@ def description(self):
         return valid_color_description
 
     def validate_coerce(self, v, should_raise=True):
-        if v is None:
-            # Pass None through
+        if is_none_or_typed_array_spec(v):
             pass
         elif self.array_ok and is_homogeneous_array(v):
             v = copy_to_readonly_numpy_array(v)
@@ -1518,8 +1523,7 @@ def description(self):
 
     def validate_coerce(self, v):
 
-        if v is None:
-            # Pass None through
+        if is_none_or_typed_array_spec(v):
             pass
         elif is_array(v):
             validated_v = [
@@ -1724,8 +1728,7 @@ def description(self):
         return desc
 
     def validate_coerce(self, v):
-        if v is None:
-            # Pass None through
+        if is_none_or_typed_array_spec(v):
             pass
         elif self.array_ok and is_homogeneous_array(v):
             try:
@@ -1913,6 +1916,9 @@ def validate_coerce(self, v):
         if v is None:
             # Pass None through
             pass
+        if is_typed_array_spec(v):
+            # Pass typed array spec through
+            pass
         elif self.array_ok and is_array(v):
 
             # Coerce individual strings
@@ -1969,8 +1975,7 @@ def description(self):
         return desc
 
     def validate_coerce(self, v):
-        if v is None:
-            # Pass None through
+        if is_none_or_typed_array_spec(v):
             pass
         elif self.array_ok and is_homogeneous_array(v):
             v = copy_to_readonly_numpy_array(v, kind="O")
@@ -2178,8 +2183,7 @@ def validate_element_with_indexed_name(self, val, validator, inds):
         return val
 
     def validate_coerce(self, v):
-        if v is None:
-            # Pass None through
+        if is_none_or_typed_array_spec(v):
             return None
         elif not is_array(v):
             self.raise_invalid_val(v)
@@ -2242,7 +2246,7 @@ def validate_coerce(self, v):
         return v
 
     def present(self, v):
-        if v is None:
+        if is_none_or_typed_array_spec(v):
             return None
         else:
             if (

packages/python/plotly/_plotly_utils/utils.py

@@ -1,11 +1,116 @@
+import base64
 import decimal
 import json as _json
 import sys
 import re
 from functools import reduce
 
 from _plotly_utils.optional_imports import get_module
-from _plotly_utils.basevalidators import ImageUriValidator
+from _plotly_utils.basevalidators import (
+    ImageUriValidator,
+    copy_to_readonly_numpy_array,
+    is_homogeneous_array,
+)
+
+
+int8min = -128
+int8max = 127
+int16min = -32768
+int16max = 32767
+int32min = -2147483648
+int32max = 2147483647
+
+uint8max = 255
+uint16max = 65535
+uint32max = 4294967295
+
+plotlyjsShortTypes = {
+    "int8": "i1",
+    "uint8": "u1",
+    "int16": "i2",
+    "uint16": "u2",
+    "int32": "i4",
+    "uint32": "u4",
+    "float32": "f4",
+    "float64": "f8",
+}
+
+
+def to_typed_array_spec(v):
+    """
+    Convert numpy array to plotly.js typed array spec
+    If not possible return the original value
+    """
+    v = copy_to_readonly_numpy_array(v)
+
+    np = get_module("numpy", should_load=False)
+    if not isinstance(v, np.ndarray):
+        return v
+
+    dtype = str(v.dtype)
+
+    # convert default Big Ints until we could support them in plotly.js
+    if dtype == "int64":
+        max = v.max()
+        min = v.min()
+        if max <= int8max and min >= int8min:
+            v = v.astype("int8")
+        elif max <= int16max and min >= int16min:
+            v = v.astype("int16")
+        elif max <= int32max and min >= int32min:
+            v = v.astype("int32")
+        else:
+            return v
+
+    elif dtype == "uint64":
+        max = v.max()
+        min = v.min()
+        if max <= uint8max and min >= 0:
+            v = v.astype("uint8")
+        elif max <= uint16max and min >= 0:
+            v = v.astype("uint16")
+        elif max <= uint32max and min >= 0:
+            v = v.astype("uint32")
+        else:
+            return v
+
+    dtype = str(v.dtype)
+
+    if dtype in plotlyjsShortTypes:
+        arrObj = {
+            "dtype": plotlyjsShortTypes[dtype],
+            "bdata": base64.b64encode(v).decode("ascii"),
+        }
+
+        if v.ndim > 1:
+            arrObj["shape"] = str(v.shape)[1:-1]
+
+        return arrObj
+
+    return v
+
+
+def is_skipped_key(key):
+    """
+    Return whether any keys in the parent hierarchy are in the list of keys that
+    are skipped for conversion to the typed array spec
+    """
+    skipped_keys = ["geojson", "layer", "range"]
+    return any(skipped_key in key for skipped_key in skipped_keys)
+
+
+def convert_to_base64(obj):
+    if isinstance(obj, dict):
+        for key, value in obj.items():
+            if is_skipped_key(key):
+                continue
+            elif is_homogeneous_array(value):
+                obj[key] = to_typed_array_spec(value)
+            else:
+                convert_to_base64(value)
+    elif isinstance(obj, list) or isinstance(obj, tuple):
+        for value in obj:
+            convert_to_base64(value)
 
 
 def cumsum(x):

packages/python/plotly/plotly/basedatatypes.py

@@ -15,6 +15,7 @@
     display_string_positions,
     chomp_empty_strings,
     find_closest_string,
+    convert_to_base64,
 )
 from _plotly_utils.exceptions import PlotlyKeyError
 from .optional_imports import get_module
@@ -3310,6 +3311,9 @@ def to_dict(self):
         if frames:
             res["frames"] = frames
 
+        # Add base64 conversion before sending to the front-end
+        convert_to_base64(res)
+
         return res
 
     def to_plotly_json(self):

packages/python/plotly/plotly/io/_utils.py

@@ -24,6 +24,7 @@ def validate_coerce_fig_to_dict(fig, validate):
                 typ=type(fig), v=fig
             )
         )
+
     return fig_dict