Lazy imports of graph object hierarchy for Python 3.7+

This involved splitting validators/graph object classes back into separate files

Author: jonmmease

Diff for: packages/python/plotly/_plotly_utils/importers.py

@@ -0,0 +1,49 @@
+import importlib
+
+
+def relative_import(parent_name, rel_modules=(), rel_classes=()):
+    """
+    Helper function to import submodules lazily in Pythong 3.7+
+
+    Parameters
+    ----------
+    rel_modules: list of str
+        list of submodules to import, of the form .submodule
+    rel_classes: list of str
+        list of submodule classes/variables to import, of the form ._submodule.Foo
+
+    Returns
+    -------
+    tuple
+        Tuple that should be assigned to __all__, __getattr__ in the caller
+    """
+    module_names = {rel_module.split(".")[-1]: rel_module for rel_module in rel_modules}
+    class_names = {rel_path.split(".")[-1]: rel_path for rel_path in rel_classes}
+
+    def __getattr__(import_name):
+        # In Python 3.7+, lazy import submodules
+
+        # Check for submodule
+        if import_name in module_names:
+            # print(parent_name, import_name)
+            rel_import = module_names[import_name]
+            return importlib.import_module(rel_import, parent_name)
+
+        # Check for submodule class
+        if import_name in class_names:
+            # print(parent_name, import_name)
+            rel_path_parts = class_names[import_name].split(".")
+            rel_module = ".".join(rel_path_parts[:-1])
+            class_name = import_name
+            class_module = importlib.import_module(rel_module, parent_name)
+            return getattr(class_module, class_name)
+
+        raise AttributeError(
+            "module {__name__!r} has no attribute {name!r}".format(
+                name=import_name, __name__=parent_name
+            )
+        )
+
+    __all__ = list(module_names) + list(class_names)
+
+    return __all__, __getattr__

Diff for: packages/python/plotly/codegen/__init__.py

@@ -1,4 +1,5 @@
 import json
+import os
 import os.path as opath
 import shutil
 import subprocess
@@ -17,6 +18,7 @@
     FrameNode,
     write_init_py,
     ElementDefaultsNode,
+    build_from_imports_py,
 )
 from codegen.validators import (
     write_validator_py,
@@ -190,14 +192,6 @@ def perform_codegen():
     # -------------------
     for node in all_compound_nodes:
         write_datatype_py(outdir, node)
-        alls.setdefault(node.path_parts, [])
-        alls[node.path_parts].extend(
-            c.name_datatype_class for c in node.child_compound_datatypes
-        )
-        if node.parent_path_parts == ():
-            # Add top-level classes to alls
-            alls.setdefault((), [])
-            alls[node.parent_path_parts].append(node.name_datatype_class)
 
     # ### Deprecated ###
     # These are deprecated legacy datatypes like graph_objs.Marker
@@ -219,54 +213,86 @@ def perform_codegen():
         layout_array_nodes,
     )
 
+    # Write validator __init__.py files
+    # ---------------------------------
+    # ### Write __init__.py files for each validator package ###
+    validator_rel_class_imports = {}
+    for node in all_datatype_nodes:
+        if node.is_mapped:
+            continue
+        key = node.parent_path_parts
+        validator_rel_class_imports.setdefault(key, []).append(
+            f"._{node.name_property}.{node.name_validator_class}"
+        )
+
+    # Add Data validator
+    root_validator_pairs = validator_rel_class_imports[()]
+    root_validator_pairs.append("._data.DataValidator")
+
+    # Output validator __init__.py files
+    validators_pkg = opath.join(outdir, "validators")
+    for path_parts, rel_classes in validator_rel_class_imports.items():
+        write_init_py(validators_pkg, path_parts, [], rel_classes)
+
     # Write datatype __init__.py files
     # --------------------------------
-    # ### Build mapping from parent package to datatype class ###
-    path_to_datatype_import_info = {}
+    datatype_rel_class_imports = {}
+    datatype_rel_module_imports = {}
+
     for node in all_compound_nodes:
         key = node.parent_path_parts
 
+        # class import
+        datatype_rel_class_imports.setdefault(key, []).append(
+            f"._{node.name_undercase}.{node.name_datatype_class}"
+        )
+
         # submodule import
         if node.child_compound_datatypes:
-
-            path_to_datatype_import_info.setdefault(key, []).append(
-                (f"plotly.graph_objs{node.parent_dotpath_str}", node.name_undercase)
+            datatype_rel_module_imports.setdefault(key, []).append(
+                f".{node.name_undercase}"
             )
-            alls[node.parent_path_parts].append(node.name_undercase)
 
     # ### Write plotly/graph_objs/graph_objs.py ###
     # This if for backward compatibility. It just imports everything from
     # graph_objs/__init__.py
     write_graph_objs_graph_objs(outdir)
 
     # ### Add Figure and FigureWidget ###
-    root_datatype_imports = path_to_datatype_import_info[()]
-    root_datatype_imports.append(("._figure", "Figure"))
-    alls[()].append("Figure")
+    root_datatype_imports = datatype_rel_class_imports[()]
+    root_datatype_imports.append("._figure.Figure")
 
     # ### Add deprecations ###
-    root_datatype_imports.append(("._deprecations", DEPRECATED_DATATYPES.keys()))
-    alls[()].extend(DEPRECATED_DATATYPES.keys())
-
-    # Sort alls
-    for k, v in alls.items():
-        alls[k] = list(sorted(v))
+    for dep_clas in DEPRECATED_DATATYPES:
+        root_datatype_imports.append(f"._deprecations.{dep_clas}")
 
+    validate_import = "from .._validate import validate\n"
     optional_figure_widget_import = f"""
-__all__ = {alls[()]}
-try:
-    import ipywidgets
-    from distutils.version import LooseVersion
-    if LooseVersion(ipywidgets.__version__) >= LooseVersion('7.0.0'):
-        from ._figurewidget import FigureWidget
-        __all__.append("FigureWidget")
-    del LooseVersion
-    del ipywidgets
-except ImportError:
-    pass
+if sys.version_info < (3, 7):
+    try:
+        import ipywidgets
+        from distutils.version import LooseVersion
+        if LooseVersion(ipywidgets.__version__) >= LooseVersion('7.0.0'):
+            from ..graph_objs._figurewidget import FigureWidget
+        del LooseVersion
+        del ipywidgets
+    except ImportError:
+        pass
+else:
+    orig_getattr = __getattr__
+    def __getattr__(import_name):
+        if import_name == "FigureWidget":
+            try:
+                import ipywidgets
+                from distutils.version import LooseVersion
+                if LooseVersion(ipywidgets.__version__) >= LooseVersion('7.0.0'):
+                    from ..graph_objs._figurewidget import FigureWidget
+                    return FigureWidget
+            except ImportError:
+                    pass
+        
+        return orig_getattr(import_name)
 """
-    root_datatype_imports.append(optional_figure_widget_import)
-
     # ### __all__ ###
     for path_parts, class_names in alls.items():
         if path_parts and class_names:
@@ -276,18 +302,34 @@ def perform_codegen():
 
     # ### Output datatype __init__.py files ###
     graph_objs_pkg = opath.join(outdir, "graph_objs")
-    for path_parts, import_pairs in path_to_datatype_import_info.items():
-        write_init_py(graph_objs_pkg, path_parts, import_pairs)
+    for path_parts in datatype_rel_class_imports:
+        rel_classes = datatype_rel_class_imports[path_parts]
+        rel_modules = datatype_rel_module_imports.get(path_parts, [])
+        if path_parts == ():
+            init_extra = validate_import + optional_figure_widget_import
+        else:
+            init_extra = ""
+        write_init_py(graph_objs_pkg, path_parts, rel_modules, rel_classes, init_extra)
 
     # ### Output graph_objects.py alias
-    graph_objects_path = opath.join(outdir, "graph_objects.py")
+    graph_objects_rel_classes = [
+        "..graph_objs." + rel_path.split(".")[-1]
+        for rel_path in datatype_rel_class_imports[()]
+    ]
+    graph_objects_rel_modules = [
+        "..graph_objs." + rel_module.split(".")[-1]
+        for rel_module in datatype_rel_module_imports[()]
+    ]
+
+    graph_objects_init_source = build_from_imports_py(
+        graph_objects_rel_modules,
+        graph_objects_rel_classes,
+        init_extra=validate_import + optional_figure_widget_import,
+    )
+    graph_objects_path = opath.join(outdir, "graph_objects", "__init__.py")
+    os.makedirs(opath.join(outdir, "graph_objects"), exist_ok=True)
     with open(graph_objects_path, "wt") as f:
-        f.write(
-            f"""\
-from __future__ import absolute_import
-from plotly.graph_objs import *
-__all__ = {alls[()]}"""
-        )
+        f.write(graph_objects_init_source)
 
     # ### Run black code formatter on output directories ###
     subprocess.call(["black", "--target-version=py27", validators_pkgdir])

Diff for: packages/python/plotly/codegen/datatypes.py

@@ -153,10 +153,22 @@ def _subplot_re_match(self, prop):
 """
         )
 
-    # ### Property definitions ###
     child_datatype_nodes = node.child_datatypes
-
     subtype_nodes = child_datatype_nodes
+    valid_props_list = sorted(
+        [node.name_property for node in subtype_nodes + literal_nodes]
+    )
+    buffer.write(
+        f"""
+    # class properties
+    # --------------------
+    _parent_path_str = '{node.parent_path_str}'
+    _path_str = '{node.path_str}'
+    _valid_props = {{"{'", "'.join(valid_props_list)}"}}
+"""
+    )
+
+    # ### Property definitions ###
     for subtype_node in subtype_nodes:
         if subtype_node.is_array_element:
             prop_type = (
@@ -243,15 +255,9 @@ def {literal_node.name_property}(self):
         )
 
     # ### Private properties descriptions ###
+    valid_props = {node.name_property for node in subtype_nodes}
     buffer.write(
         f"""
-
-    # property parent name
-    # --------------------
-    @property
-    def _parent_path_str(self):
-        return '{node.parent_path_str}'
-
     # Self properties description
     # ---------------------------
     @property
@@ -309,6 +315,23 @@ def __init__(self"""
         f"""
         super({datatype_class}, self).__init__('{node.name_property}')
 
+        if '_parent' in kwargs:
+            self._parent = kwargs['_parent']
+            return
+"""
+    )
+
+    if datatype_class == "Layout":
+        buffer.write(
+            f"""
+        # Override _valid_props for instance so that instance can mutate set
+        # to support subplot properties (e.g. xaxis2)
+        self._valid_props = {{"{'", "'.join(valid_props_list)}"}}
+"""
+        )
+
+    buffer.write(
+        f"""
         # Validate arg
         # ------------
         if arg is None:
@@ -326,23 +349,8 @@ def __init__(self"""
         # Handle skip_invalid
         # -------------------
         self._skip_invalid = kwargs.pop('skip_invalid', False)
-        
-        # Import validators
-        # -----------------
-        from plotly.validators{node.parent_dotpath_str} import (
-            {undercase} as v_{undercase}) 
-
-        # Initialize validators
-        # ---------------------"""
+        """
     )
-    for subtype_node in subtype_nodes:
-        if not subtype_node.is_mapped:
-            sub_name = subtype_node.name_property
-            sub_validator = subtype_node.name_validator_class
-            buffer.write(
-                f"""
-        self._validators['{sub_name}'] = v_{undercase}.{sub_validator}()"""
-            )
 
     buffer.write(
         f"""
@@ -352,27 +360,13 @@ def __init__(self"""
     )
     for subtype_node in subtype_nodes:
         name_prop = subtype_node.name_property
-        if name_prop == "template" or subtype_node.is_mapped:
-            # Special handling for layout.template to avoid infinite
-            # recursion.  Only initialize layout.template object if non-None
-            # value specified.
-            #
-            # Same special handling for mapped nodes (e.g. layout.titlefont)
-            # to keep them for overriding mapped property with None
-            buffer.write(
-                f"""
+        buffer.write(
+            f"""
         _v = arg.pop('{name_prop}', None)
         _v = {name_prop} if {name_prop} is not None else _v
         if _v is not None:
             self['{name_prop}'] = _v"""
-            )
-        else:
-            buffer.write(
-                f"""
-        _v = arg.pop('{name_prop}', None)
-        self['{name_prop}'] = {name_prop} \
-if {name_prop} is not None else _v"""
-            )
+        )
 
     # ### Literals ###
     if literal_nodes:
@@ -381,18 +375,14 @@ def __init__(self"""
 
         # Read-only literals
         # ------------------
-        from _plotly_utils.basevalidators import LiteralValidator"""
+"""
         )
         for literal_node in literal_nodes:
             lit_name = literal_node.name_property
-            lit_parent = literal_node.parent_path_str
             lit_val = repr(literal_node.node_data)
             buffer.write(
                 f"""
         self._props['{lit_name}'] = {lit_val}
-        self._validators['{lit_name}'] =\
-LiteralValidator(plotly_name='{lit_name}',\
-    parent_name='{lit_parent}', val={lit_val})
         arg.pop('{lit_name}', None)"""
             )
 
@@ -609,13 +599,15 @@ def write_datatype_py(outdir, node):
 
     # Build file path
     # ---------------
-    filepath = opath.join(outdir, "graph_objs", *node.parent_path_parts, "__init__.py")
+    # filepath = opath.join(outdir, "graph_objs", *node.parent_path_parts, "__init__.py")
+    filepath = opath.join(
+        outdir, "graph_objs", *node.parent_path_parts, "_" + node.name_undercase + ".py"
+    )
 
     # Generate source code
     # --------------------
     datatype_source = build_datatype_py(node)
 
     # Write file
     # ----------
-
     write_source_py(datatype_source, filepath, leading_newlines=2)