Merge pull request #29 from delsim/doc_live_updating

Documentation for live updating

Author: GibbsConsulting

demo/demo/plotly_apps.py

@@ -134,6 +134,7 @@ def callback_c(*args, **kwargs):
     html.Div(id='button_local_counter', children="Press any button to start"),
     ], className="")
 
+#pylint: disable=too-many-arguments
 @liveIn.expanded_callback(
     dash.dependencies.Output('button_local_counter', 'children'),
     [dash.dependencies.Input('red-button', 'n_clicks'),
@@ -186,10 +187,10 @@ def callback_liveIn_button_press(red_clicks, blue_clicks, green_clicks,
                                                                                              datetime.fromtimestamp(0.001*timestamp))
 
 liveOut = DjangoDash("LiveOutput",
-                    )#serve_locally=True)
+                     )#serve_locally=True)
 
 def _get_cache_key(state_uid):
-    return "demo-liveout-s4-%s" % state_uid
+    return "demo-liveout-s6-%s" % state_uid
 
 def generate_liveOut_layout():
     'Generate the layout per-app, generating each tine a new uuid for the state_uid argument'
@@ -210,6 +211,7 @@ def generate_liveOut_layout():
 
 liveOut.layout = generate_liveOut_layout
 
+#pylint: disable=unused-argument
 #@liveOut.expanded_callback(
 @liveOut.callback(
     dash.dependencies.Output('internal_state', 'children'),
@@ -242,6 +244,18 @@ def callback_liveOut_pipe_in(named_count, state_uid, **kwargs):
             colour_set = [(None, 0, 100) for i in range(5)]
 
         _, last_ts, prev = colour_set[-1]
+
+        # Loop over all existing timestamps and find the latest one
+        if not click_timestamp or click_timestamp < 1:
+            click_timestamp = 0
+
+            for _, the_colour_set in state.items():
+                _, lts, _ = the_colour_set[-1]
+                if lts > click_timestamp:
+                    click_timestamp = lts
+
+            click_timestamp = click_timestamp + 1000
+
         if click_timestamp > last_ts:
             colour_set.append((user, click_timestamp, prev * random.lognormvariate(0.0, 0.1)),)
             colour_set = colour_set[-100:]
@@ -268,23 +282,28 @@ def callback_show_timeseries(internal_state_string, state_uid, **kwargs):
 
     colour_series = {}
 
+    colors = {'red':'#FF0000',
+              'blue':'#0000FF',
+              'green':'#00FF00',
+              'yellow': '#FFFF00',
+              'cyan': '#00FFFF',
+              'magenta': '#FF00FF',
+              'black' : '#000000',
+             }
+
     for colour, values in state.items():
         timestamps = [datetime.fromtimestamp(int(0.001*ts)) for _, ts, _ in values if ts > 0]
-        users = [user for user, ts, _ in values if ts > 0]
+        #users = [user for user, ts, _ in values if ts > 0]
         levels = [level for _, ts, level in values if ts > 0]
-        colour_series[colour] = pd.Series(levels, index=timestamps).groupby(level=0).first()
+        if colour in colors:
+            colour_series[colour] = pd.Series(levels, index=timestamps).groupby(level=0).first()
 
     df = pd.DataFrame(colour_series).fillna(method="ffill").reset_index()[-25:]
 
-    colors = {'red':'#FF0000',
-              'blue':'#0000FF',
-              'green':'#00FF00',
-             }
-
     traces = [go.Scatter(y=df[colour],
                          x=df['index'],
                          name=colour,
-                         line=dict(color=colors[colour]),
+                         line=dict(color=colors.get(colour, '#000000')),
                         ) for colour in colour_series]
 
     return {'data':traces,

demo/demo/settings.py

@@ -125,6 +125,8 @@
     "ws_route" : "ws/channel",
 
     "insert_demo_migrations" : True,  # Insert model instances used by the demo
+
+    "http_poke_enabled" : True, # Flag controlling availability of direct-to-messaging http endpoint
     }
 
 # Static files (CSS, JavaScript, Images)
@@ -165,6 +167,10 @@
 # can be useful for development especially if offline - we add in the root directory
 # of each module. This is a bit of fudge and only needed if serve_locally=True is
 # set on a DjangoDash instance.
+#
+# Note that this makes all of the python module (including .py and .pyc) files available
+# through the static route. This is not a big deal for development but at the same time
+# not particularly neat or tidy.
 
 if DEBUG:
 

demo/demo/templates/base.html

@@ -25,6 +25,9 @@
          <a class="nav-item nav-link btn btn-lg" href="{%url "demo-two"%}">Demo Two - Initial State</a>
          <a class="nav-item nav-link btn btn-lg" href="{%url "demo-three"%}">Demo Three - Enhanced Callbacks</a>
          <a class="nav-item nav-link btn btn-lg" href="{%url "demo-four"%}">Demo Four - Live Updating</a>
+         <a class="nav-item nav-link btn btn-lg"
+            target="_blank"
+            href="https://django-plotly-dash.readthedocs.io/en/latest/">Online Documentation</a>
          {%endblock%}
        </div>
      </nav>

demo/demo/templates/demo_four.html

@@ -41,4 +41,17 @@ <h1>Live Updating</h1>
     {%plotly_app slug="liveoutput-2" ratio=0.5 %}
   </div>
 </div>
+<p>
+</p>
+<p>
+  Any http command
+  can be used to send a message to the apps. This is equiavent to a press of
+  the red button. Other colours can be specified, including yellow, cyan and black in
+  addition to the three named in the LiveInput app.
+</p>
+<div class="card bg-light border-dark">
+  <div class="card-body">
+    curl http://localhost:8000/dpd/views/poke/ -d'{"channel_name":"live_button_counter","label":"named_counts","value":{"click_colour":"red"}}'
+  </div>
+</div>
 {%endblock%}

demo/demo/urls.py

@@ -28,6 +28,8 @@
 # Load demo plotly apps - this triggers their registration
 import demo.plotly_apps # pylint: disable=unused-import
 
+from django_plotly_dash.views import add_to_session
+
 urlpatterns = [
     url('^$', TemplateView.as_view(template_name='index.html'), name="home"),
     url('^demo-one$', TemplateView.as_view(template_name='demo_one.html'), name="demo-one"),
@@ -36,6 +38,8 @@
     url('^demo-four$', TemplateView.as_view(template_name='demo_four.html'), name="demo-four"),
     url('^admin/', admin.site.urls),
     url('^django_plotly_dash/', include('django_plotly_dash.urls')),
+
+    url('^demo-session-var$', add_to_session, name="session-variable-example"),
 ]
 
 # Add in static routes so daphne can serve files; these should

django_plotly_dash/routing.py

@@ -29,11 +29,19 @@
 from django.conf.urls import url
 
 from .consumers import MessageConsumer, PokePipeConsumer
-from .util import pipe_ws_endpoint_name, http_endpoint
+from .util import pipe_ws_endpoint_name, http_endpoint, http_poke_endpoint_enabled
 
 # TODO document this and discuss embedding with other routes
+
+http_routes = [
+    ]
+
+if http_poke_endpoint_enabled():
+    http_routes.append(url(http_endpoint("poke"), PokePipeConsumer))
+
+http_routes.append(url("^", AsgiHandler)) # AsgiHandler is 'the normal Django view handlers'
+
 application = ProtocolTypeRouter({
     'websocket': AuthMiddlewareStack(URLRouter([url(pipe_ws_endpoint_name(), MessageConsumer),])),
-    'http': AuthMiddlewareStack(URLRouter([url(http_endpoint("poke"), PokePipeConsumer),
-                                           url("^", AsgiHandler),])), # AsgiHandler is 'the normal Django view handlers'
+    'http': AuthMiddlewareStack(URLRouter(http_routes)),
     })

django_plotly_dash/util.py

@@ -49,3 +49,7 @@ def insert_demo_migrations():
     'Check settings and report if objects for demo purposes should be inserted during migration'
 
     return _get_settings().get('insert_demo_migrations', False)
+
+def http_poke_endpoint_enabled():
+    'Return true if the http endpoint is enabled through the settings'
+    return _get_settings().get('http_poke_enabled', True)

django_plotly_dash/views.py

@@ -106,3 +106,18 @@ def component_suites(request, resource=None, component=None, **kwargs):
         redone_url = "/static/dash/%s/%s" %(component, resource)
 
     return HttpResponseRedirect(redirect_to=redone_url)
+
+
+# pylint: disable=wrong-import-position, wrong-import-order
+from django.template.response import TemplateResponse
+
+def add_to_session(request, template_name="index.html", **kwargs):
+    'Add some info to a session in a place that django-plotly-dash can pass to a callback'
+
+    django_plotly_dash = request.session.get("django_plotly_dash", dict())
+
+    session_add_count = django_plotly_dash.get('add_counter', 0)
+    django_plotly_dash['add_counter'] = session_add_count + 1
+    request.session['django_plotly_dash'] = django_plotly_dash
+
+    return TemplateResponse(request, template_name, {})

docs/configuration.rst

@@ -16,6 +16,9 @@ below.
       # Route used for direct http insertion of pipe messages
       "http_route" : "dpd/views",
 
+      # Flag controlling existince of http poke endpoint
+      "http_poke_enabled" : True,
+
       # Insert data for the demo when migrating
       "insert_demo_migrations" : False,
   }

docs/dash_components.rst

@@ -3,10 +3,39 @@
 Dash components
 ===============
 
-The ``dpd-components`` package contains ``Dash`` components.
+The ``dpd-components`` package contains ``Dash`` components. This package is installed as a
+dependency of ``django-plotly-dash``.
 
 .. _pipe_component:
 The ``Pipe`` component
 --------------
 
-Blah
+Each ``Pipe`` component instance listens for messages on a single channel. The ``value`` member of any message on that channel whose ``label`` matches
+that of the component will be used to update the ``value`` property of the component. This property can then be used in callbacks like
+any other ``Dash`` component property.
+
+An example, from the demo application:
+
+.. code-block:: python
+
+    import dpd_components as dpd
+
+    app.layout = html.Div([
+        ...
+        dpd.Pipe(id="named_count_pipe",               # ID in callback
+                 value=None,                          # Initial value prior to any message
+                 label="named_counts",                # Label used to identify relevant messages
+                 channel_name="live_button_counter"), # Channel whose messages are to be examined
+        ...
+        ])
+
+The ``value`` of the message is sent from the server to all front ends with ``Pipe`` components listening
+on the given ``channel_name``. This means that this part of the message should be small, and it must
+be JSON serialisable. Also, there is no guarantee that any callbacks will be executed in the same Python
+process as the one that initiated the initial message from server to front end.
+
+The ``Pipe`` properties can be persisted like any other ``DashApp`` instance, although it is unlikely
+that continued persistence of state on each update of this component is likely to be useful.
+
+This component requires a bidirectional connection, such as a websocket, to the server. Inserting
+a ``plotly_message_pipe`` :ref:`template tag <plotly_message_pipe>` is sufficient.

docs/installation.rst

@@ -3,8 +3,7 @@
 Installation
 ============
 
-The package requires version 2.0 or greater of Django, essentially due to the use of the ``path`` function for
-registering routes. The minimum Python version needed is 3.5.
+The package requires version 2.0 or greater of Django, and a minimum Python version needed of 3.5.
 
 Use ``pip`` to install the package, preferably to a local ``virtualenv``::
 

docs/introduction.rst

@@ -31,4 +31,6 @@ above for stateless applications.
 Also, an enhanced version of the ``Dash`` callback is provided, giving the callback access to the current User, the current session, and also
 the model instance associated with the application's internal state.
 
-This package is compatible with version 2.0 onwards of Django, as it uses the new path registration functionality.
+This package is compatible with version 2.0 onwards of Django. Use of the :ref:`live updating <updating>` feature requires
+the Django Channels extension; in turn this requires a suitable messaging backend such as Redis.
+

docs/template_tags.rst

@@ -36,6 +36,8 @@ At least one of ``da``, ``slug`` or ``name`` must be provided. An object identif
 identified by ``name`` will be. If either of these arguments are provided, they must resolve to valid objects even if
 not used. If neither are provided, then the model instance in ``da`` will be used.
 
+.. _plotly_message_pipe:
+
 The ``plotly_message_pipe`` template tag
 ----------------------------------------
 

docs/updating.rst

@@ -3,17 +3,110 @@
 Live updating
 =============
 
-Live updating blah.
+Live updating is supported using additional ``Dash`` :ref:`components <dash_components>` and
+leveraging `Django Channels <https://channels.readthedocs.io/en/latest/>`_ to provide websocket endpoints.
+
+Server-initiated messages are sent to all interested clients. The content of the message is then injected into
+the application from the client, and from that point it is handled like any other value passed to a callback function.
+The messages are constrained to be JSON serialisable, as that is how they are transmitted to and from the clients, and should
+also be as small as possible given that they travel from the server, to each interested client, and then back to the
+server again as an argument to one or more callback functions.
+
+The round-trip of the message is a deliberate design choice, in order to enable the value within the message to be treated
+as much as possible like any other piece of data within a ``Dash`` application. This data is essentially stored
+on the client side of the client-server split, and passed to the server when each callback is invoked; note that this also
+encourages designs that keep the size of in-application data small. An
+alternative approach, such as directly invoking
+a callback in the server, would require the server to maintain its own copy of the application state.
+
+Live updating requires a server setup that is considerably more
+complex than the alternative, namely use of the built-in `Interval <https://dash.plot.ly/live-updates>`_ component. However, live
+updating can be used to reduce server load (as callbacks are only made when needed) and application latency (as callbacks are
+invoked as needed, not on the tempo of the Interval component).
 
 Message channels
 ----------------
 
-Blah
+Messages are passed through named channels, and each message consists
+of a ``label`` and ``value`` pair. A :ref:`Pipe <pipe_component>` component is provided that listens for messages and makes
+them available to ``Dash`` callbacks. Each message is sent through a message channel to all ``Pipe`` components that have
+registered their interest in that channel, and in turn the components will select messages by ``label``.
+
+A message channel exists as soon as a component signals that it is listening for messages on it. The
+message delivery requirement is 'hopefully at least once'. In other words, applications should be robust against both the failure
+of a message to be delivered, and also for a message to be delivered multiple times. A design approach that has messages
+of the form 'you should look at X and see if something should be done' is strongly encouraged. The accompanying demo has
+messages of the form 'button X at time T', for example.
+
+Sending messages from within Django
+-----------------------------------
+
+Messages can be easily sent from within Django, provided that they are within the ASGI server.
+
+.. code-block:: python
+
+   from django_plotly_dash.consumers import send_to_pipe_channel
+
+   # Send a message
+   #
+   # This function may return *before* the message has been sent
+   # to the pipe channel.
+   #
+   send_to_pipe_channel(channel_name="live_button_counter",
+                        label="named_counts",
+                        value=value)
+
+   # Send a message asynchronously
+   #
+   await async_send_to_pipe_channel(channel_name="live_button_counter",
+                                    label="named_counts",
+                                    value=value)
+
+In general, making assumptions about the ordering of code between message sending and receiving is
+unsafe. The ``send_to_pipe`` function uses the Django Channels ``async_to_sync`` wrapper around
+a call to ``async_send_to_pipe`` and therefore may return before the asynchronous call is made (perhaps
+on a different thread). Furthermore, the transit of the message through the channels backend
+introduces another indeterminacy.
+
+HTTP Endpoint
+-------------
+
+There is an HTTP endpoint, :ref:`configured <configuration>` with
+the ``http_route`` option, that allows direct insertion of messages into a message
+channel. It is a
+direct equivalent of calling the ``send_to_pipe_channel`` function, and
+expects the ``channel_name``, ``label`` and ``value`` arguments to be provided in a JSON-encoded
+dictionary.
+
+.. code-block:: bash
+
+   curl -d '{"channel_name":"live_button_counter",
+             "label":"named_counts",
+             "value":{"click_colour":"cyan"}}'
+             http://localhost:8000/dpd/views/poke/
+
+This will cause the (JSON-encoded) ``value`` argument to be sent on the ``channel_name`` channel with
+the given ``label``.
+
+The provided endpoint skips any CSRF checks
+and does not perform any security checks such as authentication or authorisation, and should
+be regarded as a starting point for a more complete implementation if exposing this functionality is desired. On the
+other hand, if this endpoint is restricted so that it is only available from trusted sources such as the server
+itself, it does provide a mechanism for Django code running outside of the ASGI server, such as in a WSGI process or
+Celery worker, to push a message out to running applications.
 
-Pipes
------
+The ``http_poke_enabled`` flag controls the availability of the endpoint. If false, then it is not registered at all and
+all requests will receive a 404 HTTP error code.
 
-A :ref:`Pipe <pipe_component>` component is provided.
+Deployment
+----------
 
+The live updating feature needs both Redis, as it is the only supported backend at present for v2.0 and up of
+Channels, and Daphne or any other ASGI server for production use. It is also good practise to place the server(s) behind
+a reverse proxy such as Nginx; this can then also be configured to serve Django's static files.
 
+A further consideration is the use of a WSGI server, such as Gunicorn, to serve the non-asynchronous subset of the http
+routes, albeit at the expense of having to separately manage ASGI and WSGI servers. This can be easily achieved through selective
+routing at the reverse proxy level, and is the driver behind the ``ws_route`` configuration option.
 
+In passing, note that the demo also uses Redis as the caching backend for Django.