Replace long callback interval with request polling handled in renderer.

Author: T4rk1n

Diff for: dash/_callback.py

@@ -1,12 +1,14 @@
 import collections
 from functools import wraps
 
+import flask
+
 from .dependencies import (
     handle_callback_args,
     handle_grouped_callback_args,
     Output,
 )
-from .exceptions import PreventUpdate
+from .exceptions import PreventUpdate, WildcardInLongCallback, DuplicateCallback
 
 from ._grouping import (
     flatten_grouping,
@@ -17,9 +19,11 @@
     create_callback_id,
     stringify_id,
     to_json,
+    coerce_to_list,
 )
 
 from . import _validate
+from .long_callback.managers import BaseLongCallbackManager
 
 
 class NoUpdate:
@@ -30,15 +34,28 @@ def to_plotly_json(self):  # pylint: disable=no-self-use
 
     @staticmethod
     def is_no_update(obj):
-        return obj == {"_dash_no_update": "_dash_no_update"}
+        return isinstance(obj, NoUpdate) or obj == {
+            "_dash_no_update": "_dash_no_update"
+        }
 
 
 GLOBAL_CALLBACK_LIST = []
 GLOBAL_CALLBACK_MAP = {}
 GLOBAL_INLINE_SCRIPTS = []
 
 
-def callback(*_args, **_kwargs):
+def callback(
+    *_args,
+    long=False,
+    long_interval=1000,
+    long_progress=None,
+    long_progress_default=None,
+    long_running=None,
+    long_cancel=None,
+    long_manager=None,
+    long_cache_args_to_ignore=None,
+    **_kwargs,
+):
     """
     Normally used as a decorator, `@dash.callback` provides a server-side
     callback relating the values of one or more `Output` items to one or
@@ -56,15 +73,79 @@ def callback(*_args, **_kwargs):
     not to fire when its outputs are first added to the page. Defaults to
     `False` and unlike `app.callback` is not configurable at the app level.
     """
+
+    long_spec = None
+
+    if long:
+        long_spec = {
+            "interval": long_interval,
+        }
+
+        if long_manager:
+            long_spec["manager"] = long_manager
+
+        if long_progress:
+            long_spec["progress"] = coerce_to_list(long_progress)
+            validate_long_inputs(long_spec["progress"])
+
+        if long_progress_default:
+            long_spec["progressDefault"] = coerce_to_list(long_progress_default)
+
+            if not len(long_spec["progress"]) == len(long_spec["progressDefault"]):
+                raise Exception(
+                    "Progress and progress default needs to be of same length"
+                )
+
+        if long_running:
+            long_spec["running"] = coerce_to_list(long_running)
+            validate_long_inputs(x[0] for x in long_spec["running"])
+
+        if long_cancel:
+            cancel_inputs = coerce_to_list(long_cancel)
+            validate_long_inputs(cancel_inputs)
+
+            cancels_output = [Output(c.component_id, "id") for c in cancel_inputs]
+
+            try:
+
+                @callback(cancels_output, cancel_inputs, prevent_initial_call=True)
+                def cancel_call(*_):
+                    job_ids = flask.request.args.getlist("cancelJob")
+                    manager = long_manager or flask.g.long_callback_manager
+                    if job_ids:
+                        for job_id in job_ids:
+                            manager.terminate_job(int(job_id))
+                    return NoUpdate()
+
+            except DuplicateCallback:
+                pass  # Already a callback to cancel, will get the proper jobs from the store.
+
+            long_spec["cancel"] = [c.to_dict() for c in cancel_inputs]
+
+        if long_cache_args_to_ignore:
+            long_spec["cache_args_to_ignore"] = long_cache_args_to_ignore
+
     return register_callback(
         GLOBAL_CALLBACK_LIST,
         GLOBAL_CALLBACK_MAP,
         False,
         *_args,
         **_kwargs,
+        long=long_spec,
     )
 
 
+def validate_long_inputs(deps):
+    for dep in deps:
+        if dep.has_wildcard():
+            raise WildcardInLongCallback(
+                f"""
+                long callbacks does not support dependencies with
+                pattern-matching ids
+                    Received: {repr(dep)}\n"""
+            )
+
+
 def clientside_callback(clientside_function, *args, **kwargs):
     return register_clientside_callback(
         GLOBAL_CALLBACK_LIST,
@@ -87,6 +168,7 @@ def insert_callback(
     state,
     inputs_state_indices,
     prevent_initial_call,
+    long=None,
 ):
     if prevent_initial_call is None:
         prevent_initial_call = config_prevent_initial_callbacks
@@ -98,19 +180,26 @@ def insert_callback(
         "state": [c.to_dict() for c in state],
         "clientside_function": None,
         "prevent_initial_call": prevent_initial_call,
+        "long": long
+        and {
+            "interval": long["interval"],
+        },
     }
+
     callback_map[callback_id] = {
         "inputs": callback_spec["inputs"],
         "state": callback_spec["state"],
         "outputs_indices": outputs_indices,
         "inputs_state_indices": inputs_state_indices,
+        "long": long,
     }
     callback_list.append(callback_spec)
 
     return callback_id
 
 
-def register_callback(
+# pylint: disable=R0912
+def register_callback(  # pylint: disable=R0914
     callback_list, callback_map, config_prevent_initial_callbacks, *_args, **_kwargs
 ):
     (
@@ -129,6 +218,8 @@ def register_callback(
         insert_output = flatten_grouping(output)
         multi = True
 
+    long = _kwargs.get("long")
+
     output_indices = make_grouping_by_index(output, list(range(grouping_len(output))))
     callback_id = insert_callback(
         callback_list,
@@ -140,23 +231,118 @@ def register_callback(
         flat_state,
         inputs_state_indices,
         prevent_initial_call,
+        long=long,
     )
 
     # pylint: disable=too-many-locals
     def wrap_func(func):
+
+        if long is not None:
+            long_key = BaseLongCallbackManager.register_func(
+                func, long.get("progress") is not None
+            )
+
         @wraps(func)
         def add_context(*args, **kwargs):
             output_spec = kwargs.pop("outputs_list")
+            callback_manager = long.get(
+                "manager", kwargs.pop("long_callback_manager", None)
+            )
             _validate.validate_output_spec(insert_output, output_spec, Output)
 
             func_args, func_kwargs = _validate.validate_and_group_input_args(
                 args, inputs_state_indices
             )
 
-            # don't touch the comment on the next line - used by debugger
-            output_value = func(*func_args, **func_kwargs)  # %% callback invoked %%
+            response = {"multi": True}
+
+            if long is not None:
+                progress_outputs = long.get("progress")
+                cache_key = flask.request.args.get("cacheKey")
+                job_id = flask.request.args.get("job")
+
+                current_key = callback_manager.build_cache_key(
+                    func,
+                    # Inputs provided as dict is kwargs.
+                    func_args if func_args else func_kwargs,
+                    long.get("cache_args_to_ignore", []),
+                )
+
+                if not cache_key:
+                    cache_key = current_key
+
+                    job_fn = callback_manager.func_registry.get(long_key)
+
+                    job = callback_manager.call_job_fn(
+                        cache_key,
+                        job_fn,
+                        args,
+                    )
+
+                    data = {
+                        "cacheKey": cache_key,
+                        "job": job,
+                    }
+
+                    running = long.get("running")
+
+                    if running:
+                        data["running"] = {str(r[0]): r[1] for r in running}
+                        data["runningOff"] = {str(r[0]): r[2] for r in running}
+                    cancel = long.get("cancel")
+                    if cancel:
+                        data["cancel"] = cancel
+
+                    progress_default = long.get("progressDefault")
+                    if progress_default:
+                        data["progressDefault"] = {
+                            str(o): x
+                            for o, x in zip(progress_outputs, progress_default)
+                        }
+                    return to_json(data)
+                else:
+                    if progress_outputs:
+                        # Get the progress before the result as it would be erased after the results.
+                        progress = callback_manager.get_progress(cache_key)
+                        if progress:
+                            response["progress"] = {
+                                str(x): progress[i]
+                                for i, x in enumerate(progress_outputs)
+                            }
+
+                    output_value = callback_manager.get_result(cache_key, job_id)
+                    # Must get job_running after get_result since get_results terminates it.
+                    job_running = callback_manager.job_running(job_id)
+                    if not job_running and output_value is callback_manager.UNDEFINED:
+                        # Job canceled -> no output to close the loop.
+                        output_value = NoUpdate()
+
+                    elif (
+                        isinstance(output_value, dict)
+                        and "long_callback_error" in output_value
+                    ):
+                        error = output_value.get("long_callback_error")
+                        raise Exception(
+                            f"An error occurred inside a long callback: {error['msg']}\n{error['tb']}"
+                        )
+
+                    if job_running and output_value is not callback_manager.UNDEFINED:
+                        # cached results.
+                        callback_manager.terminate_job(job_id)
+
+                    if multi and isinstance(output_value, (list, tuple)):
+                        output_value = [
+                            NoUpdate() if NoUpdate.is_no_update(r) else r
+                            for r in output_value
+                        ]
+
+                    if output_value is callback_manager.UNDEFINED:
+                        return to_json(response)
+            else:
+                # don't touch the comment on the next line - used by debugger
+                output_value = func(*func_args, **func_kwargs)  # %% callback invoked %%
 
-            if isinstance(output_value, NoUpdate):
+            if NoUpdate.is_no_update(output_value):
                 raise PreventUpdate
 
             if not multi:
@@ -191,7 +377,7 @@ def add_context(*args, **kwargs):
             if not has_update:
                 raise PreventUpdate
 
-            response = {"response": component_ids, "multi": True}
+            response["response"] = component_ids
 
             try:
                 jsonResponse = to_json(response)

Diff for: dash/_utils.py

@@ -217,3 +217,9 @@ def gen_salt(chars):
     return "".join(
         secrets.choice(string.ascii_letters + string.digits) for _ in range(chars)
     )
+
+
+def coerce_to_list(obj):
+    if not isinstance(obj, (list, tuple)):
+        return [obj]
+    return obj

Diff for: dash/dash-renderer/src/actions/callbacks.ts

@@ -1,12 +1,15 @@
 import {
     concat,
     flatten,
+    intersection,
     keys,
     map,
     mergeDeepRight,
     path,
     pick,
     pluck,
+    values,
+    toPairs,
     zip
 } from 'ramda';
 
@@ -24,13 +27,18 @@ import {
     ICallbackPayload,
     IStoredCallback,
     IBlockedCallback,
-    IPrioritizedCallback
+    IPrioritizedCallback,
+    LongCallbackInfo,
+    CallbackResponse,
+    CallbackResponseData
 } from '../types/callbacks';
 import {isMultiValued, stringifyId, isMultiOutputProp} from './dependencies';
 import {urlBase} from './utils';
 import {getCSRFHeader} from '.';
 import {createAction, Action} from 'redux-actions';
 import {addHttpHeaders} from '../actions';
+import {updateProps} from './index';
+import {CallbackJobPayload} from '../reducers/callbackJobs';
 
 export const addBlockedCallbacks = createAction<IBlockedCallback[]>(
     CallbackActionType.AddBlocked
@@ -83,6 +91,9 @@ export const aggregateCallbacks = createAction<
 
 const updateResourceUsage = createAction('UPDATE_RESOURCE_USAGE');
 
+const addCallbackJob = createAction('ADD_CALLBACK_JOB');
+const removeCallbackJob = createAction('REMOVE_CALLBACK_JOB');
+
 function unwrapIfNotMulti(
     paths: any,
     idProps: any,
@@ -300,28 +311,73 @@ async function handleClientside(
     return result;
 }
 
+function sideUpdate(outputs: any, dispatch: any, paths: any) {
+    toPairs(outputs).forEach(([id, value]) => {
+        const [componentId, propName] = id.split('.');
+        const componentPath = paths.strs[componentId];
+        dispatch(
+            updateProps({
+                props: {[propName]: value},
+                itempath: componentPath
+            })
+        );
+    });
+}
+
 function handleServerside(
     dispatch: any,
     hooks: any,
     config: any,
-    payload: any
-): Promise<any> {
+    payload: any,
+    paths: any,
+    long?: LongCallbackInfo,
+    additionalArgs?: [string, string][]
+): Promise<CallbackResponse> {
     if (hooks.request_pre) {
         hooks.request_pre(payload);
     }
 
     const requestTime = Date.now();
     const body = JSON.stringify(payload);
+    let cacheKey: string;
+    let job: string;
+    let runningOff: any;
+    let progressDefault: any;
+
+    const fetchCallback = () => {
+        const headers = getCSRFHeader() as any;
+        let url = `${urlBase(config)}_dash-update-component`;
+
+        const addArg = (name: string, value: string) => {
+            let delim = '?';
+            if (url.includes('?')) {
+                delim = '&';
+            }
+            url = `${url}${delim}${name}=${value}`;
+        };
+        if (cacheKey) {
+            addArg('cacheKey', cacheKey);
+        }
+        if (job) {
+            addArg('job', job);
+        }
+
+        if (additionalArgs) {
+            additionalArgs.forEach(([key, value]) => addArg(key, value));
+        }
+
+        return fetch(
+            url,
+            mergeDeepRight(config.fetch, {
+                method: 'POST',
+                headers,
+                body
+            })
+        );
+    };
 
-    return fetch(
-        `${urlBase(config)}_dash-update-component`,
-        mergeDeepRight(config.fetch, {
-            method: 'POST',
-            headers: getCSRFHeader() as any,
-            body
-        })
-    ).then(
-        (res: any) => {
+    return new Promise((resolve, reject) => {
+        const handleOutput = (res: any) => {
             const {status} = res;
 
             function recordProfile(result: any) {
@@ -361,36 +417,86 @@ function handleServerside(
                 }
             }
 
+            const finishLine = (data: CallbackResponseData) => {
+                const {multi, response} = data;
+                if (hooks.request_post) {
+                    hooks.request_post(payload, response);
+                }
+
+                let result;
+                if (multi) {
+                    result = response as CallbackResponse;
+                } else {
+                    const {output} = payload;
+                    const id = output.substr(0, output.lastIndexOf('.'));
+                    result = {[id]: (response as CallbackResponse).props};
+                }
+
+                recordProfile(result);
+                resolve(result);
+            };
+
+            const completeJob = () => {
+                if (job) {
+                    dispatch(removeCallbackJob({jobId: job}));
+                }
+                if (runningOff) {
+                    sideUpdate(runningOff, dispatch, paths);
+                }
+                if (progressDefault) {
+                    sideUpdate(progressDefault, dispatch, paths);
+                }
+            };
+
             if (status === STATUS.OK) {
-                return res.json().then((data: any) => {
-                    const {multi, response} = data;
-                    if (hooks.request_post) {
-                        hooks.request_post(payload, response);
+                res.json().then((data: CallbackResponseData) => {
+                    if (!cacheKey && data.cacheKey) {
+                        cacheKey = data.cacheKey;
                     }
 
-                    let result;
-                    if (multi) {
-                        result = response;
-                    } else {
-                        const {output} = payload;
-                        const id = output.substr(0, output.lastIndexOf('.'));
-                        result = {[id]: response.props};
+                    if (!job && data.job) {
+                        const jobInfo: CallbackJobPayload = {
+                            jobId: data.job,
+                            cacheKey: data.cacheKey as string,
+                            cancelInputs: data.cancel,
+                            progressDefault: data.progressDefault
+                        };
+                        dispatch(addCallbackJob(jobInfo));
+                        job = data.job;
                     }
 
-                    recordProfile(result);
-                    return result;
+                    if (data.progress) {
+                        sideUpdate(data.progress, dispatch, paths);
+                    }
+                    if (data.running) {
+                        sideUpdate(data.running, dispatch, paths);
+                    }
+                    if (!runningOff && data.runningOff) {
+                        runningOff = data.runningOff;
+                    }
+                    if (!progressDefault && data.progressDefault) {
+                        progressDefault = data.progressDefault;
+                    }
+
+                    if (!long || data.response !== undefined) {
+                        completeJob();
+                        finishLine(data);
+                    } else {
+                        // Poll chain.
+                        setTimeout(handle, long.interval || 500);
+                    }
                 });
-            }
-            if (status === STATUS.PREVENT_UPDATE) {
+            } else if (status === STATUS.PREVENT_UPDATE) {
+                completeJob();
                 recordProfile({});
-                return {};
+                resolve({});
+            } else {
+                completeJob();
+                reject(res);
             }
-            throw res;
-        },
-        () => {
-            // fetch rejection - this means the request didn't return,
-            // we don't get here from 400/500 errors, only network
-            // errors or unresponsive servers.
+        };
+
+        const handleError = () => {
             if (config.ui) {
                 dispatch(
                     updateResourceUsage({
@@ -402,9 +508,14 @@ function handleServerside(
                     })
                 );
             }
-            throw new Error('Callback failed: the server did not respond.');
-        }
-    );
+            reject(new Error('Callback failed: the server did not respond.'));
+        };
+
+        const handle = () => {
+            fetchCallback().then(handleOutput, handleError);
+        };
+        handle();
+    });
 }
 
 function inputsToDict(inputs_list: any) {
@@ -443,10 +554,10 @@ export function executeCallback(
     paths: any,
     layout: any,
     {allOutputs}: any,
-    dispatch: any
+    dispatch: any,
+    getState: any
 ): IExecutingCallback {
-    const {output, inputs, state, clientside_function} = cb.callback;
-
+    const {output, inputs, state, clientside_function, long} = cb.callback;
     try {
         const inVals = fillVals(paths, layout, cb, inputs, 'Input', true);
 
@@ -518,13 +629,40 @@ export function executeCallback(
                 let newHeaders: Record<string, string> | null = null;
                 let lastError: any;
 
+                const additionalArgs: [string, string][] = [];
+                values(getState().callbackJobs).forEach(
+                    (job: CallbackJobPayload) => {
+                        if (!job.cancelInputs) {
+                            return;
+                        }
+                        const inter = intersection(
+                            job.cancelInputs,
+                            cb.callback.inputs
+                        );
+                        if (inter.length) {
+                            additionalArgs.push(['cancelJob', job.jobId]);
+                            if (job.progressDefault) {
+                                console.log(job.progressDefault);
+                                sideUpdate(
+                                    job.progressDefault,
+                                    dispatch,
+                                    paths
+                                );
+                            }
+                        }
+                    }
+                );
+
                 for (let retry = 0; retry <= MAX_AUTH_RETRIES; retry++) {
                     try {
                         const data = await handleServerside(
                             dispatch,
                             hooks,
                             newConfig,
-                            payload
+                            payload,
+                            paths,
+                            long,
+                            additionalArgs.length ? additionalArgs : undefined
                         );
 
                         if (newHeaders) {

Diff for: dash/dash-renderer/src/observers/prioritizedCallbacks.ts

@@ -110,7 +110,8 @@ const observer: IStoreObserverDefinition<IStoreState> = {
                                     paths,
                                     layout,
                                     getStash(cb, paths),
-                                    dispatch
+                                    dispatch,
+                                    getState
                                 ),
                             pickedSyncCallbacks
                         )
@@ -162,7 +163,8 @@ const observer: IStoreObserverDefinition<IStoreState> = {
                     paths,
                     layout,
                     cb,
-                    dispatch
+                    dispatch,
+                    getState
                 );
 
                 dispatch(

Diff for: dash/dash-renderer/src/reducers/callbackJobs.ts

@@ -0,0 +1,35 @@
+import {assoc, dissoc} from 'ramda';
+import {ICallbackProperty} from '../types/callbacks';
+
+type CallbackJobState = {[k: string]: CallbackJobPayload};
+
+export type CallbackJobPayload = {
+    cancelInputs?: ICallbackProperty[];
+    cacheKey: string;
+    jobId: string;
+    progressDefault?: any;
+};
+
+type CallbackJobAction = {
+    type: 'ADD_CALLBACK_JOB' | 'REMOVE_CALLBACK_JOB';
+    payload: CallbackJobPayload;
+};
+
+const setJob = (job: CallbackJobPayload, state: CallbackJobState) =>
+    assoc(job.jobId, job, state);
+const removeJob = (jobId: string, state: CallbackJobState) =>
+    dissoc(jobId, state);
+
+export default function (
+    state: CallbackJobState = {},
+    action: CallbackJobAction
+) {
+    switch (action.type) {
+        case 'ADD_CALLBACK_JOB':
+            return setJob(action.payload, state);
+        case 'REMOVE_CALLBACK_JOB':
+            return removeJob(action.payload.jobId, state);
+        default:
+            return state;
+    }
+}

Diff for: dash/dash-renderer/src/reducers/reducer.js

@@ -17,6 +17,7 @@ import isLoading from './isLoading';
 import layout from './layout';
 import loadingMap from './loadingMap';
 import paths from './paths';
+import callbackJobs from './callbackJobs';
 
 export const apiRequests = [
     'dependenciesRequest',
@@ -45,6 +46,8 @@ function mainReducer() {
         parts[r] = createApiReducer(r);
     }, apiRequests);
 
+    parts.callbackJobs = callbackJobs;
+
     return combineReducers(parts);
 }
 

Diff for: dash/dash-renderer/src/types/callbacks.ts

@@ -11,6 +11,7 @@ export interface ICallbackDefinition {
     outputs: ICallbackProperty[];
     prevent_initial_call: boolean;
     state: ICallbackProperty[];
+    long?: LongCallbackInfo;
 }
 
 export interface ICallbackProperty {
@@ -75,7 +76,29 @@ export interface ICallbackPayload {
 }
 
 export type CallbackResult = {
-    data?: any;
+    data?: CallbackResponse;
     error?: Error;
     payload: ICallbackPayload | null;
 };
+
+export type LongCallbackInfo = {
+    interval?: number;
+    progress?: any;
+    running?: any;
+};
+
+export type CallbackResponse = {
+    [k: string]: any;
+};
+
+export type CallbackResponseData = {
+    response?: CallbackResponse;
+    multi?: boolean;
+    cacheKey?: string;
+    job?: string;
+    progressDefault?: CallbackResponse;
+    progress?: CallbackResponse;
+    running?: CallbackResponse;
+    runningOff?: CallbackResponse;
+    cancel?: ICallbackProperty[];
+};

Diff for: dash/dash.py

@@ -27,9 +27,7 @@
 from .fingerprint import build_fingerprint, check_fingerprint
 from .resources import Scripts, Css
 from .dependencies import (
-    handle_grouped_callback_args,
     Output,
-    State,
     Input,
 )
 from .development.base_component import ComponentRegistry
@@ -61,7 +59,7 @@
 from . import _watch
 from . import _get_app
 
-from ._grouping import flatten_grouping, map_grouping, grouping_len, update_args_group
+from ._grouping import map_grouping, grouping_len, update_args_group
 
 from . import _pages
 from ._pages import (
@@ -1118,6 +1116,7 @@ def clientside_callback(self, clientside_function, *args, **kwargs):
             **kwargs,
         )
 
+    # pylint: disable=R0201
     def callback(self, *_args, **_kwargs):
         """
         Normally used as a decorator, `@app.callback` provides a server-side
@@ -1132,16 +1131,27 @@ def callback(self, *_args, **_kwargs):
 
 
         """
-        return _callback.register_callback(
-            self._callback_list,
-            self.callback_map,
-            self.config.prevent_initial_callbacks,
+        return _callback.callback(
             *_args,
             **_kwargs,
         )
 
-    def long_callback(self, *_args, **_kwargs):  # pylint: disable=too-many-statements
+    # pylint: disable=R0201
+    def long_callback(
+        self,
+        *_args,
+        manager=None,
+        interval=None,
+        running=None,
+        cancel=None,
+        progress=None,
+        progress_default=None,
+        cache_args_to_ignore=None,
+        **_kwargs,
+    ):  # pylint: disable=too-many-statements
         """
+        Deprecation Notice, use long
+
         Normally used as a decorator, `@app.long_callback` is an alternative to
         `@app.callback` designed for callbacks that take a long time to run,
         without locking up the Dash app or timing out.
@@ -1200,242 +1210,19 @@ def long_callback(self, *_args, **_kwargs):  # pylint: disable=too-many-statemen
                 this should be a list of argument names as strings. Otherwise,
                 this should be a list of argument indices as integers.
         """
-        # pylint: disable-next=import-outside-toplevel
-        from dash._callback_context import callback_context
-
-        # pylint: disable-next=import-outside-toplevel
-        from dash.exceptions import WildcardInLongCallback
-
-        # Get long callback manager
-        callback_manager = _kwargs.pop("manager", self._long_callback_manager)
-        if callback_manager is None:
-            raise ValueError(
-                "The @app.long_callback decorator requires a long callback manager\n"
-                "instance.  This may be provided to the app using the \n"
-                "long_callback_manager argument to the dash.Dash constructor, or\n"
-                "it may be provided to the @app.long_callback decorator as the \n"
-                "manager argument"
-            )
-
-        # Extract special long_callback kwargs
-        running = _kwargs.pop("running", ())
-        cancel = _kwargs.pop("cancel", ())
-        progress = _kwargs.pop("progress", ())
-        progress_default = _kwargs.pop("progress_default", None)
-        interval_time = _kwargs.pop("interval", 1000)
-        cache_args_to_ignore = _kwargs.pop("cache_args_to_ignore", [])
-
-        # Parse remaining args just like app.callback
-        (
-            output,
-            flat_inputs,
-            flat_state,
-            inputs_state_indices,
-            prevent_initial_call,
-        ) = handle_grouped_callback_args(_args, _kwargs)
-        inputs_and_state = flat_inputs + flat_state
-        args_deps = map_grouping(lambda i: inputs_and_state[i], inputs_state_indices)
-        multi_output = isinstance(output, (list, tuple)) and len(output) > 1
-
-        # Disallow wildcard dependencies
-        for deps in [output, flat_inputs, flat_state]:
-            for dep in flatten_grouping(deps):
-                if dep.has_wildcard():
-                    raise WildcardInLongCallback(
-                        f"""
-                        @app.long_callback does not support dependencies with
-                        pattern-matching ids
-                            Received: {repr(dep)}\n"""
-                    )
-
-        # Get unique id for this long_callback definition.  This increment is not
-        # thread safe, but it doesn't need to be because callback definitions
-        # happen on the main thread before the app starts
-        self._long_callback_count += 1
-        long_callback_id = self._long_callback_count
-
-        # Create Interval and Store for long callback and add them to the app's
-        # _extra_components list
-        interval_id = f"_long_callback_interval_{long_callback_id}"
-        interval_component = dcc.Interval(
-            id=interval_id, interval=interval_time, disabled=prevent_initial_call
-        )
-        store_id = f"_long_callback_store_{long_callback_id}"
-        store_component = dcc.Store(id=store_id, data={})
-        error_id = f"_long_callback_error_{long_callback_id}"
-        error_store_component = dcc.Store(id=error_id)
-        error_dummy = f"_long_callback_error_dummy_{long_callback_id}"
-        self._extra_components.extend(
-            [
-                interval_component,
-                store_component,
-                error_store_component,
-                dcc.Store(id=error_dummy),
-            ]
-        )
-
-        # Compute full component plus property name for the cancel dependencies
-        cancel_prop_ids = tuple(
-            ".".join([dep.component_id, dep.component_property]) for dep in cancel
+        return _callback.callback(
+            *_args,
+            long=True,
+            long_manager=manager,
+            long_interval=interval,
+            long_progress=progress,
+            long_progress_default=progress_default,
+            long_running=running,
+            long_cancel=cancel,
+            long_cache_args_to_ignore=cache_args_to_ignore,
+            **_kwargs,
         )
 
-        def wrapper(fn):
-            background_fn = callback_manager.make_job_fn(fn, bool(progress), args_deps)
-
-            def callback(_triggers, user_store_data, user_callback_args):
-                # Build result cache key from inputs
-                pending_key = callback_manager.build_cache_key(
-                    fn, user_callback_args, cache_args_to_ignore
-                )
-                current_key = user_store_data.get("current_key", None)
-                pending_job = user_store_data.get("pending_job", None)
-
-                should_cancel = pending_key == current_key or any(
-                    trigger["prop_id"] in cancel_prop_ids
-                    for trigger in callback_context.triggered
-                )
-
-                # Compute grouping of values to set the progress component's to
-                # when cleared
-                if progress_default is None:
-                    clear_progress = (
-                        map_grouping(lambda x: None, progress) if progress else ()
-                    )
-                else:
-                    clear_progress = progress_default
-
-                if should_cancel:
-                    user_store_data["current_key"] = None
-                    user_store_data["pending_key"] = None
-                    user_store_data["pending_job"] = None
-
-                    callback_manager.terminate_job(pending_job)
-
-                    return dict(
-                        user_callback_output=map_grouping(lambda x: no_update, output),
-                        interval_disabled=True,
-                        in_progress=[val for (_, _, val) in running],
-                        progress=clear_progress,
-                        user_store_data=user_store_data,
-                        error=no_update,
-                    )
-
-                # Look up progress value if a job is in progress
-                if pending_job:
-                    progress_value = callback_manager.get_progress(pending_key)
-                else:
-                    progress_value = None
-
-                if callback_manager.result_ready(pending_key):
-                    result = callback_manager.get_result(pending_key, pending_job)
-                    # Set current key (hash of data stored in client)
-                    # to pending key (hash of data requested by client)
-                    user_store_data["current_key"] = pending_key
-
-                    if isinstance(result, dict) and result.get("long_callback_error"):
-                        error = result.get("long_callback_error")
-                        print(
-                            result["long_callback_error"]["tb"],
-                            file=sys.stderr,
-                        )
-                        return dict(
-                            error=f"An error occurred inside a long callback: {error['msg']}\n"
-                            + error["tb"],
-                            user_callback_output=no_update,
-                            in_progress=[val for (_, _, val) in running],
-                            interval_disabled=pending_job is None,
-                            progress=clear_progress,
-                            user_store_data=user_store_data,
-                        )
-
-                    if _callback.NoUpdate.is_no_update(result):
-                        result = no_update
-
-                    if multi_output and isinstance(result, (list, tuple)):
-                        result = [
-                            no_update if _callback.NoUpdate.is_no_update(r) else r
-                            for r in result
-                        ]
-
-                    # Disable interval if this value was pulled from cache.
-                    # If this value was the result of a background calculation, don't
-                    # disable yet. If no other calculations are in progress,
-                    # interval will be disabled in should_cancel logic above
-                    # the next time the interval fires.
-                    interval_disabled = pending_job is None
-                    return dict(
-                        user_callback_output=result,
-                        interval_disabled=interval_disabled,
-                        in_progress=[val for (_, _, val) in running],
-                        progress=clear_progress,
-                        user_store_data=user_store_data,
-                        error=no_update,
-                    )
-                if progress_value:
-                    return dict(
-                        user_callback_output=map_grouping(lambda x: no_update, output),
-                        interval_disabled=False,
-                        in_progress=[val for (_, val, _) in running],
-                        progress=progress_value or {},
-                        user_store_data=user_store_data,
-                        error=no_update,
-                    )
-
-                # Check if there is a running calculation that can now
-                # be canceled
-                old_pending_key = user_store_data.get("pending_key", None)
-                if (
-                    old_pending_key
-                    and old_pending_key != pending_key
-                    and callback_manager.job_running(pending_job)
-                ):
-                    callback_manager.terminate_job(pending_job)
-
-                user_store_data["pending_key"] = pending_key
-                callback_manager.terminate_unhealthy_job(pending_job)
-                if not callback_manager.job_running(pending_job):
-                    user_store_data["pending_job"] = callback_manager.call_job_fn(
-                        pending_key, background_fn, user_callback_args
-                    )
-
-                return dict(
-                    user_callback_output=map_grouping(lambda x: no_update, output),
-                    interval_disabled=False,
-                    in_progress=[val for (_, val, _) in running],
-                    progress=clear_progress,
-                    user_store_data=user_store_data,
-                    error=no_update,
-                )
-
-            self.clientside_callback(
-                "function (error) {throw new Error(error)}",
-                Output(error_dummy, "data"),
-                [Input(error_id, "data")],
-                prevent_initial_call=True,
-            )
-
-            return self.callback(
-                inputs=dict(
-                    _triggers=dict(
-                        n_intervals=Input(interval_id, "n_intervals"),
-                        cancel=cancel,
-                    ),
-                    user_store_data=State(store_id, "data"),
-                    user_callback_args=args_deps,
-                ),
-                output=dict(
-                    user_callback_output=output,
-                    interval_disabled=Output(interval_id, "disabled"),
-                    in_progress=[dep for (dep, _, _) in running],
-                    progress=progress,
-                    user_store_data=Output(store_id, "data"),
-                    error=Output(error_id, "data"),
-                ),
-                prevent_initial_call=prevent_initial_call,
-            )(callback)
-
-        return wrapper
-
     def dispatch(self):
         body = flask.request.get_json()
 
@@ -1455,6 +1242,7 @@ def dispatch(self):
         flask.g.state_values = inputs_to_dict(  # pylint: disable=assigning-non-slot
             state
         )
+        flask.g.long_callback_manager = self._long_callback_manager  # pylint: disable=E0237
         changed_props = body.get("changedPropIds", [])
         flask.g.triggered_inputs = [  # pylint: disable=assigning-non-slot
             {"prop_id": x, "value": input_values.get(x)} for x in changed_props
@@ -1517,7 +1305,14 @@ def dispatch(self):
         except KeyError as missing_callback_function:
             msg = f"Callback function not found for output '{output}', perhaps you forgot to prepend the '@'?"
             raise KeyError(msg) from missing_callback_function
-        response.set_data(func(*args, outputs_list=outputs_list))
+        # noinspection PyArgumentList
+        response.set_data(
+            func(
+                *args,
+                outputs_list=outputs_list,
+                long_callback_manager=self._long_callback_manager,
+            )
+        )
         return response
 
     def _setup_server(self):

Diff for: dash/long_callback/managers/__init__.py

@@ -4,12 +4,29 @@
 
 
 class BaseLongCallbackManager(ABC):
+    UNDEFINED = object()
+
+    # Keep a ref to all the ref to register every callback to every manager.
+    managers = []
+
+    # Keep every function for late registering.
+    functions = []
+
     def __init__(self, cache_by):
         if cache_by is not None and not isinstance(cache_by, list):
             cache_by = [cache_by]
 
         self.cache_by = cache_by
 
+        BaseLongCallbackManager.managers.append(self)
+
+        self.func_registry = {}
+
+        # Register all funcs that were added before instantiation.
+        # Ensure all celery task are registered.
+        for fdetails in self.functions:
+            self.register(*fdetails)
+
     def terminate_job(self, job):
         raise NotImplementedError
 
@@ -19,7 +36,7 @@ def terminate_unhealthy_job(self, job):
     def job_running(self, job):
         raise NotImplementedError
 
-    def make_job_fn(self, fn, progress, args_deps):
+    def make_job_fn(self, fn, progress):
         raise NotImplementedError
 
     def call_job_fn(self, key, job_fn, args):
@@ -58,6 +75,31 @@ def build_cache_key(self, fn, args, cache_args_to_ignore):
 
         return hashlib.sha1(str(hash_dict).encode("utf-8")).hexdigest()
 
+    def register(self, key, fn, progress):
+        self.func_registry[key] = self.make_job_fn(fn, progress)
+
+    @staticmethod
+    def register_func(fn, progress):
+        key = BaseLongCallbackManager.hash_function(fn)
+        BaseLongCallbackManager.functions.append(
+            (
+                key,
+                fn,
+                progress,
+            )
+        )
+
+        for manager in BaseLongCallbackManager.managers:
+            manager.register(key, fn, progress)
+
+        return key
+
     @staticmethod
     def _make_progress_key(key):
         return key + "-progress"
+
+    @staticmethod
+    def hash_function(fn):
+        fn_source = inspect.getsource(fn)
+        fn_str = fn_source
+        return hashlib.sha1(fn_str.encode("utf-8")).hexdigest()

Diff for: dash/long_callback/managers/celery_manager.py

@@ -5,7 +5,6 @@
 
 from _plotly_utils.utils import PlotlyJSONEncoder
 
-from dash._callback import NoUpdate
 from dash.exceptions import PreventUpdate
 from dash.long_callback.managers import BaseLongCallbackManager
 
@@ -74,8 +73,8 @@ def job_running(self, job):
             "PROGRESS",
         )
 
-    def make_job_fn(self, fn, progress, args_deps):
-        return _make_job_fn(fn, self.handle, progress, args_deps)
+    def make_job_fn(self, fn, progress):
+        return _make_job_fn(fn, self.handle, progress)
 
     def get_task(self, job):
         if job:
@@ -105,7 +104,7 @@ def get_result(self, key, job):
         # Get result value
         result = self.handle.backend.get(key)
         if result is None:
-            return None
+            return self.UNDEFINED
 
         result = json.loads(result)
 
@@ -122,7 +121,7 @@ def get_result(self, key, job):
         return result
 
 
-def _make_job_fn(fn, celery_app, progress, args_deps):
+def _make_job_fn(fn, celery_app, progress):
     cache = celery_app.backend
 
     # Hash function source and module to create a unique (but stable) celery task name
@@ -133,19 +132,28 @@ def _make_job_fn(fn, celery_app, progress, args_deps):
     @celery_app.task(name=f"long_callback_{fn_hash}")
     def job_fn(result_key, progress_key, user_callback_args, fn=fn):
         def _set_progress(progress_value):
+            if not isinstance(progress_value, (list, tuple)):
+                progress_value = [progress_value]
+
             cache.set(progress_key, json.dumps(progress_value, cls=PlotlyJSONEncoder))
 
         maybe_progress = [_set_progress] if progress else []
 
         try:
-            if isinstance(args_deps, dict):
+            if isinstance(user_callback_args, dict):
                 user_callback_output = fn(*maybe_progress, **user_callback_args)
-            elif isinstance(args_deps, (list, tuple)):
+            elif isinstance(user_callback_args, (list, tuple)):
                 user_callback_output = fn(*maybe_progress, *user_callback_args)
             else:
                 user_callback_output = fn(*maybe_progress, user_callback_args)
         except PreventUpdate:
-            cache.set(result_key, json.dumps(NoUpdate(), cls=PlotlyJSONEncoder))
+            # Put NoUpdate dict directly to avoid circular imports.
+            cache.set(
+                result_key,
+                json.dumps(
+                    {"_dash_no_update": "_dash_no_update"}, cls=PlotlyJSONEncoder
+                ),
+            )
         except Exception as err:  # pylint: disable=broad-except
             cache.set(
                 result_key,

Diff for: dash/long_callback/managers/diskcache_manager.py

@@ -1,7 +1,6 @@
 import traceback
 
 from . import BaseLongCallbackManager
-from ..._callback import NoUpdate
 from ...exceptions import PreventUpdate
 
 _pending_value = "__$pending__"
@@ -48,16 +47,18 @@ def __init__(self, cache=None, cache_by=None, expire=None):
                 )
             self.handle = cache
 
-        super().__init__(cache_by)
-        self.expire = expire
         self.lock = diskcache.Lock(self.handle, "long-callback-lock")
+        self.expire = expire
+        super().__init__(cache_by)
 
     def terminate_job(self, job):
         import psutil  # pylint: disable=import-outside-toplevel,import-error
 
         if job is None:
             return
 
+        job = int(job)
+
         # Use diskcache transaction so multiple process don't try to kill the
         # process at the same time
         with self.handle.transact():
@@ -83,6 +84,8 @@ def terminate_job(self, job):
     def terminate_unhealthy_job(self, job):
         import psutil  # pylint: disable=import-outside-toplevel,import-error
 
+        job = int(job)
+
         if job and psutil.pid_exists(job):
             if not self.job_running(job):
                 self.terminate_job(job)
@@ -93,13 +96,15 @@ def terminate_unhealthy_job(self, job):
     def job_running(self, job):
         import psutil  # pylint: disable=import-outside-toplevel,import-error
 
+        job = int(job)
+
         if job and psutil.pid_exists(job):
             proc = psutil.Process(job)
             return proc.status() != psutil.STATUS_ZOMBIE
         return False
 
-    def make_job_fn(self, fn, progress, args_deps):
-        return _make_job_fn(fn, self.handle, progress, args_deps, self.lock)
+    def make_job_fn(self, fn, progress):
+        return _make_job_fn(fn, self.handle, progress, self.lock)
 
     def clear_cache_entry(self, key):
         self.handle.delete(key)
@@ -122,9 +127,9 @@ def result_ready(self, key):
 
     def get_result(self, key, job):
         # Get result value
-        result = self.handle.get(key)
-        if result is None:
-            return None
+        result = self.handle.get(key, self.UNDEFINED)
+        if result is self.UNDEFINED:
+            return self.UNDEFINED
 
         # Clear result if not caching
         if self.cache_by is None:
@@ -135,28 +140,32 @@ def get_result(self, key, job):
 
         self.clear_cache_entry(self._make_progress_key(key))
 
-        self.terminate_job(job)
+        if job:
+            self.terminate_job(job)
         return result
 
 
-def _make_job_fn(fn, cache, progress, args_deps, lock):
+def _make_job_fn(fn, cache, progress, lock):
     def job_fn(result_key, progress_key, user_callback_args):
         def _set_progress(progress_value):
+            if not isinstance(progress_value, (list, tuple)):
+                progress_value = [progress_value]
+
             with lock:
                 cache.set(progress_key, progress_value)
 
         maybe_progress = [_set_progress] if progress else []
 
         try:
-            if isinstance(args_deps, dict):
+            if isinstance(user_callback_args, dict):
                 user_callback_output = fn(*maybe_progress, **user_callback_args)
-            elif isinstance(args_deps, (list, tuple)):
+            elif isinstance(user_callback_args, (list, tuple)):
                 user_callback_output = fn(*maybe_progress, *user_callback_args)
             else:
                 user_callback_output = fn(*maybe_progress, user_callback_args)
         except PreventUpdate:
             with lock:
-                cache.set(result_key, NoUpdate())
+                cache.set(result_key, {"_dash_no_update": "_dash_no_update"})
         except Exception as err:  # pylint: disable=broad-except
             with lock:
                 cache.set(

Diff for: tests/integration/long_callback/test_basic_long_callback.py

@@ -451,9 +451,14 @@ def click_n_wait():
         dash_duo.wait_for_text_to_equal("#output", "Clicked 1 times")
 
         click_n_wait()
-        dash_duo.wait_for_contains_text(
-            ".dash-fe-error__title", "An error occurred inside a long callback:"
+        dash_duo.wait_for_element(".dash-fe-error__title").click()
+
+        dash_duo.driver.switch_to.frame(dash_duo.find_element("iframe"))
+        assert (
+            "Exception: An error occurred inside a long callback:"
+            in dash_duo.wait_for_element(".errormsg").text
         )
+        dash_duo.driver.switch_to.default_content()
 
         click_n_wait()
         dash_duo.wait_for_text_to_equal("#output", "Clicked 3 times")