multi_caching.js

/** @module cacheManager/multiCaching */
var async = require('async');
var CallbackFiller = require('./callback_filler');
var utils = require('./utils');
var isObject = utils.isObject;
var parseWrapArguments = utils.parseWrapArguments;

/**
 * Module that lets you specify a hierarchy of caches.
 *
 * @param {array} caches - Array of caching objects.
 * @param {object} [options]
 * @param {function} [options.isCacheableValue] - A callback function which is called
 *   with every value returned from cache or from a wrapped function. This lets you specify
 *   which values should and should not be cached. If the function returns true, it will be
 *   stored in cache. By default it caches everything except undefined.
 *
 *   If an underlying cache specifies its own isCacheableValue function, that function will
 *   be used instead of the multiCaching's _isCacheableValue function.
 */
var multiCaching = function(caches, options) {
    var self = {};
    options = options || {};

    var Promise = options.promiseDependency || global.Promise;

    if (!Array.isArray(caches)) {
        throw new Error('multiCaching requires an array of caches');
    }

    var callbackFiller = new CallbackFiller();
    var backgroundQueue = new Set();

    if (typeof options.isCacheableValue === 'function') {
        self._isCacheableValue = options.isCacheableValue;
    } else {
        self._isCacheableValue = function(value) {
            return value !== undefined;
        };
    }

    /**
     * If the underlying cache specifies its own isCacheableValue function (such
     * as how node-cache-manager-redis does), use that function, otherwise use
     * self._isCacheableValue function.
     */
    function getIsCacheableValueFunction(cache) {
        if (cache.store && typeof cache.store.isCacheableValue === 'function') {
            return cache.store.isCacheableValue.bind(cache.store);
        } else {
            return self._isCacheableValue;
        }
    }

    function getFromHighestPriorityCachePromise() {
        var args = Array.prototype.slice.apply(arguments).filter(function(v) {
            return typeof v !== 'undefined';
        });

        return new Promise(function(resolve, reject) {
            var cb = function(err, result) {
                if (err) {
                    return reject(err);
                }
                resolve(result);
            };
            args.push(cb);
            getFromHighestPriorityCache.apply(null, args);
        });
    }

    function getFromHighestPriorityCache() {
        var args = Array.prototype.slice.apply(arguments).filter(function(v) {
            return typeof v !== 'undefined';
        });
        var cb;
        var options = {};

        if (typeof args[args.length - 1] === 'function') {
            cb = args.pop();
        }

        if (!cb) {
            return getFromHighestPriorityCachePromise.apply(this, args);
        }

        if (isObject(args[args.length - 1])) {
            options = args.pop();
        }

        /**
         * Keep a copy of the keys to retrieve
         */
        var keys = Array.prototype.slice.apply(args);
        var multi = keys.length > 1;

        /**
         * Then put back the options in the args Array
         */
        args.push(options);

        if (multi) {
            /**
             * Keep track of the keys left to fetch accross the caches
             */
            var keysToFetch = Array.prototype.slice.apply(keys);

            /**
             * Hash to save our multi keys result
             */
            var mapResult = {};
        }

        var i = 0;
        async.eachSeries(caches, function(cache, next) {
            var callback = function(err, result) {
                if (err) {
                    return next(err);
                }

                var _isCacheableValue = getIsCacheableValueFunction(cache);

                if (multi) {
                    addResultToMap(result, _isCacheableValue);

                    if (keysToFetch.length === 0 || i === caches.length - 1) {
                        // Return an Array with the values merged from all the caches
                        return cb(null, keys.map(function(k) {
                            return mapResult[k] || undefined;
                        }), i);
                    }
                } else if (_isCacheableValue(result)) {
                    // break out of async loop.
                    return cb(err, result, i);
                }

                i += 1;
                next();
            };

            if (multi) {
                if (typeof cache.store.mget !== 'function') {
                    /**
                     * Silently fail for store that don't support mget()
                     */
                    return callback(null, []);
                }
                var _args = Array.prototype.slice.apply(keysToFetch);
                _args.push(options);
                _args.push(callback);
                cache.store.mget.apply(cache.store, _args);
            } else {
                try {
                    cache.store.get(args[0], options, callback);
                } catch (err) {
                    callback(err);
                }
            }
        }, function(err, result) {
            return cb(err, result);
        });

        function addResultToMap(result, isCacheable) {
            var key;
            var diff = 0;

            /**
             * We loop through the result and if the value
             * is cacheable we add it to the mapResult hash
             * and remove the key to fetch from the "keysToFetch" array
             */
            result.forEach(function(res, i) {
                if (isCacheable(res)) {
                    key = keysToFetch[i - diff];

                    // Add the result to our map
                    mapResult[key] = res;

                    // delete key from our keysToFetch array
                    keysToFetch.splice(i - diff, 1);
                    diff += 1;
                }
            });
        }
    }

    function setInMultipleCachesPromise() {
        var args = Array.prototype.slice.apply(arguments);

        return new Promise(function(resolve, reject) {
            var cb = function(err, result) {
                if (err) {
                    return reject(err);
                }
                resolve(result);
            };
            args.push(cb);
            setInMultipleCaches.apply(null, args);
        });
    }

    function setInMultipleCaches() {
        var args = Array.prototype.slice.apply(arguments);
        var _caches = Array.isArray(args[0]) ? args.shift() : caches;

        var cb;
        var options = {};

        if (typeof args[args.length - 1] === 'function') {
            cb = args.pop();
        }

        if (!cb) {
            return setInMultipleCachesPromise.apply(this, args);
        }

        if (args.length % 2 > 0 && isObject(args[args.length - 1])) {
            options = args.pop();
        }

        var length = args.length;
        var multi = length > 2;
        var i;

        async.each(_caches, function(cache, next) {
            var _isCacheableValue = getIsCacheableValueFunction(cache);
            var keysValues = Array.prototype.slice.apply(args);

            /**
             * We filter out the keys *not* cacheable
             */
            for (i = 0; i < length; i += 2) {
                if (!_isCacheableValue(keysValues[i + 1])) {
                    keysValues.splice(i, 2);
                }
            }

            if (keysValues.length === 0) {
                return next();
            }

            var cacheOptions = options;
            if (typeof options.ttl === 'function') {
                /**
                 * Dynamically set the ttl by context depending of the store
                 */
                cacheOptions = {};
                cacheOptions.ttl = options.ttl(keysValues, cache.store.name);
            }

            if (multi) {
                if (typeof cache.store.mset !== 'function') {
                    /**
                     * Silently fail for store that don't support mset()
                     */
                    return next();
                }
                keysValues.push(cacheOptions);
                keysValues.push(next);

                cache.store.mset.apply(cache.store, keysValues);
            } else {
                cache.store.set(keysValues[0], keysValues[1], cacheOptions, next);
            }
        }, function(err, result) {
            cb(err, result);
        });
    }

    function getAndPassUpPromise(key) {
        return new Promise(function(resolve, reject) {
            self.getAndPassUp(key, function(err, result) {
                if (err) {
                    return reject(err);
                }
                resolve(result);
            });
        });
    }

    /**
     * Looks for an item in cache tiers.
     * When a key is found in a lower cache, all higher levels are updated.
     *
     * @param {string} key
     * @param {function} cb
     */
    self.getAndPassUp = function(key, cb) {
        if (!cb) {
            return getAndPassUpPromise(key);
        }

        getFromHighestPriorityCache(key, function(err, result, index) {
            if (err) {
                return cb(err);
            }

            if (index) {
                var cachesToUpdate = caches.slice(0, index);
                async.each(cachesToUpdate, function(cache, next) {
                    var _isCacheableValue = getIsCacheableValueFunction(cache);
                    if (_isCacheableValue(result)) {
                        // We rely on the cache module's default TTL
                        cache.set(key, result, next);
                    }
                });
            }

            return cb(err, result);
        });
    };

    function wrapPromise(key, promise, options) {
        return new Promise(function(resolve, reject) {
            self.wrap(key, function(cb) {
                Promise.resolve()
                    .then(promise)
                    .then(function(result) {
                        cb(null, result);
                    })
                    .catch(cb);
            }, options, function(err, result) {
                if (err) {
                    return reject(err);
                }
                resolve(result);
            });
        });
    }

    function handleBackgroundRefresh(caches, index, key, work, options) {
        if (caches[index].refreshThreshold && !backgroundQueue.has(key)) {
            backgroundQueue.add(key);
            caches[index].checkRefreshThreshold(key, function(err, isExpiring) {
                if (err) {
                    backgroundQueue.delete(key);
                    return;
                }
                if (isExpiring) {
                    work(function(workErr, workData) {
                        if (workErr || !self._isCacheableValue(workData)) {
                            backgroundQueue.delete(key);
                            return;
                        }
                        var args = [caches, key, workData, options, function() {
                            backgroundQueue.delete(key);
                        }];
                        setInMultipleCaches.apply(null, args);
                    });
                } else {
                    backgroundQueue.delete(key);
                }
            });
        }
    }

    /**
     * Wraps a function in one or more caches.
     * Has same API as regular caching module.
     *
     * If a key doesn't exist in any cache, it gets set in all caches.
     * If a key exists in a high-priority (e.g., first) cache, it gets returned immediately
     * without getting set in other lower-priority caches.
     * If a key doesn't exist in a higher-priority cache but exists in a lower-priority
     * cache, it gets set in all higher-priority caches.
     * You can pass any number of keys as long as the wrapped function returns
     * an array with the same number of values and in the same order.
     *
     * @function
     * @name wrap
     *
     * @param {string} key - The cache key to use in cache operations. Can be one or many.
     * @param {function} work - The function to wrap
     * @param {object} [options] - options passed to `set` function
     * @param {function} cb
     */
    self.wrap = function() {
        var parsedArgs = parseWrapArguments(Array.prototype.slice.apply(arguments));
        var keys = parsedArgs.keys;
        var work = parsedArgs.work;
        var options = parsedArgs.options;
        var cb = parsedArgs.cb;

        if (!cb) {
            keys.push(work);
            keys.push(options);
            return wrapPromise.apply(this, keys);
        }

        if (keys.length > 1) {
            /**
             * Handle more than 1 key
             */
            return wrapMultiple(keys, work, options, cb);
        }

        var key = keys[0];

        var hasKey = callbackFiller.has(key);
        callbackFiller.add(key, {cb: cb});
        if (hasKey) { return; }

        getFromHighestPriorityCache(key, function(err, result, index) {
            if (err) {
                return callbackFiller.fill(key, err);
            } else if (self._isCacheableValue(result)) {
                handleBackgroundRefresh(caches, index, key, work, options);
                var cachesToUpdate = caches.slice(0, index);
                var args = [cachesToUpdate, key, result, options, function(err) {
                    callbackFiller.fill(key, err, result);
                }];

                setInMultipleCaches.apply(null, args);
            } else {
                work(function(err, data) {
                    if (err) {
                        return callbackFiller.fill(key, err);
                    }

                    if (!self._isCacheableValue(data)) {
                        return callbackFiller.fill(key, err, data);
                    }

                    var args = [caches, key, data, options, function(err) {
                        callbackFiller.fill(key, err, data);
                    }];

                    setInMultipleCaches.apply(null, args);
                });
            }
        });
    };

    function wrapMultiple(keys, work, options, cb) {
        /**
         * We create a unique key for the multiple keys
         * by concatenating them
         */
        var combinedKey = keys.reduce(function(acc, k) {
            return acc + k;
        }, '');

        var hasKey = callbackFiller.has(combinedKey);
        callbackFiller.add(combinedKey, {cb: cb});
        if (hasKey) { return; }

        keys.push(options);
        keys.push(onResult);

        /**
         * Get from all the caches. If multiple keys have been passed,
         * we'll go through all the caches and merge the result
         */
        getFromHighestPriorityCache.apply(this, keys);

        function onResult(err, result, index) {
            if (err) {
                return done(err);
            }

            /**
             * If all the values returned are cacheable we don't need
             * to call our "work" method and the values returned by the cache
             * are valid. If one or more of the values is not cacheable
             * the cache result is not valid.
             */
            var cacheOK = result.filter(function(_result) {
                return self._isCacheableValue(_result);
            }).length === result.length;

            if (!cacheOK) {
                /**
                 * We need to fetch the data first
                 */
                return work(workCallback);
            }

            var cachesToUpdate = caches.slice(0, index);

            /**
             * Prepare arguments to set the values in
             * higher priority caches
             */
            var _args = [cachesToUpdate];

            /**
             * Add the {key, value} pair
             */
            result.forEach(function(value, i) {
                _args.push(keys[i]);
                _args.push(value);
            });

            /**
             * Add options and final callback
             */
            _args.push(options);
            _args.push(function(err) {
                done(err, result);
            });

            return setInMultipleCaches.apply(null, _args);

            /**
             * Wrapped function callback
             */
            function workCallback(err, data) {
                if (err) {
                    return done(err);
                }

                /**
                 * Prepare arguments for "setInMultipleCaches"
                 */
                var _args;

                _args = [];
                data.forEach(function(value, i) {
                    /**
                     * Add the {key, value} pair to the args
                     * array that we will send to mset()
                     */
                    if (self._isCacheableValue(value)) {
                        _args.push(keys[i]);
                        _args.push(value);
                    }
                });
                // If no key,value --> exit
                if (_args.length === 0) {
                    return done(null);
                }

                /**
                 * Add options and final callback
                 */
                _args.push(options);
                _args.push(function(err) {
                    done(err, data);
                });

                setInMultipleCaches.apply(null, _args);
            }

            /**
             * Final callback
             */
            function done(err, data) {
                callbackFiller.fill(combinedKey, err, data);
            }
        }
    }

    /**
     * Set value in all caches
     *
     * @function
     * @name set
     *
     * @param {string} key
     * @param {*} value
     * @param {object} [options] to pass to underlying set function.
     * @param {function} [cb]
     */
    self.set = setInMultipleCaches;

    /**
     * Set multiple values in all caches
     * Accepts an unlimited pair of {key, value}
     *
     * @function
     * @name mset
     *
     * @param {string} key
     * @param {*} value
     * @param {string} [key2]
     * @param {*} [value2]
     * @param {object} [options] to pass to underlying set function.
     * @param {function} [cb]
     */
    self.mset = setInMultipleCaches;

    /**
     * Get value from highest level cache that has stored it.
     *
     * @function
     * @name get
     *
     * @param {string} key
     * @param {object} [options] to pass to underlying get function.
     * @param {function} cb
     */
    self.get = getFromHighestPriorityCache;

    /**
     * Get multiple value from highest level cache that has stored it.
     * If some values are not found, the next highest cache is used
     * until either all keys are found or all caches have been fetched.
     * Accepts an unlimited number of keys.
     *
     * @function
     * @name mget
     *
     * @param {string} key key to get (any number)
     * @param {object} [options] to pass to underlying get function.
     * @param {function} cb optional callback
     */
    self.mget = getFromHighestPriorityCache;

    /**
     * Delete value from all caches.
     *
     * @function
     * @name del
     *
     * @param {string} key
     * @param {object} [options] to pass to underlying del function.
     * @param {function} cb
     */
    self.del = function() {
        var args = Array.prototype.slice.apply(arguments);
        var cb;
        var options = {};

        if (typeof args[args.length - 1] === 'function') {
            cb = args.pop();
        }

        if (isObject(args[args.length - 1])) {
            options = args.pop();
        }

        args.push(options);
        async.each(caches, function(cache, next) {
            var _args = Array.prototype.slice.apply(args);
            _args.push(next);
            cache.store.del.apply(cache.store, _args);
        }, cb);
    };

    /**
     * Reset all caches.
     *
     * @function
     * @name reset
     *
     * @param {function} cb
     */
    self.reset = function(cb) {
        async.each(caches, function(cache, next) {
            cache.store.reset(next);
        }, cb);
    };

    return self;
};

module.exports = multiCaching;