Merge branch 'dev' into persistence

Author: alexcjohnson

Diff for: CONTRIBUTING.md

@@ -38,7 +38,7 @@ As of Dash 1.2, the renderer bundle and its peer dependencies can be packed and
 3.  `renderer bundles` parses the locked version JSON, copies all the peer dependencies into dash_renderer folder, bundles the renderer assets, and generates an `__init__.py` to map all the resources. There are also a list of helpful `scripts` property defined in `package.json` you might need to do some handy tasks like linting, syntax format with prettier, etc.
 4.  `renderer digest` computes the content hash of each asset in `dash_renderer` folder, prints out the result in logs, and dumps into a JSON file `digest.json`. Use this when you have a doubt about the current assets in `dash_renderer`, and compare it with previous result in one shot by this command.
 5.  `renderer build` runs 1, 2, 3, 4 in sequence as a complete build process from scratch.
-6.  `renderer watch` runs the webpack in watch mode, so any source code change triggers a rebuild. Use this if you are actively updating renderer code and you want to test your changes immediately.
+6.  `renderer build local` runs the same order as in 5 and also generates source maps for debugging purposes. You also need to install dash-renderer with editable mode: `pip install -e .`.
 
 When a change in renderer code doesn't reflect in your browser as expected, this could be: confused bundle generation, caching issue in a browser, python package not in `editable` mode, etc. The new tool reduces the risk of bundle assets by adding the digest to help compare asset changes.
 

Diff for: dash-renderer/CHANGELOG.md

@@ -2,6 +2,11 @@
 All notable changes to this project will be documented in this file.
 This project adheres to [Semantic Versioning](http://semver.org/).
 
+## Unreleased
+
+### Fixed
+- Reduced about 55% of the dash-renderer packages size on **PyPI** by removing the source maps. To do more advanced debugging, the source maps needs to be generated from source code with `npm run build:local` and pip install in editable mode, i.e. `pip install -e .` [#910](https://github.com/plotly/dash/pull/910)
+
 ## [1.0.1] - 2019-08-27
 - Clean all the binary assets in dash-renderer repo, add tool to build all the required bundles from fresh source code to avoid confusion of the assets and improve the release process. [#874](https://github.com/plotly/dash/pull/874)
 

Diff for: dash-renderer/init.template

@@ -45,8 +45,6 @@ _js_dist = [
     {
         "relative_package_path": "{}.min.js.map".format(__name__),
         "dev_package_path": "{}.dev.js.map".format(__name__),
-        "external_url": "https://unpkg.com/dash-renderer@$version"
-        "/dash_renderer/dash_renderer.min.js.map",
         "namespace": "dash_renderer",
         "dynamic": True,
     },

Diff for: dash-renderer/package.json

@@ -6,8 +6,9 @@
   "scripts": {
     "lint": "./node_modules/.bin/eslint --quiet --fix .",
     "lint:test": "./node_modules/.bin/eslint .",
-    "build:js": "webpack --mode development && webpack --mode production",
-    "build:dev": "webpack --mode development --watch",
+    "build:js": "webpack --build release",
+    "build:dev": "webpack --build local",
+    "build:local": "renderer build local",
     "build": "renderer build",
     "start": "webpack-serve ./webpack.serve.config.js",
     "format": "prettier --config .prettierrc --write \"src/**/*.js\"",

Diff for: dash-renderer/webpack.config.js

@@ -1,87 +1,84 @@
 const webpack = require('webpack');
+const R = require('ramda');
 const path = require('path');
 const packagejson = require('./package.json');
-
 const dashLibraryName = packagejson.name.replace(/-/g, '_');
 
-module.exports = (env, argv) => {
-    let mode;
-
-    // if user specified mode flag take that value
-    if (argv && argv.mode) {
-        mode = argv.mode;
-    }
-
-    // else if configuration object is already set (module.exports) use that value
-    else if (module.exports && module.exports.mode) {
-        mode = module.exports = mode;
-    }
-
-    // else take webpack default
-    else {
-        mode = 'production';
-    }
-    return {
-        entry: {main: ['@babel/polyfill', 'whatwg-fetch', './src/index.js']},
-        output: {
-            path: path.resolve(__dirname, dashLibraryName),
-            filename:
-                mode === 'development'
-                    ? `${dashLibraryName}.dev.js`
-                    : `${dashLibraryName}.min.js`,
-            library: dashLibraryName,
-            libraryTarget: 'window',
-        },
-        devtool: 'source-map',
-        externals: {
-            react: 'React',
-            'react-dom': 'ReactDOM',
-            'plotly.js': 'Plotly',
-            'prop-types': 'PropTypes'
-        },
-        module: {
-            rules: [
-                {
-                    test: /\.js$/,
-                    exclude: /node_modules/,
-                    use: {
-                        loader: 'babel-loader',
-                    },
-                },
-                {
-                    test: /\.css$/,
-                    use: [
-                        {
-                            loader: 'style-loader',
-                        },
-                        {
-                            loader: 'css-loader',
-                        },
-                    ],
+const defaultOptions = {
+    mode: 'development',
+    devtool: 'none',
+    entry: {
+        main: ['@babel/polyfill', 'whatwg-fetch', './src/index.js'],
+    },
+    output: {
+        path: path.resolve(__dirname, dashLibraryName),
+        filename: `${dashLibraryName}.dev.js`,
+        library: dashLibraryName,
+        libraryTarget: 'window',
+    },
+    externals: {
+        react: 'React',
+        'react-dom': 'ReactDOM',
+        'plotly.js': 'Plotly',
+        'prop-types': 'PropTypes',
+    },
+    plugins: [],
+    module: {
+        rules: [
+            {
+                test: /\.js$/,
+                exclude: /node_modules/,
+                use: {
+                    loader: 'babel-loader',
                 },
-                {
-                    test: /\.svg$/,
-                    use: ['@svgr/webpack']
-                },
-                {
-                    test: /\.txt$/i,
-                    use: 'raw-loader',
+            },
+            {
+                test: /\.css$/,
+                use: [
+                    {
+                        loader: 'style-loader',
+                    },
+                    {
+                        loader: 'css-loader',
+                    },
+                ],
+            },
+            {
+                test: /\.svg$/,
+                use: ['@svgr/webpack'],
+            },
+            {
+                test: /\.txt$/i,
+                use: 'raw-loader',
+            },
+        ],
+    },
+};
 
+module.exports = (_, argv) => {
+    const devtool = argv.build === 'local' ? 'source-map' : 'none';
+    return [
+        R.mergeDeepLeft({devtool}, defaultOptions),
+        R.mergeDeepLeft(
+            {
+                devtool: devtool,
+                mode: 'production',
+                output: {
+                    filename: `${dashLibraryName}.min.js`,
                 },
-            ],
-        },
-        plugins: [
-            new webpack.NormalModuleReplacementPlugin(
-                /(.*)GlobalErrorContainer.react(\.*)/,
-                function(resource) {
-                    if (mode === 'production') {
-                        resource.request = resource.request.replace(
-                            /GlobalErrorContainer.react/,
-                            'GlobalErrorContainerPassthrough.react'
-                        );
-                    }
-                }
-            ),
-        ],
-    };
+                plugins: [
+                    new webpack.NormalModuleReplacementPlugin(
+                        /(.*)GlobalErrorContainer.react(\.*)/,
+                        function(resource) {
+                            resource.request = resource.request.replace(
+                                /GlobalErrorContainer.react/,
+                                'GlobalErrorContainerPassthrough.react'
+                            );
+                        }
+                    ),
+                ],
+            },
+            defaultOptions
+        ),
+    ];
 };

Diff for: dash/CHANGELOG.md

@@ -1,7 +1,13 @@
 ## Unreleased
 
+### Added
+
+- [#918](https://github.com/plotly/dash/pull/918) Adds `wait_for_element_by_id` and `visit_and_snapshot` APIs in browser, adds `raw_command` option (it also has higher priority than
+the default waitress one) and optional `start_timeout` argument to handle large application within process runner
+
 ### Fixed
 
+- [#915](https://github.com/plotly/dash/issues/915) Fixes `dash-generate-components` on Windows
 - [#829](https://github.com/plotly/dash/issues/829) Fixes the `--remote` pytest argument which was not effective in the code, adding a new argument `--remote-url` to support the selenium grid usage in the cloud.
 
 ## [1.2.0] - 2019-08-27

Diff for: dash/dash.py

@@ -441,6 +441,7 @@ def layout(self, value):
                 'or a function that returns '
                 'a dash component.')
 
+        self._cached_layout = None
         self._layout = value
 
     @property

Diff for: dash/development/build_process.py

@@ -77,10 +77,10 @@ def watch(self):
         os.system("npm run build:dev")
 
     @job("run the whole building process in sequence")
-    def build(self):
+    def build(self, build=None):
         self.clean()
         self.npm()
-        self.bundles()
+        self.bundles(build)
         self.digest()
 
     @job("compute the hash digest for assets")
@@ -108,7 +108,7 @@ def digest(self):
         )
 
     @job("copy and generate the bundles")
-    def bundles(self):
+    def bundles(self, build=None):
         if not os.path.exists(self.build_folder):
             try:
                 os.makedirs(self.build_folder)
@@ -142,9 +142,10 @@ def bundles(self):
                 self._concat(self.build_folder, target),
             )
 
-        logger.info("run `npm run build:js`")
+        _script = 'build:dev' if build == 'local' else 'build:js'
+        logger.info("run `npm run %s`", _script)
         os.chdir(self.main)
-        run_command_with_process("npm run build:js")
+        run_command_with_process("npm run {}".format(_script))
 
         logger.info("generate the `__init__.py` from template and verisons")
         with open(self._concat(self.main, "init.template")) as fp:

Diff for: dash/development/component_generator.py

@@ -58,8 +58,9 @@ def generate_components(
     reserved_patterns = "|".join("^{}$".format(p) for p in reserved_words)
 
     os.environ["NODE_PATH"] = "node_modules"
+
     cmd = shlex.split(
-        "node {} {} {} {}".format(
+        'node {} "{}" "{}" {}'.format(
             extract_path, ignore, reserved_patterns, components_source
         ),
         posix=not is_windows,

Diff for: dash/extract-meta.js

@@ -5,7 +5,7 @@ const path = require('path');
 const reactDocs = require('react-docgen');
 
 const componentPaths = process.argv.slice(4);
-const ignorePattern = new RegExp(process.argv[2]);
+const ignorePattern = new RegExp(process.argv[2].split('"').join(''));
 const reservedPatterns = process.argv[3].split('|').map(part => new RegExp(part));
 
 let failed = false;

Diff for: dash/testing/application_runners.py

@@ -179,23 +179,39 @@ def __init__(self, keep_open=False, stop_timeout=3):
         self.proc = None
 
     # pylint: disable=arguments-differ
-    def start(self, app_module, application_name="app", port=8050):
+    def start(
+        self,
+        app_module=None,
+        application_name="app",
+        raw_command=None,
+        port=8050,
+        start_timeout=3,
+    ):
         """Start the server with waitress-serve in process flavor """
-        entrypoint = "{}:{}.server".format(app_module, application_name)
+        if not (app_module or raw_command):  # need to set a least one
+            logging.error(
+                "the process runner needs to start with"
+                " at least one valid command"
+            )
+            return
         self.port = port
-
         args = shlex.split(
-            "waitress-serve --listen=0.0.0.0:{} {}".format(port, entrypoint),
+            raw_command
+            if raw_command
+            else "waitress-serve --listen=0.0.0.0:{} {}:{}.server".format(
+                port, app_module, application_name
+            ),
             posix=not self.is_windows,
         )
+
         logger.debug("start dash process with %s", args)
 
         try:
             self.proc = subprocess.Popen(
                 args, stdout=subprocess.PIPE, stderr=subprocess.PIPE
             )
             # wait until server is able to answer http request
-            wait.until(lambda: self.accessible(self.url), timeout=3)
+            wait.until(lambda: self.accessible(self.url), timeout=start_timeout)
 
         except (OSError, ValueError):
             logger.exception("process server has encountered an error")
@@ -233,7 +249,7 @@ def __init__(self, keep_open=False, stop_timeout=3):
         self.proc = None
 
     # pylint: disable=arguments-differ
-    def start(self, app):
+    def start(self, app, start_timeout=2):
         """Start the server with waitress-serve in process flavor """
 
         # app is a R string chunk
@@ -267,7 +283,7 @@ def start(self, app):
                 args, stdout=subprocess.PIPE, stderr=subprocess.PIPE
             )
             # wait until server is able to answer http request
-            wait.until(lambda: self.accessible(self.url), timeout=2)
+            wait.until(lambda: self.accessible(self.url), timeout=start_timeout)
 
         except (OSError, ValueError):
             logger.exception("process server has encountered an error")

Diff for: dash/testing/browser.py

@@ -173,6 +173,20 @@ def wait_for_element_by_css_selector(self, selector, timeout=None):
             ),
         )
 
+    def wait_for_element_by_id(self, element_id, timeout=None):
+        """explicit wait until the element is present,
+        timeout if not set, equals to the fixture's `wait_timeout`
+        shortcut to `WebDriverWait` with `EC.presence_of_element_located`
+        """
+        return self._wait_for(
+            EC.presence_of_element_located,
+            ((By.ID, element_id),),
+            timeout,
+            "timeout {}s => waiting for element id {}".format(
+                timeout if timeout else self._wait_timeout, element_id
+            ),
+        )
+
     def wait_for_style_to_equal(self, selector, style, val, timeout=None):
         """explicit wait until the element's style has expected `value`
         timeout if not set, equals to the fixture's `wait_timeout`
@@ -433,6 +447,16 @@ def reset_log_timestamp(self):
             if entries:
                 self._last_ts = entries[-1]["timestamp"]
 
+    def visit_and_snapshot(self, resource_path, hook_id):
+        try:
+            self.driver.get(self.server_url + resource_path)
+            self.wait_for_element_by_id(hook_id)
+            self.percy_snapshot(resource_path)
+            self.driver.back()
+        except WebDriverException as e:
+            logger.exception("snapshot at resource %s error", resource_path)
+            raise e
+
     @property
     def driver(self):
         """expose the selenium webdriver as fixture property"""

Diff for: tests/unit/test_app_runners.py

@@ -25,7 +25,7 @@ def test_threaded_server_smoke(dash_thread_server):
     sys.version_info < (3,), reason="requires python3 for process testing"
 )
 def test_process_server_smoke(dash_process_server):
-    dash_process_server("simple_app")
+    dash_process_server('simple_app')
     r = requests.get(dash_process_server.url)
     assert r.status_code == 200, "the server is reachable"
     assert 'id="react-entry-point"' in r.text, "the entrypoint is present"