From b6d5e176cea36f15262a56c0f4c7128892e89d76 Mon Sep 17 00:00:00 2001 From: gsaibhar Date: Thu, 11 Jun 2020 19:00:21 +0530 Subject: [PATCH 1/9] Adding support for jupyterlab statusbar-extension #36 --- nbresuse/__init__.py | 6 +++- nbresuse/api.py | 52 +++++++++++++++++++++++++++++ nbresuse/static/main.js | 72 +++++++++++++++++------------------------ 3 files changed, 86 insertions(+), 44 deletions(-) create mode 100644 nbresuse/api.py diff --git a/nbresuse/__init__.py b/nbresuse/__init__.py index e4d6a4f..18cf1e6 100644 --- a/nbresuse/__init__.py +++ b/nbresuse/__init__.py @@ -1,5 +1,7 @@ +from notebook.utils import url_path_join from tornado import ioloop +from nbresuse.api import ApiHandler from nbresuse.config import ResourceUseDisplay from nbresuse.metrics import PSUtilMetricsLoader from nbresuse.prometheus import PrometheusHandler @@ -32,7 +34,9 @@ def load_jupyter_server_extension(nbapp): """ resuseconfig = ResourceUseDisplay(parent=nbapp) nbapp.web_app.settings["nbresuse_display_config"] = resuseconfig + route_pattern = url_path_join(nbapp.web_app.settings['base_url'], '/api/nbresuse/v1') + nbapp.web_app.add_handlers('.*', [(route_pattern, ApiHandler)]) callback = ioloop.PeriodicCallback( PrometheusHandler(PSUtilMetricsLoader(nbapp)), 1000 ) - callback.start() + callback.start() \ No newline at end of file diff --git a/nbresuse/api.py b/nbresuse/api.py new file mode 100644 index 0000000..8e7f854 --- /dev/null +++ b/nbresuse/api.py @@ -0,0 +1,52 @@ +import json + +from notebook.base.handlers import IPythonHandler +from tornado import web + +try: + # since psutil is an optional dependency + import psutil +except ImportError: + psutil = None + +try: + # Traitlets >= 4.3.3 + from traitlets import Callable +except ImportError: + from .utils import Callable + + +class ApiHandler(IPythonHandler): + + @web.authenticated + async def get(self): + """ + Calculate and return current resource usage metrics + """ + config = self.settings['nbresuse_display_config'] + + if psutil: + cur_process = psutil.Process() + all_processes = [cur_process] + cur_process.children(recursive=True) + + # Get memory information + rss = sum([p.memory_info().rss for p in all_processes]) + + if callable(config.mem_limit): + mem_limit = config.mem_limit(rss=rss) + else: # mem_limit is an Int + mem_limit = config.mem_limit + + limits = {'memory': { + 'rss': mem_limit + }} + if config.mem_limit: + limits['memory']['warn'] = (mem_limit - rss) < ( + mem_limit * config.mem_warning_threshold) + + metrics = { + 'rss': rss, + 'limits': limits, + } + + self.write(json.dumps(metrics)) \ No newline at end of file diff --git a/nbresuse/static/main.js b/nbresuse/static/main.js index a1facca..52ee5ad 100644 --- a/nbresuse/static/main.js +++ b/nbresuse/static/main.js @@ -5,13 +5,13 @@ define([ function setupDOM() { $('#maintoolbar-container').append( $('
').attr('id', 'nbresuse-display') - .addClass('btn-group') - .addClass('pull-right') - .append( - $('').text('Memory: ') - ).append( + .addClass('btn-group') + .addClass('pull-right') + .append( + $('').text('Memory: ') + ).append( $('').attr('id', 'nbresuse-mem') - .attr('title', 'Actively used Memory (updates every 5s)') + .attr('title', 'Actively used Memory (updates every 5s)') ) ); // FIXME: Do something cleaner to get styles in here? @@ -24,49 +24,35 @@ define([ } function humanFileSize(size) { - var i = Math.floor( Math.log(size) / Math.log(1024) ); - return ( size / Math.pow(1024, i) ).toFixed(1) * 1 + ' ' + ['B', 'kB', 'MB', 'GB', 'TB'][i]; + var i = Math.floor(Math.log(size) / Math.log(1024)); + return (size / Math.pow(1024, i)).toFixed(1) * 1 + ' ' + ['B', 'kB', 'MB', 'GB', 'TB'][i]; } - - function metric(metric_name, text, multiple=false) { - var regex = new RegExp("^" + metric_name + "\{?([^ \}]*)\}? (.*)$", "gm"); - var matches = []; - var match; - - do{ - match = regex.exec(text); - if (match){ - matches.push(match) - } - } - while (match); - - if (!multiple) { - if (matches.length > 0) - return matches[0]; - return null; - }else - return matches; - } - - var displayMetrics = function() { + var displayMetrics = function () { if (document.hidden) { // Don't poll when nobody is looking return; } - $.ajax({ - url: utils.get_body_data('baseUrl') + 'metrics', - success: function(data) { - let totalMemoryUsage = metric("total_memory_usage", data); - let maxMemoryUsage = metric("max_memory_usage", data); + $.getJSON({ + url: utils.get_body_data('baseUrl') + 'api/nbresuse/v1', + success: function (data) { + totalMemoryUsage = humanFileSize(data['rss']); + + var limits = data['limits']; + var display = totalMemoryUsage; - if (maxMemoryUsage[2] <= 0) - return; - totalMemoryUsage = humanFileSize(parseFloat(totalMemoryUsage[2])); - maxMemoryUsage = humanFileSize(parseFloat(maxMemoryUsage[2])); + if (limits['memory']) { + if (limits['memory']['rss']) { + maxMemoryUsage = humanFileSize(limits['memory']['rss']); + display += "/" + maxMemoryUsage + } + if (limits['memory']['warn']) { + $('#nbresuse-display').addClass('nbresuse-warn'); + } else { + $('#nbresuse-display').removeClass('nbresuse-warn'); + } + } - var display = totalMemoryUsage + "/" + maxMemoryUsage; $('#nbresuse-mem').text(display); } }); @@ -78,7 +64,7 @@ define([ // Update every five seconds, eh? setInterval(displayMetrics, 1000 * 5); - document.addEventListener("visibilitychange", function() { + document.addEventListener("visibilitychange", function () { // Update instantly when user activates notebook tab // FIXME: Turn off update timer completely when tab not in focus if (!document.hidden) { @@ -90,4 +76,4 @@ define([ return { load_ipython_extension: load_ipython_extension, }; -}); +}); \ No newline at end of file From ef69cad1175fa92543fce085cd46a9ec990fa55b Mon Sep 17 00:00:00 2001 From: Jeremy Tuloup Date: Fri, 12 Jun 2020 08:49:02 +0200 Subject: [PATCH 2/9] Add back the /metrics endpoint --- nbresuse/__init__.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/nbresuse/__init__.py b/nbresuse/__init__.py index 18cf1e6..0349981 100644 --- a/nbresuse/__init__.py +++ b/nbresuse/__init__.py @@ -34,9 +34,15 @@ def load_jupyter_server_extension(nbapp): """ resuseconfig = ResourceUseDisplay(parent=nbapp) nbapp.web_app.settings["nbresuse_display_config"] = resuseconfig - route_pattern = url_path_join(nbapp.web_app.settings['base_url'], '/api/nbresuse/v1') - nbapp.web_app.add_handlers('.*', [(route_pattern, ApiHandler)]) + base_url = nbapp.web_app.settings["base_url"] + nbapp.web_app.add_handlers( + ".*", + [ + (url_path_join(base_url, "/api/nbresuse/v1"), ApiHandler), + (url_path_join(base_url, "/metrics"), ApiHandler), + ], + ) callback = ioloop.PeriodicCallback( PrometheusHandler(PSUtilMetricsLoader(nbapp)), 1000 ) - callback.start() \ No newline at end of file + callback.start() From f9f7c520ec2a7ee655e9f9254d1e7cfeed5eacee Mon Sep 17 00:00:00 2001 From: Jeremy Tuloup Date: Fri, 12 Jun 2020 08:53:01 +0200 Subject: [PATCH 3/9] Add back psutil as a dependency --- CONTRIBUTING.md | 8 +++----- README.md | 4 +--- nbresuse/api.py | 45 +++++++++++++++++---------------------------- setup.py | 7 +++---- 4 files changed, 24 insertions(+), 40 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index efdf877..ada4f16 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -17,7 +17,7 @@ We recommend using [pipenv](https://docs.pipenv.org/) to make development easier ``` 2. Create an environment that will hold our dependencies. - + ```bash cd nbresuse pipenv --python 3.6 @@ -32,11 +32,9 @@ We recommend using [pipenv](https://docs.pipenv.org/) to make development easier 4. Do a dev install of nbresuse and its dependencies ```bash - pip install --editable .[resources] + pip install --editable .[dev] ``` - To test the behavior of NBResuse without `psutil` installed, run `pip install --editable .` instead. - 5. Install and enable the nbextension for use with Jupyter Classic Notebook. ```bash @@ -73,7 +71,7 @@ the pre-commit hook should take care of how it should look. Here is how to set u ```bash pre-commit run ``` - + which should run any autoformatting on your code and tell you about any errors it couldn't fix automatically. You may also install [black integration](https://github.com/ambv/black#editor-integration) diff --git a/README.md b/README.md index e67ee7f..dba1e4c 100644 --- a/README.md +++ b/README.md @@ -22,11 +22,9 @@ main toolbar in the notebook itself, refreshing every 5s. You can currently install this package from PyPI. ```bash -pip install nbresuse[resources] +pip install nbresuse ``` -The above command will install NBResuse along with `psutil` Python package (which is used for getting hardware usage information from the system). If you would like to install NBResuse _without_ `psutil` (in which case NBResuse does essentially nothing), run `pip install nbresuse` instead. - **If your notebook version is < 5.3**, you need to enable the extension manually. ``` diff --git a/nbresuse/api.py b/nbresuse/api.py index 8e7f854..f1cf50f 100644 --- a/nbresuse/api.py +++ b/nbresuse/api.py @@ -1,14 +1,9 @@ import json +import psutil from notebook.base.handlers import IPythonHandler from tornado import web -try: - # since psutil is an optional dependency - import psutil -except ImportError: - psutil = None - try: # Traitlets >= 4.3.3 from traitlets import Callable @@ -17,36 +12,30 @@ class ApiHandler(IPythonHandler): - @web.authenticated async def get(self): """ Calculate and return current resource usage metrics """ - config = self.settings['nbresuse_display_config'] + config = self.settings["nbresuse_display_config"] - if psutil: - cur_process = psutil.Process() - all_processes = [cur_process] + cur_process.children(recursive=True) + cur_process = psutil.Process() + all_processes = [cur_process] + cur_process.children(recursive=True) - # Get memory information - rss = sum([p.memory_info().rss for p in all_processes]) + # Get memory information + rss = sum([p.memory_info().rss for p in all_processes]) - if callable(config.mem_limit): - mem_limit = config.mem_limit(rss=rss) - else: # mem_limit is an Int - mem_limit = config.mem_limit + if callable(config.mem_limit): + mem_limit = config.mem_limit(rss=rss) + else: # mem_limit is an Int + mem_limit = config.mem_limit - limits = {'memory': { - 'rss': mem_limit - }} - if config.mem_limit: - limits['memory']['warn'] = (mem_limit - rss) < ( - mem_limit * config.mem_warning_threshold) + limits = {"memory": {"rss": mem_limit}} + if config.mem_limit: + limits["memory"]["warn"] = (mem_limit - rss) < ( + mem_limit * config.mem_warning_threshold + ) - metrics = { - 'rss': rss, - 'limits': limits, - } + metrics = {"rss": rss, "limits": limits} - self.write(json.dumps(metrics)) \ No newline at end of file + self.write(json.dumps(metrics)) diff --git a/setup.py b/setup.py index 92cbfa2..9195dc0 100644 --- a/setup.py +++ b/setup.py @@ -23,10 +23,9 @@ "Programming Language :: Python :: 3", ], packages=setuptools.find_packages(), - install_requires=["notebook>=5.6.0", "prometheus_client"], - extras_require={ - "resources": ["psutil>=5.6.0"], - "dev": ["autopep8", "black", "pytest", "flake8", "pytest-cov>=2.6.1", "mock"], + install_requires=["notebook>=5.6.0", "prometheus_client", "psutil>=5.6.0"], + extres_require={ + "dev": ["autopep8", "black", "pytest", "flake8", "pytest-cov>=2.6.1", "mock"] }, data_files=[ ("share/jupyter/nbextensions/nbresuse", glob("nbresuse/static/*")), From 4b9ce14d92fea6a36ecaed1321bb6388e051f5dd Mon Sep 17 00:00:00 2001 From: Jeremy Tuloup Date: Fri, 12 Jun 2020 11:54:33 +0200 Subject: [PATCH 4/9] Fix setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9195dc0..09b7b9d 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ ], packages=setuptools.find_packages(), install_requires=["notebook>=5.6.0", "prometheus_client", "psutil>=5.6.0"], - extres_require={ + extras_require={ "dev": ["autopep8", "black", "pytest", "flake8", "pytest-cov>=2.6.1", "mock"] }, data_files=[ From 7fbf4917d3fb8d84680204cb06580175e68ae749 Mon Sep 17 00:00:00 2001 From: Jeremy Tuloup Date: Fri, 12 Jun 2020 12:01:29 +0200 Subject: [PATCH 5/9] Mock base_url in the tests for now --- nbresuse/tests/test_basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nbresuse/tests/test_basic.py b/nbresuse/tests/test_basic.py index 5d1b6cd..a761ba9 100644 --- a/nbresuse/tests/test_basic.py +++ b/nbresuse/tests/test_basic.py @@ -25,7 +25,7 @@ def test_import_serverextension(self): # mock a notebook app nbapp_mock = MagicMock() - nbapp_mock.web_app.settings = {} + nbapp_mock.web_app.settings = {"base_url": ""} # mock these out for unit test with patch("tornado.ioloop.PeriodicCallback") as periodic_callback_mock, patch( From 7292577c8af757ef1721984535ece90aeebd9dc2 Mon Sep 17 00:00:00 2001 From: Jeremy Tuloup Date: Fri, 12 Jun 2020 12:44:09 +0200 Subject: [PATCH 6/9] Restore the previous (released) version number --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 09b7b9d..90c04bd 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ setuptools.setup( name="nbresuse", - version="0.4.0", + version="0.3.4", url="https://github.com/yuvipanda/nbresuse", author="Yuvi Panda", description="Simple Jupyter extension to show how much resources (RAM) your notebook is using", From f42ca5481d016cc84e30f5ed02400f4342b62bda Mon Sep 17 00:00:00 2001 From: Jeremy Tuloup Date: Fri, 12 Jun 2020 13:39:24 +0200 Subject: [PATCH 7/9] Add back the optional CPU tracking for the endpoint --- nbresuse/api.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/nbresuse/api.py b/nbresuse/api.py index f1cf50f..b2d7f23 100644 --- a/nbresuse/api.py +++ b/nbresuse/api.py @@ -1,8 +1,10 @@ import json +from concurrent.futures import ThreadPoolExecutor import psutil from notebook.base.handlers import IPythonHandler from tornado import web +from tornado.concurrent import run_on_executor try: # Traitlets >= 4.3.3 @@ -12,6 +14,9 @@ class ApiHandler(IPythonHandler): + + executor = ThreadPoolExecutor(max_workers=5) + @web.authenticated async def get(self): """ @@ -38,4 +43,30 @@ async def get(self): metrics = {"rss": rss, "limits": limits} + # Optionally get CPU information + if config.track_cpu_percent: + cpu_count = psutil.cpu_count() + cpu_percent = await self._get_cpu_percent(all_processes) + + if config.cpu_limit != 0: + limits["cpu"] = {"cpu": config.cpu_limit} + if config.cpu_warning_threshold != 0: + limits["cpu"]["warn"] = (config.cpu_limit - self.cpu_percent) < ( + config.cpu_limit * config.cpu_warning_threshold + ) + + metrics.update(cpu_percent=cpu_percent, cpu_count=cpu_count) + self.write(json.dumps(metrics)) + + @run_on_executor + def _get_cpu_percent(self, all_processes): + def get_cpu_percent(p): + try: + return p.cpu_percent(interval=0.05) + # Avoid littering logs with stack traces complaining + # about dead processes having no CPU usage + except: + return 0 + + return sum([get_cpu_percent(p) for p in all_processes]) From 419ee8641b239a1b2a0725300283ba1c26d4ac6e Mon Sep 17 00:00:00 2001 From: Jeremy Tuloup Date: Fri, 12 Jun 2020 13:51:09 +0200 Subject: [PATCH 8/9] Add space around / in the nbextension display --- nbresuse/static/main.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nbresuse/static/main.js b/nbresuse/static/main.js index 52ee5ad..8589d9e 100644 --- a/nbresuse/static/main.js +++ b/nbresuse/static/main.js @@ -44,7 +44,7 @@ define([ if (limits['memory']) { if (limits['memory']['rss']) { maxMemoryUsage = humanFileSize(limits['memory']['rss']); - display += "/" + maxMemoryUsage + display += " / " + maxMemoryUsage } if (limits['memory']['warn']) { $('#nbresuse-display').addClass('nbresuse-warn'); @@ -76,4 +76,4 @@ define([ return { load_ipython_extension: load_ipython_extension, }; -}); \ No newline at end of file +}); From 8aa32c83d34c6cde2be2d9abb109ce42acd2a872 Mon Sep 17 00:00:00 2001 From: Jeremy Tuloup Date: Fri, 12 Jun 2020 14:33:20 +0200 Subject: [PATCH 9/9] Remove /api/nbresuse/v1 for now --- nbresuse/__init__.py | 6 +----- nbresuse/static/main.js | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/nbresuse/__init__.py b/nbresuse/__init__.py index 0349981..f25bc29 100644 --- a/nbresuse/__init__.py +++ b/nbresuse/__init__.py @@ -36,11 +36,7 @@ def load_jupyter_server_extension(nbapp): nbapp.web_app.settings["nbresuse_display_config"] = resuseconfig base_url = nbapp.web_app.settings["base_url"] nbapp.web_app.add_handlers( - ".*", - [ - (url_path_join(base_url, "/api/nbresuse/v1"), ApiHandler), - (url_path_join(base_url, "/metrics"), ApiHandler), - ], + ".*", [(url_path_join(base_url, "/metrics"), ApiHandler)] ) callback = ioloop.PeriodicCallback( PrometheusHandler(PSUtilMetricsLoader(nbapp)), 1000 diff --git a/nbresuse/static/main.js b/nbresuse/static/main.js index 8589d9e..581a775 100644 --- a/nbresuse/static/main.js +++ b/nbresuse/static/main.js @@ -34,7 +34,7 @@ define([ return; } $.getJSON({ - url: utils.get_body_data('baseUrl') + 'api/nbresuse/v1', + url: utils.get_body_data('baseUrl') + 'metrics', success: function (data) { totalMemoryUsage = humanFileSize(data['rss']);