Remove jupyter_client, use KernelFinder in Gateway

Author: kevin-bates

jupyter_server/gateway/handlers.py

@@ -16,7 +16,7 @@
 from tornado.escape import url_escape, json_decode, utf8
 
 from ipython_genutils.py3compat import cast_unicode
-from jupyter_client.session import Session
+from jupyter_protocol.session import Session, new_id
 from traitlets.config.configurable import LoggingConfigurable
 
 from .managers import GatewayClient
@@ -58,7 +58,7 @@ def authenticate(self):
 
     def initialize(self):
         self.log.debug("Initializing websocket connection %s", self.request.path)
-        self.session = Session(config=self.config)
+        self.session = Session(key=new_id())
         self.gateway = GatewayWebSocketClient(gateway_url=GatewayClient.instance().url)
 
     @gen.coroutine

jupyter_server/gateway/managers.py

@@ -3,7 +3,9 @@
 
 import os
 import json
+import logging
 
+from jupyter_kernel_mgmt.discovery import KernelFinder
 from socket import gaierror
 from tornado import gen, web
 from tornado.escape import json_encode, json_decode, url_escape
@@ -12,7 +14,6 @@
 from ..services.kernels.kernelmanager import MappingKernelManager
 from ..services.sessions.sessionmanager import SessionManager
 
-from jupyter_client.kernelspec import KernelSpecManager
 from ..utils import url_path_join
 
 from traitlets import Instance, Unicode, Float, Bool, default, validate, TraitError
@@ -496,14 +497,16 @@ def shutdown_all(self, now=False):
             self.remove_kernel(kernel_id)
 
 
-class GatewayKernelSpecManager(KernelSpecManager):
-
-    def __init__(self, **kwargs):
-        super(GatewayKernelSpecManager, self).__init__(**kwargs)
+class GatewayKernelFinder(KernelFinder):
+    def __init__(self, parent, providers=[]):
+        super(GatewayKernelFinder, self).__init__(providers=providers)
         self.base_endpoint = url_path_join(GatewayClient.instance().url,
                                            GatewayClient.instance().kernelspecs_endpoint)
         self.base_resource_endpoint = url_path_join(GatewayClient.instance().url,
                                                     GatewayClient.instance().kernelspecs_resource_endpoint)
+        # Because KernelFinder is not a taitlet/Configurable, we need to simulate a configurable
+        self.parent = parent
+        self.log = logging.getLogger(__name__)
 
     def _get_kernelspecs_endpoint_url(self, kernel_name=None):
         """Builds a url for the kernels endpoint
@@ -517,6 +520,11 @@ def _get_kernelspecs_endpoint_url(self, kernel_name=None):
 
         return self.base_endpoint
 
+    @gen.coroutine
+    def find_kernels(self):
+        remote_kspecs = yield self.get_all_specs()
+        raise gen.Return(remote_kspecs)
+
     @gen.coroutine
     def get_all_specs(self):
         fetched_kspecs = yield self.list_kernel_specs()

jupyter_server/serverapp.py

@@ -74,7 +74,7 @@
 from .services.contents.filemanager import FileContentsManager
 from .services.contents.largefilemanager import LargeFileManager
 from .services.sessions.sessionmanager import SessionManager
-from .gateway.managers import GatewayKernelManager, GatewayKernelSpecManager, GatewaySessionManager, GatewayClient
+from .gateway.managers import GatewayKernelManager, GatewayKernelFinder, GatewaySessionManager, GatewayClient
 
 from .auth.login import LoginHandler
 from .auth.logout import LogoutHandler
@@ -86,8 +86,7 @@
     JupyterApp, base_flags, base_aliases,
 )
 from jupyter_core.paths import jupyter_config_path
-from jupyter_client import KernelManager
-from jupyter_client.session import Session
+from jupyter_protocol.session import Session
 from jupyter_kernel_mgmt.discovery import KernelFinder
 from nbformat.sign import NotebookNotary
 from traitlets import (
@@ -528,7 +527,6 @@ def start(self):
     'ip': 'ServerApp.ip',
     'port': 'ServerApp.port',
     'port-retries': 'ServerApp.port_retries',
-    'transport': 'KernelManager.transport',
     'keyfile': 'ServerApp.keyfile',
     'certfile': 'ServerApp.certfile',
     'client-ca': 'ServerApp.client_ca',
@@ -554,9 +552,8 @@ class ServerApp(JupyterApp):
     flags = flags
 
     classes = [
-        KernelManager, Session, MappingKernelManager,
-        ContentsManager, FileContentsManager, NotebookNotary,
-        GatewayKernelManager, GatewayKernelSpecManager, GatewaySessionManager, GatewayClient,
+        Session, MappingKernelManager,
+        ContentsManager, FileContentsManager, NotebookNotary, GatewayClient,
     ]
     flags = Dict(flags)
     aliases = Dict(aliases)
@@ -1219,6 +1216,7 @@ def parse_command_line(self, argv=None):
 
     def init_configurables(self):
 
+
         # If gateway server is configured, replace appropriate managers to perform redirection.  To make
         # this determination, instantiate the GatewayClient config singleton.
         self.gateway_config = GatewayClient.instance(parent=self)
@@ -1228,15 +1226,12 @@ def init_configurables(self):
             self.session_manager_class = 'jupyter_server.gateway.managers.GatewaySessionManager'
 # FIXME - no more kernel-spec-manager!
 #           self.kernel_spec_manager_class = 'jupyter_server.gateway.managers.GatewayKernelSpecManager'
-#
-#        self.kernel_spec_manager = self.kernel_spec_manager_class(
-#            parent=self,
-#        )
-
-        if self.kernel_providers:
-            self.kernel_finder = KernelFinder(self.kernel_providers)
+            self.kernel_finder = GatewayKernelFinder(parent=self)  # no providers here, alway go remote
         else:
-            self.kernel_finder = KernelFinder.from_entrypoints()
+            if self.kernel_providers:
+                self.kernel_finder = KernelFinder(self.kernel_providers)
+            else:
+                self.kernel_finder = KernelFinder.from_entrypoints()
 
         self.kernel_manager = self.kernel_manager_class(
             parent=self,
@@ -1578,7 +1573,7 @@ def cleanup_kernels(self):
         """Shutdown all kernels.
 
         The kernels will shutdown themselves when this process no longer exists,
-        but explicit shutdown allows the KernelManagers to cleanup the connection files.
+        but explicit shutdown allows the Kernel Providers to cleanup the connection files.
         """
         n_kernels = len(self.kernel_manager.list_kernel_ids())
         kernel_msg = trans.ngettext('Shutting down %d kernel', 'Shutting down %d kernels', n_kernels)

jupyter_server/services/contents/handlers.py

@@ -11,7 +11,7 @@
 from tornado import gen, web
 
 from jupyter_server.utils import url_path_join, url_escape, maybe_future
-from jupyter_client.jsonutil import date_default
+from jupyter_protocol.jsonutil import date_default
 
 from jupyter_server.base.handlers import (
     JupyterHandler, APIHandler, path_regex,

jupyter_server/services/kernels/handlers.py

@@ -16,7 +16,7 @@
 from tornado.ioloop import IOLoop
 from tornado.websocket import WebSocketHandler
 
-from jupyter_client.jsonutil import date_default
+from jupyter_protocol.jsonutil import date_default
 from jupyter_protocol.messages import Message
 from ipython_genutils.py3compat import cast_unicode
 from jupyter_server.utils import url_path_join, url_escape, maybe_future

jupyter_server/services/kernels/kernelmanager.py

@@ -165,10 +165,6 @@ def stop_buffering(self):
 class MappingKernelManager(LoggingConfigurable):
     """A KernelManager that handles notebook mapping and HTTP error handling"""
 
-    @default('kernel_manager_class')
-    def _default_kernel_manager_class(self):
-        return "jupyter_client.ioloop.IOLoopKernelManager"
-
     default_kernel_name = Unicode('pyimport/kernel', config=True,
         help="The name of the default kernel to start"
     )

jupyter_server/services/kernels/ws_serialize.py

@@ -11,7 +11,8 @@
 from dateutil.tz import tzutc
 
 from ipython_genutils.py3compat import cast_unicode
-from jupyter_client.jsonutil import date_default, extract_dates
+from jupyter_protocol.jsonutil import date_default, extract_dates
+
 
 def serialize_message(msg, channel):
     """Serialize a message from the kernel, using JSON.
@@ -32,6 +33,7 @@ def serialize_message(msg, channel):
         smsg = json.dumps(d, default=date_default)
         return cast_unicode(smsg)
 
+
 def serialize_binary_message(msg_dict, buffers):
     """serialize a message as a binary blob
 
@@ -64,6 +66,7 @@ def serialize_binary_message(msg_dict, buffers):
     buffers.insert(0, offsets_buf)
     return b''.join(buffers)
 
+
 def deserialize_message(msg):
     """Deserialize a websocket message, return a dict.
 
@@ -79,6 +82,7 @@ def deserialize_message(msg):
     msg['parent_header'] = convert_tz_utc(extract_dates(msg['parent_header']))
     return msg
 
+
 def deserialize_binary_message(bmsg):
     """deserialize a message from a binary blob
 
@@ -105,6 +109,7 @@ def deserialize_binary_message(bmsg):
     msg['buffers'] = bufs[1:]
     return msg
 
+
 def convert_tz_utc(obj):
     """Replace dateutil tzutc objects with stdlib datetime utc objects"""
     if isinstance(obj, dict):

jupyter_server/services/kernelspecs/handlers.py

@@ -63,7 +63,9 @@ def get(self):
         model = {}
         model['default'] = km.default_kernel_name
         model['kernelspecs'] = specs = {}
-        for kernel_name, kernel_info in kf.find_kernels():
+
+        found_specs = yield maybe_future(kf.find_kernels())
+        for kernel_name, kernel_info in found_specs.items():
             try:
                 if is_kernelspec_model(kernel_info):
                     d = kernel_info
@@ -85,7 +87,8 @@ def get(self, kernel_name):
         kf = self.kernel_finder
         # TODO: Do we actually want all kernel type names to be case-insensitive?
         kernel_name = kernel_name.lower()
-        for name, info in kf.find_kernels():
+        found_specs = yield maybe_future(kf.find_kernels())
+        for name, info in found_specs.items():
             if name == kernel_name:
                 model = kernelspec_model(self, kernel_name, info,
                                          info['resource_dir'])

jupyter_server/services/sessions/handlers.py

@@ -11,9 +11,9 @@
 from tornado import gen, web
 
 from ...base.handlers import APIHandler
-from jupyter_client.jsonutil import date_default
+from jupyter_protocol.jsonutil import date_default
 from jupyter_server.utils import maybe_future, url_path_join
-from jupyter_client.kernelspec import NoSuchKernel
+from jupyter_kernel_mgmt.kernelspec import NoSuchKernel
 
 
 class SessionRootHandler(APIHandler):

jupyter_server/tests/test_gateway.py

@@ -6,18 +6,12 @@
 from tornado import gen
 from tornado.httpclient import HTTPRequest, HTTPResponse, HTTPError
 from ipython_genutils.py3compat import str_to_unicode
+from unittest.mock import patch, Mock
+from io import StringIO
+
 from .launchserver import ServerTestBase
 from jupyter_server.gateway.managers import GatewayClient
 
-try:
-    from unittest.mock import patch, Mock
-except ImportError:
-    from mock import patch, Mock # py2
-
-try:
-    from io import StringIO
-except ImportError:
-    import StringIO
 
 import nose.tools as nt
 
@@ -175,7 +169,7 @@ def test_gateway_class_mappings(self):
         # Ensure appropriate class mappings are in place.
         nt.assert_equal(self.server.kernel_manager_class.__name__, 'GatewayKernelManager')
         nt.assert_equal(self.server.session_manager_class.__name__, 'GatewaySessionManager')
-        nt.assert_equal(self.server.kernel_spec_manager_class.__name__, 'GatewayKernelSpecManager')
+        nt.assert_equal(self.server.kernel_finder.__class__.__name__, 'GatewayKernelFinder')
 
     def test_gateway_get_kernelspecs(self):
         # Validate that kernelspecs come from gateway.

setup.py

@@ -81,7 +81,6 @@
         'ipython_genutils',
         'traitlets>=4.2.1',
         'jupyter_core>=4.4.0',
-        'jupyter_client>=5.3.1',
         'jupyter_kernel_mgmt>=0.4',
         'jupyter_protocol',
         'nbformat',