Merge pull request #105 from mcg1969/discovery

Jupyter 6.0 kernel provider mechanism

.travis.yml

@@ -39,6 +39,7 @@ install:
   # difficult to work with for non-ASCII content we're using env files.
   - conda env create -f conda-recipe/testenv1.yaml
   - conda env create -f conda-recipe/testenv2.yaml
+  - rm -f $HOME/.conda/environments.txt
   - conda info -a
 
 script:

README.md

@@ -79,10 +79,7 @@ This package introduces two additional configuration options:
 1. Install [Anaconda](https://www.anaconda.com/download/) or
    [Miniconda](https://conda.io/miniconda.html).
 
-2. Create a development environment. Node.JS packages
-   [PhantomJS](http://phantomjs.org) and
-   [CasperJS](http://casperjs.org) are used for testing,
-   so installation requires both `conda` and `npm`:
+2. Create a development environment.
 
    ```shell
    conda create -n nb_conda_kernels python=YOUR_FAVORITE_PYTHON
@@ -92,8 +89,6 @@ This package introduces two additional configuration options:
    activate nb_conda_kernels
    # Install the package and test dependencies
    conda install --file requirements.txt
-   # Install PhantomJS and CasperJS
-   npm install
    ```
 
 3. Install the source package in development mode.
@@ -107,40 +102,34 @@ This package introduces two additional configuration options:
    `--prefix` argument to the installer.
 
 4. In order to properly exercise the package, the
-   tests assume the existence of `ipykernel` in the
-   base/root conda environment, and at least one conda
-   environment with the `R` kernel. For example:
-
+   tests assume a number of requirements:
+   - `ipykernel` in the base/root environment
+   - one additional environment with `ipykernel`
+   - one environment with `r-irkernel`
+   - one environment with a space in the name
+   - one environment with a non-ASCII character in the name
+
+   An easy way to accomplish this is to use the environment
+   specifications in the `conda-recipe` directory:
    ```shell
    conda install -n root ipykernel
-   conda create -n nbrtest r-irkernel
-   ```
-
-5. To run all of the tests, the Node environment must be
-   activated. The easiest way to do this is to use our
-   `npm` test command:
-
-   ```shell
-   npm run test
+   conda env create -f conda-recipe/testenv1.yaml
+   conda env create -f conda-recipe/testenv2.yaml
    ```
 
-   If you prefer to skip the Node-based tests, you can run
-   `nose` directly, skipping the `test_notebook` module:
-
-   ```
-   nosetests --exclude=test_notebook
-   ```
+5. To run all of the tests, run the command `nosetests`.
 
 ## Changelog
 
-### 2.2.0 (in development)
+### 2.2.0
 
 - Perform full activation of kernel conda environments
 - Discover kernels from their kernel specs, enabling the use
   of kernels besides Python and R
 - Support for spaces and accented characters in environment
   paths, with properly validating kernel names
 - Configurable format for kernel display names
+- Remove NodeJS-based testing
 
 ### 2.1.1
 

appveyor.yml

@@ -29,28 +29,26 @@ environment:
 install:
     # Because the specifics of activation need to be tested here, we are
     # fixing the version of conda *before* doing the full activation.
+    - mkdir C:\Users\appveyor\.conda
     - call %MINICONDA%\Scripts\conda install conda%CONDA_SPEC% --yes
     - call %MINICONDA%\Scripts\activate.bat
     - conda config --set always_yes yes --set changeps1 no --set auto_update_conda no %SAFETY_CHECKS%
     # The tests expect to see the Python kernel in the root environment
-    - conda install conda-verify conda-build ipykernel anaconda-client
-    - if %ERRORLEVEL% neq 0 exit 1
+    # Apparent bug in conda-build 3.16.2 prevents tests from being run
+    - conda install conda-verify "conda-build<3.16" ipykernel anaconda-client
     # We need to create additional environments to fully test the logic,
     # including an R kernel, a Python kernel, and environment names with
     # at least one non-ASCII character and one space. Because AppVeyor is
     # difficult to work with for non-ASCII content we're using env files.
     - conda env create -f conda-recipe\testenv1.yaml
-    - if %ERRORLEVEL% neq 0 exit 1
     - conda env create -f conda-recipe\testenv2.yaml
-    - if %ERRORLEVEL% neq 0 exit 1
     - conda info -a
 
 # Skip .NET project specific build phase.
 build: off
 
 test_script:
     - conda build conda-recipe --python=%PYTHON_VERSION%
-    - if %ERRORLEVEL% neq 0 exit 1
 
 deploy_script:
     - if not "%CONDA_SPEC%" == "" exit 0

conda-recipe/meta.yaml

@@ -18,22 +18,21 @@ requirements:
     - setuptools
   run:
     - python
+    - pywin32 # [win]
     - jupyter_client >=4.2
 
 test:
   source_files:
     - setup.cfg
   requires:
-    - notebook
-    - r-irkernel
     - nose
     - coverage
-    - mock
+    - ipykernel
+    - notebook
     - requests
-    - nodejs # [py3k and not win]
+    - mock
   commands:
-    - npx -p phantomjs -p casperjs nosetests nb_conda_kernels # [py3k and not win]
-    - nosetests nb_conda_kernels --exclude=test_notebook # [py2k or win]
+    - nosetests nb_conda_kernels
 
 about:
   home: https://github.com/Anaconda-Platform/nb_conda_kernels

conda-recipe/testenv2.yaml

@@ -1,4 +1,4 @@
-name: "t\u00ebst env2"
+name: "t\u00c6st env2"
 dependencies:
   - ipykernel
   - python=3

nb_conda_kernels/__main__.py

@@ -1,5 +1,5 @@
 from jupyter_client import kernelspec
 from .manager import CondaKernelSpecManager
 kernelspec.KernelSpecManager = CondaKernelSpecManager
-from jupyter_client.kernelspecapp import KernelSpecApp
+from jupyter_client.kernelspecapp import KernelSpecApp  # noqa
 KernelSpecApp.launch_instance()

nb_conda_kernels/discovery.py

@@ -0,0 +1,28 @@
+# Initial support for the kernel provider mechanism
+# to be introduced in jupyter_client 6.0; see
+# https://jupyter-client.readthedocs.io/en/latest/kernel_providers.html
+
+try:
+    from jupyter_client.discovery import KernelProviderBase
+except ImportError:
+    # Silently fail for version of jupyter_client that do not
+    # yet have the discovery module. This allows us to do some
+    # simple testing of this code even with jupyter_client<6
+    KernelProviderBase = object
+
+from jupyter_client.manager import KernelManager
+from .manager import CondaKernelSpecManager
+
+
+class CondaKernelProvider(KernelProviderBase):
+    id = 'conda'
+
+    def __init__(self):
+        self.cksm = CondaKernelSpecManager(conda_only=True)
+
+    def find_kernels(self):
+        for name, data in self.cksm.get_all_specs().items():
+            yield name, data['spec']
+
+    def make_manager(self, name):
+        return KernelManager(kernel_spec_manager=self.cksm, kernel_name=name)

nb_conda_kernels/install.py

@@ -4,10 +4,12 @@
 import os
 import sys
 
-from os.path import exists, join, abspath
+from os.path import join, abspath, exists
+from pkg_resources import iter_entry_points
 
 from traitlets.config.manager import BaseJSONConfigManager
 from jupyter_core.paths import jupyter_config_path
+from jupyter_client import __version__ as jc_version
 
 
 log = logging.getLogger(__name__)
@@ -52,13 +54,11 @@
 KSMC = "kernel_spec_manager_class"
 JNC = "jupyter_notebook_config"
 JNCJ = JNC + ".json"
+JCKP = "jupyter_client.kernel_providers"
+NCKDCKP = "nb_conda_kernels.discovery:CondaKernelProvider"
 ENDIS = ['disabled', 'enabled']
 
 
-def pretty(it):
-    return json.dumps(it, indent=2)
-
-
 def install(enable=False, disable=False, status=None, prefix=None, path=None, verbose=False):
     """Installs the nb_conda_kernels configuration data.
 
@@ -100,29 +100,40 @@ def install(enable=False, disable=False, status=None, prefix=None, path=None, ve
     else:
         log.info("{}ing nb_conda_kernels...".format(ENDIS[enable][:-2].capitalize()))
 
+    is_enabled_entry = False
+    has_entrypoints = int(jc_version.split('.', 1)[0]) >= 6
+    log.debug('Entry points:')
+    for ep in iter_entry_points(group=JCKP):
+        log.debug('  - {}'.format(ep))
+        if str(ep).split('=', 1)[-1].strip() == NCKDCKP:
+            is_enabled_entry = True
+    if not is_enabled_entry and has_entrypoints:
+        log.error(('NOTE: nb_conda_kernels is missing its entry point '
+                   'for jupyter_client.kernel_providers, which is needed '
+                   'for correct operation with Jupyter 6.0.'))
+    if is_enabled_entry and not has_entrypoints:
+        log.debug('  NOTE: entry points not used in Jupyter {}'.format(jc_version))
+        is_enabled_entry = False
+
     all_paths = jupyter_config_path()
     if path or prefix:
         if prefix:
-        	path = join(prefix, 'etc', 'jupyter')
+            path = join(prefix, 'etc', 'jupyter')
         if path not in all_paths:
             log.warn('WARNING: the requested path\n    {}\n'
                      'is not on the Jupyter config path'.format(path))
     else:
         prefix_s = sys.prefix + os.sep
-        for path in all_paths:
+        for path in all_paths[::-1]:
             if path.startswith(prefix_s):
                 break
         else:
             log.warn('WARNING: no path within sys.prefix was found')
-            path = all_paths[0]
-    path = abspath(path)
-    log.debug('Path: {}'.format(path))
 
     cfg = BaseJSONConfigManager(config_dir=path).get(JNC)
-    log.debug("Local configuration ({}):\n{}".format(join(path, JNCJ), pretty(cfg)))
     is_enabled_local = cfg.get(NBA, {}).get(KSMC, None) == CKSM
 
-    if not status and is_enabled_local != enable:
+    if not status and is_enabled_local != (enable and not is_enabled_entry):
         if enable:
             log.debug('Adding to local configuration')
             cfg.setdefault(NBA, {})[KSMC] = CKSM
@@ -139,21 +150,21 @@ def install(enable=False, disable=False, status=None, prefix=None, path=None, ve
     # app does: by looking through jupyter_notebook_config.json in
     # every directory in jupyter_config_path(), in reverse order.
     all_paths = jupyter_config_path()
-    log.debug('Searching configuration path:')
+    log.debug('{} entries:'.format(JNCJ))
     is_enabled_all = False
-    for path_g in all_paths[::-1]:
+    search_paths = all_paths[::-1]
+    if path not in all_paths:
+        search_paths.append(path)
+    for path_g in search_paths:
         cfg_g = BaseJSONConfigManager(config_dir=path_g).get(JNC)
-        if not cfg_g:
-        	value = 'no data'
-        elif NBA not in cfg_g:
-        	value = 'no {} entry'.format(NBA)
-        elif KSMC not in cfg_g[NBA]:
-        	value = 'no {}.{} entry'.format(NBA, KSMC)
+        flag = '-' if path != path_g else ('*' if path in all_paths else 'x')
+        if exists(join(path_g, JNCJ)):
+            value =  '\n    '.join(json.dumps(cfg_g, indent=2).splitlines())
+            if NBA in cfg_g and KSMC in cfg_g[NBA]:
+                is_enabled_all = cfg_g[NBA][KSMC] == CKSM
         else:
-        	value = cfg_g[NBA][KSMC]
-        	is_enabled_all = value == CKSM
-        	value = '\n        {}: {}'.format(KSMC, value)
-        log.debug('  - {}: {}'.format(path_g, value))
+            value = '<no file>'
+        log.debug('  {} {}: {}'.format(flag, path_g, value))
 
     if is_enabled_all != is_enabled_local:
         logsev = log.warn if status else log.error
@@ -170,6 +181,7 @@ def install(enable=False, disable=False, status=None, prefix=None, path=None, ve
         if not status:
             return 1
 
+    is_enabled_all = is_enabled_all or (has_entrypoints and is_enabled_entry)
     log.info('Status: {}'.format(ENDIS[is_enabled_all]))
     return 0
 

nb_conda_kernels/manager.py

@@ -8,7 +8,7 @@
 
 import os
 from os.path import join, split, dirname, basename, abspath
-from traitlets import Unicode
+from traitlets import Unicode, Bool
 
 from jupyter_client.kernelspec import KernelSpecManager, KernelSpec, NoSuchKernel
 
@@ -21,10 +21,13 @@ class CondaKernelSpecManager(KernelSpecManager):
     """ A custom KernelSpecManager able to search for conda environments and
         create kernelspecs for them.
     """
+    conda_only = Bool(False,
+                      help="Include only the kernels not visible from Jupyter normally")
+
     env_filter = Unicode(None, config=True, allow_none=True,
                          help="Do not list environment names that match this regex")
 
-    name_format = Unicode('{0} [conda env:{1}]', config=True, 
+    name_format = Unicode('{0} [conda env:{1}]', config=True,
                           help="String name format; '{{0}}' = Language, '{{1}}' = Kernel")
 
     def __init__(self, **kwargs):
@@ -111,7 +114,6 @@ def _all_envs(self):
             if self.env_filter is not None:
                 if self._env_filter_regex.search(env_path):
                     continue
-
             if env_path == sys.prefix:
                 continue
             elif env_path == base_prefix:
@@ -204,13 +206,16 @@ def _conda_kspecs(self):
         self._conda_kernels_cache = kspecs
         return kspecs
 
-    def find_kernel_specs(self):
+    def find_kernel_specs(self, skip_base=False):
         """ Returns a dict mapping kernel names to resource directories.
 
             The update process also adds the resource dir for the conda
             environments.
         """
-        kspecs = super(CondaKernelSpecManager, self).find_kernel_specs()
+        if self.conda_only:
+            kspecs = {}
+        else:
+            kspecs = super(CondaKernelSpecManager, self).find_kernel_specs()
 
         # add conda envs kernelspecs
         if self.whitelist:
@@ -228,8 +233,10 @@ def get_kernel_spec(self, kernel_name):
             accordingly with the detected envitonments.
         """
 
-        return (self._conda_kspecs.get(kernel_name) or
-                super(CondaKernelSpecManager, self).get_kernel_spec(kernel_name))
+        res = self._conda_kspecs.get(kernel_name)
+        if res is None and not self.conda_only:
+            res = super(CondaKernelSpecManager, self).get_kernel_spec(kernel_name)
+        return res
 
     def get_all_specs(self):
         """ Returns a dict mapping kernel names to dictionaries with two