From a1c22d47b0def49292683daed193ee034d77f27d Mon Sep 17 00:00:00 2001 From: Thorsten Klein Date: Fri, 8 May 2020 12:54:16 +0200 Subject: [PATCH] Use auto-port choosing on singleuser side ... if the kubespawner/port: auto annotation is set Signed-off-by: Thorsten Klein --- .gitignore | 3 ++ kubespawner/__init__.py | 2 + kubespawner/api.py | 28 +++++++++++++ kubespawner/autoport.py | 24 ++++++++++++ kubespawner/objects.py | 66 +++++++++++++++++++------------ kubespawner/spawner.py | 76 ++++++++++++++++++++++-------------- scripts/kubespawner-autoport | 6 +++ setup.py | 7 +++- tests/test_spawner.py | 2 + tox.ini | 8 ---- 10 files changed, 159 insertions(+), 63 deletions(-) create mode 100644 kubespawner/api.py create mode 100644 kubespawner/autoport.py create mode 100755 scripts/kubespawner-autoport delete mode 100644 tox.ini diff --git a/.gitignore b/.gitignore index 4df85f66..527771b0 100644 --- a/.gitignore +++ b/.gitignore @@ -75,3 +75,6 @@ docs/source/build/ # PyBuilder target/ + +# Pipenv +Pipfile* \ No newline at end of file diff --git a/kubespawner/__init__.py b/kubespawner/__init__.py index c4c80c90..7405f9e6 100644 --- a/kubespawner/__init__.py +++ b/kubespawner/__init__.py @@ -13,6 +13,8 @@ # instead of the more verbose import kubespawner.spawner.KubeSpawner. from kubespawner.spawner import KubeSpawner +from . import api +from . import autoport __version__ = '0.14.2.dev' __all__ = [KubeSpawner] diff --git a/kubespawner/api.py b/kubespawner/api.py new file mode 100644 index 00000000..8b3c6b57 --- /dev/null +++ b/kubespawner/api.py @@ -0,0 +1,28 @@ +import json +from tornado import web +from jupyterhub.apihandlers import APIHandler, default_handlers + +class KubeSpawnerAPIHandler(APIHandler): + @web.authenticated + def post(self): + """POST set user spawner data""" + if hasattr(self, 'current_user'): + # Jupyterhub compatability, (september 2018, d79a99323ef1d) + user = self.current_user + else: + # Previous jupyterhub, 0.9.4 and before. + user = self.get_current_user() + token = self.get_auth_token() + spawner = None + for s in user.spawners.values(): + if s.api_token == token: + spawner = s + break + data = self.get_json_body() + for key, value in data.items(): + if hasattr(spawner, key): + setattr(spawner, key, value) + self.finish(json.dumps({"message": "KubeSpawner data configured"})) + self.set_status(201) + +default_handlers.append((r"/api/kubespawner", KubeSpawnerAPIHandler)) \ No newline at end of file diff --git a/kubespawner/autoport.py b/kubespawner/autoport.py new file mode 100644 index 00000000..b6c54bce --- /dev/null +++ b/kubespawner/autoport.py @@ -0,0 +1,24 @@ +import os +import sys + +from runpy import run_path +from shutil import which + +from jupyterhub.utils import random_port, url_path_join +from jupyterhub.services.auth import HubAuth + +def main(argv=None): + port = random_port() + hub_auth = HubAuth() + hub_auth.client_ca = os.environ.get('JUPYTERHUB_SSL_CLIENT_CA', '') + hub_auth.certfile = os.environ.get('JUPYTERHUB_SSL_CERTFILE', '') + hub_auth.keyfile = os.environ.get('JUPYTERHUB_SSL_KEYFILE', '') + hub_auth._api_request(method='POST', + url=url_path_join(hub_auth.api_url, 'kubespawner'), + json={'port' : port}) + cmd_path = which(sys.argv[1]) + sys.argv = sys.argv[1:] + ['--port={}'.format(port)] + run_path(cmd_path, run_name="__main__") + +if __name__ == "__main__": + main() diff --git a/kubespawner/objects.py b/kubespawner/objects.py index 54c79884..cc093862 100644 --- a/kubespawner/objects.py +++ b/kubespawner/objects.py @@ -5,22 +5,26 @@ import re from urllib.parse import urlparse +from kubernetes.client.models import (V1Affinity, V1Container, V1ContainerPort, + V1EndpointAddress, V1EndpointPort, + V1Endpoints, V1EndpointSubset, V1EnvVar, + V1Lifecycle, V1LocalObjectReference, + V1NodeAffinity, V1NodeSelector, + V1NodeSelectorRequirement, + V1NodeSelectorTerm, V1ObjectMeta, + V1PersistentVolumeClaim, + V1PersistentVolumeClaimSpec, V1Pod, + V1PodAffinity, V1PodAffinityTerm, + V1PodAntiAffinity, V1PodSecurityContext, + V1PodSpec, V1PreferredSchedulingTerm, + V1ResourceRequirements, + V1SecurityContext, V1Service, + V1ServicePort, V1ServiceSpec, + V1Toleration, V1Volume, V1VolumeMount, + V1WeightedPodAffinityTerm) + from kubespawner.utils import get_k8s_model, update_k8s_model -from kubernetes.client.models import ( - V1Pod, V1PodSpec, V1PodSecurityContext, - V1ObjectMeta, - V1LocalObjectReference, - V1Volume, V1VolumeMount, - V1Container, V1ContainerPort, V1SecurityContext, V1EnvVar, V1ResourceRequirements, V1Lifecycle, - V1PersistentVolumeClaim, V1PersistentVolumeClaimSpec, - V1Endpoints, V1EndpointSubset, V1EndpointAddress, V1EndpointPort, - V1Service, V1ServiceSpec, V1ServicePort, - V1Toleration, - V1Affinity, - V1NodeAffinity, V1NodeSelector, V1NodeSelectorTerm, V1PreferredSchedulingTerm, V1NodeSelectorRequirement, - V1PodAffinity, V1PodAntiAffinity, V1WeightedPodAffinityTerm, V1PodAffinityTerm, -) def make_pod( name, @@ -298,11 +302,17 @@ def make_pod( prepared_env.append(get_k8s_model(V1EnvVar, v)) else: prepared_env.append(V1EnvVar(name=k, value=v)) + # port == 0: do not create a port object + if port == 0: + ports = [] + else: + ports=[V1ContainerPort(name='notebook-port', container_port=port)] + notebook_container = V1Container( name='notebook', image=image, working_dir=working_dir, - ports=[V1ContainerPort(name='notebook-port', container_port=port)], + ports=ports, env=prepared_env, args=cmd, image_pull_policy=image_pull_policy, @@ -502,18 +512,24 @@ def make_ingress( try: from kubernetes.client.models import ( - ExtensionsV1beta1Ingress, ExtensionsV1beta1IngressSpec, ExtensionsV1beta1IngressRule, - ExtensionsV1beta1HTTPIngressRuleValue, ExtensionsV1beta1HTTPIngressPath, - ExtensionsV1beta1IngressBackend, - ) + ExtensionsV1beta1HTTPIngressPath, + ExtensionsV1beta1HTTPIngressRuleValue, ExtensionsV1beta1Ingress, + ExtensionsV1beta1IngressBackend, ExtensionsV1beta1IngressRule, + ExtensionsV1beta1IngressSpec) except ImportError: - from kubernetes.client.models import ( - V1beta1Ingress as ExtensionsV1beta1Ingress, V1beta1IngressSpec as ExtensionsV1beta1IngressSpec, - V1beta1IngressRule as ExtensionsV1beta1IngressRule, - V1beta1HTTPIngressRuleValue as ExtensionsV1beta1HTTPIngressRuleValue, - V1beta1HTTPIngressPath as ExtensionsV1beta1HTTPIngressPath, + from kubernetes.client.models import \ + V1beta1HTTPIngressPath as ExtensionsV1beta1HTTPIngressPath + from kubernetes.client.models import \ + V1beta1HTTPIngressRuleValue as \ + ExtensionsV1beta1HTTPIngressRuleValue + from kubernetes.client.models import \ + V1beta1Ingress as ExtensionsV1beta1Ingress + from kubernetes.client.models import \ V1beta1IngressBackend as ExtensionsV1beta1IngressBackend - ) + from kubernetes.client.models import \ + V1beta1IngressRule as ExtensionsV1beta1IngressRule + from kubernetes.client.models import \ + V1beta1IngressSpec as ExtensionsV1beta1IngressSpec meta = V1ObjectMeta( name=name, diff --git a/kubespawner/spawner.py b/kubespawner/spawner.py index 3367055c..df2f80bc 100644 --- a/kubespawner/spawner.py +++ b/kubespawner/spawner.py @@ -5,45 +5,39 @@ implementation that should be used by JupyterHub. """ -from functools import partial # noqa -from datetime import datetime, timedelta import asyncio import json +import multiprocessing import os -import sys import string -import multiprocessing -from concurrent.futures import ThreadPoolExecutor +import sys import warnings +from asyncio import sleep +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timedelta +from functools import partial # noqa -from tornado import gen -from tornado.ioloop import IOLoop -from tornado.concurrent import run_on_executor -from tornado import web -from traitlets import ( - Bool, - Dict, - Integer, - List, - Unicode, - Union, - default, - observe, - validate, -) +import escapism +from async_generator import async_generator, yield_ +from jinja2 import BaseLoader, Environment from jupyterhub.spawner import Spawner -from jupyterhub.utils import exponential_backoff from jupyterhub.traitlets import Command -from kubernetes.client.rest import ApiException +from jupyterhub.utils import exponential_backoff from kubernetes import client -import escapism -from jinja2 import Environment, BaseLoader +from kubernetes.client.rest import ApiException +from slugify import slugify +from tornado import gen, web +from tornado.concurrent import run_on_executor +from tornado.ioloop import IOLoop -from .clients import shared_client -from kubespawner.traitlets import Callable from kubespawner.objects import make_pod, make_pvc from kubespawner.reflector import NamespacedResourceReflector -from slugify import slugify +from kubespawner.traitlets import Callable +from traitlets import (Bool, Dict, Integer, List, Unicode, Union, default, + observe, validate) + +from .clients import shared_client + class PodReflector(NamespacedResourceReflector): """ @@ -467,7 +461,7 @@ def _deprecated_changed(self, change): ) image = Unicode( - 'jupyterhub/singleuser:latest', + 'jupytertest:local', config=True, help=""" Docker image to use for spawning user's containers. @@ -1482,10 +1476,26 @@ async def get_pod_manifest(self): labels = self._build_pod_labels(self._expand_all(self.extra_labels)) annotations = self._build_common_annotations(self._expand_all(self.extra_annotations)) + # FIXME: use a real config option instead of an annotation + if "jupyterhub/port" in self.extra_annotations: + if self.extra_annotations["jupyterhub/port"] == "auto": + self.log.info(f"Letting pod {self.pod_name} choose the port itself") + self.port = 0 + if real_cmd: + for arg in real_cmd: + if arg.startswith("--port="): + self.log.debug(f"Removing '--port' flag from cmd for pod {self.pod_name}, which chooses the port itself") + real_cmd.remove(arg) + real_cmd = ["kubespawner-autoport"] + real_cmd # FIXME: add configuration option to specify the path to the executable + else: + real_cmd = ["kubespawner-autoport"] + self.log.debug(f"Full CMD for pod {self.pod_name} is '{real_cmd}'") + port_selection = self.port + return make_pod( name=self.pod_name, cmd=real_cmd, - port=self.port, + port=port_selection, image=self.image, image_pull_policy=self.image_pull_policy, image_pull_secrets=self.image_pull_secrets, @@ -1955,6 +1965,14 @@ async def _start(self): ] ), ) + + if self.port == 0: + self.log.info(f"Pod {self.pod_name} has port set to 0, so we wait for it to set the real port itself") + while self.port == 0: + self.log.debug(f"Waiting for {self.pod_name} to send the real port number...") + yield gen.sleep(1) + self.log.info(f"Pod {self.pod_name} is listening on port {self.port}") + return (pod["status"]["podIP"], self.port) async def _make_delete_pod_request(self, pod_name, delete_options, grace_seconds, request_timeout): diff --git a/scripts/kubespawner-autoport b/scripts/kubespawner-autoport new file mode 100755 index 00000000..bbca5ee6 --- /dev/null +++ b/scripts/kubespawner-autoport @@ -0,0 +1,6 @@ +#!/usr/bin/env python3 + +from kubespawner.autoport import main + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/setup.py b/setup.py index 0b952603..77a2d841 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,10 @@ from __future__ import print_function -from setuptools import setup, find_packages + +import os import sys +from glob import glob + +from setuptools import find_packages, setup v = sys.version_info if v[:2] < (3, 6): @@ -20,6 +24,7 @@ 'kubernetes>=10.1.0', 'urllib3', 'pyYAML', + 'notebook>=4.0' ], python_requires='>=3.6', extras_require={ diff --git a/tests/test_spawner.py b/tests/test_spawner.py index 7ce3b138..2a8099a2 100644 --- a/tests/test_spawner.py +++ b/tests/test_spawner.py @@ -101,6 +101,8 @@ def set_id(spawner): @pytest.mark.asyncio async def test_spawn(kube_ns, kube_client, config): spawner = KubeSpawner(hub=Hub(), user=MockUser(), config=config) + spawner.extra_annotations = {"jupyterhub/port": "auto"} + # empty spawner isn't running status = await spawner.poll() assert isinstance(status, int) diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 423408a7..00000000 --- a/tox.ini +++ /dev/null @@ -1,8 +0,0 @@ -[tox] -minversion = 1.6 -skipsdist = True - -envlist = py35,py36 - -[flake8] -max-line-length = 100