Merge pull request #670 from OpenSourceBrain/release/0.6.0

Release/0.6.0

applications/backoffice/src/pages/HomePage.tsx

@@ -29,6 +29,13 @@ export default (props: any) => {
   const [ repositories, setRepositories ] = React.useState<any>(null);
   const [ error, setError ] = React.useState<any>(null);
 
+  let realm = "osb2"
+  if (window.location.hostname.includes("local")) {
+    realm = "osblocal"
+  } else if (window.location.hostname.includes("dev")) {
+    realm = "osb2dev"
+  }
+
   const fetchInfo = () => {
     // Initialise APIs with token
     const token = Cookies.get('accessToken');
@@ -83,17 +90,10 @@ export default (props: any) => {
       }
   }
 
-  const getKeyCloakProfile = () => {
-    const hostname = window.location.hostname;
-    let realm = "osbv2"
-    if (hostname.includes("local")) {
-      realm = "osblocal"
-    }
-    else if (hostname.includes("dev")) {
-      realm = "osb2dev"
-    }
+  const keycloakBaseUrl = React.useMemo(() => {
+
     return "/auth/admin/master/console/#/realms/" + realm + "/users/";
-  }
+  }, [realm]);
 
   // for links to profiles
   const osbProfile = "/user/";
@@ -118,7 +118,12 @@ export default (props: any) => {
 
   const dataColumns: GridColDef[] = [
     {
-      field: 'id', headerName: 'Profile', renderCell: (param) => {return <><Link href={`${getHostname("")}${osbProfile}${param.value}`} target="_blank"> OSB </Link>&nbsp;|&nbsp;<Link href={`${getHostname("accounts")}${getKeyCloakProfile()}${param.value}`} target="_blank"> KeyCloak </Link></>},
+      field: 'id', headerName: 'Profile', renderCell: (param: any) =>
+      <>
+        <Link href={`${getHostname("")}${osbProfile}${param.value}`} target="_blank"> OSB </Link>
+        &nbsp;|&nbsp;
+        <Link href={`${getHostname("accounts")}${keycloakBaseUrl}${param.value}`} target="_blank"> KeyCloak </Link>
+      </>,
       minWidth: 50, flex: 2,
     },
     {

applications/jupyterhub/deploy/values-prod.yaml

@@ -1,3 +1,23 @@
+harness:
+  quotas:
+    # default for User Quotas
+    # user quotas can be set as Keycloak attributes on Groups and Users
+    # quotas set on User level will always overrule Group quotas
+    # the following keys are valid for Jupyterhub pods (workspaces)
+
+    # sets the maximum number of (included named) servers open concurrently (int)
+    quota-ws-open: 3
+    # sets the cpu guaranteed on a single workspace in CPU units (float)
+    quota-ws-guaranteecpu: 0.2
+    # sets the cpu limit on a single workspace in CPU units (float)
+    quota-ws-maxcpu: 1
+    # sets the memory guaranteed on a single workspace in Gb units (float)
+    quota-ws-guaranteemem: 0.5
+    # sets the memory limit on a single workspace in Gb units (float)
+    quota-ws-maxmem: 1.5
+    # sets the storage dedicated to the user data in Gb units (float)
+    quota-storage-max: 2
+
 singleuser:
   storage:
     capacity: 2Gi
@@ -6,4 +26,6 @@ singleuser:
     guarantee: 0.2
   memory:
     limit: 1.5G
-    guarantee: 0.5G
+    guarantee: 0.5G
+legacyusermax: 445
+legacyworkspacemax: 465

applications/jupyterhub/deploy/values.yaml

@@ -10,6 +10,25 @@ harness:
   dependencies:
     build:
       - cloudharness-base
+  quotas:
+    # default for User Quotas
+    # user quotas can be set as Keycloak attributes on Groups and Users
+    # quotas set on User level will always overrule Group quotas
+    # the following keys are valid for Jupyterhub pods (workspaces)
+
+    # sets the maximum number of (included named) servers open concurrently (int)
+    quota-ws-open: 3
+    # sets the cpu guaranteed on a single workspace in CPU units (float)
+    quota-ws-guaranteecpu: 0.2
+    # sets the cpu limit on a single workspace in CPU units (float)
+    quota-ws-maxcpu: 1
+    # sets the memory guaranteed on a single workspace in Gb units (float)
+    quota-ws-guaranteemem: 0.5
+    # sets the memory limit on a single workspace in Gb units (float)
+    quota-ws-maxmem: 1
+    # sets the storage dedicated to the user data in Gb units (float)
+    quota-storage-max : 1.25
+
 hub:
   config:
     JupyterHub:
@@ -20,21 +39,33 @@ hub:
       c.Spawner.port = 8000
       c.Spawner.http_timeout = 300
       c.Spawner.start_timeout = 300
-      c.JupyterHub.tornado_settings = { "headers": { "Content-Security-Policy": "frame-ancestors 'self' localhost localhost:3000 *.osb.local *.opensourcebrain.org *.v2.opensourcebrain.org"}}
+      c.JupyterHub.tornado_settings = { "headers": { "Content-Security-Policy": "frame-ancestors 'self' localhost localhost:3000 *.osb.local *.metacell.us *.opensourcebrain.org *.v2.opensourcebrain.org"}}
     spawner: >-
       c.Spawner.args = []
 singleuser:
   storage:
     type: dynamic
     capacity: 2Mi
+
     dynamic:
       pvcNameTemplate: osb-user-{userid}
       volumeNameTemplate: osb-user-{userid}
+      storageClass: "{{namespace}}-nfs-client"
+      storageAccessModes:
+      - "ReadWriteMany"
     homeMountPath: /opt/user
     extraLabels: {}
   cpu:
     limit: 1
     guarantee: 0.2
   memory:
     limit: 1G
-    guarantee: 0.5G
+    guarantee: 0.5G
+prePuller:
+
+  # hook relates to the hook-image-awaiter Job and hook-image-puller DaemonSet
+  hook:
+    enabled: false
+  continuous:
+    enabled: false
+  pullProfileListImages: false

applications/jupyterhub/hub/jupyter_notebook_config.py

@@ -20,7 +20,7 @@
 print(c.NotebookApp.tornado_settings)
 c.NotebookApp.tornado_settings = {
     'headers': {
-        'Content-Security-Policy': "frame-ancestors 'self' localhost:3000 localhost:* localhost *.osb.local *.opensourcebrain.org",
+        'Content-Security-Policy': "frame-ancestors 'self' localhost:3000 localhost:* localhost *.osb.local *.opensourcebrain.org *.metacell.us",
     }
 }
 print(c.NotebookApp.tornado_settings)

applications/jupyterhub/src/osb_jupyter/osb_jupyter/jupyterhub.py

@@ -36,7 +36,7 @@ def change_pod_manifest(self: KubeSpawner):
     Returns:
         -
     """
-
+    print("OSB change pod manifest")
     # get the workspace cookie to determine the workspace id
 
     def get_from_cookie(cookie_name):
@@ -46,6 +46,13 @@ def get_from_cookie(cookie_name):
                 "Required cookie not found. Check that the cookie named '%s' is set." % cookie_name)
         return cookie.value
 
+    def user_volume_is_legacy(user_id):
+        print("User id", user_id, "max", self.config['apps']['jupyterhub'].get('legacyusermax', 0))
+        return int(user_id) < self.config['apps']['jupyterhub'].get('legacyusermax', 0)
+
+    def workspace_volume_is_legacy(workspace_id):
+        return int(workspace_id) < self.config['apps']['jupyterhub'].get('legacyworkspacemax', 0)
+
     try:
         workspace_id = get_from_cookie('workspaceId')
         volume_name = f'workspace-{workspace_id}'
@@ -58,9 +65,6 @@ def get_from_cookie(cookie_name):
             'name': volume_name,
             'persistentVolumeClaim': {
                 'claimName': volume_name,
-                'spec': {
-                    'accessModes': ['ReadWriteOnce', 'ReadOnlyMany']
-                }
             }
         }
 
@@ -78,13 +82,18 @@ def get_from_cookie(cookie_name):
 
         self.common_labels = labels
         self.extra_labels = labels
-
-        self.pod_affinity_required.append(affinity_spec('user', self.user.name))
+        self.storage_class = f'{self.config["namespace"]}-nfs-client'
+        if not user_volume_is_legacy(self.user.id):
+            # User pod affinity is by default added by cloudharness
+            self.pod_affinity_required = []
+
         write_access = has_user_write_access(
             workspace_id, self.user, workspace_owner)
-        if write_access:
+        if workspace_volume_is_legacy(workspace_id):
             # Pods with write access must be on the same node
             self.pod_affinity_required.append(affinity_spec('workspace', workspace_id))
+        from pprint import pprint
+        pprint(self.volumes)
         if not [v for v in self.volume_mounts if v['name'] == volume_name]:
             self.volume_mounts.append({
                 'name': volume_name,
@@ -95,6 +104,7 @@ def get_from_cookie(cookie_name):
         log.error('Change pod manifest failed due to an error.', exc_info=True)
 
 
+
 def has_user_write_access(workspace_id, user: User, workspace_owner: str):
     print('name:', user.name, workspace_owner)
     if workspace_owner == user.name:

applications/jupyterlab-minimal/Dockerfile

@@ -0,0 +1,3 @@
+FROM jupyter/base-notebook:hub-1.4.2
+
+COPY hub/jupyter_notebook_config.py /etc/jupyter/jupyter_notebook_config.py

applications/jupyterlab-minimal/deploy/values.yaml

@@ -0,0 +1,24 @@
+harness:
+  subdomain: notebooks
+  service:
+    auto: false
+    port: 80
+    name: proxy-public
+  deployment:
+    auto: false
+  dependencies:
+    build:
+      - cloudharness-base
+    hard:
+      - jupyterhub
+  jupyterhub:
+    args: ["--debug", "--NotebookApp.default_url=/lab", "--NotebookApp.notebook_dir=/opt/workspace"]
+    applicationHook: "osb_jupyter.change_pod_manifest"
+    extraConfig:
+      timing: |
+        c.Spawner.port = 8000
+        c.Spawner.http_timeout = 300
+        c.Spawner.start_timeout = 300
+
+        c.JupyterHub.tornado_settings = { "headers": { "Content-Security-Policy": "frame-ancestors 'self' localhost:3000 *.osb.local osb.local localhost *.metacell.us *.opensourcebrain.org"}}
+

applications/jupyterlab-minimal/hub/jupyter_notebook_config.py

@@ -0,0 +1,66 @@
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+from jupyter_core.paths import jupyter_data_dir
+import subprocess
+import os
+import errno
+import stat
+
+c = get_config()
+c.NotebookApp.ip = '0.0.0.0'
+c.NotebookApp.port = 8888
+c.NotebookApp.open_browser = False
+
+# https://github.com/jupyter/notebook/issues/3130
+c.FileContentsManager.delete_to_trash = False
+
+print('*'*80)
+import notebook
+print(c.NotebookApp.tornado_settings)
+c.NotebookApp.tornado_settings = {
+    'headers': {
+        'Content-Security-Policy': "frame-ancestors 'self' localhost:3000 localhost *.osb.local *.opensourcebrain.org",
+    }
+}
+print(c.NotebookApp.tornado_settings)
+print('*'*80)
+
+# Generate a self-signed certificate
+if 'GEN_CERT' in os.environ:
+    dir_name = jupyter_data_dir()
+    pem_file = os.path.join(dir_name, 'notebook.pem')
+    try:
+        os.makedirs(dir_name)
+    except OSError as exc:  # Python >2.5
+        if exc.errno == errno.EEXIST and os.path.isdir(dir_name):
+            pass
+        else:
+            raise
+
+    # Generate an openssl.cnf file to set the distinguished name
+    cnf_file = os.path.join(os.getenv('CONDA_DIR', '/usr/lib'), 'ssl', 'openssl.cnf')
+    if not os.path.isfile(cnf_file):
+        with open(cnf_file, 'w') as fh:
+            fh.write('''\
+[req]
+distinguished_name = req_distinguished_name
+[req_distinguished_name]
+''')
+
+    # Generate a certificate if one doesn't exist on disk
+    subprocess.check_call(['openssl', 'req', '-new',
+                           '-newkey', 'rsa:2048',
+                           '-days', '365',
+                           '-nodes', '-x509',
+                           '-subj', '/C=XX/ST=XX/L=XX/O=generated/CN=generated',
+                           '-keyout', pem_file,
+                           '-out', pem_file])
+    # Restrict access to the file
+    os.chmod(pem_file, stat.S_IRUSR | stat.S_IWUSR)
+    c.NotebookApp.certfile = pem_file
+
+# Change default umask for all subprocesses of the notebook server if set in
+# the environment
+if 'NB_UMASK' in os.environ:
+    os.umask(int(os.environ['NB_UMASK'], 8))

applications/jupyterlab/Dockerfile

@@ -1,4 +1,4 @@
-FROM jupyter/base-notebook:hub-1.4.2
+FROM jupyter/base-notebook:hub-1.5.0
 USER root
 
 
@@ -41,7 +41,7 @@ RUN jupyter labextension install plotlywidget
 # this will cause the libs to get updated and use the cached dependencies
 
 # Invalidate the cache
-ARG NOCACHE
+# ARG NOCACHE
 
 USER root
 # LFPy

applications/jupyterlab/deploy/values.yaml

@@ -20,5 +20,5 @@ harness:
         c.Spawner.http_timeout = 300
         c.Spawner.start_timeout = 300
 
-        c.JupyterHub.tornado_settings = { "headers": { "Content-Security-Policy": "frame-ancestors 'self' localhost:3000 *.osb.local osb.local localhost *.opensourcebrain.org"}}
+        c.JupyterHub.tornado_settings = { "headers": { "Content-Security-Policy": "frame-ancestors 'self' localhost:3000 *.osb.local osb.local localhost *.metacell.us *.opensourcebrain.org"}}
 

applications/jupyterlab/hub/jupyter_notebook_config.py

@@ -20,7 +20,7 @@
 print(c.NotebookApp.tornado_settings)
 c.NotebookApp.tornado_settings = {
     'headers': {
-        'Content-Security-Policy': "frame-ancestors 'self' localhost:3000 localhost *.osb.local *.opensourcebrain.org",
+        'Content-Security-Policy': "frame-ancestors 'self' localhost:3000 localhost *.osb.local *.metacell.us *.opensourcebrain.org",
     }
 }
 print(c.NotebookApp.tornado_settings)

applications/jupyterlab/requirements.txt

@@ -49,7 +49,7 @@ protobuf==3.17.0
 git+https://github.com/SheffieldML/GPy.git@devel
 #modeci_mdf==0.3.3 # big jump in size of image...
 
-sklearn # Required for some Neuromatch Academy material
+scikit-learn # Required for some Neuromatch Academy material
 fasttext # Required for some Neuromatch Academy material
 
 

applications/netpyne/Dockerfile

@@ -2,7 +2,7 @@ FROM node:13.14 as jsbuild
 ENV REPO=https://github.com/MetaCell/NetPyNE-UI.git
 ENV BRANCH_TAG=osb2-dev
 ENV FOLDER=netpyne
-RUN echo "no-cache 2022-08-26"
+RUN echo "no-cache 2023-1-9"
 RUN git clone $REPO -b $BRANCH_TAG $FOLDER
 RUN rm -Rf .git
 

applications/netpyne/deploy/values.yaml

@@ -12,7 +12,7 @@ harness:
         c.Spawner.port = 8000
         c.Spawner.http_timeout = 300
         c.Spawner.start_timeout = 300
-        c.JupyterHub.tornado_settings = { "headers": { "Content-Security-Policy": "frame-ancestors 'self' localhost:3000 *.osb.local localhost *.opensourcebrain.org"}}
+        c.JupyterHub.tornado_settings = { "headers": { "Content-Security-Policy": "frame-ancestors 'self' localhost:3000 *.osb.local localhost *.metacell.us *.opensourcebrain.org"}}
   dependencies:
     hard:
     - jupyterhub

applications/netpyne/overrides/hub/jupyter_notebook_config.py

@@ -19,7 +19,7 @@
 print(c.NotebookApp.tornado_settings)
 c.NotebookApp.tornado_settings = {
     'headers': {
-        'Content-Security-Policy': "frame-ancestors 'self' localhost:3000 localhost:* localhost *.osb.local *.opensourcebrain.org",
+        'Content-Security-Policy': "frame-ancestors 'self' localhost:3000 localhost:* localhost *.osb.local *.opensourcebrain.org *.metacell.us",
     }
 }
 print(c.NotebookApp.tornado_settings)