CELE-32 merge: Merge with develop

.github/workflows/biome.yml

@@ -0,0 +1,17 @@
+name: Check JS/TS/TSX/JSX format
+
+on: [push, pull_request]
+
+jobs:
+  format:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Biome
+        uses: biomejs/setup-biome@v2
+        with:
+          version: 1.8.3
+
+      - name: Run Biome format check
+        working-directory: applications/visualizer/frontend
+        run: biome format .

.github/workflows/black.yml

@@ -1,4 +1,4 @@
-name: Python Lint
+name: Python Format
 
 on: [push, pull_request]
 

.gitignore

@@ -2,4 +2,6 @@ skaffold.yaml
 docker-compose.yaml
 # deployment/
 cloud-harness/
-.vscode/
+.vscode/
+node_modules
+secret.json

README.md

@@ -88,18 +88,23 @@ There is two modes, either you can populate the dev DB, which is basically a sql
 
 ### Populate the Dev DB
 
-Run the following script that will take all the data from `raw-data`, process them and insert them in the sqlite3 db.
+Run the following script that will take all the data from `data/db-raw-data`, process them and insert them in the sqlite3 db.
 
 ```
 bash applications/visualizer/backend/populate-db.bash
 ```
 
 This command will apply the necessary migrations to the file db and populate it.
 
+### 3D Data and Data for Local Development
+
+Data files that are related to the 3D artifacts, segmentation, EM Images, ..., can be placed in local in the `data` folder of this repository.
+The development settings are configured to automatically fetch files from this folder instead of the bucket (which is configured for the docker compose and k8s deployment).
+
 ### Populate the Docker Compose DB
 
 First, you need to have your docker compose services running (ensure you are running the `docker compose -f deployment/docker-compose.yaml up` command).
-Then, run the following script that will take all the data from `raw-data`, process them and insert them in the postgresql db
+Then, run the following script that will take all the data from `data/db-raw-data`, process them and insert them in the postgresql db
 
 ```
 bash applications/visualizer/backend/populate-db.bash compose

applications/visualizer/backend/README.md

@@ -13,7 +13,7 @@ backend
 │   │   └── commands
 │   │       └── populatedb.py   # the custom command to run the DB population
 │   ├── models.py               # the Django model (DB model)
-│   ├── populatedb.py           # dedicated function to take the raw-data and create the django entities
+│   ├── populatedb.py           # dedicated function to take the database raw data and create the django entities
 │   └── schemas.py              # the django-ninja Schemas
 ├── openapi
 │   └── openapi.json         # The openapi.json generated from the routes defined in "api.py"

applications/visualizer/backend/populate-db.bash

@@ -8,12 +8,20 @@ cd "${PARENT_PATH}"
 case "$1" in
     "compose")
       ENV="visualizer"
-      echo "In postgresql db"
+      DB_DATA="../../../data/db-raw-data/"
+      echo "In postgresql db in docker compose"
+      ;;
+    "k8s")
+      ENV="visualizer"
+      # Need copy using gsutil if we keep this script to trigger DB ingestion
+      DB_DATA="/mnt/data/db-raw-data/"
+      echo "In postgresql db in k8s"
       ;;
     *)
+      DB_DATA="../../../data/db-raw-data/"
       echo "In local sqlite3 db"
     ;;
 esac
 
 CH_CURRENT_APP_NAME=${ENV} python manage.py migrate
-CH_CURRENT_APP_NAME=${ENV} python manage.py populatedb ../../../raw-data/
+CH_CURRENT_APP_NAME=${ENV} python manage.py populatedb ${DB_DATA}

applications/visualizer/backend/requirements.txt

@@ -1,6 +1,6 @@
 django==5.0.3
 django-cors-headers==4.3.1
-django-ninja==1.1.0
+django-ninja==1.2.2
 uvicorn==0.29.0
 ruamel.yaml==0.18.6
 psycopg[binary]==3.1.18

applications/visualizer/backend/visualizer/settings/common.py

@@ -137,6 +137,7 @@
 STATIC_ROOT = BASE_DIR / "static"
 STATIC_URL = "static/"
 
+
 # Default primary key field type
 # https://docs.djangoproject.com/en/5.0/ref/settings/#default-auto-field
 

applications/visualizer/backend/visualizer/settings/development.py

@@ -3,6 +3,8 @@
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = True
 
+GCS_BUCKET_URL = BASE_DIR.parent.parent.parent / "data"
+
 INSTALLED_APPS += ["django_extensions"]
 
 # Database

applications/visualizer/backend/visualizer/settings/production.py

@@ -6,6 +6,9 @@
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = False
 
+GCS_BUCKET = "celegans"
+GCS_BUCKET_URL = f"https://storage.googleapis.com/{GCS_BUCKET}"
+
 
 class Config(dict):
     def __getattr__(self, key):

applications/visualizer/backend/visualizer/urls.py

@@ -21,11 +21,12 @@
 from django.conf.urls.static import static
 
 from api.api import api
-from .views import index
+from .views import index, access_bucket_artifact
 
 urlpatterns = [
     path("admin/", admin.site.urls),
     path("api/", api.urls),
     *static(settings.STATIC_URL, document_root=settings.STATIC_ROOT),
+    re_path(r"resources/(?P<path>.+)", access_bucket_artifact, name="resources"),
     re_path(r"(?P<path>.*)", index, name="index"),
 ]

applications/visualizer/backend/visualizer/views.py

@@ -3,6 +3,7 @@
 
 from django.conf import settings
 from django.http import FileResponse, HttpResponseRedirect
+from django.shortcuts import redirect
 from django.urls import reverse
 from django.utils._os import safe_join
 
@@ -22,3 +23,17 @@ def index(request, path=""):
     except FileNotFoundError:
         return index(request, "")  # index.html
     return FileResponse(fullpath.open("rb"), content_type=content_type)
+
+
+def access_bucket_artifact(request, path):
+    resource_folder = Path(settings.GCS_BUCKET_URL)
+    if resource_folder.exists():
+        full_path = resource_folder / path
+        content_type, _ = mimetypes.guess_type(str(full_path))
+        content_type = content_type or "application/octet-stream"
+        try:
+            return FileResponse(full_path.open("rb"), content_type=content_type)
+        except FileNotFoundError:
+            return index(request, "")  # index.html
+
+    return redirect(f"{settings.GCS_BUCKET_URL}/{path}")

applications/visualizer/frontend/additional.d.ts

@@ -1,6 +1,6 @@
-declare module '*.svg' {
-    import React = require('react');
-    export const ReactComponent: React.FC<React.SVGProps<SVGSVGElement>>;
-    const src: string;
-    export default src;
-}
+declare module "*.svg" {
+  import React = require("react");
+  export const ReactComponent: React.FC<React.SVGProps<SVGSVGElement>>;
+  const src: string;
+  export default src;
+}

applications/visualizer/frontend/package.json

@@ -6,9 +6,10 @@
   "scripts": {
     "dev": "vite",
     "build": "tsc && vite build",
-    "lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0",
-    "lint-fix": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0 --fix",
     "preview": "vite preview",
+    "format": "biome format --write .",
+    "lint": "biome lint .",
+    "lint-fix": "biome lint --write .",
     "generate-client": "openapi --input ../backend/openapi/openapi.json --output ./src/rest --useOptions --useUnionTypes --client xhr",
     "start": "DOMAIN=http://localhost:8000 vite",
     "start:dev": "DOMAIN=https://visualizer.celegans.metacell.us vite",
@@ -20,32 +21,29 @@
     "@metacell/geppetto-meta-client": "^1.2.8",
     "@metacell/geppetto-meta-core": "^1.2.8",
     "@metacell/geppetto-meta-ui": "^1.2.8",
-    "@mui/icons-material": "^5.15.15",
-    "@mui/material": "^5.15.15",
+    "@mui/base": "^5.0.0-beta.40",
+    "@mui/icons-material": "^5.16.4",
+    "@mui/material": "^5.16.4",
     "@react-three/drei": "^9.105.4",
     "@react-three/fiber": "^8.16.2",
     "@reduxjs/toolkit": "^2.2.3",
     "@types/cytoscape": "^3.21.1",
-    "@types/three": "^0.164.0",
+    "@types/three": "^0.166.0",
     "cytoscape": "^3.29.2",
     "cytoscape-dagre": "^2.5.0",
     "cytoscape-fcose": "^2.2.0",
     "file-saver": "^2.0.5",
-    "react": "^18.2.0",
+    "react": "^18.3.1",
     "react-color": "^2.19.3",
-    "react-dom": "^18.2.0",
+    "react-dom": "^18.3.1",
     "react-redux": "^9.1.1",
-    "three": "^0.163.0"
+    "three": "^0.166.1"
   },
   "devDependencies": {
+    "@biomejs/biome": "1.8.3",
     "@types/react": "^18.2.66",
     "@types/react-dom": "^18.2.22",
-    "@typescript-eslint/eslint-plugin": "^7.2.0",
-    "@typescript-eslint/parser": "^7.2.0",
     "@vitejs/plugin-react": "^4.2.1",
-    "eslint": "^8.57.0",
-    "eslint-plugin-react-hooks": "^4.6.0",
-    "eslint-plugin-react-refresh": "^0.4.6",
     "openapi-typescript-codegen": "^0.29.0",
     "sass": "^1.75.0",
     "typescript": "^5.2.2",

applications/visualizer/frontend/src/App.tsx

@@ -1,48 +1,42 @@
-import { Provider } from "react-redux";
-import { ThemeProvider } from "@mui/material/styles";
 import { Box, CssBaseline } from "@mui/material";
+import { ThemeProvider } from "@mui/material/styles";
+import { Provider } from "react-redux";
 import theme from "./theme/index.tsx";
 import "./App.css";
-import { useGlobalContext } from "./contexts/GlobalContext.tsx";
 import AppLauncher from "./components/AppLauncher.tsx";
 import WorkspaceComponent from "./components/WorkspaceComponent.tsx";
-import { ViewMode } from "./models/models.ts";
+import { useGlobalContext } from "./contexts/GlobalContext.tsx";
+import { ViewMode } from "./models";
 
 function App() {
-    const {
-        workspaces,
-        currentWorkspaceId,
-        viewMode,
-        selectedWorkspacesIds,
-    } = useGlobalContext();
-
-
-    const hasLaunched = currentWorkspaceId != undefined
-
-    return (
-        <>
-            <ThemeProvider theme={theme}>
-                <CssBaseline />
-                {hasLaunched ? (
-                    <Box className={"layout-manager-container"}>
-                        {viewMode === ViewMode.Compare ?
-                            Array.from(selectedWorkspacesIds).map(id => (
-                                <Provider key={id} store={workspaces[id].store}>
-                                    <WorkspaceComponent />
-                                </Provider>
-                            ))
-                            :
-                            <Provider store={workspaces[currentWorkspaceId].store}>
-                                <WorkspaceComponent />
-                            </Provider>
-                        }
+  const { workspaces, currentWorkspaceId, viewMode, selectedWorkspacesIds } = useGlobalContext();
 
-                    </Box>
+  const hasLaunched = currentWorkspaceId !== undefined;
 
-                ) : <AppLauncher />}
-            </ThemeProvider>
-        </>
-    )
+  return (
+    <>
+      <ThemeProvider theme={theme}>
+        <CssBaseline />
+        {hasLaunched ? (
+          <Box className={"layout-manager-container"}>
+            {viewMode === ViewMode.Compare ? (
+              Array.from(selectedWorkspacesIds).map((id) => (
+                <Provider key={id} store={workspaces[id].store}>
+                  <WorkspaceComponent />
+                </Provider>
+              ))
+            ) : (
+              <Provider store={workspaces[currentWorkspaceId].store}>
+                <WorkspaceComponent />
+              </Provider>
+            )}
+          </Box>
+        ) : (
+          <AppLauncher />
+        )}
+      </ThemeProvider>
+    </>
+  );
 }
 
 export default App;