🏗️ Integrate lnschema-core into lamindb

docs/hub-cloud/01-init-local-instance.ipynb

@@ -48,7 +48,7 @@
    "outputs": [],
    "source": [
     "from pathlib import Path\n",
-    "from lnschema_core.models import Storage\n",
+    "from lamindb.models import Storage\n",
     "\n",
     "assert ln_setup.settings.instance.storage.type_is_cloud == False\n",
     "assert ln_setup.settings.instance.owner == ln_setup.settings.user.handle\n",

docs/hub-cloud/03-add-managed-storage.ipynb

@@ -326,7 +326,7 @@
     "from laminhub_rest.core.instance.collaborator import InstanceCollaboratorHandler\n",
     "from laminhub_rest.core.account.user import UserAccountHandler\n",
     "from lamindb_setup.core._hub_client import connect_hub_with_auth\n",
-    "from lnschema_core.models import User\n",
+    "from lamindb.models import User\n",
     "\n",
     "admin_hub = connect_hub_with_auth()\n",
     "testuser2 = UserAccountHandler(admin_hub).get_by_handle(\"testuser2\")\n",

docs/hub-cloud/08-test-multi-session.ipynb

@@ -52,7 +52,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from lnschema_core.models import User"
+    "from lamindb.models import User"
    ]
   },
   {

docs/hub-prod/test-connect-anonymously.ipynb

@@ -31,7 +31,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "ln_setup.connect(\"laminlabs/lamindata\")"
+    "ln_setup.connect(\"laminlabs/lamin-site-assets\")"
    ]
   },
   {

lamindb_setup/_check_setup.py

@@ -27,6 +27,7 @@ class InstanceNotSetupError(DefaultMessageException):
 
 
 CURRENT_ISETTINGS: InstanceSettings | None = None
+IS_LOADING: bool = False
 
 
 def _get_current_instance_settings() -> InstanceSettings | None:
@@ -54,16 +55,11 @@ def _get_current_instance_settings() -> InstanceSettings | None:
 
 # we make this a private function because in all the places it's used,
 # users should not see it
-def _check_instance_setup(
-    from_lamindb: bool = False, from_module: str | None = None
-) -> bool:
-    reload_module = from_lamindb or from_module is not None
-    from ._init_instance import get_schema_module_name, reload_schema_modules
-
+def _check_instance_setup(from_module: str | None = None) -> bool:
     if django.IS_SETUP:
         # reload logic here because module might not yet have been imported
         # upon first setup
-        if from_module is not None:
+        if from_module is not None and from_module != "lamindb":
             il.reload(il.import_module(from_module))
         return True
     silence_loggers()
@@ -75,18 +71,21 @@ def _check_instance_setup(
         return True
     isettings = _get_current_instance_settings()
     if isettings is not None:
-        if reload_module and settings.auto_connect:
-            if not django.IS_SETUP:
+        if (
+            from_module is not None
+            and settings.auto_connect
+            and not django.IS_SETUP
+            and not IS_LOADING
+        ):
+            if not from_module == "lamindb":
+                import lamindb
+
+                il.reload(il.import_module(from_module))
+            else:
                 django.setup_django(isettings)
-                if from_module is not None:
-                    # this only reloads `from_module`
-                    il.reload(il.import_module(from_module))
-                else:
-                    # this bulk reloads all schema modules
-                    reload_schema_modules(isettings)
                 logger.important(f"connected lamindb: {isettings.slug}")
         return django.IS_SETUP
     else:
-        if reload_module and settings.auto_connect:
+        if from_module is not None and settings.auto_connect:
             logger.warning(InstanceNotSetupError.default_message)
         return False

lamindb_setup/_connect_instance.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
+import importlib
 import os
-import sys
 from typing import TYPE_CHECKING
 from uuid import UUID
 
@@ -257,6 +257,14 @@
         if _test:
             return None
         silence_loggers()
+        # migrate away from lnschema-core
+        no_lnschema_core_file = (
+            settings_dir / f"no_lnschema_core-{isettings.slug.replace('/', '--')}"
+        )
+        if not no_lnschema_core_file.exists():
+            migrate_lnschema_core(
+                isettings, no_lnschema_core_file, write_file=_write_settings
+            )
         check, msg = isettings._load_db()
         if not check:
             local_db = (
@@ -292,21 +300,13 @@
         #     except ProgrammingError:
         #         pass
         load_from_isettings(isettings, user=_user, write_settings=_write_settings)
+        importlib.reload(importlib.import_module("lamindb"))
     except Exception as e:
         if isettings is not None:
             if _write_settings:
                 isettings._get_settings_file().unlink(missing_ok=True)  # type: ignore
             settings._instance_settings = None
         raise e
-    # rename lnschema_bionty to bionty for sql tables
-    if "bionty" in isettings.schema:
-        no_lnschema_bionty_file = (
-            settings_dir / f"no_lnschema_bionty-{isettings.slug.replace('/', '')}"
-        )
-        if not no_lnschema_bionty_file.exists():
-            migrate_lnschema_bionty(
-                isettings, no_lnschema_bionty_file, write_file=_write_settings
-            )
     return None
 
 
@@ -322,13 +322,10 @@
     return result
 
 
-def migrate_lnschema_bionty(
-    isettings: InstanceSettings, no_lnschema_bionty_file: Path, write_file: bool = True
+def migrate_lnschema_core(
+    isettings: InstanceSettings, no_lnschema_core_file: Path, write_file: bool = True
 ):
-    """Migrate lnschema_bionty tables to bionty tables if bionty_source doesn't exist.
-
-    :param db_uri: str, database URI (e.g., 'sqlite:///path/to/db.sqlite' or 'postgresql://user:password@host:port/dbname')
-    """
+    """Migrate lnschema_core tables to lamindb tables."""
     from urllib.parse import urlparse
 
     parsed_uri = urlparse(isettings.db)
@@ -348,60 +345,62 @@
     cur = conn.cursor()
 
     try:
-        # check if bionty_source table exists
         if db_type == "sqlite":
             cur.execute(
-                "SELECT name FROM sqlite_master WHERE type='table' AND name='bionty_source'"
+                "SELECT name FROM sqlite_master WHERE type='table' AND name='lamindb_user'"
             )
             migrated = cur.fetchone() is not None
 
             # tables that need to be renamed
             cur.execute(
-                "SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'lnschema_bionty_%'"
+                "SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'lnschema_core_%'"
             )
             tables_to_rename = [
-                row[0][len("lnschema_bionty_") :] for row in cur.fetchall()
+                row[0][len("lnschema_core_") :] for row in cur.fetchall()
             ]
         else:  # postgres
             cur.execute(
-                "SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = 'bionty_source')"
+                "SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = 'lamindb_user')"
             )
             migrated = cur.fetchone()[0]
 
             # tables that need to be renamed
             cur.execute(
-                "SELECT table_name FROM information_schema.tables WHERE table_name LIKE 'lnschema_bionty_%'"
+                "SELECT table_name FROM information_schema.tables WHERE table_name LIKE 'lnschema_core_%'"
             )
             tables_to_rename = [
-                row[0][len("lnschema_bionty_") :] for row in cur.fetchall()
+                row[0][len("lnschema_core_") :] for row in cur.fetchall()
             ]
 
         if migrated:
             if write_file:
-                no_lnschema_bionty_file.touch(exist_ok=True)
+                no_lnschema_core_file.touch(exist_ok=True)
         else:
             try:
-                # rename tables only if bionty_source doesn't exist and there are tables to rename
+                response = input(
+                    f"Do you want to migrate to lamindb 0.78 (integrate lnschema_core into lamindb)? (y/n) -- Will rename {tables_to_rename}"
+                )
+                if response != "y":
+                    print("Aborted.")
+                    quit()
                 for table in tables_to_rename:
                     if db_type == "sqlite":
                         cur.execute(
-                            f"ALTER TABLE lnschema_bionty_{table} RENAME TO bionty_{table}"
+                            f"ALTER TABLE lnschema_core_{table} RENAME TO lamindb_{table}"
                         )
                     else:  # postgres
                         cur.execute(
-                            f"ALTER TABLE lnschema_bionty_{table} RENAME TO bionty_{table};"
+                            f"ALTER TABLE lnschema_core_{table} RENAME TO lamindb_{table};"
                         )
 
-                # update django_migrations table
                 cur.execute(
-                    "UPDATE django_migrations SET app = 'bionty' WHERE app = 'lnschema_bionty'"
+                    "UPDATE django_migrations SET app = 'lamindb' WHERE app = 'lnschema_core'"
                 )
-
-                logger.warning(
-                    "Please uninstall lnschema-bionty via `pip uninstall lnschema-bionty`!"
+                print(
+                    "Renaming tables finished.\nNow, *please* call: lamin migrate deploy"
                 )
                 if write_file:
-                    no_lnschema_bionty_file.touch(exist_ok=True)
+                    no_lnschema_core_file.touch(exist_ok=True)
             except Exception:
                 # read-only users can't rename tables
                 pass

lamindb_setup/_django.py

@@ -5,24 +5,24 @@
 
 
 def django(command: str, package_name: str | None = None, **kwargs):
-    r"""Manage migrations.
+    r"""Call Django commands.
 
     Examples:
 
     Reset auto-incrementing primary integer ids after a database import:
 
     >>> import lamindb as ln
-    >>> ln.setup.django("sqlsequencereset", "lnschema_core")
+    >>> ln.setup.django("sqlsequencereset", "lamindb")
     BEGIN;
-    SELECT setval(pg_get_serial_sequence('"lnschema_core_user"','id'), coalesce(max("id"), 1), max("id") IS NOT null) FROM "lnschema_core_user";  # noqa
-    SELECT setval(pg_get_serial_sequence('"lnschema_core_storage"','id'), coalesce(max("id"), 1), max("id") IS NOT null) FROM "lnschema_core_storage";  # noqa
+    SELECT setval(pg_get_serial_sequence('"lamindb_user"','id'), coalesce(max("id"), 1), max("id") IS NOT null) FROM "lamindb_user";  # noqa
+    SELECT setval(pg_get_serial_sequence('"lamindb_storage"','id'), coalesce(max("id"), 1), max("id") IS NOT null) FROM "lamindb_storage";  # noqa
     COMMIT;
 
     You can then run the SQL output that you'll see like so:
 
     >>> sql = \"\"\"BEGIN;
-        SELECT setval(pg_get_serial_sequence('"lnschema_core_user"','id'), coalesce(max("id"), 1), max("id") IS NOT null) FROM "lnschema_core_user";  # noqa
-        SELECT setval(pg_get_serial_sequence('"lnschema_core_storage"','id'), coalesce(max("id"), 1), max("id") IS NOT null) FROM "lnschema_core_storage";  # noqa
+        SELECT setval(pg_get_serial_sequence('"lamindb_user"','id'), coalesce(max("id"), 1), max("id") IS NOT null) FROM "lamindb_user";  # noqa
+        SELECT setval(pg_get_serial_sequence('"lamindb_storage"','id'), coalesce(max("id"), 1), max("id") IS NOT null) FROM "lamindb_storage";  # noqa
         COMMIT;\"\"\"
     >>> from django.db import connection
     >>> with connection.cursor() as cursor:

lamindb_setup/_exportdb.py

@@ -4,7 +4,7 @@
 from pathlib import Path
 
 MODELS = {
-    "core": {
+    "lamindb": {
         "Collection": False,
         "Artifact": False,
         "Transform": False,

lamindb_setup/_init_instance.py

@@ -2,7 +2,6 @@
 
 import importlib
 import os
-import sys
 import uuid
 from typing import TYPE_CHECKING, Literal
 from uuid import UUID
@@ -29,6 +28,8 @@
 def get_schema_module_name(schema_name, raise_import_error: bool = True) -> str | None:
     import importlib.util
 
+    if schema_name == "core":
+        return "lamindb"
     name_attempts = [f"lnschema_{schema_name.replace('-', '_')}", schema_name]
     for name in name_attempts:
         module_spec = importlib.util.find_spec(name)
@@ -42,8 +43,8 @@
 
 
 def register_storage_in_instance(ssettings: StorageSettings):
-    from lnschema_core.models import Storage
-    from lnschema_core.users import current_user_id
+    from lamindb.base.users import current_user_id
+    from lamindb.models import Storage
 
     from .core.hashing import hash_and_encode_as_b62
 
@@ -71,7 +72,7 @@
 
 
 def register_user(usettings):
-    from lnschema_core.models import User
+    from lamindb.models import User
 
     try:
         # need to have try except because of integer primary key migration
@@ -99,35 +100,6 @@
         logger.warning(f"instance seems not set up ({error})")
 
 
-def reload_schema_modules(isettings: InstanceSettings, include_core: bool = True):
-    schema_names = ["core"] if include_core else []
-    # schema_names += list(isettings.schema)
-    schema_module_names = [get_schema_module_name(n) for n in schema_names]
-
-    for schema_module_name in schema_module_names:
-        if schema_module_name in sys.modules:
-            schema_module = importlib.import_module(schema_module_name)
-            importlib.reload(schema_module)
-
-
-def reload_lamindb_itself(isettings) -> bool:
-    reloaded = False
-    if "lamindb" in sys.modules:
-        import lamindb
-
-        importlib.reload(lamindb)
-        reloaded = True
-    return reloaded
-
-
-def reload_lamindb(isettings: InstanceSettings):
-    log_message = settings.auto_connect
-    if not reload_lamindb_itself(isettings):
-        log_message = True
-    if log_message:
-        logger.important(f"connected lamindb: {isettings.slug}")
-
-
 ERROR_SQLITE_CACHE = """
 Your cached local SQLite file exists, while your cloud SQLite file ({}) doesn't.
 Either delete your cache ({}) or add it back to the cloud (if delete was accidental).
@@ -329,6 +301,8 @@
             update_schema_in_hub(access_token=access_token)
         if _write_settings:
             settings.auto_connect = True
+        importlib.reload(importlib.import_module("lamindb"))
+        logger.important(f"initialized lamindb: {isettings.slug}")
     except Exception as e:
         from ._delete import delete_by_isettings
         from .core._hub_core import delete_instance_record, delete_storage_record
@@ -338,16 +312,17 @@
                 delete_by_isettings(isettings)
             else:
                 settings._instance_settings = None
-            if (
-                user_handle != "anonymous" or access_token is not None
-            ) and isettings.is_on_hub:
-                delete_instance_record(isettings._id, access_token=access_token)
         if (
             ssettings is not None
             and (user_handle != "anonymous" or access_token is not None)
             and ssettings.is_on_hub
         ):
             delete_storage_record(ssettings._uuid, access_token=access_token)  # type: ignore
+        if isettings is not None:
+            if (
+                user_handle != "anonymous" or access_token is not None
+            ) and isettings.is_on_hub:
+                delete_instance_record(isettings._id, access_token=access_token)
         raise e
     return None
 
@@ -378,7 +353,6 @@
         if not isettings._get_settings_file().exists():
             register_user(user)
     isettings._persist(write_to_disk=write_settings)
-    reload_lamindb(isettings)
 
 
 def validate_sqlite_state(isettings: InstanceSettings) -> None: