🏗️ Integrate lnschema-core into lamindb (#2309)

.github/workflows/build.yml

@@ -36,7 +36,7 @@ jobs:
           filters: |
             curator:
               - 'lamindb/curators/**'
-              - 'tests/core/test_curator.py'
+              - 'tests/curators/test_curator.py'
 
       - uses: actions/setup-python@v5
         if: ${{ matrix.group != 'curator' || steps.changes.outputs.curator == 'true' }}

.gitignore

@@ -12,7 +12,7 @@ docs/guide/bionty.ipynb
 docs/guide/lnschema-core.ipynb
 docs/paradisi05_laminopathic_nuclei.jpg
 bionty_docs/
-lnschema_core_docs/
+lamindb_docs/
 _build
 mydata/
 lamin-intro/

.gitmodules

@@ -1,6 +1,3 @@
-[submodule "lnschema-core"]
-	path = sub/lnschema-core
-	url = https://github.com/laminlabs/lnschema-core.git
 [submodule "sub/lamindb-setup"]
 	path = sub/lamindb-setup
 	url = https://github.com/laminlabs/lamindb-setup
@@ -19,6 +16,3 @@
 [submodule "sub/ourprojects"]
 	path = sub/ourprojects
 	url = https://github.com/laminlabs/ourprojects
-[submodule "sub/cellregistry"]
-	path = sub/cellregistry
-	url = https://github.com/laminlabs/cellregistry

.pre-commit-config.yaml

@@ -47,13 +47,13 @@ repos:
               tests/core/notebooks/basic-r-notebook.Rmd.cleaned.html
             )
       - id: check-case-conflict
-  - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.7.1
-    hooks:
-      - id: mypy
-        args: [--no-strict-optional, --ignore-missing-imports]
-        additional_dependencies: ["types-requests", "types-attrs"]
-        exclude: |
-          (?x)(
-              test_notebooks.py|script-to-test-versioning.py|tests/storage/conftest.py
-          )
+  # - repo: https://github.com/pre-commit/mirrors-mypy
+  #   rev: v1.7.1
+  #   hooks:
+  #     - id: mypy
+  #       args: [--no-strict-optional, --ignore-missing-imports]
+  #       additional_dependencies: ["types-requests", "types-attrs"]
+  #       exclude: |
+  #         (?x)(
+  #             test_notebooks.py|script-to-test-versioning.py|tests/storage/conftest.py
+  #         )

docs/cellregistry.md

@@ -1,5 +1,3 @@
 # `cellregistry`
 
-```{eval-rst}
-.. automodule:: cellregistry
-```
+Place holder.

docs/faq/validate-fields.ipynb

@@ -28,7 +28,7 @@
    "outputs": [],
    "source": [
     "import ourprojects as pj\n",
-    "from lnschema_core.validation import FieldValidationError"
+    "from lamindb.base.validation import FieldValidationError"
    ]
   },
   {

docs/schemas.ipynb

@@ -36,7 +36,7 @@
    "id": "5141b1a0",
    "metadata": {},
    "source": [
-    "## Basic data management: `lnschema_core`"
+    "## Basic data management: `lamindb`"
    ]
   },
   {
@@ -53,9 +53,9 @@
    "id": "b614a6ea",
    "metadata": {},
    "source": [
-    "The core schema module is built into the core LaminDB API with its own Python package `lnschema_core` with repository [here](https://github.com/laminlabs/lnschema-core).\n",
+    "The core schema module is built into the core LaminDB API with its own Python package `lamindb` with repository [here](https://github.com/laminlabs/lnschema-core).\n",
     "\n",
-    "All of LaminDB's core classes, like {class}`~lamindb.Artifact`, {class}`~lamindb.Transform`, {class}`~lamindb.User`, etc. are registries defined in the core schema. You can see their source code [here](https://github.com/laminlabs/lnschema-core/blob/main/lnschema_core/models.py)."
+    "All of LaminDB's core classes, like {class}`~lamindb.Artifact`, {class}`~lamindb.Transform`, {class}`~lamindb.User`, etc. are registries defined in the core schema. You can see their source code [here](https://github.com/laminlabs/lnschema-core/blob/main/lamindb/models.py)."
    ]
   },
   {

docs/setup.ipynb

@@ -344,7 +344,7 @@
     "Your database is not up to date with your installed Python library.\n",
     "\n",
     "The database misses the following migrations:\n",
-    "[<Migration lnschema_core.0014_rename_ref_field_featureset_registry>, <Migration lnschema_core.0015_artifact_initial_version_artifact_version>]\n",
+    "[<Migration lamindb.0014_rename_ref_field_featureset_registry>, <Migration lamindb.0015_artifact_initial_version_artifact_version>]\n",
     "\n",
     "Only if you are an admin and manage migrations manually, deploy them to the database:\n",
     "lamin migrate deploy\n",

lamindb/__init__.py

@@ -43,11 +43,8 @@
 """
 
 # denote a release candidate for 0.1.0 with 0.1rc1, 0.1a1, 0.1b1, etc.
-__version__ = "0.77.3"
+__version__ = "0.78a1"
 
-import os as _os
-
-import lamindb_setup as _lamindb_setup
 from lamindb_setup._check_setup import InstanceNotSetupError as _InstanceNotSetupError
 from lamindb_setup._check_setup import _check_instance_setup
 from lamindb_setup._connect_instance import connect
@@ -60,22 +57,8 @@ def __getattr__(name):
     raise _InstanceNotSetupError()
 
 
-if _check_instance_setup(from_module="lnschema_core"):
-    del _InstanceNotSetupError
-    del __getattr__  # delete so that imports work out
-    from lnschema_core.models import (
-        Artifact,
-        Collection,
-        Feature,
-        FeatureSet,
-        Param,
-        Run,
-        Storage,
-        Transform,
-        ULabel,
-        User,
-    )
-
+if _check_instance_setup(from_module="lamindb"):
+    del __getattr__  # so that imports work out
     from . import core  # isort: split
     from . import (
         _artifact,
@@ -97,10 +80,21 @@ def __getattr__(name):
     from .core._context import context
     from .core._settings import settings
     from .curators import Curator
+    from .models import (
+        Artifact,
+        Collection,
+        Feature,
+        FeatureSet,
+        Param,
+        Run,
+        Storage,
+        Transform,
+        ULabel,
+        User,
+    )
 
-    track = context.track  # simple access because these are so common
-    finish = context.finish  # simple access because these are so common
-    Curate = Curator  # backward compat
+    track = context.track  # simple access
+    finish = context.finish  # simple access
     settings.__doc__ = """Global settings (:class:`~lamindb.core.Settings`)."""
     context.__doc__ = """Global run context (:class:`~lamindb.core.Context`).
 

lamindb/_artifact.py

@@ -23,10 +23,11 @@
     get_stat_dir_cloud,
     get_stat_file_cloud,
 )
-from lnschema_core.models import Artifact, FeatureManager, ParamManager, Run, Storage
-from lnschema_core.types import (
+
+from lamindb.base.types import (
     VisibilityChoice,
 )
+from lamindb.models import Artifact, FeatureManager, ParamManager, Run, Storage
 
 from ._parents import view_lineage
 from ._utils import attach_func_to_class_method

lamindb/_can_curate.py

@@ -8,7 +8,8 @@
 from django.core.exceptions import FieldDoesNotExist
 from lamin_utils import colors, logger
 from lamindb_setup.core._docs import doc_args
-from lnschema_core import CanCurate, Record
+
+from lamindb.models import CanCurate, Record
 
 from ._from_values import _format_values, _has_organism_field, get_or_create_records
 from ._record import _queryset, get_name_field
@@ -18,9 +19,9 @@
 if TYPE_CHECKING:
     from django.db.models import QuerySet
     from lamin_utils._inspect import InspectResult
-    from lnschema_core.types import ListLike, StrField
 
     from lamindb._query_set import RecordList
+    from lamindb.base.types import ListLike, StrField
 
 
 # from_values doesn't apply for QuerySet or Manager

lamindb/_collection.py

@@ -13,14 +13,14 @@
 from lamin_utils import logger
 from lamindb_setup.core._docs import doc_args
 from lamindb_setup.core.hashing import hash_set
-from lnschema_core.models import (
+
+from lamindb.base.types import VisibilityChoice
+from lamindb.models import (
     Collection,
     CollectionArtifact,
     FeatureSet,
 )
-from lnschema_core.types import VisibilityChoice
 
-from . import Artifact, Run
 from ._parents import view_lineage
 from ._record import init_self_from_db, update_attributes
 from ._utils import attach_func_to_class_method
@@ -35,6 +35,7 @@
 from .core._mapped_collection import MappedCollection
 from .core._settings import settings
 from .core.versioning import process_revises
+from .models import Artifact, Run
 
 if TYPE_CHECKING:
     from collections.abc import Iterable

lamindb/_feature.py

@@ -6,11 +6,11 @@
 import pandas as pd
 from lamin_utils import logger
 from lamindb_setup.core._docs import doc_args
-from lnschema_core.models import Artifact, Feature, Record
-from lnschema_core.types import FeatureDtype
 from pandas.api.types import CategoricalDtype, is_string_dtype
 
+from lamindb.base.types import FeatureDtype
 from lamindb.core.exceptions import ValidationError
+from lamindb.models import Artifact, Feature, Record
 
 from ._query_set import RecordList
 from ._utils import attach_func_to_class_method
@@ -20,9 +20,10 @@
 if TYPE_CHECKING:
     from collections.abc import Iterable
 
-    from lnschema_core.types import FieldAttr
     from pandas.core.dtypes.base import ExtensionDtype
 
+    from lamindb.base.types import FieldAttr
+
 
 FEATURE_DTYPES = set(get_args(FeatureDtype))
 

lamindb/_feature_set.py

@@ -7,8 +7,10 @@
 from lamin_utils import logger
 from lamindb_setup.core._docs import doc_args
 from lamindb_setup.core.hashing import hash_set
-from lnschema_core import Feature, FeatureSet, Record, ids
-from lnschema_core.types import FieldAttr, ListLike
+
+from lamindb.base import ids
+from lamindb.base.types import FieldAttr, ListLike
+from lamindb.models import Feature, FeatureSet, Record
 
 from ._feature import convert_pandas_dtype_to_lamin_dtype
 from ._record import init_self_from_db

lamindb/_finish.py

@@ -13,7 +13,7 @@
 if TYPE_CHECKING:
     from pathlib import Path
 
-    from lnschema_core import Run, Transform
+    from lamindb import Run, Transform
 
     from ._query_set import QuerySet
 
@@ -129,12 +129,11 @@ def save_context_core(
     ignore_non_consecutive: bool | None = None,
     from_cli: bool = False,
 ) -> str | None:
-    from lnschema_core.models import (
+    import lamindb as ln
+    from lamindb.models import (
         format_field_value,  # needs to come after lamindb was imported because of CLI use
     )
 
-    import lamindb as ln
-
     from .core._context import context, is_run_from_ipython
 
     ln.settings.verbosity = "success"

lamindb/_from_values.py

@@ -5,16 +5,16 @@
 import pandas as pd
 from django.core.exceptions import FieldDoesNotExist
 from lamin_utils import colors, logger
-from lnschema_core.models import Record
 
 from lamindb._query_set import RecordList
+from lamindb.models import Record
 
 from .core._settings import settings
 
 if TYPE_CHECKING:
     from collections.abc import Iterable
 
-    from lnschema_core.types import ListLike, StrField
+    from lamindb.base.types import ListLike, StrField
 
 
 # The base function for `from_values`

lamindb/_is_versioned.py

@@ -3,7 +3,8 @@
 import lamindb_setup as ln_setup
 from lamin_utils import logger
 from lamindb_setup.core.upath import UPath
-from lnschema_core.models import IsVersioned
+
+from lamindb.models import IsVersioned
 
 from ._utils import attach_func_to_class_method
 from .core.versioning import create_uid, get_new_path_from_uid

lamindb/_parents.py

@@ -5,15 +5,22 @@
 
 import lamindb_setup as ln_setup
 from lamin_utils import logger
-from lnschema_core import Artifact, Collection, Record, Run, Transform
-from lnschema_core.models import HasParents, format_field_value
+
+from lamindb.models import (
+    Artifact,
+    Collection,
+    HasParents,
+    Record,
+    Run,
+    Transform,
+    format_field_value,
+)
 
 from ._record import get_name_field
 from ._utils import attach_func_to_class_method
 
 if TYPE_CHECKING:
-    from lnschema_core.types import StrField
-
+    from lamindb.base.types import StrField
     from lamindb.core import QuerySet
 
 LAMIN_GREEN_LIGHTER = "#10b981"

lamindb/_query_manager.py

@@ -5,13 +5,14 @@
 from django.db import models
 from lamin_utils import logger
 from lamindb_setup.core._docs import doc_args
-from lnschema_core.models import Record
+
+from lamindb.models import Record
 
 from .core._feature_manager import get_feature_set_by_slot_
 from .core._settings import settings
 
 if TYPE_CHECKING:
-    from lnschema_core.types import StrField
+    from lamindb.base.types import StrField
 
 
 class QueryManager(models.Manager):

lamindb/_query_set.py

@@ -12,7 +12,8 @@
 from django.db.models.fields.related import ForeignObjectRel
 from lamin_utils import logger
 from lamindb_setup.core._docs import doc_args
-from lnschema_core.models import (
+
+from lamindb.models import (
     Artifact,
     CanCurate,
     Collection,
@@ -32,7 +33,7 @@
 if TYPE_CHECKING:
     from collections.abc import Iterable
 
-    from lnschema_core.types import ListLike, StrField
+    from lamindb.base.types import ListLike, StrField
 
 
 class MultipleResultsFound(Exception):