Skip to content

Commit

Permalink
Change butler.collections to be a ButlerCollections sequence
Browse files Browse the repository at this point in the history
  • Loading branch information
timj committed Aug 14, 2024
1 parent 6ba7992 commit 276c136
Show file tree
Hide file tree
Showing 7 changed files with 64 additions and 10 deletions.
12 changes: 11 additions & 1 deletion python/lsst/daf/butler/_butler.py
Original file line number Diff line number Diff line change
Expand Up @@ -1428,7 +1428,17 @@ def collection_chains(self) -> ButlerCollections:

@property
@abstractmethod
def collections(self) -> Sequence[str]:
def collections(self) -> ButlerCollections:
"""Object with methods for modifying and querying collections
(`~lsst.daf.butler.ButlerCollections`).
Use of this object is preferred over `registry` wherever possible.
"""
raise NotImplementedError()

@property
@abstractmethod
def _collections(self) -> Sequence[str]:
"""The collections to search by default, in order
(`~collections.abc.Sequence` [ `str` ]).
"""
Expand Down
29 changes: 27 additions & 2 deletions python/lsst/daf/butler/_butler_collections.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,37 @@
__all__ = ("ButlerCollections",)

from abc import ABC, abstractmethod
from collections.abc import Iterable
from collections.abc import Iterable, Sequence
from typing import Any, overload


class ButlerCollections(ABC):
class ButlerCollections(ABC, Sequence):
"""Methods for working with collections stored in the Butler."""

@overload
def __getitem__(self, index: int) -> str: ...

@overload
def __getitem__(self, index: slice) -> Sequence[str]: ...

def __getitem__(self, index: int | slice) -> str | Sequence[str]:
return self.defaults[index]

def __len__(self) -> int:
return len(self.defaults)

def __eq__(self, other: Any) -> bool:
# Do not try to compare registry instances.
if not isinstance(other, type(self)):
return False
return self.defaults == other.defaults

@property
@abstractmethod
def defaults(self) -> Sequence[str]:
"""Collection defaults associated with this butler."""
raise NotImplementedError("Defaults must be implemented by a subclass")

@abstractmethod
def extend_chain(self, parent_collection_name: str, child_collection_names: str | Iterable[str]) -> None:
"""Add children to the end of a CHAINED collection.
Expand Down
9 changes: 7 additions & 2 deletions python/lsst/daf/butler/direct_butler/_direct_butler.py
Original file line number Diff line number Diff line change
Expand Up @@ -296,7 +296,7 @@ def __reduce__(self) -> tuple:
DirectButler._unpickle,
(
self._config,
self.collections,
self.collections.defaults,
self.run,
dict(self._registry.defaults.dataId.required),
self._registry.isWriteable(),
Expand Down Expand Up @@ -2154,7 +2154,12 @@ def collection_chains(self) -> DirectButlerCollections:
return DirectButlerCollections(self._registry)

@property
def collections(self) -> Sequence[str]:
def collections(self) -> DirectButlerCollections:
"""Object with methods for modifying collection chains."""
return DirectButlerCollections(self._registry)

@property
def _collections(self) -> Sequence[str]:
"""The collections to search by default, in order
(`~collections.abc.Sequence` [ `str` ]).
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@

__all__ = ("DirectButlerCollections",)

from collections.abc import Iterable
from collections.abc import Iterable, Sequence

from lsst.utils.iteration import ensure_iterable

Expand All @@ -49,6 +49,10 @@ class DirectButlerCollections(ButlerCollections):
def __init__(self, registry: SqlRegistry):
self._registry = registry

@property
def defaults(self) -> Sequence[str]:
return self._registry.defaults.collections

def extend_chain(self, parent_collection_name: str, child_collection_names: str | Iterable[str]) -> None:
return self._registry._managers.collections.extend_collection_chain(
parent_collection_name, list(ensure_iterable(child_collection_names))
Expand Down
10 changes: 8 additions & 2 deletions python/lsst/daf/butler/remote_butler/_remote_butler.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@
from ._collection_args import convert_collection_arg_to_glob_string_list
from ._query_driver import RemoteQueryDriver
from ._ref_utils import apply_storage_class_override, normalize_dataset_type_name, simplify_dataId
from ._remote_butler_collections import RemoteButlerCollections
from .server_models import (
CollectionList,
FindDatasetRequestModel,
Expand Down Expand Up @@ -145,7 +146,12 @@ def isWriteable(self) -> bool:
@property
def collection_chains(self) -> ButlerCollections:
"""Object with methods for modifying collection chains."""
raise NotImplementedError()
return RemoteButlerCollections(self._registry)

@property
def collections(self) -> ButlerCollections:
"""Object with methods for modifying collection chains."""
return RemoteButlerCollections(self._registry)

@property
def dimensions(self) -> DimensionUniverse:
Expand Down Expand Up @@ -511,7 +517,7 @@ def validateConfiguration(
raise NotImplementedError()

@property
def collections(self) -> Sequence[str]:
def _collections(self) -> Sequence[str]:
# Docstring inherited.
return self._registry_defaults.collections

Expand Down
6 changes: 5 additions & 1 deletion python/lsst/daf/butler/tests/hybrid_butler.py
Original file line number Diff line number Diff line change
Expand Up @@ -322,7 +322,7 @@ def validateConfiguration(
return self._direct_butler.validateConfiguration(logFailures, datasetTypeNames, ignore)

@property
def collections(self) -> Sequence[str]:
def _collections(self) -> Sequence[str]:
return self._remote_butler.collections

@property
Expand Down Expand Up @@ -459,3 +459,7 @@ def _extract_all_dimension_records_from_data_ids(
@property
def collection_chains(self) -> ButlerCollections:
return self._direct_butler.collection_chains

@property
def collections(self) -> ButlerCollections:
return self._remote_butler.collection_chains
2 changes: 1 addition & 1 deletion tests/test_butler.py
Original file line number Diff line number Diff line change
Expand Up @@ -652,7 +652,7 @@ def testConstructor(self) -> None:
self.assertEqual(collections, {special_run})

butler2 = Butler.from_config(butler=butler, collections=["other"])
self.assertEqual(butler2.collections, ("other",))
self.assertEqual(butler2.collections.defaults, ("other",))
self.assertIsNone(butler2.run)
self.assertEqual(type(butler._datastore), type(butler2._datastore))
self.assertEqual(butler._datastore.config, butler2._datastore.config)
Expand Down

0 comments on commit 276c136

Please sign in to comment.