Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix garbage collection #1119

Merged
merged 40 commits into from
Jul 22, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
40 commits
Select commit Hold shift + click to select a range
a9a10af
Make creation of aligned mapping lazy
ivirshup Sep 4, 2023
ca0759a
Use weakref for filemanager
ivirshup Sep 4, 2023
92615d8
Merge branch 'main' into remove-backrefs
ilan-gold Jun 3, 2024
cb53b77
(chore): add benchmark
ilan-gold Jun 3, 2024
2df03b5
(fix): use right `gen_adata` function
ilan-gold Jun 3, 2024
38d218f
(fix): change name
ilan-gold Jun 3, 2024
a9eba7a
(chore): fewer runs
ilan-gold Jun 3, 2024
be8ae55
(fix): benchmark test name
ilan-gold Jun 4, 2024
82fc74c
Merge branch 'main' into remove-backrefs
ilan-gold Jun 4, 2024
e0bff0e
(fix): return `cls` for `None` obj
ilan-gold Jun 4, 2024
ccbdaf3
Merge branch 'remove-backrefs' of https://github.com/ivirshup/anndata…
ilan-gold Jun 4, 2024
6ed9963
(fix): try `track_peakmem`
ilan-gold Jun 4, 2024
ce4e2d8
(fix): remove `track_` name
ilan-gold Jun 5, 2024
0c9e563
(fix): docs
ilan-gold Jun 5, 2024
c971dfb
(fix): do custom peakmem track
ilan-gold Jun 5, 2024
9e08151
(fix): `n` -> `runs`
ilan-gold Jun 5, 2024
c771079
(fix): comment
ilan-gold Jun 5, 2024
9d3caad
(fix): `vals` typing for aligned mappings
ilan-gold Jun 5, 2024
eeb8865
(fix): don't use mutable argument
ilan-gold Jun 5, 2024
5267c2d
(fix): change profiler name to `track_peakmem_garbage_collection`
ilan-gold Jun 5, 2024
13d134a
(chore): delete rogue comment
ilan-gold Jun 5, 2024
4dec616
clarify reference use
flying-sheep Jun 7, 2024
5361188
consistency
flying-sheep Jun 7, 2024
b61549d
remove boilerplate
flying-sheep Jun 7, 2024
81d5933
Merge branch 'main' into remove-backrefs
ilan-gold Jun 7, 2024
1a85497
add types, undo moves
flying-sheep Jun 7, 2024
f750954
Add type hints for properties
flying-sheep Jun 7, 2024
ab4cd79
Merge branch 'main' into remove-backrefs
ilan-gold Jun 18, 2024
cf4377c
Merge branch 'main' into pr/ivirshup/1119
flying-sheep Jun 21, 2024
48ae69d
fmt
flying-sheep Jul 2, 2024
480b0a4
cleanup
flying-sheep Jul 2, 2024
24b89a0
more fmt
flying-sheep Jul 4, 2024
05ebee9
dedupe and test
flying-sheep Jul 4, 2024
866abbd
Simplify copy
flying-sheep Jul 4, 2024
3595197
Merge branch 'main' into pr/ivirshup/1119
flying-sheep Jul 8, 2024
015d6ad
docs and typing
flying-sheep Jul 22, 2024
3a1a007
fix parent_mapping type
flying-sheep Jul 22, 2024
73fc94c
Fix I
flying-sheep Jul 22, 2024
ba61e2b
fix docs
flying-sheep Jul 22, 2024
e138fc9
fix 3.9
flying-sheep Jul 22, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 24 additions & 15 deletions src/anndata/_core/aligned_mapping.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,19 +3,17 @@
import warnings
from abc import ABC, abstractmethod
from collections import abc as cabc
from collections.abc import Iterator, Mapping, MutableMapping, Sequence
from copy import copy
from typing import (
TYPE_CHECKING,
ClassVar,
Generic,
Literal,
TypeVar,
Union,
)

import numpy as np
import pandas as pd
from scipy.sparse import spmatrix

from anndata._warnings import ExperimentalFeatureWarning, ImplicitModificationWarning
from anndata.compat import AwkArray
Expand All @@ -32,16 +30,23 @@
from .views import as_view, view_update

if TYPE_CHECKING:
from collections.abc import Iterable, Iterator, Mapping, MutableMapping, Sequence

import numpy as np
from scipy.sparse import spmatrix

from .anndata import AnnData
from .raw import Raw

OneDIdx = Union[Sequence[int], Sequence[bool], slice]
TwoDIdx = tuple[OneDIdx, OneDIdx]

OneDIdx = Union[Sequence[int], Sequence[bool], slice]
TwoDIdx = tuple[OneDIdx, OneDIdx]
I = TypeVar("I", OneDIdx, TwoDIdx, covariant=True)
# TODO: pd.DataFrame only allowed in AxisArrays?
V = Union[pd.DataFrame, spmatrix, np.ndarray]

I = TypeVar("I", OneDIdx, TwoDIdx, covariant=True)
# TODO: pd.DataFrame only allowed in AxisArrays?
V = Union[pd.DataFrame, spmatrix, np.ndarray]
# Used in `Generic[T]`
T = TypeVar("T")


class AlignedMapping(cabc.MutableMapping, ABC):
Expand Down Expand Up @@ -335,7 +340,7 @@
self,
parent: AnnData,
*,
axis: tuple[int, int] = (0, 1),
axis: tuple[Literal[0], Literal[1]] = (0, 1),
store: MutableMapping[str, V],
):
assert axis == (0, 1), axis
Expand Down Expand Up @@ -419,15 +424,17 @@
PairwiseArraysBase._actual_class = PairwiseArrays


class AlignedMappingProperty(property):
def __init__(self, name, cls, axis):
class AlignedMappingProperty(property, Generic[T]):
def __init__(
self, name: str, cls: T, axis: Literal[0, 1] | tuple[Literal[0], Literal[1]]
flying-sheep marked this conversation as resolved.
Show resolved Hide resolved
):
self.name = name
self.axis = axis
self.cls = cls

def __get__(self, obj, objtype=None):
def __get__(self, obj: None | AnnData, objtype: type | None = None) -> T:
if obj is None: # needs to return a `property`, e.g. for Sphinx
return self
return self # type: ignore

Check warning on line 437 in src/anndata/_core/aligned_mapping.py

View check run for this annotation

Codecov / codecov/patch

src/anndata/_core/aligned_mapping.py#L437

Added line #L437 was not covered by tests
if obj.is_view:
parent_anndata = obj._adata_ref
idxs = (obj._oidx, obj._vidx)
Expand All @@ -438,12 +445,14 @@
else:
return self.cls(obj, axis=self.axis, store=getattr(obj, "_" + self.name))

def __set__(self, obj, value):
def __set__(
self, obj: AnnData, value: Mapping[str, V] | Iterable[tuple[str, V]]
) -> None:
value = convert_to_dict(value)
_ = self.cls(obj, axis=self.axis, store=value) # Validate
if obj.is_view:
obj._init_as_actual(obj.copy())
setattr(obj, "_" + self.name, value)

def __delete__(self, obj):
def __delete__(self, obj) -> None:
setattr(obj, self.name, dict())
106 changes: 52 additions & 54 deletions src/anndata/_core/anndata.py
Original file line number Diff line number Diff line change
Expand Up @@ -662,26 +662,6 @@ def X(self, value: np.ndarray | sparse.spmatrix | SpArray | None):
def X(self):
self.X = None

obsm = AlignedMappingProperty("obsm", AxisArrays, 0)
"""\
Multi-dimensional annotation of observations
(mutable structured :class:`~numpy.ndarray`).

Stores for each key a two or higher-dimensional :class:`~numpy.ndarray`
of length `n_obs`.
Is sliced with `data` and `obs` but behaves otherwise like a :term:`mapping`.
"""

varm = AlignedMappingProperty("varm", AxisArrays, 1)
"""\
Multi-dimensional annotation of variables/features
(mutable structured :class:`~numpy.ndarray`).

Stores for each key a two or higher-dimensional :class:`~numpy.ndarray`
of length `n_vars`.
Is sliced with `data` and `var` but behaves otherwise like a :term:`mapping`.
"""

layers = AlignedMappingProperty("layers", Layers, (0, 1))
"""\
Dictionary-like object with values of the same dimensions as :attr:`X`.
Expand Down Expand Up @@ -709,26 +689,6 @@ def X(self):
adata.layers.keys()
"""

obsp = AlignedMappingProperty("obsp", PairwiseArrays, 0)
"""\
Pairwise annotation of observations,
a mutable mapping with array-like values.

Stores for each key a two or higher-dimensional :class:`~numpy.ndarray`
whose first two dimensions are of length `n_obs`.
Is sliced with `data` and `obs` but behaves otherwise like a :term:`mapping`.
"""

varp = AlignedMappingProperty("varp", PairwiseArrays, 1)
"""\
Pairwise annotation of variables/features,
a mutable mapping with array-like values.

Stores for each key a two or higher-dimensional :class:`~numpy.ndarray`
whose first two dimensions are of length `n_var`.
Is sliced with `data` and `var` but behaves otherwise like a :term:`mapping`.
"""

@property
def raw(self) -> Raw:
"""\
Expand Down Expand Up @@ -910,6 +870,46 @@ def uns(self, value: MutableMapping):
def uns(self):
self.uns = OrderedDict()

obsm = AlignedMappingProperty("obsm", AxisArrays, 0)
"""\
Multi-dimensional annotation of observations
(mutable structured :class:`~numpy.ndarray`).

Stores for each key a two or higher-dimensional :class:`~numpy.ndarray`
of length `n_obs`.
Is sliced with `data` and `obs` but behaves otherwise like a :term:`mapping`.
"""

varm = AlignedMappingProperty("varm", AxisArrays, 1)
"""\
Multi-dimensional annotation of variables/features
(mutable structured :class:`~numpy.ndarray`).

Stores for each key a two or higher-dimensional :class:`~numpy.ndarray`
of length `n_vars`.
Is sliced with `data` and `var` but behaves otherwise like a :term:`mapping`.
"""

obsp = AlignedMappingProperty("obsp", PairwiseArrays, 0)
"""\
Pairwise annotation of observations,
a mutable mapping with array-like values.

Stores for each key a two or higher-dimensional :class:`~numpy.ndarray`
whose first two dimensions are of length `n_obs`.
Is sliced with `data` and `obs` but behaves otherwise like a :term:`mapping`.
"""

varp = AlignedMappingProperty("varp", PairwiseArrays, 1)
"""\
Pairwise annotation of variables/features,
a mutable mapping with array-like values.

Stores for each key a two or higher-dimensional :class:`~numpy.ndarray`
whose first two dimensions are of length `n_var`.
Is sliced with `data` and `var` but behaves otherwise like a :term:`mapping`.
"""

def obs_keys(self) -> list[str]:
"""List keys of observation annotation :attr:`obs`."""
return self._obs.keys().tolist()
Expand All @@ -920,11 +920,11 @@ def var_keys(self) -> list[str]:

def obsm_keys(self) -> list[str]:
"""List keys of observation annotation :attr:`obsm`."""
return list(self._obsm.keys())
return list(self.obsm.keys())

def varm_keys(self) -> list[str]:
"""List keys of variable annotation :attr:`varm`."""
return list(self._varm.keys())
return list(self.varm.keys())

def uns_keys(self) -> list[str]:
"""List keys of unstructured annotation."""
Expand Down Expand Up @@ -1203,10 +1203,10 @@ def transpose(self) -> AnnData:
obs=self.var,
var=self.obs,
uns=self._uns,
obsm=self._varm,
varm=self._obsm,
obsp=self._varp,
varp=self._obsp,
obsm=self.varm,
varm=self.obsm,
obsp=self.varp,
varp=self.obsp,
filename=self.filename,
)

Expand Down Expand Up @@ -1763,24 +1763,22 @@ def _check_dimensions(self, key=None):
else:
key = {key}
if "obsm" in key:
obsm = self._obsm
if (
not all([axis_len(o, 0) == self.n_obs for o in obsm.values()])
and len(obsm.dim_names) != self.n_obs
not all([axis_len(o, 0) == self.n_obs for o in self.obsm.values()])
and len(self.obsm.dim_names) != self.n_obs
):
raise ValueError(
"Observations annot. `obsm` must have number of rows of `X`"
f" ({self.n_obs}), but has {len(obsm)} rows."
f" ({self.n_obs}), but has {len(self.obsm)} rows."
)
if "varm" in key:
varm = self._varm
if (
not all([axis_len(v, 0) == self.n_vars for v in varm.values()])
and len(varm.dim_names) != self.n_vars
not all([axis_len(v, 0) == self.n_vars for v in self.varm.values()])
and len(self.varm.dim_names) != self.n_vars
):
raise ValueError(
"Variables annot. `varm` must have number of columns of `X`"
f" ({self.n_vars}), but has {len(varm)} rows."
f" ({self.n_vars}), but has {len(self.varm)} rows."
)

def write_h5ad(
Expand Down
Loading