diff --git a/pyproject.toml b/pyproject.toml index 6f5f7a73..e7152f5e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,15 +43,14 @@ dependencies = [ "requests-toolbelt==0.10.1", "python-dateutil>=2.8", "GitPython==3.1.40", - # for old latch develop, to be removed "aioconsole==0.6.1", "asyncssh==2.13.2", "websockets==11.0.3", "watchfiles==0.19.0", - # marshmallow_jsonschema depends on setuptools but doesn't specify it so we have to do it for them yay :D "setuptools>=75.3.0", + "pyxattr>=0.8.1", ] classifiers = [ "Development Status :: 4 - Beta", @@ -85,7 +84,6 @@ Repository = "https://github.com/latchbio/latch" Issues = "https://github.com/latchbio/latch/issues" Changelog = "https://github.com/latchbio/latch/blob/main/CHANGELOG.md" - [dependency-groups] dev = ["ruff>=0.7.0", "pytest>=8.3.3"] docs = [ diff --git a/src/latch/ldata/path.py b/src/latch/ldata/path.py index 3615342a..24cdc1da 100644 --- a/src/latch/ldata/path.py +++ b/src/latch/ldata/path.py @@ -4,6 +4,7 @@ import shutil from dataclasses import dataclass, field from pathlib import Path +import sys from typing import Iterator, Optional, Type import gql @@ -18,6 +19,7 @@ ) from flytekit.extend import TypeEngine, TypeTransformer from typing_extensions import Self +import xattr from latch.ldata.type import LatchPathError, LDataNodeType from latch_cli.utils import urljoins @@ -52,6 +54,7 @@ class _Cache: size: Optional[int] = None dir_size: Optional[int] = None content_type: Optional[str] = None + version_id: Optional[str] = None @dataclass(frozen=True) @@ -101,6 +104,7 @@ def fetch_metadata(self) -> None: ldataObjectMeta { contentSize contentType + versionId } } } @@ -131,6 +135,8 @@ def fetch_metadata(self) -> None: None if meta["contentSize"] is None else int(meta["contentSize"]) ) self._cache.content_type = meta["contentType"] + self._cache.version_id = meta["versionId"] + def _clear_cache(self): self._cache.path = None @@ -140,6 +146,7 @@ def _clear_cache(self): self._cache.size = None self._cache.dir_size = None self._cache.content_type = None + self._cache.version_id = None def node_id(self, *, load_if_missing: bool = True) -> Optional[str]: match = node_id_regex.match(self.path) @@ -188,6 +195,11 @@ def content_type(self, *, load_if_missing: bool = True) -> Optional[str]: self.fetch_metadata() return self._cache.content_type + def version_id(self, *, load_if_missing: bool = True) -> Optional[str]: + if self._cache.version_id is None and load_if_missing: + self.fetch_metadata() + return self._cache.version_id + def is_dir(self, *, load_if_missing: bool = True) -> bool: return self.type(load_if_missing=load_if_missing) in _dir_types @@ -291,7 +303,7 @@ def upload_from(self, src: Path, *, show_progress_bar: bool = False) -> None: self._clear_cache() def download( - self, dst: Optional[Path] = None, *, show_progress_bar: bool = False + self, dst: Optional[Path] = None, *, show_progress_bar: bool = False, cache: bool = False ) -> Path: """Download the file at this instance's path to the given destination. @@ -306,7 +318,18 @@ def download( _download_idx += 1 tmp_dir.mkdir(parents=True, exist_ok=True) atexit.register(lambda p: shutil.rmtree(p), tmp_dir) - dst = tmp_dir / self.name() + name = self.name() + if name is None: + raise Exception("unable get name of ldata node") + dst = tmp_dir / name + + not_windows = sys.platform != "win32" + dst_str = str(dst) + + self._clear_cache() + version_id = self.version_id() + if not_windows and cache and dst.exists() and version_id == xattr.getxattr(dst_str, 'user.version_id').decode(): + return dst _download( self.path, @@ -315,6 +338,10 @@ def download( verbose=False, confirm_overwrite=False, ) + + if not_windows: + xattr.setxattr(dst_str, 'user.version_id', version_id) + return dst def __truediv__(self, other: object) -> "LPath": diff --git a/uv.lock b/uv.lock index 62525821..5d5f8a01 100644 --- a/uv.lock +++ b/uv.lock @@ -1023,6 +1023,7 @@ dependencies = [ { name = "paramiko" }, { name = "pyjwt" }, { name = "python-dateutil" }, + { name = "pyxattr" }, { name = "requests" }, { name = "requests-toolbelt" }, { name = "scp" }, @@ -1075,6 +1076,7 @@ requires-dist = [ { name = "pulp", marker = "extra == 'snakemake'", specifier = ">=2.0,<2.8" }, { name = "pyjwt", specifier = ">=0.2.0" }, { name = "python-dateutil", specifier = ">=2.8" }, + { name = "pyxattr", specifier = ">=0.8.1" }, { name = "requests", specifier = ">=2.28.1" }, { name = "requests-toolbelt", specifier = "==0.10.1" }, { name = "scp", specifier = ">=0.14.0" }, @@ -1963,6 +1965,12 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/de/3d/8161f7711c017e01ac9f008dfddd9410dff3674334c233bde66e7ba65bbf/pywin32_ctypes-0.2.3-py3-none-any.whl", hash = "sha256:8a1513379d709975552d202d942d9837758905c8d01eb82b8bcc30918929e7b8", size = 30756 }, ] +[[package]] +name = "pyxattr" +version = "0.8.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/97/d1/7b85f2712168dfa26df6471082403013f3f815f3239aee3def17b6fd69ee/pyxattr-0.8.1.tar.gz", hash = "sha256:48c578ecf8ea0bd4351b1752470e301a90a3761c7c21f00f953dcf6d6fa6ee5a", size = 38443 } + [[package]] name = "pyyaml" version = "6.0.2"