cache ldata download

pyproject.toml

@@ -43,15 +43,14 @@ dependencies = [
   "requests-toolbelt==0.10.1",
   "python-dateutil>=2.8",
   "GitPython==3.1.40",
-
   # for old latch develop, to be removed
   "aioconsole==0.6.1",
   "asyncssh==2.13.2",
   "websockets==11.0.3",
   "watchfiles==0.19.0",
-
   # marshmallow_jsonschema depends on setuptools but doesn't specify it so we have to do it for them yay :D
   "setuptools>=75.3.0",
+  "pyxattr>=0.8.1",
 ]
 classifiers = [
   "Development Status :: 4 - Beta",
@@ -87,7 +86,10 @@ Changelog = "https://github.com/latchbio/latch/blob/main/CHANGELOG.md"
 
 
 [dependency-groups]
-dev = ["ruff>=0.7.0", "pytest>=8.3.3"]
+dev = [
+    "ruff>=0.7.0",
+    "pytest>=8.3.3",
+]
 docs = [
   "sphinx",
   "sphinx-book-theme",

src/latch/ldata/path.py

@@ -18,6 +18,7 @@
 )
 from flytekit.extend import TypeEngine, TypeTransformer
 from typing_extensions import Self
+import xattr
 
 from latch.ldata.type import LatchPathError, LDataNodeType
 from latch_cli.utils import urljoins
@@ -52,6 +53,7 @@ class _Cache:
     size: Optional[int] = None
     dir_size: Optional[int] = None
     content_type: Optional[str] = None
+    version_id: Optional[str] = None
 
 
 @dataclass(frozen=True)
@@ -101,6 +103,7 @@ def fetch_metadata(self) -> None:
                             ldataObjectMeta {
                                 contentSize
                                 contentType
+                                versionId
                             }
                         }
                     }
@@ -132,6 +135,7 @@ def fetch_metadata(self) -> None:
             )
             self._cache.content_type = meta["contentType"]
 
+
     def _clear_cache(self):
         self._cache.path = None
         self._cache.node_id = None
@@ -188,6 +192,11 @@ def content_type(self, *, load_if_missing: bool = True) -> Optional[str]:
             self.fetch_metadata()
         return self._cache.content_type
 
+    def version_id(self) -> Optional[str]:
+        # note: must always be fetched to keep in sync with db
+        self.fetch_metadata()
+        return self._cache.version_id
+
     def is_dir(self, *, load_if_missing: bool = True) -> bool:
         return self.type(load_if_missing=load_if_missing) in _dir_types
 
@@ -291,7 +300,7 @@ def upload_from(self, src: Path, *, show_progress_bar: bool = False) -> None:
         self._clear_cache()
 
     def download(
-        self, dst: Optional[Path] = None, *, show_progress_bar: bool = False
+        self, dst: Optional[Path] = None, *, show_progress_bar: bool = False, cache: bool = False
     ) -> Path:
         """Download the file at this instance's path to the given destination.
 
@@ -306,7 +315,15 @@ def download(
             _download_idx += 1
             tmp_dir.mkdir(parents=True, exist_ok=True)
             atexit.register(lambda p: shutil.rmtree(p), tmp_dir)
-            dst = tmp_dir / self.name()
+            name = self.name()
+            if name is None:
+                raise Exception("unable get name of ldata node")
+            dst = tmp_dir / name
+
+        dst_str = str(dst)
+        version_id = self.version_id
+        if cache and dst.exists() and version_id == xattr.getxattr(dst_str, 'user.version_id'):
+            return dst
 
         _download(
             self.path,
@@ -315,6 +332,8 @@ def download(
             verbose=False,
             confirm_overwrite=False,
         )
+        xattr.setxattr(dst_str, 'user.version_id', version_id)
+
         return dst
 
     def __truediv__(self, other: object) -> "LPath":