feat(BA-680): Impl noop storage backend

changes/3629.feature.md

@@ -0,0 +1 @@
+Implement noop storage backend

src/ai/backend/common/defs.py

@@ -44,3 +44,6 @@
 
 DEFAULT_VFOLDER_PERMISSION_MODE: Final[int] = 0o755
 VFOLDER_GROUP_PERMISSION_MODE: Final[int] = 0o775
+
+NOOP_STORAGE_VOLUME_NAME: Final[str] = "noop"
+NOOP_STORAGE_BACKEND_TYPE: Final[str] = "noop"

src/ai/backend/storage/config.py

@@ -15,6 +15,7 @@
     override_with_env,
     read_from_file,
 )
+from ai.backend.common.defs import NOOP_STORAGE_BACKEND_TYPE, NOOP_STORAGE_VOLUME_NAME
 from ai.backend.common.etcd import AsyncEtcd, ConfigScopes
 from ai.backend.logging.config import logging_config_iv
 
@@ -139,6 +140,14 @@ def load_local_config(config_path: Path | None, debug: bool = False) -> dict[str
     if debug:
         override_key(raw_cfg, ("debug", "enabled"), True)
 
+    if NOOP_STORAGE_VOLUME_NAME in raw_cfg["volume"]:
+        raise ValueError(f"Volume name {NOOP_STORAGE_VOLUME_NAME} is not allowed")
+
+    raw_cfg["volume"][NOOP_STORAGE_VOLUME_NAME] = {
+        "path": ".",
+        "backend": NOOP_STORAGE_BACKEND_TYPE,
+    }
+
     try:
         local_config = check(raw_cfg, local_config_iv)
         local_config["_src"] = cfg_src_path

src/ai/backend/storage/context.py

@@ -40,6 +40,7 @@
 from .volumes.dellemc import DellEMCOneFSVolume
 from .volumes.gpfs import GPFSVolume
 from .volumes.netapp import NetAppVolume
+from .volumes.noop import NoopVolume
 from .volumes.purestorage import FlashBladeVolume
 from .volumes.vast import VASTVolume
 from .volumes.vfs import BaseVolume
@@ -65,6 +66,7 @@
     CephFSVolume.name: CephFSVolume,
     VASTVolume.name: VASTVolume,
     EXAScalerFSVolume.name: EXAScalerFSVolume,
+    NoopVolume.name: NoopVolume,
 }
 
 

src/ai/backend/storage/volumes/noop/BUILD

@@ -0,0 +1 @@
+python_sources()

src/ai/backend/storage/volumes/noop/__init__.py

@@ -0,0 +1,251 @@
+from collections.abc import Sequence
+from pathlib import Path, PurePosixPath
+from typing import Any, AsyncIterator, Optional
+
+from ai.backend.common.defs import DEFAULT_VFOLDER_PERMISSION_MODE, NOOP_STORAGE_BACKEND_TYPE
+from ai.backend.common.types import BinarySize, HardwareMetadata, QuotaScopeID
+
+from ...types import (
+    CapacityUsage,
+    DirEntry,
+    FSPerfMetric,
+    QuotaConfig,
+    QuotaUsage,
+    TreeUsage,
+    VFolderID,
+)
+from ..abc import AbstractFSOpModel, AbstractQuotaModel, AbstractVolume
+
+
+class NoopQuotaModel(AbstractQuotaModel):
+    def __init__(self) -> None:
+        return
+
+    def mangle_qspath(self, ref: VFolderID | QuotaScopeID | str | None) -> Path:
+        return Path()
+
+    async def create_quota_scope(
+        self,
+        quota_scope_id: QuotaScopeID,
+        options: Optional[QuotaConfig] = None,
+        extra_args: Optional[dict[str, Any]] = None,
+    ) -> None:
+        raise NotImplementedError
+
+    async def describe_quota_scope(
+        self,
+        quota_scope_id: QuotaScopeID,
+    ) -> Optional[QuotaUsage]:
+        raise NotImplementedError
+
+    async def update_quota_scope(
+        self,
+        quota_scope_id: QuotaScopeID,
+        config: QuotaConfig,
+    ) -> None:
+        raise NotImplementedError
+
+    async def unset_quota(
+        self,
+        quota_scope_id: QuotaScopeID,
+    ) -> None:
+        raise NotImplementedError
+
+    async def delete_quota_scope(
+        self,
+        quota_scope_id: QuotaScopeID,
+    ) -> None:
+        raise NotImplementedError
+
+
+class NoopFSOpModel(AbstractFSOpModel):
+    def __init__(self) -> None:
+        return
+
+    async def copy_tree(
+        self,
+        src_path: Path,
+        dst_path: Path,
+    ) -> None:
+        raise NotImplementedError
+
+    async def move_tree(
+        self,
+        src_path: Path,
+        dst_path: Path,
+    ) -> None:
+        raise NotImplementedError
+
+    async def delete_tree(
+        self,
+        path: Path,
+    ) -> None:
+        raise NotImplementedError
+
+    def scan_tree(
+        self,
+        path: Path,
+        *,
+        recursive: bool = True,
+    ) -> AsyncIterator[DirEntry]:
+        raise NotImplementedError
+
+    async def scan_tree_usage(
+        self,
+        path: Path,
+    ) -> TreeUsage:
+        raise NotImplementedError
+
+    async def scan_tree_size(
+        self,
+        path: Path,
+    ) -> BinarySize:
+        raise NotImplementedError
+
+
+class NoopVolume(AbstractVolume):
+    name = NOOP_STORAGE_BACKEND_TYPE
+
+    async def create_quota_model(self) -> AbstractQuotaModel:
+        return NoopQuotaModel()
+
+    async def create_fsop_model(self) -> AbstractFSOpModel:
+        return NoopFSOpModel()
+
+    # ------ volume operations -------
+
+    async def get_capabilities(self) -> frozenset[str]:
+        return frozenset()
+
+    async def get_hwinfo(self) -> HardwareMetadata:
+        return {
+            "status": "healthy",
+            "status_info": None,
+            "metadata": {},
+        }
+
+    async def create_vfolder(
+        self,
+        vfid: VFolderID,
+        exist_ok: bool = False,
+        mode: int = DEFAULT_VFOLDER_PERMISSION_MODE,
+    ) -> None:
+        return None
+
+    async def delete_vfolder(self, vfid: VFolderID) -> None:
+        return None
+
+    async def clone_vfolder(
+        self,
+        src_vfid: VFolderID,
+        dst_vfid: VFolderID,
+    ) -> None:
+        return None
+
+    async def get_vfolder_mount(self, vfid: VFolderID, subpath: str) -> Path:
+        return Path()
+
+    async def put_metadata(self, vfid: VFolderID, payload: bytes) -> None:
+        raise NotImplementedError
+
+    async def get_metadata(self, vfid: VFolderID) -> bytes:
+        raise NotImplementedError
+
+    async def get_performance_metric(self) -> FSPerfMetric:
+        raise NotImplementedError
+
+    async def get_fs_usage(self) -> CapacityUsage:
+        return CapacityUsage(0, 0)
+
+    async def get_usage(
+        self,
+        vfid: VFolderID,
+        relpath: PurePosixPath = PurePosixPath("."),
+    ) -> TreeUsage:
+        return TreeUsage(0, 0)
+
+    async def get_used_bytes(self, vfid: VFolderID) -> BinarySize:
+        return BinarySize(0)
+
+    # ------ vfolder operations -------
+
+    def scandir(
+        self,
+        vfid: VFolderID,
+        relpath: PurePosixPath,
+        *,
+        recursive: bool = True,
+    ) -> AsyncIterator[DirEntry]:
+        raise NotImplementedError
+
+    async def mkdir(
+        self,
+        vfid: VFolderID,
+        relpath: PurePosixPath,
+        *,
+        parents: bool = False,
+        exist_ok: bool = False,
+    ) -> None:
+        raise NotImplementedError
+
+    async def rmdir(
+        self,
+        vfid: VFolderID,
+        relpath: PurePosixPath,
+        *,
+        recursive: bool = False,
+    ) -> None:
+        raise NotImplementedError
+
+    async def move_file(
+        self,
+        vfid: VFolderID,
+        src: PurePosixPath,
+        dst: PurePosixPath,
+    ) -> None:
+        raise NotImplementedError
+
+    async def move_tree(
+        self,
+        vfid: VFolderID,
+        src: PurePosixPath,
+        dst: PurePosixPath,
+    ) -> None:
+        raise NotImplementedError
+
+    async def copy_file(
+        self,
+        vfid: VFolderID,
+        src: PurePosixPath,
+        dst: PurePosixPath,
+    ) -> None:
+        raise NotImplementedError
+
+    async def prepare_upload(self, vfid: VFolderID) -> str:
+        raise NotImplementedError
+
+    async def add_file(
+        self,
+        vfid: VFolderID,
+        relpath: PurePosixPath,
+        payload: AsyncIterator[bytes],
+    ) -> None:
+        raise NotImplementedError
+
+    def read_file(
+        self,
+        vfid: VFolderID,
+        relpath: PurePosixPath,
+        *,
+        chunk_size: int = 0,
+    ) -> AsyncIterator[bytes]:
+        raise NotImplementedError
+
+    async def delete_files(
+        self,
+        vfid: VFolderID,
+        relpaths: Sequence[PurePosixPath],
+        *,
+        recursive: bool = False,
+    ) -> None:
+        raise NotImplementedError